From 91c65c4eedd3bab7adf6f0b9fe0a74b2607ec59d Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 8 Jun 2021 17:15:08 +0300 Subject: [PATCH 1/3] Improvements in the parsing of data from the MessageBroker Handling of edge case where the threshold is determined to be over the maximum value / under the minimum value of an unbounded monitoring attribute Renaming of the configuration file to be used in Docker deployments --- morphemic-slo-severity-calculator/Dockerfile | 2 +- .../AttributeSubscription.java | 14 ++++---- .../utilities/SLOViolationCalculator.java | 2 +- .../PredictedMonitoringAttribute.java | 33 +++++++++++++++++-- ...hemic.sloseverityconfiguration.properties} | 4 +-- .../src/main/resources/input_data.properties | 8 ++--- .../UnboundedMonitoringAttributeTests.java | 12 +++---- 7 files changed, 52 insertions(+), 23 deletions(-) rename morphemic-slo-severity-calculator/src/main/resources/{docker_configuration.properties => eu.morphemic.sloseverityconfiguration.properties} (64%) diff --git a/morphemic-slo-severity-calculator/Dockerfile b/morphemic-slo-severity-calculator/Dockerfile index b7842a17..ee84b2c9 100644 --- a/morphemic-slo-severity-calculator/Dockerfile +++ b/morphemic-slo-severity-calculator/Dockerfile @@ -1,6 +1,6 @@ FROM openjdk:11 RUN mkdir -p /home/src/main/resources/ -COPY src/main/resources/docker_configuration.properties /home/src/main/resources/input_data.properties +COPY src/main/resources/eu.morphemic.sloseverityconfiguration.properties /home/src/main/resources/input_data.properties COPY src/main/resources/amq_messaging_library.properties /home/src/main/resources/amq_messaging_library.properties COPY target/SLOSeverityCalculator-1.0-SNAPSHOT.jar /home/SLOSeverityCalculator-1.0-SNAPSHOT.jar WORKDIR /home diff --git a/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java b/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java index 408d1726..f7fcfca3 100644 --- a/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java +++ b/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java @@ -3,6 +3,7 @@ package metric_retrieval; import eu.melodic.event.brokerclient.BrokerSubscriber; import eu.melodic.event.brokerclient.templates.EventFields; import eu.melodic.event.brokerclient.templates.TopicNames; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -37,7 +38,7 @@ public class AttributeSubscription { BiFunction function = (topic, message) ->{ synchronized (RealtimeMonitoringAttribute.getMonitoring_attributes().get(topic)) { try { - update_monitoring_attribute_value(topic,(Double)((JSONObject)new JSONParser().parse(message)).get("metricValue")); + update_monitoring_attribute_value(topic,((Number)((JSONObject)new JSONParser().parse(message)).get("metricValue")).doubleValue()); Logger.getAnonymousLogger().log(info_logging_level,"RECEIVED message with value for "+topic+" equal to "+(((JSONObject)new JSONParser().parse(message)).get("metricValue"))); } catch (ParseException e) { @@ -75,11 +76,12 @@ public class AttributeSubscription { String predicted_attribute_name = topic.replaceFirst("prediction\\.",EMPTY); HashMap> predicted_attributes = getPredicted_monitoring_attributes(); try { - double forecasted_value = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.metric_value)); - double probability_confidence = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.probability)); - double confidence_interval = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.confidence_interval)); - long timestamp = ((Long)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.timestamp)); - long targeted_prediction_time = ((Long)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.prediction_time)); + double forecasted_value = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.metric_value)).doubleValue(); + double probability_confidence = 100*((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.probability)).doubleValue(); + JSONArray json_array_confidence_interval = ((JSONArray)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.confidence_interval)); + double confidence_interval = ((Number)json_array_confidence_interval.get(1)).doubleValue() - ((Number)json_array_confidence_interval.get(0)).doubleValue(); + long timestamp = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.timestamp)).longValue(); + long targeted_prediction_time = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.prediction_time)).longValue(); Logger.getAnonymousLogger().log(info_logging_level,"RECEIVED message with predicted value for "+predicted_attribute_name+" equal to "+ forecasted_value); synchronized (Main.can_modify_slo_rules) { diff --git a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java index d65900a1..69f0f9ca 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java +++ b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java @@ -46,7 +46,7 @@ public class SLOViolationCalculator { //TODO take into account the confidence interval! public static double get_Severity_prconf_delta_method(PredictedMonitoringAttribute predictionAttribute){ - double severity_sum = (predictionAttribute.getDelta()*predictionAttribute.getProbability_confidence()*(100-predictionAttribute.getConfidence_interval_width()/100))/(100*100*100); //dividing by 10000 to normalize; + double severity_sum = (predictionAttribute.getDelta()*predictionAttribute.getProbability_confidence()*(100-predictionAttribute.getNormalizedConfidenceIntervalWidth()/100))/(100*100*100); //dividing by 10000 to normalize; Logger.getAnonymousLogger().log(info_logging_level,"The prconf-delta attribute severity for " + predictionAttribute.getName() + " based on a (prconf,delta,confidence_interval) triplet of (" + predictionAttribute.getProbability_confidence() + "," + predictionAttribute.getDelta() +","+predictionAttribute.getConfidence_interval_width()+ ") is " + severity_sum); return severity_sum; } diff --git a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java index 5396f136..2d9ed65b 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java +++ b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java @@ -48,11 +48,38 @@ public class PredictedMonitoringAttribute { } this.rate_of_change = getRateOfChange(forecasted_value, current_value,name); if (greater_than_rule) { - this.delta = 100*(forecasted_value - threshold)/(getMonitoring_attributes_statistics().get(name).getUpper_bound()-threshold); - //this.previous_delta = 100*Math.abs(current_value - threshold)/(getMonitoring_attributes_statistics().get(name).getUpper_bound()-threshold); + + if(getMonitoring_attributes_statistics().get(name).getUpper_bound()>threshold){ + this.delta = 100*(forecasted_value - threshold)/(getMonitoring_attributes_statistics().get(name).getUpper_bound()-threshold); + }else /*if (getMonitoring_attributes_statistics().get(name).getUpper_bound()<=threshold)*/{ + if (forecasted_value>threshold){ + this.delta = 100; + }else if (forecasted_value==threshold){ + this.delta = 0; + }else{ + this.delta = -100; + } + } + + //this.previous_delta = 100*Math.abs(current_value - threshold)/(getMonitoring_attributes_statistics().get(name).getUpper_bound()-threshold); }else{ this.rate_of_change = -this.rate_of_change; //inversion necessary, as when a rate of change is positive, it means that the metric is increasing and thus not directed towards the interval in which a less-than rule is fired. - this.delta = 100*(threshold-forecasted_value)/(threshold-getMonitoring_attributes_statistics().get(name).getLower_bound()); + + if(threshold>getMonitoring_attributes_statistics().get(name).getLower_bound()) { + + this.delta = 100 * (threshold - forecasted_value) / (threshold - getMonitoring_attributes_statistics().get(name).getLower_bound()); + + //this.previous_delta = 100*Math.abs(current_value-threshold)/(threshold-getMonitoring_attributes_statistics().get(name).getLower_bound()); + }else{ + if (threshold>forecasted_value){ + this.delta = 100; + }else if (threshold==forecasted_value){ + this.delta = 0; + }else{ + this.delta = -100; + } + } + //this.previous_delta = 100*Math.abs(current_value-threshold)/(threshold-getMonitoring_attributes_statistics().get(name).getLower_bound()); } this.probability_confidence = probability_confidence; diff --git a/morphemic-slo-severity-calculator/src/main/resources/docker_configuration.properties b/morphemic-slo-severity-calculator/src/main/resources/eu.morphemic.sloseverityconfiguration.properties similarity index 64% rename from morphemic-slo-severity-calculator/src/main/resources/docker_configuration.properties rename to morphemic-slo-severity-calculator/src/main/resources/eu.morphemic.sloseverityconfiguration.properties index e15623a9..8edfe4de 100644 --- a/morphemic-slo-severity-calculator/src/main/resources/docker_configuration.properties +++ b/morphemic-slo-severity-calculator/src/main/resources/eu.morphemic.sloseverityconfiguration.properties @@ -1,11 +1,11 @@ self_publish_rule_file = false -metrics_bounds = custom1;unbounded;unbounded,custom2;0;3 +metrics_bounds = AvgResponseTime;unbounded;unbounded,custom2;0;3 roc_calculation_mode = prototype slo_rules_topic = metrics.metric_list single_slo_rule_active = true -broker_ip_url = tcp://192.168.224.1:61616 +broker_ip_url = tcp://192.168.224.1:61616?wireFormat.maxInactivityDuration=0 broker_username = admin broker_password = admin diff --git a/morphemic-slo-severity-calculator/src/main/resources/input_data.properties b/morphemic-slo-severity-calculator/src/main/resources/input_data.properties index 37b4042a..6edb2af1 100644 --- a/morphemic-slo-severity-calculator/src/main/resources/input_data.properties +++ b/morphemic-slo-severity-calculator/src/main/resources/input_data.properties @@ -21,7 +21,7 @@ disk = 5,2,10,20,8,\ 10,50,20,0,50,\ 10,10,20,50,30 -metrics_bounds = custom1;unbounded;unbounded,custom2;0;3 +metrics_bounds = AvgResponseTime;unbounded;unbounded,custom2;0;3 roc_calculation_mode = prototype confidence_interval = 10 @@ -32,9 +32,9 @@ slo_rules_topic = metrics.metric_list single_slo_rule_active = true metrics_list = cpu,ram,disk stored_values_per_metric = 5 -broker_ip_url = tcp://localhost:61616 -broker_username = admin -broker_password = admin +broker_ip_url = tcp://147.102.17.76:61616?wireFormat.maxInactivityDuration=0 +broker_username = aaa +broker_password = 111 slo_violation_determination_method = all-metrics time_horizon_seconds = 10 diff --git a/morphemic-slo-severity-calculator/src/test/java/UnboundedMonitoringAttributeTests.java b/morphemic-slo-severity-calculator/src/test/java/UnboundedMonitoringAttributeTests.java index c1acf19f..d3cd7f98 100644 --- a/morphemic-slo-severity-calculator/src/test/java/UnboundedMonitoringAttributeTests.java +++ b/morphemic-slo-severity-calculator/src/test/java/UnboundedMonitoringAttributeTests.java @@ -82,7 +82,7 @@ public class UnboundedMonitoringAttributeTests { BiFunction function = (topic, message) -> { synchronized (RealtimeMonitoringAttribute.getMonitoring_attributes().get(topic)) { try { - update_monitoring_attribute_value(topic, (Double) ((JSONObject) new JSONParser().parse(message)).get("metricValue")); + update_monitoring_attribute_value(topic, ((Number) ((JSONObject) new JSONParser().parse(message)).get("metricValue")).doubleValue()); Logger.getAnonymousLogger().log(info_logging_level, "RECEIVED message with value for " + topic + " equal to " + (((JSONObject) new JSONParser().parse(message)).get("metricValue"))); } catch (ParseException e) { @@ -105,11 +105,11 @@ public class UnboundedMonitoringAttributeTests { String predicted_attribute_name = topic.replaceFirst("prediction\\.",EMPTY); HashMap> predicted_attributes = getPredicted_monitoring_attributes(); try { - double forecasted_value = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.metric_value)); - double probability_confidence = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.probability)); - double confidence_interval = ((Double)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.confidence_interval)); - long timestamp = ((Long)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.timestamp)); - long targeted_prediction_time = ((Long)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.prediction_time)); + double forecasted_value = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.metric_value)).doubleValue(); + double probability_confidence = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.probability)).doubleValue(); + double confidence_interval = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.confidence_interval)).doubleValue(); + long timestamp = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.timestamp)).longValue(); + long targeted_prediction_time = ((Number)((JSONObject)new JSONParser().parse(message)).get(EventFields.PredictionMetricEventFields.prediction_time)).longValue(); Logger.getAnonymousLogger().log(info_logging_level,"RECEIVED message with predicted value for "+predicted_attribute_name+" equal to "+ forecasted_value); synchronized (Main.can_modify_slo_rules) { -- GitLab From 1bc592d319865052228747e7e7d6a5fa24ae3d09 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 8 Jun 2021 17:24:43 +0300 Subject: [PATCH 2/3] Improvements in the parsing of data by each BrokerSubscriber --- amq-message-java-library/pom.xml | 5 +++++ .../event/brokerclient/BrokerSubscriber.java | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/amq-message-java-library/pom.xml b/amq-message-java-library/pom.xml index e0120e9b..25044787 100644 --- a/amq-message-java-library/pom.xml +++ b/amq-message-java-library/pom.xml @@ -46,6 +46,11 @@ commons-lang3 3.8.1 + + com.googlecode.json-simple + json-simple + 1.1.1 + diff --git a/amq-message-java-library/src/main/java/eu/melodic/event/brokerclient/BrokerSubscriber.java b/amq-message-java-library/src/main/java/eu/melodic/event/brokerclient/BrokerSubscriber.java index f6b41004..e3dbadbd 100644 --- a/amq-message-java-library/src/main/java/eu/melodic/event/brokerclient/BrokerSubscriber.java +++ b/amq-message-java-library/src/main/java/eu/melodic/event/brokerclient/BrokerSubscriber.java @@ -7,6 +7,15 @@ import javax.jms.*; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; +import java.util.logging.Logger; + +import org.apache.activemq.command.ActiveMQBytesMessage; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.apache.activemq.command.ActiveMQObjectMessage; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +import static java.util.logging.Level.INFO; @Slf4j @@ -38,7 +47,18 @@ public class BrokerSubscriber { client.receiveEvents(url, topic, stop_signal,message -> { try { if (message!=null) { - function.apply(topic, ((TextMessage) message).getText()); + if (message instanceof TextMessage){ + function.apply(topic, ((TextMessage) message).getText()); + }else if (message instanceof ActiveMQBytesMessage) { + try { + String json_string = (((ActiveMQBytesMessage) message).readUTF()); + Logger.getAnonymousLogger().log(INFO, json_string); + function.apply(topic, json_string); + } + catch (Exception e){ + e.printStackTrace(); + } + } } } catch (JMSException j) { log.info("Shutting down subscriber..."); -- GitLab From 4b1022d28e750bb890955c18e7d8e54b42cb4622 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Wed, 9 Jun 2021 10:26:12 +0300 Subject: [PATCH 3/3] Change of Docker image used for the deployment of the slo-severity-calculator --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8c14e71..281fbe60 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,12 +134,14 @@ deploy:amq-message-java-library: deploy:slo-severity-calculator: stage: deploy - image: $MAVEN_IMAGE + image: $DOCKER_DIND_IMAGE only: - master - morphemic-rc1.5 dependencies: - build:slo-severity-calculator + services: + - $DOCKER_DIND_SERVICE script: - cd morphemic-slo-severity-calculator - docker build -t slo_severity_calculator -f ./Dockerfile . -- GitLab