From 5e0d648e6ec0fa66848c7485562ebc8e73dcc3cf Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Fri, 11 Feb 2022 17:54:34 +0200 Subject: [PATCH 1/5] Removed extraneous command from python dependencies installation during creation of Docker container --- .../src/prepare_python_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphemic-forecasting-exponentialsmoothing/src/prepare_python_dependencies.sh b/morphemic-forecasting-exponentialsmoothing/src/prepare_python_dependencies.sh index 26c3247d..4d9f294f 100644 --- a/morphemic-forecasting-exponentialsmoothing/src/prepare_python_dependencies.sh +++ b/morphemic-forecasting-exponentialsmoothing/src/prepare_python_dependencies.sh @@ -26,7 +26,7 @@ pip install esm_forecaster-$EXPONENTIAL_SMOOTHING_VERSION.tar.gz tar -xzvf esm_forecaster-$EXPONENTIAL_SMOOTHING_VERSION.tar.gz # Install other dependencies -cd esm_forecaster-$EXPONENTIAL_SMOOTHING_VERSION/src +#cd esm_forecaster-$EXPONENTIAL_SMOOTHING_VERSION/src pip3 install -r requirements.txt cd /home/r_predictions \ No newline at end of file -- GitLab From 6229b5d36e855f69773c5356e169d125a9d70613 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Tue, 15 Mar 2022 17:52:08 +0200 Subject: [PATCH 2/5] Remove unused code and TODO items --- .../src/main/java/runtime/Main.java | 3 +- .../src/main/java/slo_processing/SLORule.java | 1 - .../utilities/SLOViolationCalculator.java | 1 - .../PredictedMonitoringAttribute.java | 33 ------------------- 4 files changed, 1 insertion(+), 37 deletions(-) diff --git a/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java b/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java index a05968c8..1ba92336 100644 --- a/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java +++ b/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java @@ -51,7 +51,7 @@ public class Main { public static void main(String[] args) { - //The input of this program is the type of the SLO violations which are monitored, and the predicted metric values which are evaluated. + //The input of this program is the type of the SLO violations which are monitored, and the predicted metric values which are evaluated. Its output are events which produce an estimate of the probability of an adaptation. //The SLO violations which are monitored need to mention the following data: // - The name of the predicted metrics which are monitored as part of the rule // - The threshold and whether it is a more-than or less-than threshold @@ -77,7 +77,6 @@ public class Main { } prop.load(inputStream); String slo_rules_topic = prop.getProperty("slo_rules_topic"); - //roc_calculation_mode = prop.getProperty("roc_calculation_mode"); kept_values_per_metric = Integer.parseInt(prop.getProperty("stored_values_per_metric","5")); self_publish_rule_file = Boolean.parseBoolean(prop.getProperty("self_publish_rule_file")); single_slo_rule_active = Boolean.parseBoolean(prop.getProperty("single_slo_rule_active")); diff --git a/morphemic-slo-severity-calculator/src/main/java/slo_processing/SLORule.java b/morphemic-slo-severity-calculator/src/main/java/slo_processing/SLORule.java index bb27b06f..728998b2 100644 --- a/morphemic-slo-severity-calculator/src/main/java/slo_processing/SLORule.java +++ b/morphemic-slo-severity-calculator/src/main/java/slo_processing/SLORule.java @@ -18,7 +18,6 @@ import static utility_beans.PredictedMonitoringAttribute.getPredicted_monitoring public class SLORule { private ArrayList slo_subrules = new ArrayList<>(6); - //TODO evaluate if the monitoring attributes need to be associated with the real monitoring attributes private ArrayList monitoring_attributes = new ArrayList<>(); private JSONObject rule_representation; //double SLO_severity; diff --git a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java index 04391830..0b05a205 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java +++ b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java @@ -46,7 +46,6 @@ public class SLOViolationCalculator { } } - //TODO take into account the confidence interval! public static double get_Severity_prconf_delta_method(PredictedMonitoringAttribute predictionAttribute){ double severity_sum = (predictionAttribute.getDelta()*predictionAttribute.getProbability_confidence()*(100-predictionAttribute.getNormalizedConfidenceIntervalWidth()/100))/(100*100*100); //dividing by 10000 to normalize; diff --git a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java index 44d53772..76c5f3bc 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java +++ b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java @@ -12,17 +12,12 @@ public class PredictedMonitoringAttribute { private static HashMap attributes_maximum_rate_of_change = new HashMap<>(); //initialization private static HashMap attributes_minimum_rate_of_change = new HashMap<>(); - private static HashMap attributes_rate_of_change = new HashMap<>(); private static HashMap predicted_monitoring_attribute_values = new HashMap<>(); private static HashMap > predicted_monitoring_attributes = new HashMap<>(); private String name; - private Double forecasted_value; private boolean initialized = false; - private int associated_subrule_id; - private boolean greater_than_rule; private double delta; - //private double previous_delta; //delta-like metric calculated by using the current metric value instead of the forecasted value private double threshold; private double rate_of_change; // the rate of change for the metric private double probability_confidence; //the probability confidence for the prediction @@ -39,7 +34,6 @@ public class PredictedMonitoringAttribute { this.initialized = true; this.name = name; this.threshold = threshold; - this.forecasted_value = forecasted_value; double current_value = RealtimeMonitoringAttribute.get_metric_value(name); if (Double.isNaN(current_value)){ Logger.getAnonymousLogger().log(info_logging_level,"Detected NaN value for metric "+name+". Thus we cannot compute severity although a predicted value of "+forecasted_value+" has arrived"); @@ -86,7 +80,6 @@ public class PredictedMonitoringAttribute { this.probability_confidence = probability_confidence; this.confidence_interval_width = confidence_interval_width; //actual_metric_values = get_last_n_actual_values(Constants.elements_considered_in_prediction, MonitoringAttribute.get_monitoring_attributes_values_map().get(name),true); - this.associated_subrule_id = associated_subrule_id; this.timestamp = timestamp; this.targeted_prediction_time = targeted_prediction_time; } @@ -133,32 +126,6 @@ public class PredictedMonitoringAttribute { } } - /* - else if (roc_calculation_mode.equals("simple")){ - if (isZero(maximum_rate_of_change-minimum_rate_of_change)){ - //TODO clarify if/else-if - if (actual_value <= getMonitoring_attributes_statistics().get(name).getLower_bound()) { - rate_of_change = 1.0; - normalized_rate_of_change = 100*rate_of_change; - } else if (actual_value >= getMonitoring_attributes_statistics().get(name).getUpper_bound()){ - rate_of_change = 0.0; - normalized_rate_of_change = 100*rate_of_change; - } - else { - rate_of_change = 1 * Math.max(Math.min(((forecasted_value-actual_value)) / Math.abs(actual_value), roc_limit),-roc_limit); - normalized_rate_of_change = 100*rate_of_change; - } - } - else{ - if (forecasted_value>actual_value){ - rate_of_change = 1 * Math.min(((forecasted_value - actual_value)) / Math.abs(maximum_rate_of_change), roc_limit); - }else{ - rate_of_change = 1 * Math.min(((forecasted_value - actual_value)) / Math.abs(minimum_rate_of_change), -roc_limit); - } - normalized_rate_of_change = 100*rate_of_change; - } - } - */ else{ Logger.getAnonymousLogger().log(severe_logging_level,"Effectively disabling rate of change (ROC) metric, setting it to 0, as an invalid roc_calculation_mode has been chosen"); rate_of_change = 0; -- GitLab From 6beabfa6fef1a1b6e9f72de1d9be48683f550d3e Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Wed, 16 Mar 2022 18:28:13 +0200 Subject: [PATCH 3/5] Corrected a float to integer (using the floor function) conversion Enabled persisting attempts to connect to the broker Added logging to show that subscription was successful on metrics to predict and start/stop forecasting topics --- .../src/runtime/Predictor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py b/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py index c840c17a..7a4690bf 100644 --- a/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py +++ b/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py @@ -135,7 +135,7 @@ def calculate_and_publish_predictions(prediction_horizon,maximum_time_required_f #Below we subtract one reconfiguration interval, as we cannot send a prediction for a time point later than one prediction_horizon interval wait_time = State.next_prediction_time - prediction_horizon - time.time() - print("Waiting for "+str((wait_time*100).__floor__()/100)+" seconds") + print("Waiting for "+str((int(wait_time*100))/100)+" seconds") if (wait_time>0): time.sleep(wait_time) if(not Listener.start_forecasting): @@ -290,12 +290,16 @@ if __name__ == "__main__": State.connection.connect() State.disconnected = False State.connection.topic("metrics_to_predict", id) + print("Successfully connected to metrics_to_predict") State.connection.topic("start_forecasting.exponentialsmoothing", id) + print("Successfully connected to start_forecasting.exponentialsmoothing") State.connection.topic("stop_forecasting.exponentialsmoothing", id) + print("Successfully connected to stop_forecasting.exponentialsmoothing") except Exception as e: print("Encountered exception while trying to connect to broker") print(traceback.format_exc()) State.disconnected = True + continue State.disconnection_handler.acquire() State.disconnection_handler.wait() State.disconnection_handler.release() -- GitLab From 88b62831edaa8931537ba2211bb639bd1dc2b787 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Thu, 17 Mar 2022 15:20:21 +0200 Subject: [PATCH 4/5] Corrected disconnection process Improved logging location in the Dockerfile of the exponential smoothing predictor --- morphemic-forecasting-exponentialsmoothing/Dockerfile | 2 +- .../src/runtime/Predictor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/morphemic-forecasting-exponentialsmoothing/Dockerfile b/morphemic-forecasting-exponentialsmoothing/Dockerfile index 5007c74b..26aee870 100644 --- a/morphemic-forecasting-exponentialsmoothing/Dockerfile +++ b/morphemic-forecasting-exponentialsmoothing/Dockerfile @@ -66,4 +66,4 @@ COPY ./src/r_predictors/forecasting_real_workload.R /home/r_predictions/ WORKDIR /home/r_predictions/esm_forecaster-0.1.0 -CMD ["/bin/sh","-c","python3 /home/r_predictions/esm_forecaster-0.1.0/runtime/Predictor.py /home/r_predictions/esm_forecaster-0.1.0/r_predictors/prediction_configuration.properties > /home/r_predictions/exponential_smoothing.log"] \ No newline at end of file +CMD ["/bin/sh","-c","python3 /home/r_predictions/esm_forecaster-0.1.0/runtime/Predictor.py /home/r_predictions/esm_forecaster-0.1.0/r_predictors/prediction_configuration.properties > $LOG_FILE"] \ No newline at end of file diff --git a/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py b/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py index 7a4690bf..aad3fbbf 100644 --- a/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py +++ b/morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py @@ -310,4 +310,4 @@ if __name__ == "__main__": # connection.topic("stop_forecasting",id) # connection.topic("start_forecasting",id) - m.disconnect() + State.connection.disconnect() -- GitLab From 8fc0b2e93caa5ed98c90eb100e903492a8082c84 Mon Sep 17 00:00:00 2001 From: Andreas Tsagkaropoulos Date: Thu, 17 Mar 2022 15:30:05 +0200 Subject: [PATCH 5/5] Redirect all process output to a specific logfile (assuming bash shell) --- morphemic-forecasting-exponentialsmoothing/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphemic-forecasting-exponentialsmoothing/Dockerfile b/morphemic-forecasting-exponentialsmoothing/Dockerfile index 26aee870..8db981a4 100644 --- a/morphemic-forecasting-exponentialsmoothing/Dockerfile +++ b/morphemic-forecasting-exponentialsmoothing/Dockerfile @@ -66,4 +66,4 @@ COPY ./src/r_predictors/forecasting_real_workload.R /home/r_predictions/ WORKDIR /home/r_predictions/esm_forecaster-0.1.0 -CMD ["/bin/sh","-c","python3 /home/r_predictions/esm_forecaster-0.1.0/runtime/Predictor.py /home/r_predictions/esm_forecaster-0.1.0/r_predictors/prediction_configuration.properties > $LOG_FILE"] \ No newline at end of file +CMD ["/bin/sh","-c","python3 /home/r_predictions/esm_forecaster-0.1.0/runtime/Predictor.py /home/r_predictions/esm_forecaster-0.1.0/r_predictors/prediction_configuration.properties > $LOG_FILE 2>&1"] \ No newline at end of file -- GitLab