Merge remote-tracking branch 'actual-morphemic/morphemic-rc2.0' into...

Merge remote-tracking branch 'actual-morphemic/morphemic-rc2.0' into exponential_smoothing_predictor

# Conflicts:
#	morphemic-forecasting-exponentialsmoothing/morphemic-forecasting-exponentialsmoothing.iml
#	morphemic-forecasting-exponentialsmoothing/src/r_predictors/forecasting_real_workload.R
#	morphemic-forecasting-exponentialsmoothing/src/r_predictors/prediction_configuration.properties
#	morphemic-forecasting-exponentialsmoothing/src/r_predictors/r_commands.R
#	morphemic-forecasting-exponentialsmoothing/src/runtime/Predictor.py
#	morphemic-forecasting-exponentialsmoothing/src/runtime/operational_status/State.py
#	morphemic-forecasting-exponentialsmoothing/src/runtime/utilities/Utilities.py
#	morphemic-forecasting-exponentialsmoothing/src/test/benchmark_config.properties
#	morphemic-forecasting-exponentialsmoothing/src/test/benchmarking.py
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_ETPercentile.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_EstimatedRemainingTimeContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_MinimumCores.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_MinimumCoresContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_NotFinished.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_NotFinishedOnTime.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_NotFinishedOnTimeContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_RemainingSimulationTimeMetric.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_SimulationElapsedTime.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_SimulationLeftNumber.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_TotalCores.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/output_WillFinishTooSoonContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_ETPercentile.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_EstimatedRemainingTimeContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_MinimumCores.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_MinimumCoresContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_NotFinished.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_NotFinishedOnTime.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_NotFinishedOnTimeContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_RemainingSimulationTimeMetric.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_SimulationElapsedTime.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_SimulationLeftNumber.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_TotalCores.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome12/prediction_lists_WillFinishTooSoonContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_ETPercentile.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_EstimatedRemainingTimeContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_MinimumCores.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_MinimumCoresContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_NotFinished.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_NotFinishedOnTime.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_NotFinishedOnTimeContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_RemainingSimulationTimeMetric.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_SimulationElapsedTime.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_SimulationLeftNumber.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_TotalCores.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/output_WillFinishTooSoonContext.csv
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_ETPercentile.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_EstimatedRemainingTimeContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_MinimumCores.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_MinimumCoresContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_NotFinished.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_NotFinishedOnTime.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_NotFinishedOnTimeContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_RemainingSimulationTimeMetric.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_SimulationElapsedTime.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_SimulationLeftNumber.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_TotalCores.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/datasets/genome18/prediction_lists_WillFinishTooSoonContext.pkl
#	morphemic-forecasting-exponentialsmoothing/src/test/forecasting_real_workload.R

forecasting_gluonts/gluonts_forecaster.py

@@ -36,10 +36,10 @@ import statistics
 import math
 
 directory_path = "/morphemic_project/"
-
+pd.options.mode.chained_assignment = None
 
 def train(metric):
-    
+
     data_file_path = os.path.join(os.environ.get("DATA_PATH", "./"), f'{os.environ.get("APP_NAME", "demo")}.csv')
     #while (not os.path.isfile(data_file_path)):
     	#sleep(30)
@@ -59,12 +59,12 @@ def train(metric):
     for j in range(i + 1, (len(dataset[metric]) - 1)):
         if (dataset[metric][j] == "None" and dataset[metric][j + 1] != "None" and dataset[metric][j - 1] != "None"):
             myds.append(dataset['ems_time'][j])
-            myy.append((dataset[metric][j - 1] + dataset[metric][j + 1]) / 2)
+            myy.append((float(dataset[metric][j - 1]) + (float(dataset[metric][j + 1]) )/ 2))
         elif (dataset[metric][j] == "None" and (dataset[metric][j + 1] == "None" or dataset[metric][j - 1] == "None")):
             j = j + 1
         elif (dataset[metric][j] != "None"):
             myds.append(dataset['ems_time'][j])
-            myy.append(dataset[metric][j])
+            myy.append(float(dataset[metric][j]))
             j = j + 1
 
     if (dataset[metric][len(dataset[metric]) - 1] != "None"):
@@ -271,3 +271,4 @@ def predict(model, number_of_forward_predictions, prediction_horizon, epoch_star
     returnDict = {'mins': mins, 'maxs': maxs, 'values': values}
 
     return returnDict
+ 

forecasting_gluonts/gluonts_listener.py

@@ -40,10 +40,10 @@ def worker(self,body,metric):
     epoch_start= body["epoch_start"]
     predictionTimes[metric] = epoch_start
     
-    while (not os.path.isfile(directory_path+'models/gluonts_'+metric+".pkl")):  
+    while (not os.path.isfile(directory_path+'models/gluonts_'+metric+".pkl")):
         sleep(30)
         logging.debug("Waiting for the trained model for metric: " + metric)
-       
+
     while(True):
         #if flags[metric] == 0:
             #epoch_start = predictionTimes[metric]
@@ -139,13 +139,13 @@ class Gluonts(morphemic.handler.ModelHandler,messaging.listener.MorphemicListene
         dataset_preprocessor = CSVData(APP_NAME)
         data_file_path = os.path.join(os.environ.get("DATA_PATH", "./"), f'{os.environ.get("APP_NAME", "demo")}.csv')
         dataset_preprocessor.prepare_csv()
-        
+
         while (not os.path.isfile(data_file_path)):
             logging.debug("Waiting for dataset to be loaded")
-    	    sleep(30)
+            sleep(30)
             dataset_preprocessor.prepare_csv()
-            
-    	    
+
+
         logging.debug("DATASET DOWNLOADED")
         
         for r in body:

forecasting_prophet/docker_image/run_prophet.sh

@@ -2,7 +2,7 @@
 sudo docker rmi -f $(sudo docker images)
 
 # Build the image from dockerfile and clone the latest version of my code
-docker build -t gitlab.ow2.org:4567/melodic/morphemic-preprocessor/prophet:morphemic-rc1.5 -f ./forecasting_prophet/docker_image/Dockerfile .
+docker build -t gitlab.ow2.org:4567/melodic/morphemic-preprocessor/prophet:morphemic-rc2.0 -f ./forecasting_prophet/docker_image/Dockerfile .
 
 
 # Test the image
@@ -13,3 +13,4 @@ sudo docker login gitlab.ow2.org:4567
 sudo docker push gitlab.ow2.org:4567/melodic/morphemic-preprocessor/prophet:morphemic-rc2.0
 
 
+

forecasting_prophet/prophet_forecaster.py

@@ -65,12 +65,12 @@ def train(metric):
         for j in range(i + 1, (len(dataset[metric]) - 1)):
             if (dataset[metric][j] == "None" and dataset[metric][j + 1] != "None" and dataset[metric][j - 1] != "None"):
                 myds.append(dataset['ems_time'][j])
-                myy.append((dataset[metric][j - 1] + dataset[metric][j + 1]) / 2)
+                myy.append((float(dataset[metric][j - 1]) + (float(dataset[metric][j + 1]) )/ 2))
             elif(dataset[metric][j] == "None" and (dataset[metric][j + 1] == "None" or dataset[metric][j - 1] == "None")):
                 j = j + 1
             elif(dataset[metric][j] != "None"):
                 myds.append(dataset['ems_time'][j])
-                myy.append(dataset[metric][j])
+                myy.append(float(dataset[metric][j]))
                 j = j + 1
 
         if (dataset[metric][len(dataset[metric]) - 1] != "None"):

forecasting_prophet/prophet_listener.py

@@ -45,7 +45,7 @@ def worker(self,body,metric):
     predictionTimes[metric] = epoch_start
     messages=list()
     f=0
-    while (not os.path.isfile(directory_path+'models/prophet_'+metric+".pkl")):  
+    while (not os.path.isfile(directory_path+'models/prophet_'+metric+".pkl")):
         sleep(30)
         logging.debug("Waiting for the trained model for metric: " + metric)
 
@@ -138,10 +138,10 @@ class Prophet(morphemic.handler.ModelHandler,messaging.listener.MorphemicListene
         dataset_preprocessor = CSVData(APP_NAME)
         data_file_path = os.path.join(os.environ.get("DATA_PATH", "./"), f'{os.environ.get("APP_NAME", "demo")}.csv')
         dataset_preprocessor.prepare_csv()
-        
+
         while (not os.path.isfile(data_file_path)):
             logging.debug("Waiting for dataset to be loaded")
-    	    sleep(30)
+            sleep(30)
             dataset_preprocessor.prepare_csv()
         
         for r in body:

morphemic-forecasting-exponentialsmoothing/src/README.md

+# Exponential Smoothing Predictor
+
+## Introduction
+
+The exponential smoothing predictor is based on the use of the Holt-Winters [1] and the ETS [2] libraries. It is coded using Python and R. The core predictive functionality is available as an R script while the Python implementation is responsible for organizing the communication to/from the Morphemic Forecasting module broker and the Dataset provider (InfluxDB).
+
+Apart from standard R and Python libraries, the `jproperties` library should be available for the Python code to be successfully executed. Moreover the `rapportools`,`gbutils`,`forecast`,`ggplot2`,`properties`,`xts`,`anytime` and `purrr` R libraries should be available.  
+
+## Configuration
+
+The predictor comes with two configuration files which can be used to specify the behaviour of the component. The two files are located in the src/r_predictors directory of the project.
+The options which will most probably need to be changed before deployment are the `broker_address`,the `horizon`, the `number_of_days_to_aggregate_data_from` and the `number_of_seconds_to_aggregate_on`. 
+
+| Option | Description |
+-------- |------------ |
+| realtime_mode | Whether we are producing predictions at realtime or are simply using the R script to make predictions with a given dataset  |
+| broker_address | The IP address of the broker (or localhost)  |
+|broker_port| The port which should be used for the broker|
+|prediction_method| The name of the prediction method which will be used to create the predictions, either 'Holt-Winters' or 'ETS' (unquoted)|
+|forecasting_data_slicing_mode| 'none' (or any string value) to indicate that the whole of the data should be used, or 'percentage' to indicate that only some of the data should be used|
+|forecasting_data_offset| The percentage of (initial) data points which should not be used if forecasting_data_slicing mode has been set to 'percentage'|
+|forecasting_data_limit| The percentage of (last) data points which should not be used if forecasting_data_slicing mode has been set to 'percentage'|
+|forecasting_data_used_for_training| The percentage of data (following its trimming using forecasting_data_offset and forecasting_data_limit) which should be used for training, and assuming that no forecasting horizon has been set and realtime_mode is False|
+|horizon| The number of seconds which should be forecasted into the future|
+|path_to_datasets|The absolute path to the datasets, **not** including the final slash ('/') character.|
+|application_name|The application name to be used when creating a dataset|
+|number_of_seconds_to_aggregate_on| The duration of the monitoring data interval in seconds. Monitoring data inside this interval will be aggregated to a single value (the mean value of all per-second observated/interpolated values) |
+|number_of_days_to_aggregate_data_from| The number of days which will be used to retrieve monitoring data from when creating a dataset|
+|prediction_processing_time_safety_margin_seconds| The number of seconds which will be used as a buffer when performing a prediction in order not to delay predictions and yet use as much data as possible|
+|testing_prediction_functionality| If set to 'True', then it will not send a prediction time for which predictions will be requested, but will rather allow the horizon setting to be used to create predictions|
+|try_to_optimize_parameters| If set to 'True' it will enable custom search for the optimal (alpha,beta,gamma) parameters, using a number of steps which are defined inside the code. This option may be unstable when used in conjunction with the 'ETS' method and will need more time to finish|
+|generate_prediction_png_output| A boolean variable, which if set to 'True' will trigger the creation of a png illustrating the prediction |
+|png_output_file| The absolute file path for the png file which should be generated|
+
+## Simple execution of the Predictor component with full Morphemic functionality
+
+When the component is started from the Python initializer script, it will listen to the Forecasting module broker topics which are relevant to it with the exception of the `training_models` event (as the predictor is currently retraining each time it is run). Furthermore, it will be producing monitoring events published under the 'intermediate_prediction.exponentialsmoothing' topic. 
+To start the predictor in this mode, it is necessary to start the src/runtime/predictions/Prediction.py script, with the location of the configuration file given as an argument.
+When the predictor has been started, it is ready to accept messages from the broker, provided it has been started as well.
+
+## Simple usage of the core prediction functionality
+
+In order to simply test the prediction functionality, it is sufficient to use the `Rscript` command, followed by the full path of   the `forecasting_real_workload.R` file which resides in the src/r_predictors directory, providing as an argument the prediction time for which a prediction is requested, and optionally the alpha/beta values for the prediction method which will be used (gamma will be estimated.)
+
+
+## Dockerized execution of the component with full Morphemic functionality
+
+TBD
\ No newline at end of file

scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/application/deployment/PASchedulerGateway.java

@@ -324,7 +324,7 @@ public class PASchedulerGateway {
         restSmartProxy = SchedulerConnectionHelper.disconnect();
     }
 
-    private void reconnectIfDisconnected() {
+    private synchronized void reconnectIfDisconnected() {
         if (!restSmartProxy.isConnected()) {
             try {
                 LOGGER.warn("WARNING: Not connected to the scheduler. Reconnecting to Scheduler ...");
@@ -334,6 +334,9 @@ public class PASchedulerGateway {
                 LOGGER.error("ERROR: Not able to reconnect to Scheduler due to: " + Arrays.toString(e.getStackTrace()));
             }
         }
+        else{
+            LOGGER.info("The Scheduler is already connected.");
+        }
     }
 
     private void renewSession() {