diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index feb2933fc6fda30c6d39b0670d87b5d2bc7aa492..97521136140b5f89b8e96ada6de021be9fdbe587 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,7 @@ build:prediction_orchestrator: script: - $PREDICTON_ORCHESTRATOR_CLI -Pwithout-docker clean install artifacts: + expire_in: 1 week paths: - /builds/melodic/morphemic-preprocessor/maven_repo/eu/morphemic/prediction_orchestrator/ @@ -104,7 +105,7 @@ deploy:performance-model: - docker tag performance_model:latest $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH - docker push $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH -deploy:persistent-storage-database: +deploy:persistent-storage: stage: deploy image: $DOCKER_DIND_IMAGE only: @@ -114,13 +115,13 @@ deploy:persistent-storage-database: - $DOCKER_DIND_SERVICE script: - cd morphemic-persistent-storage/database - - docker build -t persistent_storage_database -f ./Dockerfile . + - docker build -t persistent_storage -f ./Dockerfile . - docker image ls - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin - - docker tag persistent_storage_database:latest $CI_REGISTRY_IMAGE/persistent_storage_database:$CI_COMMIT_BRANCH - - docker push $CI_REGISTRY_IMAGE/persistent_storage_database:$CI_COMMIT_BRANCH + - docker tag persistent_storage:latest $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH + - docker push $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH -deploy:persistent-storage-activemq: +deploy:forecaster-cnn: stage: deploy image: $DOCKER_DIND_IMAGE only: @@ -129,12 +130,12 @@ deploy:persistent-storage-activemq: services: - $DOCKER_DIND_SERVICE script: - - cd morphemic-persistent-storage/example - - docker build -t persistent_storage_activemq -f ./Dockerfile . + - cd forecaster-cnn + - docker build -t cnn -f ./Dockerfile . - docker image ls - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin - - docker tag persistent_storage_activemq:latest $CI_REGISTRY_IMAGE/persistent_storage_activemq:$CI_COMMIT_BRANCH - - docker push $CI_REGISTRY_IMAGE/persistent_storage_activemq:$CI_COMMIT_BRANCH + - docker tag cnn:latest $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH + - docker push $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH deploy:scheduling-abstraction-layer: stage: deploy diff --git a/amq-message-python-library/MorphemicConnection.py b/amq-message-python-library/MorphemicConnection.py index 0994383d5cf913a46150ad452866a34942089769..0c15fbf766ec30c44b521b740061a28773e438de 100644 --- a/amq-message-python-library/MorphemicConnection.py +++ b/amq-message-python-library/MorphemicConnection.py @@ -12,11 +12,13 @@ class Connection: def __init__(self, username, password, host='localhost', port=61613, - debug=False): + debug=False, + **kwargs): self.username = username self.password = password self.hosts = [(host, port)] - self.conn = stomp.Connection(host_and_ports=self.hosts, auto_content_length=False) + self.conn = stomp.Connection(host_and_ports=self.hosts, auto_content_length=False, + timeout=kwargs.get('timeout',180000),keepalive=kwargs.get('keepalive', True)) if debug: logging.debug("Enabling debug") diff --git a/deployment/arima/.~lock.demo.csv# b/deployment/arima/.~lock.demo.csv# new file mode 100644 index 0000000000000000000000000000000000000000..07cbfa7de1074723757c6642d9a245df9fe18428 --- /dev/null +++ b/deployment/arima/.~lock.demo.csv# @@ -0,0 +1 @@ +,awarno,bulls-ThinkPad-T480,28.09.2021 09:11,file:///home/awarno/.config/libreoffice/4; \ No newline at end of file diff --git a/deployment/arima/env b/deployment/arima/env index 150f8c5eea3783eb32376e5738c6d15b08689080..1d3b68b4ee14a2de71cd64d909ad4fbcc84aedf4 100644 --- a/deployment/arima/env +++ b/deployment/arima/env @@ -1,15 +1,12 @@ AMQ_HOSTNAME=localhost -AMQ_USER=morphemic -AMQ_PASSWORD=morphemic -AMQ_HOST=147.102.17.76 -AMQ_PORT_BROKER=61610 -APP_NAME=default_application +AMQ_USER=admin +AMQ_PASSWORD=admin +AMQ_PORT=61613 +APP_NAME=demo METHOD=arima DATA_PATH=./ -INFLUXDB_HOSTNAME=147.102.17.76 +INFLUXDB_HOSTNAME=localhost INFLUXDB_PORT=8086 INFLUXDB_USERNAME=morphemic INFLUXDB_PASSWORD=password INFLUXDB_DBNAME=morphemic - - diff --git a/deployment/arima/main.py b/deployment/arima/main.py index de065605b1e95b7047e6e4832415499dc57eeef7..7908bfe1c47e55d2dbf0151b9d684c329ece6356 100644 --- a/deployment/arima/main.py +++ b/deployment/arima/main.py @@ -3,17 +3,20 @@ import stomp import json from amq_message_python_library import * # python amq-message-python-library import logging +import time from datetime import datetime from pytz import timezone -from datetime import datetime import time +import setproctitle + +# from src.log import logger AMQ_USER = os.environ.get("AMQ_USER", "admin") AMQ_PASSWORD = os.environ.get("AMQ_PASSWORD", "admin") AMQ_HOST = os.environ.get("AMQ_HOST", "localhost") AMQ_PORT_BROKER = os.environ.get("AMQ_PORT_BROKER", "61613") START_APP_TOPIC = "metrics_to_predict" -METHOD = os.environ.get("METHOD", "nbeats") +METHOD = os.environ.get("METHOD", "model") START_TOPIC = f"start_forecasting.{METHOD}" TZ = os.environ.get("TIME_ZONE", "Europe/Vienna") @@ -75,8 +78,9 @@ class Msg(object): def main(): + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}") logging.basicConfig( - filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", + filename=f"logs/{os.environ.get('METHOD', 'model')}.out", level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S", format="START %(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", @@ -108,11 +112,11 @@ def main(): ) # msg1 = Msg() - # msg1.body = '[{"metric": "value", "level": 3, "publish_rate": 30000}]' + # msg1.body = '[{"metric": "MinimumCores", "level": 3, "publish_rate": 30000}, {"metric": "EstimatedRemainingTimeContext", "level": 3, "publish_rate": 30000}, {"metric": "NotFinishedOnTimeContext", "level": 3, "publish_rate": 30000}]' # msg2 = Msg() # msg2.body = ( # "{" - # + f'"metrics": ["value"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' + # + f'"metrics": ["MinimumCores", "EstimatedRemainingTimeContext", "NotFinishedOnTimeContext"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' # + "}" # ) @@ -121,6 +125,7 @@ def main(): # StartForecastingListener(start_conn.conn, START_APP_TOPIC).on_message(msg2) while True: + time.sleep(60) pass diff --git a/deployment/arima/predict.py b/deployment/arima/predict.py index 4c44195786f3221f7b406322007c128bdbe2d930..c87915b9bfd82bb6131f1bd5c8b294d760b6c86a 100644 --- a/deployment/arima/predict.py +++ b/deployment/arima/predict.py @@ -122,9 +122,11 @@ def main(): for metric in predicted_metrics: prediction_msgs = predict( metric, - prediction_length, + prediction_lengths[metric], + single_prediction_points_length=prediction_points_horizons[metric], prediction_hor=prediction_horizon, timestamp=time_0, + publish_rate=metrics_info[metric]["publish_rate"], ) if prediction_msgs: @@ -137,31 +139,37 @@ def main(): start_conn.send_to_topic(dest, message[metric]) influxdb_conn.send_to_influxdb(metric, message) + current_time = int(time.time()) + logging.info( + f"metric: {metric}, Time difference between current and last horizon: {prediction_msgs[-1][metric]['predictionTime'] - current_time} seconds. TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + print( + f"metric: {metric}, Time difference between current and last horizon: {prediction_msgs[-1][metric]['predictionTime'] - current_time} seconds. TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + end_time = int(time.time()) time_0 = time_0 + prediction_cycle - time_to_wait = prediction_cycle - (end_time - start_time) + time_to_wait = time_0 - end_time if time_to_wait < 0: - time_to_wait = prediction_cycle - (time_to_wait % prediction_cycle) + time_to_skip = (end_time - time_0) // prediction_cycle + time_0 = time_0 + (time_to_skip + 1) * prediction_cycle + time_to_wait = time_0 - end_time logging.info( f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - time_0 = time_0 + prediction_cycle - + print( + f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) time.sleep(time_to_wait) if __name__ == "__main__": logging.basicConfig( - filename=f"logs/{os.environ.get('METHOD', 'arima')}.out", + filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - format="START %(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", ) - logging.Formatter.converter = lambda *args: datetime.now( - tz=timezone(TZ) - ).timetuple() msg = json.loads(sys.argv[1]) metrics_info = { m["metric"]: { @@ -173,15 +181,19 @@ if __name__ == "__main__": time_0 = msg["epoch_start"] prediction_horizon = msg["prediction_horizon"] * 1000 - prediction_points_horizon = msg["prediction_horizon"] * 1000 / msg["publish_rate"] + prediction_points_horizons = { + metric["metric"]: msg["prediction_horizon"] * 1000 // metric["publish_rate"] + for metric in msg["all_metrics"] + } predicted_metrics = set(msg["metrics"]) prediction_cycle = msg["prediction_horizon"] - prediction_length = ( - msg["prediction_horizon"] + prediction_lengths = { + metric["metric"]: msg["prediction_horizon"] * 1000 - // msg["publish_rate"] + // metric["publish_rate"] * msg["number_of_forward_predictions"] - ) + for metric in msg["all_metrics"] + } logging.info(f"Predicted metrics: {predicted_metrics}") number_of_forward_predictions = { metric: msg["number_of_forward_predictions"] for metric in predicted_metrics diff --git a/deployment/arima/requirements.txt b/deployment/arima/requirements.txt index e9ce5c82c1d44e0065e7dbf17360137a45989e48..c9a336864f83b46c96967f4d7528c779079775a1 100644 --- a/deployment/arima/requirements.txt +++ b/deployment/arima/requirements.txt @@ -5,5 +5,4 @@ filelock==3.0.12 pyyaml influxdb python-slugify - - +setproctitle diff --git a/deployment/arima/retrain.py b/deployment/arima/retrain.py index 2a6e654d17e71abb3ec2aac8cdc632634c556200..4fe0dfbb80afb89e3eceec7bda5f20e62f021eec 100644 --- a/deployment/arima/retrain.py +++ b/deployment/arima/retrain.py @@ -36,7 +36,11 @@ def main(predicted_metrics, prediction_horizon): while True: start_time = int(time.time()) for metric in predicted_metrics: - retrain_msg = train(metric, prediction_horizon) + retrain_msg = train( + metric, + prediction_horizon, + publish_rate=metrics_info[metric]["publish_rate"], + ) if retrain_msg: logging.info( f"Training completed for {metric} metric TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" diff --git a/deployment/arima/src/example.py b/deployment/arima/src/example.py new file mode 100644 index 0000000000000000000000000000000000000000..75ff52b0674ea7b5066c82ba11db7ed8c17276ed --- /dev/null +++ b/deployment/arima/src/example.py @@ -0,0 +1,2 @@ +def square(x): + return x * x \ No newline at end of file diff --git a/deployment/arima/src/model_predict.py b/deployment/arima/src/model_predict.py index 9ffa970c066efe6dc34fd99d6f957dcd21e621f8..dc191c0ffb0e55a9872445bc92ea4bbb2f7e033a 100644 --- a/deployment/arima/src/model_predict.py +++ b/deployment/arima/src/model_predict.py @@ -21,9 +21,11 @@ TZ = os.environ.get("TIME_ZONE", "Europe/Vienna") def predict( target_column, prediction_length, + single_prediction_points_length=1, yaml_file="model.yaml", prediction_hor=60, timestamp=0, + publish_rate=10000, ): with open(yaml_file) as file: params = yaml.load(file, Loader=yaml.FullLoader) @@ -39,54 +41,80 @@ def predict( return None dataset = pd.read_csv(data_path) - new_ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + new_ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"], publish_rate=publish_rate) if new_ts_dataset.dropped_recent_series: # series with recent data was too short logging.info( - f"Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC: {target_column} Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + print( + f"METRIC: {target_column} Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - print("Not enough fresh data, unable to predict TIME:") return None dataset = pd.read_csv(data_path) - ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"], publish_rate=publish_rate) + if ts_dataset.dataset.shape[0] < 1: + logging.info( + f"METRIC: {target_column} Not enough fresh preprocessed data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + print( + f"METRIC: {target_column} Not enough fresh preprocessed data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + return None + ts_dataset = ts_dataset.dataset[ ts_dataset.dataset.series == ts_dataset.dataset.series.max() - ] + ].tail(1000) pred_len = params["dataset"]["prediction_length"] best_model_aic = None - for order in [(2, 1, 2), (3, 0, 1), (1, 0, 1), (3, 1, 0)]: - sarima = sm.tsa.statespace.SARIMAX(ts_dataset[target_column], order=order) - model = sarima.fit() - if best_model_aic: - if model.aic < best_model_aic: + for order in [(2, 1, 2), (3, 0, 1), (1, 0, 1), (3, 1, 0), (1, 0, 0), (4, 0, 2)]: + try: + sarima = sm.tsa.statespace.SARIMAX( + ts_dataset[target_column], order=order, enforce_stationarity=False + ) + model = sarima.fit() + if best_model_aic: + if model.aic < best_model_aic: + predictions = model.get_forecast( + pred_len, return_conf_int=True, alpha=0.05 + ) + best_model_aic = model.aic + else: predictions = model.get_forecast( pred_len, return_conf_int=True, alpha=0.05 ) best_model_aic = model.aic - else: - predictions = model.get_forecast(pred_len, return_conf_int=True, alpha=0.05) - best_model_aic = model.aic + except np.linalg.LinAlgError: + logging.info( + f"METRIC: {target_column} SARIMAX model order: {order} failed for metric {target_column} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + if not best_model_aic: + logging.info( + f"METRIC: {target_column} All SARIMAX failed for metric: {target_column}, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + return None msgs = [] for i in range(pred_len): - msg = { - target_column: { - "metricValue": predictions.predicted_mean.values[i], - "level": 1, # TODO - "timestamp": int(time.time()), - "probability": 0.95, - "confidence_interval": [ - predictions.conf_int(alpha=0.05).iloc[i].values[0], - predictions.conf_int(alpha=0.05).iloc[i].values[1], - ], # quantiles difference - "predictionTime": timestamp + (i + 1) * (prediction_hor // 1000), - "refersTo": "TODO", - "cloud": "TODO", - "provider": "TODO", + if ((i + 1) % single_prediction_points_length) == 0: + msg = { + target_column: { + "metricValue": predictions.predicted_mean.values[i], + "level": 1, # TODO + "timestamp": int(time.time()), + "probability": 0.95, + "confidence_interval": [ + predictions.conf_int(alpha=0.05).iloc[i].values[0], + predictions.conf_int(alpha=0.05).iloc[i].values[1], + ], # quantiles difference + "predictionTime": timestamp + (i + 1) * (prediction_hor // 1000), + "refersTo": "TODO", + "cloud": "TODO", + "provider": "TODO", + } } - } - msgs.append(msg) + msgs.append(msg) return msgs diff --git a/deployment/arima/src/preprocess_dataset.py b/deployment/arima/src/preprocess_dataset.py index a39aedeb8a93880dc0fd4044661f31ca7e9f2396..789b21320f1697e0bbde4308de6d25afa81e3970 100644 --- a/deployment/arima/src/preprocess_dataset.py +++ b/deployment/arima/src/preprocess_dataset.py @@ -1,6 +1,7 @@ import pandas as pd import numpy as np import logging +import time pd.options.mode.chained_assignment = None @@ -13,6 +14,7 @@ class Dataset(object): self, dataset, target_column="value", + time_column="ems_time", tv_unknown_reals=[], known_reals=[], tv_unknown_cat=[], @@ -20,12 +22,14 @@ class Dataset(object): classification=0, context_length=40, prediction_length=5, + publish_rate=10000, ): self.max_missing_values = ( 20 # max consecutive missing values allowed per series ) self.target_column = target_column + self.time_column = time_column self.tv_unknown_cat = tv_unknown_cat self.known_reals = known_reals self.tv_unknown_reals = tv_unknown_reals @@ -33,17 +37,25 @@ class Dataset(object): self.classification = classification self.context_length = context_length self.prediction_length = prediction_length + self.publish_rate = publish_rate self.dataset = dataset - self.check_gap() + self.dropped_recent_series = True # default set to be true + if self.dataset.shape[0] > 0: + self.check_gap() self.n = dataset.shape[0] def cut_nan_start(self, dataset): dataset.index = range(dataset.shape[0]) first_not_nan_index = dataset[self.target_column].first_valid_index() - return dataset[dataset.index > first_not_nan_index] + if first_not_nan_index == first_not_nan_index: # check is if it;s not np.nan + if first_not_nan_index is not None: + if first_not_nan_index > -1: + return dataset[dataset.index > first_not_nan_index] + else: + return dataset.dropna() def fill_na(self, dataset): - dataset = dataset.replace("None", np.nan) + dataset = dataset.replace(np.inf, np.nan) dataset = dataset.ffill(axis="rows") return dataset @@ -55,56 +67,131 @@ class Dataset(object): for name in self.tv_unknown_cat: dataset[name] = dataset[name].astype(str) - - dataset["series"] = dataset["series"].astype(str) return dataset + def convert_time_to_ms(self): + if self.dataset.shape[0] > 0: + digit_len = len(str(int(self.dataset[self.time_column].values[0]))) + if digit_len >= 13: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(str(x)[:13]) + ) + else: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(int(str(x)[:digit_len]) * 10 ** (13 - digit_len)) + ) + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(x // 1e4 * 1e4) + ) + def add_obligatory_columns(self, dataset): - dataset["series"] = 0 - dataset["split"] = "train" n = dataset.shape[0] - dataset["split"][int(n * 0.8) :] = "val" - dataset["time_idx"] = range(n) + dataset["time_idx"] = range(n) # TODO check time gaps return dataset + def get_time_difference_current(self): + if self.dataset.shape[0] > 0: + last_timestamp_database = self.dataset[self.time_column].values[-1] + current_time = int(time.time()) + print( + f"Time difference between last timestamp and current time: {current_time - last_timestamp_database / 1000}" + ) + logging.info( + f"Time difference between last timestamp and current time: {current_time - last_timestamp_database / 1000}" + ) + def check_gap(self): - max_gap = self.dataset["time"].diff().abs().max() - logging.info(f"Max time gap in series {max_gap}") - print(f"Max time gap in series {max_gap}") - series_freq = self.dataset["time"].diff().value_counts().index.values[0] - logging.info(f"Detected series with {series_freq} frequency") - print(f"Detected series with {series_freq} frequency") - # check series length - series = np.split( - self.dataset, - *np.where( - self.dataset["time"].diff().abs().fillna(0).astype(int) - >= np.abs(self.max_missing_values * series_freq) - ), - ) - logging.info(f"{len(series)} series found") - print(f"{len(series)} series found") - preprocessed_series = [] - for i, s in enumerate(series): - s = self.fill_na(s) - s = self.cut_nan_start(s) - s = self.add_obligatory_columns(s) - s = self.convert_formats(s) - s["split"] = "train" - if s.shape[0] > self.prediction_length * 2 + self.context_length: - s["series"] = i - preprocessed_series.append(s) - if i == len(series) - 1: + print(self.dataset) + if self.dataset.shape[0] > 0: + self.dataset = self.dataset.groupby(by=[self.time_column]).min() + self.dataset[self.time_column] = self.dataset.index + self.dataset.index = range(self.dataset.shape[0]) + self.convert_time_to_ms() + print(self.dataset) + self.dataset[self.target_column] = pd.to_numeric( + self.dataset[self.target_column], errors="coerce" + ).fillna(np.nan) + self.dataset = self.dataset.replace(np.inf, np.nan) + self.dataset = self.dataset.dropna(subset=[self.target_column]) + if self.dataset.shape[0] > 0: + max_gap = self.dataset[self.time_column].diff().abs().max() logging.info( - f"Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" + f"Metric: {self.target_column} Max time gap in series {max_gap}" + ) + print(f" Metric: {self.target_column} Max time gap in series {max_gap}") + print(self.dataset[self.time_column].diff().fillna(0).value_counts()) + series_freq = ( + (self.dataset[self.time_column]) + .diff() + .fillna(0) + .value_counts() + .index.values[0] ) - logging.info(f"{len(preprocessed_series)} long enough series found") - print(f"{len(preprocessed_series)} long enough series found") - # logging.info(f"") - self.dataset = pd.concat(preprocessed_series) - if self.dataset["series"].max() != len(series) - 1: - self.dropped_recent_series = True + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + if series_freq != self.publish_rate: + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + + # check series length + series = np.split( + self.dataset, + *np.where( + self.dataset[self.time_column] + .diff() + .abs() + .fillna(0) + .astype(int) + >= np.abs(self.max_missing_values * self.publish_rate) + ), + ) + logging.info(f"Metric: {self.target_column} {len(series)} series found") + print(f"{len(series)} series found") + preprocessed_series = [] + for i, s in enumerate(series): + s = self.fill_na(s) + s = self.cut_nan_start(s) + s = self.add_obligatory_columns(s) + s["split"] = "train" + s = self.convert_formats(s) + print(s.shape) + logging.info( + f"Metric: {self.target_column} Found series {i} of length: {s.shape[0]}, required data rows: {self.prediction_length * 2 + self.context_length}" + ) + if s.shape[0] > self.prediction_length * 2 + self.context_length: + s["series"] = i + preprocessed_series.append(s) + if i == len(series) - 1: + logging.info( + f"Metric: {self.target_column} Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" + ) + + logging.info( + f"Metric: {self.target_column} {len(preprocessed_series)} long enough series found" + ) + print(f"{len(preprocessed_series)} long enough series found") + # logging.info(f"") + if preprocessed_series: + self.dataset = pd.concat(preprocessed_series) + if self.dataset["series"].max() != len(series) - 1: + self.dropped_recent_series = True + else: + self.dropped_recent_series = False + else: + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + self.dataset.index = range(self.dataset.shape[0]) else: - self.dropped_recent_series = False - self.dataset.index = range(self.dataset.shape[0]) + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + + self.get_time_difference_current() diff --git a/deployment/arima/test/model_predict_test.py b/deployment/arima/test/model_predict_test.py new file mode 100644 index 0000000000000000000000000000000000000000..bfecb97ddd0158021f119111a7d1c1d78003640a --- /dev/null +++ b/deployment/arima/test/model_predict_test.py @@ -0,0 +1,180 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_predict import predict +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = predict(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/arima/test/model_train_test.py b/deployment/arima/test/model_train_test.py new file mode 100644 index 0000000000000000000000000000000000000000..00f4ebd62dbe8ba04f7dcf0cf7a61158bd8559a2 --- /dev/null +++ b/deployment/arima/test/model_train_test.py @@ -0,0 +1,221 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_train import train +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :3 + ] + ] + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + print(df) + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :6000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(6000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = train(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/arima/test/preprocess_dataset_test.py b/deployment/arima/test/preprocess_dataset_test.py new file mode 100644 index 0000000000000000000000000000000000000000..7abb9e0d514d1b44c816e428150621e0a16c3977 --- /dev/null +++ b/deployment/arima/test/preprocess_dataset_test.py @@ -0,0 +1,214 @@ +import sys + +sys.path.append(".") + +import pytest +from src.preprocess_dataset import Dataset +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df_13(): + df = pd.DataFrame() + df["ems_time"] = 1 + for i in range(5): + df[f"metric_{i}"] = [random.random() for i in range(1000)] + return df + + +@pytest.fixture +def df_14(): + df = pd.DataFrame() + df["ems_time"] = 10 + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(1000)] + return df + + +@pytest.fixture +def df_15(): + df = pd.DataFrame() + df["ems_time"] = [i * 30 * 1e5 for i in range(500)] + [ + i * 30 * 1e5 + 10000 for i in range(500) + ] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df_16(): + df = pd.DataFrame() + df["ems_time"] = [i for i in range(500)] + [i + 10000 for i in range(500)] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +class TestDataset: + @pytest.mark.parametrize( + "df,metric", + [ + ("df_1", metric), + ("df_2", metric), + ("df_3", metric), + ("df_4", metric), + ("df_5", metric), + ("df_6", metric), + ("df_7", metric), + ("df_8", metric), + ("df_9", metric), + ("df_10", metric), + ("df_11", metric), + ("df_12", metric), + ("df_13", metric), + ("df_14", metric), + ("df_15", metric), + ("df_16", metric), + ], + indirect=True, + ) + def test_init(self, df, metric): + preprocessed_dataset = Dataset(df, metric) + assert isinstance(preprocessed_dataset, Dataset) diff --git a/deployment/nbeats/env b/deployment/nbeats/env index a83d6aec69c8f0a1d106dee9a64b527c6e667e77..6f917a734e4c548ac70f623937987315243a2947 100644 --- a/deployment/nbeats/env +++ b/deployment/nbeats/env @@ -1,15 +1,12 @@ AMQ_HOSTNAME=localhost AMQ_USER=morphemic AMQ_PASSWORD=morphemic -AMQ_HOST=147.102.17.76 -AMQ_PORT_BROKER=61610 -APP_NAME=default_application -METHOD=nbeats +AMQ_PORT=61613 +APP_NAME=demo +METHOD=tft DATA_PATH=./ -INFLUXDB_HOSTNAME=147.102.17.76 +INFLUXDB_HOSTNAME=localhost INFLUXDB_PORT=8086 INFLUXDB_USERNAME=morphemic INFLUXDB_PASSWORD=password INFLUXDB_DBNAME=morphemic -TIME_ZONE=Europe/Vienna - diff --git a/deployment/nbeats/main.py b/deployment/nbeats/main.py index 4427fce4c7b7bfdfe61af0af79b58b5e9813159c..1e3c863b9e86b3a8b712122a26093e205c5c1330 100644 --- a/deployment/nbeats/main.py +++ b/deployment/nbeats/main.py @@ -7,7 +7,8 @@ import logging import time from datetime import datetime from pytz import timezone -from datetime import datetime +import time +import setproctitle # from src.log import logger @@ -81,8 +82,9 @@ class Msg(object): def main(): + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}") logging.basicConfig( - filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", + filename=f"logs/{os.environ.get('METHOD', 'model')}.out", level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S", format="START %(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", @@ -114,11 +116,11 @@ def main(): ) # msg1 = Msg() - # msg1.body = '[{"metric": "memory", "level": 3, "publish_rate": 30000}]' + # msg1.body = '[{"metric": "MinimumCores", "level": 3, "publish_rate": 30000}, {"metric": "EstimatedRemainingTimeContext", "level": 3, "publish_rate": 30000}, {"metric": "NotFinishedOnTimeContext", "level": 3, "publish_rate": 30000}]' # msg2 = Msg() # msg2.body = ( # "{" - # + f'"metrics": ["memory"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' + # + f'"metrics": ["MinimumCores", "EstimatedRemainingTimeContext", "NotFinishedOnTimeContext"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' # + "}" # ) @@ -127,6 +129,7 @@ def main(): # StartForecastingListener(start_conn.conn, START_APP_TOPIC).on_message(msg2) while True: + time.sleep(60) pass diff --git a/deployment/nbeats/model.yaml b/deployment/nbeats/model.yaml index 95b13d0c9f232d90c6a8fc1c3e8edf6dddb3d335..af21277231e7a1ba081e8d04465ac02a48780b82 100644 --- a/deployment/nbeats/model.yaml +++ b/deployment/nbeats/model.yaml @@ -23,3 +23,5 @@ save_path: models dataloader_path: dataloader +context_length_ratio: + 10 diff --git a/deployment/nbeats/predict.py b/deployment/nbeats/predict.py index 2dd0091d5d75ef89739190384fdab82fcd1d85fb..1c40e84319fb743bd0a3a26156b2c4ab5b3d9cfb 100644 --- a/deployment/nbeats/predict.py +++ b/deployment/nbeats/predict.py @@ -14,6 +14,8 @@ from src.dataset_maker import CSVData from pytz import timezone import pytz from datetime import datetime +import random +import setproctitle METHOD = os.environ.get("METHOD", "nbeats") START_TOPIC = f"start_forecasting.{METHOD}" @@ -122,20 +124,25 @@ def main(): for metric in predicted_metrics: predictions = None for i in range(number_of_forward_predictions[metric]): - print(int((i + 1) * prediction_points_horizon), "point idx") prediction_msgs, prediction = predict( metric, - prediction_length, + prediction_lengths[metric], extra_data=None, m=i + 1, prediction_hor=prediction_horizon, timestamp=time_0 + (i + 1) * (prediction_horizon // 1000), - predicted_point_idx=int((i + 1) * prediction_points_horizon - 1), + predicted_point_idx=int( + (i + 1) * prediction_points_horizons[metric] - 1 + ), + publish_rate=metrics_info[metric]["publish_rate"], ) if i == (number_of_forward_predictions[metric] - 1): print( f"time_0 difference seconds {start_time + (i + 1) * prediction_horizon // 1000 - int(time.time())}" ) + logging.info( + f"time difference in seconds between last preiction and current time {start_time + (i + 1) * prediction_horizon // 1000 - int(time.time())}" + ) else: predictions = prediction @@ -159,21 +166,23 @@ def main(): influxdb_conn.send_to_influxdb(metric, prediction_msgs) end_time = int(time.time()) - print(f"sleeping {prediction_cycle - (end_time - start_time)} seconds") - time_0 = time_0 + prediction_cycle - time_to_wait = prediction_cycle - (end_time - start_time) + time_to_wait = time_0 - end_time if time_to_wait < 0: - time_to_wait = prediction_cycle - (time_to_wait % prediction_cycle) + time_to_skip = (end_time - time_0) // prediction_cycle + time_0 = time_0 + (time_to_skip + 1) * prediction_cycle + time_to_wait = time_0 - end_time logging.info( f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - time_0 = time_0 + prediction_cycle - + print( + f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) time.sleep(time_to_wait) if __name__ == "__main__": + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}_predict") logging.basicConfig( filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", ) @@ -189,15 +198,19 @@ if __name__ == "__main__": time_0 = msg["epoch_start"] prediction_horizon = msg["prediction_horizon"] * 1000 - prediction_points_horizon = msg["prediction_horizon"] * 1000 / msg["publish_rate"] + prediction_points_horizons = { + metric["metric"]: msg["prediction_horizon"] * 1000 // metric["publish_rate"] + for metric in msg["all_metrics"] + } predicted_metrics = set(msg["metrics"]) prediction_cycle = msg["prediction_horizon"] - prediction_length = ( - msg["prediction_horizon"] + prediction_lengths = { + metric["metric"]: msg["prediction_horizon"] * 1000 - // msg["publish_rate"] + // metric["publish_rate"] * msg["number_of_forward_predictions"] - ) + for metric in msg["all_metrics"] + } logging.info(f"Predicted metrics: {predicted_metrics}") number_of_forward_predictions = { metric: msg["number_of_forward_predictions"] for metric in predicted_metrics diff --git a/deployment/nbeats/requirements.txt b/deployment/nbeats/requirements.txt index 1940fae2ffeb2ab99665dc424196659443108911..ae603574f3dfba609a0ea37d1f9a7e0a34628c7d 100644 --- a/deployment/nbeats/requirements.txt +++ b/deployment/nbeats/requirements.txt @@ -6,5 +6,4 @@ filelock==3.0.12 influxdb python-slugify torchmetrics==0.5.0 - - +setproctitle diff --git a/deployment/nbeats/retrain.py b/deployment/nbeats/retrain.py index 55eae32bcb58f7b1e10993d3a62cf1885b390e5a..fcfb77d562d14798bddbde15806ddbb585827688 100644 --- a/deployment/nbeats/retrain.py +++ b/deployment/nbeats/retrain.py @@ -10,6 +10,7 @@ import pytz import time from pytz import timezone from datetime import datetime +import setproctitle TOPIC_NAME = "training_models" RETRAIN_CYCLE = 10 # minutes @@ -21,7 +22,7 @@ APP_NAME = os.environ.get("APP_NAME", "demo") TZ = os.environ.get("TIME_ZONE", "Europe/Vienna") -def main(predicted_metrics, prediction_horizon): +def main(predicted_metrics, prediction_horizons): start_conn = morphemic.Connection( AMQ_USER, AMQ_PASSWORD, host=AMQ_HOST, port=AMQ_PORT_BROKER ) @@ -33,7 +34,11 @@ def main(predicted_metrics, prediction_horizon): while True: start_time = int(time.time()) for metric in predicted_metrics: - retrain_msg = train(metric, prediction_horizon) + retrain_msg = train( + metric, + prediction_horizons[metric], + publish_rate=metrics_info[metric]["publish_rate"], + ) if retrain_msg: logging.info( f"Training completed for {metric} metric TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" @@ -43,7 +48,7 @@ def main(predicted_metrics, prediction_horizon): else: print("Not enough data for model training, waiting ...") logging.info( - f"Not enough data for model training, waiting ... TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC: {metric} Not enough data for model training, waiting ... TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) end_time = int(time.time()) @@ -57,6 +62,7 @@ def main(predicted_metrics, prediction_horizon): if __name__ == "__main__": + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}_retrain") logging.basicConfig( filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", ) @@ -70,14 +76,15 @@ if __name__ == "__main__": } for m in msg["all_metrics"] } - prediction_horizon = ( - msg["prediction_horizon"] + prediction_horizons = { + metric["metric"]: msg["prediction_horizon"] * 1000 - // msg["publish_rate"] + // metric["publish_rate"] * msg["number_of_forward_predictions"] - ) + for metric in msg["all_metrics"] + } predicted_metrics = set(metrics_info.keys()) logging.info( f"Predicted metrics: {predicted_metrics} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - main(predicted_metrics, prediction_horizon) + main(predicted_metrics, prediction_horizons) diff --git a/deployment/nbeats/src/model_predict.py b/deployment/nbeats/src/model_predict.py index 21f7fa2118a9ee073b02e9ef6fc612056a17110f..0ea200bf397cb326685f1862f7c7b681e6a2cd0a 100644 --- a/deployment/nbeats/src/model_predict.py +++ b/deployment/nbeats/src/model_predict.py @@ -31,12 +31,15 @@ def predict( prediction_hor=60, timestamp=0, predicted_point_idx=0, + publish_rate=10000, ): with open(yaml_file) as file: params = yaml.load(file, Loader=yaml.FullLoader) params["dataset"]["prediction_length"] = prediction_length - params["dataset"]["context_length"] = prediction_length * 12 + params["dataset"]["context_length"] = ( + prediction_length * params["context_length_ratio"] + ) model_path = os.path.join(params["save_path"], f"{target_column}.pth") @@ -55,7 +58,12 @@ def predict( return (None, None) dataset = pd.read_csv(data_path) - new_ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + new_ts_dataset = Dataset( + dataset, + target_column=target_column, + **params["dataset"], + publish_rate=publish_rate, + ) if new_ts_dataset.dropped_recent_series: # series with recent data was too short logging.info( f"Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" @@ -68,7 +76,7 @@ def predict( if extra_data is not None: dataset = pd.concat([dataset, extra_data[dataset.columns]], ignore_index=True) - lockfile = params["dataloader_path"] + ".pickle" + lockfile = f"{params['dataloader_path']}_{target_column}.pickle" lock = FileLock(lockfile + ".lock") with lock: @@ -106,7 +114,11 @@ def predict( lock = FileLock(lockfile + ".lock") with lock: - model.load_state_dict(torch.load(model_path)) + if os.path.isfile(model_path): + model.load_state_dict(torch.load(model_path)) + logging.info("Model corrupted unable to predict") + else: + return (None, None) prediction_input = ts_dataset.get_from_dataset(future_df) prediction_input = prediction_input.to_dataloader(train=False) diff --git a/deployment/nbeats/src/model_train.py b/deployment/nbeats/src/model_train.py index 785d9febd98a5617689144631103369abfb17383..cf9f8f86c10150ea6fb1073beda773140a444a47 100644 --- a/deployment/nbeats/src/model_train.py +++ b/deployment/nbeats/src/model_train.py @@ -29,13 +29,15 @@ logging.basicConfig( ) -def train(target_column, prediction_length, yaml_file="model.yaml"): +def train(target_column, prediction_length, yaml_file="model.yaml", publish_rate=10000): torch.manual_seed(12345) with open(yaml_file) as file: params = yaml.load(file, Loader=yaml.FullLoader) params["dataset"]["prediction_length"] = prediction_length - params["dataset"]["context_length"] = prediction_length * 12 + params["dataset"]["context_length"] = ( + prediction_length * params["context_length_ratio"] + ) data_path = os.path.join( os.environ.get("DATA_PATH", "./"), f'{os.environ.get("APP_NAME", "demo")}.csv' @@ -45,23 +47,35 @@ def train(target_column, prediction_length, yaml_file="model.yaml"): return None dataset = pd.read_csv(data_path) + print(dataset, "dataset downloaded from persostent storage") - if dataset.shape[0] < 14 * prediction_length: + if dataset.shape[0] < (params["context_length_ratio"] + 2) * prediction_length: logging.info( - f"dataset len: {dataset.shape[0]}, minimum points required: {14 * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC: {target_column} dataset len: {dataset.shape[0]}, minimum points required: {(params['context_length_ratio'] + 2) * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) return None - ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + ts_dataset = Dataset( + dataset, + target_column=target_column, + **params["dataset"], + publish_rate=publish_rate, + ) - lockfile = params["dataloader_path"] + ".pickle" + if ts_dataset.dataset.shape[0] < 1: + logging.info( + f"METRIC: {target_column} Preprocessed dataset len: {ts_dataset.dataset.shape[0]}, minimum points required: {(params['context_length_ratio'] + 2) * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + return None + + lockfile = f"{params['dataloader_path']}_{target_column}.pickle" lock = FileLock(lockfile + ".lock") with lock: with open(lockfile, "wb") as handle: pickle.dump(ts_dataset, handle) logging.info( - f"train dataset saved: {lockfile} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC: {target_column} train dataset saved: {lockfile} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) training = ts_dataset.ts_dataset @@ -99,6 +113,7 @@ def train(target_column, prediction_length, yaml_file="model.yaml"): widths=[32, 512], backcast_loss_ratio=1.0, reduce_on_plateau_patience=5, + loss=MAE(), ) model_path = os.path.join(params["save_path"], f"{target_column}.pth") @@ -108,8 +123,9 @@ def train(target_column, prediction_length, yaml_file="model.yaml"): if os.path.isfile(lockfile): print("downloading weigths") - with lock: - model.load_state_dict(torch.load(model_path)) + if os.path.isfile(model_path): + with lock: + model.load_state_dict(torch.load(model_path)) trainer.fit( model, diff --git a/deployment/nbeats/src/preprocess_dataset.py b/deployment/nbeats/src/preprocess_dataset.py index 9383d3d352c04a4da870943570c6d96eb749d556..1a736b25f098e24f45e789e340d5d537915c8249 100644 --- a/deployment/nbeats/src/preprocess_dataset.py +++ b/deployment/nbeats/src/preprocess_dataset.py @@ -3,6 +3,7 @@ from pytorch_forecasting import TimeSeriesDataSet from pytorch_forecasting.data import NaNLabelEncoder import numpy as np import logging +import time pd.options.mode.chained_assignment = None @@ -15,6 +16,7 @@ class Dataset(object): self, dataset, target_column="value", + time_column="ems_time", tv_unknown_reals=[], known_reals=[], tv_unknown_cat=[], @@ -22,12 +24,14 @@ class Dataset(object): classification=0, context_length=40, prediction_length=5, + publish_rate=10000, ): self.max_missing_values = ( 20 # max consecutive missing values allowed per series ) self.target_column = target_column + self.time_column = time_column self.tv_unknown_cat = tv_unknown_cat self.known_reals = known_reals self.tv_unknown_reals = tv_unknown_reals @@ -35,18 +39,27 @@ class Dataset(object): self.classification = classification self.context_length = context_length self.prediction_length = prediction_length + self.publish_rate = publish_rate self.dataset = dataset - self.check_gap() + self.dropped_recent_series = True # default set to be true + if self.dataset.shape[0] > 0: + self.check_gap() self.n = dataset.shape[0] - self.ts_dataset = self.create_time_series_dataset() + if self.dataset.shape[0] > 0: + self.ts_dataset = self.create_time_series_dataset() def cut_nan_start(self, dataset): dataset.index = range(dataset.shape[0]) first_not_nan_index = dataset[self.target_column].first_valid_index() - return dataset[dataset.index > first_not_nan_index] + if first_not_nan_index == first_not_nan_index: # check is if it;s not np.nan + if first_not_nan_index is not None: + if first_not_nan_index > -1: + return dataset[dataset.index > first_not_nan_index] + else: + return dataset.dropna() def fill_na(self, dataset): - dataset = dataset.replace("None", np.nan) + dataset = dataset.replace(np.inf, np.nan) dataset = dataset.ffill(axis="rows") return dataset @@ -58,18 +71,145 @@ class Dataset(object): for name in self.tv_unknown_cat: dataset[name] = dataset[name].astype(str) - - dataset["series"] = dataset["series"].astype(str) return dataset + def convert_time_to_ms(self): + if self.dataset.shape[0] > 0: + digit_len = len(str(int(self.dataset[self.time_column].values[0]))) + if digit_len >= 13: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(str(int(x))[:13]) + ) + else: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(int(str(int(x))[:digit_len]) * 10 ** (13 - digit_len)) + ) + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(x // 1e4 * 1e4) + ) + def add_obligatory_columns(self, dataset): - dataset["series"] = 0 - dataset["split"] = "train" n = dataset.shape[0] - dataset["split"][int(n * 0.8) :] = "val" dataset["time_idx"] = range(n) # TODO check time gaps return dataset + def get_time_difference_current(self): + if self.dataset.shape[0] > 0: + last_timestamp_database = self.dataset[self.time_column].values[-1] + current_time = int(time.time()) + print( + f"Time difference between last timestamp and current time: {current_time - last_timestamp_database}" + ) + logging.info( + f"Time difference between last timestamp and current time: {current_time - last_timestamp_database}" + ) + + def check_gap(self): + if self.dataset.shape[0] > 0: + self.dataset = self.dataset.groupby(by=[self.time_column]).min() + self.dataset[self.time_column] = self.dataset.index + self.dataset.index = range(self.dataset.shape[0]) + self.convert_time_to_ms() + self.dataset[self.target_column] = pd.to_numeric( + self.dataset[self.target_column], errors="coerce" + ).fillna(np.nan) + self.dataset = self.dataset.replace(np.inf, np.nan) + self.dataset = self.dataset.dropna(subset=[self.target_column]) + if self.dataset.shape[0] > 0: + max_gap = self.dataset[self.time_column].diff().abs().max() + logging.info( + f"Metric: {self.target_column} Max time gap in series {max_gap}" + ) + print(f" Metric: {self.target_column} Max time gap in series {max_gap}") + series_freq = ( + (self.dataset[self.time_column]) + .diff() + .fillna(0) + .value_counts() + .index.values[0] + ) + + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + if series_freq != self.publish_rate: + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + + # check series length + series = np.split( + self.dataset, + *np.where( + self.dataset[self.time_column] + .diff() + .abs() + .fillna(0) + .astype(int) + >= np.abs(self.max_missing_values * self.publish_rate) + ), + ) + logging.info(f"Metric: {self.target_column} {len(series)} series found") + print(f"{len(series)} series found") + preprocessed_series = [] + for i, s in enumerate(series): + s = self.fill_na(s) + s = self.cut_nan_start(s) + s = self.add_obligatory_columns(s) + s["split"] = "train" + s = self.convert_formats(s) + logging.info( + f"Metric: {self.target_column} Found series {i} of length: {s.shape[0]}, required data rows: {self.prediction_length * 2 + self.context_length}" + ) + if s.shape[0] > self.prediction_length * 2 + self.context_length: + s["series"] = i + preprocessed_series.append(s) + if i == len(series) - 1: + logging.info( + f"Metric: {self.target_column} Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" + ) + + logging.info( + f"Metric: {self.target_column} {len(preprocessed_series)} long enough series found" + ) + print(f"{len(preprocessed_series)} long enough series found") + # logging.info(f"") + if preprocessed_series: + self.dataset = pd.concat(preprocessed_series) + if self.dataset["series"].max() != len(series) - 1: + self.dropped_recent_series = True + else: + self.dropped_recent_series = False + else: + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + self.dataset.index = range(self.dataset.shape[0]) + else: + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + if self.dataset.shape[0] > 0: + self.get_time_difference_current() + + def inherited_dataset(self, split1, split2): + df1 = ( + self.dataset[lambda x: x.split == split1] + .groupby("series", as_index=False) + .apply(lambda x: x.iloc[-self.context_length :]) + ) # previous split fragment + df2 = self.dataset[lambda x: x.split == split2] # split part + inh_dataset = pd.concat([df1, df2]) + inh_dataset = inh_dataset.sort_values(by=["series", "time_idx"]) + inh_dataset = TimeSeriesDataSet.from_dataset( + self.ts_dataset, inh_dataset, min_prediction_idx=0, stop_randomization=True + ) + return inh_dataset + def create_time_series_dataset(self): if not self.classification: self.time_varying_unknown_reals = [ @@ -98,62 +238,6 @@ class Dataset(object): ) return ts_dataset - def check_gap(self): - max_gap = self.dataset["time"].diff().abs().max() - logging.info(f"Max time gap in series {max_gap}") - print(f"Max time gap in series {max_gap}") - series_freq = self.dataset["time"].diff().value_counts().index.values[0] - logging.info(f"Detected series with {series_freq} frequency") - print(f"Detected series with {series_freq} frequency") - # check series length - series = np.split( - self.dataset, - *np.where( - self.dataset["time"].diff().abs().fillna(0).astype(int) - >= np.abs(self.max_missing_values * series_freq) - ), - ) - logging.info(f"{len(series)} series found") - print(f"{len(series)} series found") - preprocessed_series = [] - for i, s in enumerate(series): - s = self.fill_na(s) - s = self.cut_nan_start(s) - s = self.add_obligatory_columns(s) - s = self.convert_formats(s) - s["split"] = "train" - if s.shape[0] > self.prediction_length * 2 + self.context_length: - s["series"] = i - preprocessed_series.append(s) - if i == len(series) - 1: - logging.info( - f"Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" - ) - - logging.info(f"{len(preprocessed_series)} long enough series found") - print(f"{len(preprocessed_series)} long enough series found") - # logging.info(f"") - self.dataset = pd.concat(preprocessed_series) - if self.dataset["series"].max() != len(series) - 1: - self.dropped_recent_series = True - else: - self.dropped_recent_series = False - self.dataset.index = range(self.dataset.shape[0]) - - def inherited_dataset(self, split1, split2): - df1 = ( - self.dataset[lambda x: x.split == split1] - .groupby("series", as_index=False) - .apply(lambda x: x.iloc[-self.context_length :]) - ) # previous split fragment - df2 = self.dataset[lambda x: x.split == split2] # split part - inh_dataset = pd.concat([df1, df2]) - inh_dataset = inh_dataset.sort_values(by=["series", "time_idx"]) - inh_dataset = TimeSeriesDataSet.from_dataset( - self.ts_dataset, inh_dataset, min_prediction_idx=0, stop_randomization=True - ) - return inh_dataset - def get_from_dataset(self, dataset): return TimeSeriesDataSet.from_dataset( self.ts_dataset, dataset, min_prediction_idx=0, stop_randomization=True diff --git a/deployment/nbeats/test/model_predict_test.py b/deployment/nbeats/test/model_predict_test.py new file mode 100644 index 0000000000000000000000000000000000000000..bfecb97ddd0158021f119111a7d1c1d78003640a --- /dev/null +++ b/deployment/nbeats/test/model_predict_test.py @@ -0,0 +1,180 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_predict import predict +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = predict(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/nbeats/test/model_train_test.py b/deployment/nbeats/test/model_train_test.py new file mode 100644 index 0000000000000000000000000000000000000000..00f4ebd62dbe8ba04f7dcf0cf7a61158bd8559a2 --- /dev/null +++ b/deployment/nbeats/test/model_train_test.py @@ -0,0 +1,221 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_train import train +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :3 + ] + ] + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + print(df) + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :6000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(6000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = train(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/nbeats/test/preprocess_dataset_test.py b/deployment/nbeats/test/preprocess_dataset_test.py new file mode 100644 index 0000000000000000000000000000000000000000..7abb9e0d514d1b44c816e428150621e0a16c3977 --- /dev/null +++ b/deployment/nbeats/test/preprocess_dataset_test.py @@ -0,0 +1,214 @@ +import sys + +sys.path.append(".") + +import pytest +from src.preprocess_dataset import Dataset +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df_13(): + df = pd.DataFrame() + df["ems_time"] = 1 + for i in range(5): + df[f"metric_{i}"] = [random.random() for i in range(1000)] + return df + + +@pytest.fixture +def df_14(): + df = pd.DataFrame() + df["ems_time"] = 10 + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(1000)] + return df + + +@pytest.fixture +def df_15(): + df = pd.DataFrame() + df["ems_time"] = [i * 30 * 1e5 for i in range(500)] + [ + i * 30 * 1e5 + 10000 for i in range(500) + ] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df_16(): + df = pd.DataFrame() + df["ems_time"] = [i for i in range(500)] + [i + 10000 for i in range(500)] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +class TestDataset: + @pytest.mark.parametrize( + "df,metric", + [ + ("df_1", metric), + ("df_2", metric), + ("df_3", metric), + ("df_4", metric), + ("df_5", metric), + ("df_6", metric), + ("df_7", metric), + ("df_8", metric), + ("df_9", metric), + ("df_10", metric), + ("df_11", metric), + ("df_12", metric), + ("df_13", metric), + ("df_14", metric), + ("df_15", metric), + ("df_16", metric), + ], + indirect=True, + ) + def test_init(self, df, metric): + preprocessed_dataset = Dataset(df, metric) + assert isinstance(preprocessed_dataset, Dataset) diff --git a/deployment/tft/env b/deployment/tft/env index 1a2ad8d813c13f5a3f67c5d194cf952d1ed8a683..aa4cd5c1df91db819278744698b449d4d4448274 100644 --- a/deployment/tft/env +++ b/deployment/tft/env @@ -1,15 +1,12 @@ AMQ_HOSTNAME=localhost -AMQ_USER=morphemic -AMQ_PASSWORD=morphemic -AMQ_HOST=147.102.17.76 -AMQ_PORT_BROKER=61610 -APP_NAME=default_application +AMQ_USER=admin +AMQ_PASSWORD=admin +AMQ_PORT=61613 +APP_NAME=demo METHOD=tft DATA_PATH=./ -INFLUXDB_HOSTNAME=147.102.17.76 +INFLUXDB_HOSTNAME=localhost INFLUXDB_PORT=8086 INFLUXDB_USERNAME=morphemic INFLUXDB_PASSWORD=password INFLUXDB_DBNAME=morphemic - - diff --git a/deployment/tft/main.py b/deployment/tft/main.py index 361b146d36f09411874b42157a5f166b5c8a0c77..a0e0ca67e624fb3659f464c263b2eb16d6f729bf 100644 --- a/deployment/tft/main.py +++ b/deployment/tft/main.py @@ -7,7 +7,10 @@ import logging import time from datetime import datetime from pytz import timezone -from datetime import datetime +import time +import setproctitle + +# from src.log import logger AMQ_USER = os.environ.get("AMQ_USER", "admin") AMQ_PASSWORD = os.environ.get("AMQ_PASSWORD", "admin") @@ -79,8 +82,9 @@ class Msg(object): def main(): + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}") logging.basicConfig( - filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", + filename=f"logs/{os.environ.get('METHOD', 'model')}.out", level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S", format="START %(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", @@ -112,11 +116,11 @@ def main(): ) # msg1 = Msg() - # msg1.body = '[{"metric": "memory", "level": 3, "publish_rate": 30000}]' + # msg1.body = '[{"metric": "MinimumCores", "level": 3, "publish_rate": 30000}, {"metric": "EstimatedRemainingTimeContext", "level": 3, "publish_rate": 30000}, {"metric": "NotFinishedOnTimeContext", "level": 3, "publish_rate": 30000}, {"metric": "WillFinishTooSoonContext", "level": 3, "publish_rate": 30000}]' # msg2 = Msg() # msg2.body = ( # "{" - # + f'"metrics": ["memory"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' + # + f'"metrics": ["MinimumCores", "EstimatedRemainingTimeContext", "NotFinishedOnTimeContext", "WillFinishTooSoonContext"],"timestamp": {int(time.time())}, "epoch_start": {int(time.time()) + 30}, "number_of_forward_predictions": 8,"prediction_horizon": 30' # + "}" # ) @@ -125,6 +129,7 @@ def main(): # StartForecastingListener(start_conn.conn, START_APP_TOPIC).on_message(msg2) while True: + time.sleep(60) pass diff --git a/deployment/tft/model.yaml b/deployment/tft/model.yaml index ea58a201f41e15e0e329c736cb0cb25db351f9c5..884ba526a83ea79ef5c262711e1a351083458ee1 100644 --- a/deployment/tft/model.yaml +++ b/deployment/tft/model.yaml @@ -24,3 +24,5 @@ save_path: models dataloader_path: dataloader +context_length_ratio: + 12 diff --git a/deployment/tft/predict.py b/deployment/tft/predict.py index dc057f4219a9716bd7da1cbf11ee968111531558..1c40e84319fb743bd0a3a26156b2c4ab5b3d9cfb 100644 --- a/deployment/tft/predict.py +++ b/deployment/tft/predict.py @@ -14,6 +14,8 @@ from src.dataset_maker import CSVData from pytz import timezone import pytz from datetime import datetime +import random +import setproctitle METHOD = os.environ.get("METHOD", "nbeats") START_TOPIC = f"start_forecasting.{METHOD}" @@ -122,24 +124,26 @@ def main(): for metric in predicted_metrics: predictions = None for i in range(number_of_forward_predictions[metric]): - print(int((i + 1) * prediction_points_horizon), "point idx") prediction_msgs, prediction = predict( metric, - prediction_length, + prediction_lengths[metric], extra_data=None, m=i + 1, prediction_hor=prediction_horizon, timestamp=time_0 + (i + 1) * (prediction_horizon // 1000), - predicted_point_idx=int((i + 1) * prediction_points_horizon - 1), + predicted_point_idx=int( + (i + 1) * prediction_points_horizons[metric] - 1 + ), + publish_rate=metrics_info[metric]["publish_rate"], ) if i == (number_of_forward_predictions[metric] - 1): print( f"time_0 difference seconds {start_time + (i + 1) * prediction_horizon // 1000 - int(time.time())}" ) - # if predictions is not None: - # predictions = pd.concat( - # [predictions, prediction], ignore_index=True - # ) + logging.info( + f"time difference in seconds between last preiction and current time {start_time + (i + 1) * prediction_horizon // 1000 - int(time.time())}" + ) + else: predictions = prediction @@ -162,31 +166,27 @@ def main(): influxdb_conn.send_to_influxdb(metric, prediction_msgs) end_time = int(time.time()) - print(f"sleeping {prediction_cycle - (end_time - start_time)} seconds") - time_0 = time_0 + prediction_cycle - time_to_wait = prediction_cycle - (end_time - start_time) + time_to_wait = time_0 - end_time if time_to_wait < 0: - time_to_wait = prediction_cycle - (time_to_wait % prediction_cycle) + time_to_skip = (end_time - time_0) // prediction_cycle + time_0 = time_0 + (time_to_skip + 1) * prediction_cycle + time_to_wait = time_0 - end_time logging.info( f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - time_0 = time_0 + prediction_cycle - + print( + f"Prediction time is too slow (predictions might be delayed) TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) time.sleep(time_to_wait) if __name__ == "__main__": + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}_predict") logging.basicConfig( filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - format="START %(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", ) - logging.Formatter.converter = lambda *args: datetime.now( - tz=timezone(TZ) - ).timetuple() msg = json.loads(sys.argv[1]) metrics_info = { m["metric"]: { @@ -198,15 +198,19 @@ if __name__ == "__main__": time_0 = msg["epoch_start"] prediction_horizon = msg["prediction_horizon"] * 1000 - prediction_points_horizon = msg["prediction_horizon"] * 1000 / msg["publish_rate"] + prediction_points_horizons = { + metric["metric"]: msg["prediction_horizon"] * 1000 // metric["publish_rate"] + for metric in msg["all_metrics"] + } predicted_metrics = set(msg["metrics"]) prediction_cycle = msg["prediction_horizon"] - prediction_length = ( - msg["prediction_horizon"] + prediction_lengths = { + metric["metric"]: msg["prediction_horizon"] * 1000 - // msg["publish_rate"] + // metric["publish_rate"] * msg["number_of_forward_predictions"] - ) + for metric in msg["all_metrics"] + } logging.info(f"Predicted metrics: {predicted_metrics}") number_of_forward_predictions = { metric: msg["number_of_forward_predictions"] for metric in predicted_metrics diff --git a/deployment/tft/requirements.txt b/deployment/tft/requirements.txt index c2e88571a4afef8440f44b1751deaa0e26bb8330..ae603574f3dfba609a0ea37d1f9a7e0a34628c7d 100644 --- a/deployment/tft/requirements.txt +++ b/deployment/tft/requirements.txt @@ -5,4 +5,5 @@ pytorch-forecasting==0.8.4 filelock==3.0.12 influxdb python-slugify -torchmetrics==0.5.0 \ No newline at end of file +torchmetrics==0.5.0 +setproctitle diff --git a/deployment/tft/retrain.py b/deployment/tft/retrain.py index 2a6e654d17e71abb3ec2aac8cdc632634c556200..fcfb77d562d14798bddbde15806ddbb585827688 100644 --- a/deployment/tft/retrain.py +++ b/deployment/tft/retrain.py @@ -8,7 +8,9 @@ from amq_message_python_library import * from src.dataset_maker import CSVData import pytz import time +from pytz import timezone from datetime import datetime +import setproctitle TOPIC_NAME = "training_models" RETRAIN_CYCLE = 10 # minutes @@ -19,12 +21,8 @@ AMQ_PORT_BROKER = os.environ.get("AMQ_PORT_BROKER", "61613") APP_NAME = os.environ.get("APP_NAME", "demo") TZ = os.environ.get("TIME_ZONE", "Europe/Vienna") -logging.basicConfig( - filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", level=logging.INFO -) - -def main(predicted_metrics, prediction_horizon): +def main(predicted_metrics, prediction_horizons): start_conn = morphemic.Connection( AMQ_USER, AMQ_PASSWORD, host=AMQ_HOST, port=AMQ_PORT_BROKER ) @@ -36,7 +34,11 @@ def main(predicted_metrics, prediction_horizon): while True: start_time = int(time.time()) for metric in predicted_metrics: - retrain_msg = train(metric, prediction_horizon) + retrain_msg = train( + metric, + prediction_horizons[metric], + publish_rate=metrics_info[metric]["publish_rate"], + ) if retrain_msg: logging.info( f"Training completed for {metric} metric TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" @@ -46,7 +48,7 @@ def main(predicted_metrics, prediction_horizon): else: print("Not enough data for model training, waiting ...") logging.info( - f"Not enough data for model training, waiting ... TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC: {metric} Not enough data for model training, waiting ... TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) end_time = int(time.time()) @@ -60,6 +62,11 @@ def main(predicted_metrics, prediction_horizon): if __name__ == "__main__": + setproctitle.setproctitle(f"{os.environ.get('METHOD', 'model')}_retrain") + logging.basicConfig( + filename=f"logs/{os.environ.get('METHOD', 'nbeats')}.out", + ) + logging.info(f"Training loop started") msg = json.loads(sys.argv[1]) metrics_info = { @@ -69,14 +76,15 @@ if __name__ == "__main__": } for m in msg["all_metrics"] } - prediction_horizon = ( - msg["prediction_horizon"] + prediction_horizons = { + metric["metric"]: msg["prediction_horizon"] * 1000 - // msg["publish_rate"] + // metric["publish_rate"] * msg["number_of_forward_predictions"] - ) + for metric in msg["all_metrics"] + } predicted_metrics = set(metrics_info.keys()) logging.info( f"Predicted metrics: {predicted_metrics} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - main(predicted_metrics, prediction_horizon) + main(predicted_metrics, prediction_horizons) diff --git a/deployment/tft/src/model_predict.py b/deployment/tft/src/model_predict.py index 2f7f9b5c073da5907ac44d254ce2e864b497eba1..c98c90abcd5f29a4ef24ab510c9727405680bf29 100644 --- a/deployment/tft/src/model_predict.py +++ b/deployment/tft/src/model_predict.py @@ -33,6 +33,7 @@ def predict( prediction_hor=60, timestamp=0, predicted_point_idx=0, + publish_rate=10000, ): with open(yaml_file) as file: params = yaml.load(file, Loader=yaml.FullLoader) @@ -43,7 +44,7 @@ def predict( if not os.path.isfile(model_path): # no pretrained model, unable to predict logging.info( - f"no pretrained model, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC {target_column} no pretrained model, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) print("no pretrained model, unable to predict") return (None, None) @@ -56,12 +57,17 @@ def predict( return (None, None) dataset = pd.read_csv(data_path) - new_ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + new_ts_dataset = Dataset( + dataset, + target_column=target_column, + **params["dataset"], + publish_rate=publish_rate, + ) if new_ts_dataset.dropped_recent_series: # series with recent data was too short logging.info( - f"Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"METRIC {target_column} Not enough fresh data, unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) - print("Not enough fresh data, unable to predict TIME:") + print("METRIC {target_column} Not enough fresh data, unable to predict TIME:") return (None, None) dataset = new_ts_dataset.dataset @@ -69,7 +75,7 @@ def predict( if extra_data is not None: dataset = pd.concat([dataset, extra_data[dataset.columns]], ignore_index=True) - lockfile = params["dataloader_path"] + ".pickle" + lockfile = f"{params['dataloader_path']}_{target_column}.pickle" lock = FileLock(lockfile + ".lock") with lock: @@ -110,11 +116,17 @@ def predict( model_path = os.path.join(params["save_path"], f"{target_column}.pth") if not os.path.isfile(model_path): # no pretrained model, unable to predict - logging.info(f"No pretrained model unable to predict") + logging.info( + f"METRIC {target_column} No pretrained model unable to predict TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) return (None, None) with lock: - model.load_state_dict(torch.load(model_path)) + if os.path.isfile(model_path): + model.load_state_dict(torch.load(model_path)) + logging.info("Model corrupted unable to predict") + else: + return (None, None) prediction_input = ts_dataset.get_from_dataset(future_df) prediction_input = prediction_input.to_dataloader(train=False) diff --git a/deployment/tft/src/model_train.py b/deployment/tft/src/model_train.py index 8a5e139378e9485f1db6cacb9ab5da37f6145a95..92e9312cfd1dffb3ce5dd1a954d72c3a20c84bb6 100644 --- a/deployment/tft/src/model_train.py +++ b/deployment/tft/src/model_train.py @@ -31,13 +31,15 @@ LOSSES_DICT = { } -def train(target_column, prediction_length, yaml_file="model.yaml"): +def train(target_column, prediction_length, yaml_file="model.yaml", publish_rate=10000): torch.manual_seed(12345) with open(yaml_file) as file: params = yaml.load(file, Loader=yaml.FullLoader) params["dataset"]["prediction_length"] = prediction_length - params["dataset"]["context_length"] = prediction_length * 12 + params["dataset"]["context_length"] = ( + prediction_length * params["context_length_ratio"] + ) data_path = os.path.join( os.environ.get("DATA_PATH", "./"), f'{os.environ.get("APP_NAME", "demo")}.csv' @@ -48,15 +50,26 @@ def train(target_column, prediction_length, yaml_file="model.yaml"): dataset = pd.read_csv(data_path) - if dataset.shape[0] < 14 * prediction_length: + if dataset.shape[0] < (params["context_length_ratio"] + 2) * prediction_length: logging.info( - f"dataset len: {dataset.shape[0]}, minimum points required: {14 * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + f"dataset len: {dataset.shape[0]}, minimum points required: {(params['context_length_ratio'] + 2) * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" ) return None - ts_dataset = Dataset(dataset, target_column=target_column, **params["dataset"]) + ts_dataset = Dataset( + dataset, + target_column=target_column, + **params["dataset"], + publish_rate=publish_rate, + ) - lockfile = params["dataloader_path"] + ".pickle" + if ts_dataset.dataset.shape[0] < 1: + logging.info( + f"METRIC: {target_column} Preprocessed dataset len: {ts_dataset.dataset.shape[0]}, minimum points required: {(params['context_length_ratio'] + 2) * prediction_length} TIME: {datetime.now(pytz.timezone(TZ)).strftime('%d/%m/%Y %H:%M:%S')}" + ) + return None + + lockfile = f"{params['dataloader_path']}_{target_column}.pickle" lock = FileLock(lockfile + ".lock") with lock: @@ -108,8 +121,9 @@ def train(target_column, prediction_length, yaml_file="model.yaml"): if os.path.isfile(lockfile): print("downloading weigths") - with lock: - tft.load_state_dict(torch.load(model_path)) + if os.path.isfile(model_path): + with lock: + tft.load_state_dict(torch.load(model_path)) trainer.fit( tft, diff --git a/deployment/tft/src/preprocess_dataset.py b/deployment/tft/src/preprocess_dataset.py index 1880094eb7955e8062377ca74f91928e27f02317..97a6d9a8fa3542f50e263d8b415286839f709ed8 100644 --- a/deployment/tft/src/preprocess_dataset.py +++ b/deployment/tft/src/preprocess_dataset.py @@ -3,10 +3,11 @@ from pytorch_forecasting import TimeSeriesDataSet from pytorch_forecasting.data import NaNLabelEncoder import numpy as np import logging +import time pd.options.mode.chained_assignment = None -"""Script for preparing time series dataset from pythorch-forecasting package +"""Script for preparingself.time_columnseries dataset from pythorch-forecasting package TODO: add checking whether data consists of multiple series, handle nans values""" @@ -15,6 +16,7 @@ class Dataset(object): self, dataset, target_column="value", + time_column="ems_time", tv_unknown_reals=[], known_reals=[], tv_unknown_cat=[], @@ -22,12 +24,14 @@ class Dataset(object): classification=0, context_length=40, prediction_length=5, + publish_rate=10000, ): self.max_missing_values = ( 20 # max consecutive missing values allowed per series ) self.target_column = target_column + self.time_column = time_column self.tv_unknown_cat = tv_unknown_cat self.known_reals = known_reals self.tv_unknown_reals = tv_unknown_reals @@ -35,18 +39,27 @@ class Dataset(object): self.classification = classification self.context_length = context_length self.prediction_length = prediction_length + self.publish_rate = publish_rate self.dataset = dataset - self.check_gap() + self.dropped_recent_series = True # default set to be true + if self.dataset.shape[0] > 0: + self.check_gap() self.n = dataset.shape[0] - self.ts_dataset = self.create_time_series_dataset() + if self.dataset.shape[0] > 0: + self.ts_dataset = self.create_time_series_dataset() def cut_nan_start(self, dataset): dataset.index = range(dataset.shape[0]) first_not_nan_index = dataset[self.target_column].first_valid_index() - return dataset[dataset.index > first_not_nan_index] + if first_not_nan_index == first_not_nan_index: # check is if it;s not np.nan + if first_not_nan_index is not None: + if first_not_nan_index > -1: + return dataset[dataset.index > first_not_nan_index] + else: + return dataset.dropna() def fill_na(self, dataset): - dataset = dataset.replace("None", np.nan) + dataset = dataset.replace(np.inf, np.nan) dataset = dataset.ffill(axis="rows") return dataset @@ -58,18 +71,132 @@ class Dataset(object): for name in self.tv_unknown_cat: dataset[name] = dataset[name].astype(str) - - dataset["series"] = dataset["series"].astype(str) return dataset + def convert_time_to_ms(self): + if self.dataset.shape[0] > 0: + digit_len = len(str(int(self.dataset[self.time_column].values[0]))) + if digit_len >= 13: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(str(x)[:13]) + ) + else: + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(int(str(x)[:digit_len]) * 10 ** (13 - digit_len)) + ) + self.dataset[self.time_column] = self.dataset[self.time_column].apply( + lambda x: int(x // 1e4 * 1e4) + ) + def add_obligatory_columns(self, dataset): - dataset["series"] = 0 - dataset["split"] = "train" n = dataset.shape[0] - dataset["split"][int(n * 0.8) :] = "val" - dataset["time_idx"] = range(n) # TODO check time gaps + dataset["time_idx"] = range(n) # TODO checkself.time_columngaps return dataset + def get_time_difference_current(self): + if self.dataset.shape[0] > 0: + last_timestamp_database = self.dataset[self.time_column].values[-1] + current_time = int(time.time()) + print( + f"Time difference between last timestamp and current time: {current_time * 1000 - last_timestamp_database}" + ) + logging.info( + f"Time difference between last timestamp and current time: {current_time * 1000 - last_timestamp_database}" + ) + + def check_gap(self): + if self.dataset.shape[0] > 0: + self.dataset = self.dataset.groupby(by=[self.time_column]).min() + self.dataset[self.time_column] = self.dataset.index + self.dataset.index = range(self.dataset.shape[0]) + self.convert_time_to_ms() + self.dataset[self.target_column] = pd.to_numeric( + self.dataset[self.target_column], errors="coerce" + ).fillna(np.nan) + self.dataset = self.dataset.replace(np.inf, np.nan) + self.dataset = self.dataset.dropna(subset=[self.target_column]) + if self.dataset.shape[0] > 0: + max_gap = self.dataset[self.time_column].diff().abs().max() + logging.info( + f"Metric: {self.target_column} Maxself.time_columngap in series {max_gap}" + ) + print( + f" Metric: {self.target_column} Maxself.time_columngap in series {max_gap}" + ) + series_freq = ( + (self.dataset[self.time_column]) + .diff() + .fillna(0) + .value_counts() + .index.values[0] + ) + + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency" + ) + if series_freq != self.publish_rate: + logging.info( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + print( + f"Metric: {self.target_column} Detected series with {series_freq} frequency, but the frequency should be: {self.publish_rate}!" + ) + + # check series length + series = np.split( + self.dataset, + *np.where( + self.dataset[self.time_column] + .diff() + .abs() + .fillna(0) + .astype(int) + >= np.abs(self.max_missing_values * self.publish_rate) + ), + ) + logging.info(f"Metric: {self.target_column} {len(series)} series found") + print(f"{len(series)} series found") + preprocessed_series = [] + for i, s in enumerate(series): + s = self.fill_na(s) + s = self.cut_nan_start(s) + s = self.add_obligatory_columns(s) + s["split"] = "train" + s = self.convert_formats(s) + logging.info( + f"Metric: {self.target_column} Found series {i} of length: {s.shape[0]}, required data rows: {self.prediction_length * 2 + self.context_length}" + ) + if s.shape[0] > self.prediction_length * 2 + self.context_length: + s["series"] = i + preprocessed_series.append(s) + if i == len(series) - 1: + logging.info( + f"Metric: {self.target_column} Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" + ) + + logging.info( + f"Metric: {self.target_column} {len(preprocessed_series)} long enough series found" + ) + print(f"{len(preprocessed_series)} long enough series found") + # logging.info(f"") + if preprocessed_series: + self.dataset = pd.concat(preprocessed_series) + if self.dataset["series"].max() != len(series) - 1: + self.dropped_recent_series = True + else: + self.dropped_recent_series = False + else: + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + self.dataset.index = range(self.dataset.shape[0]) + else: + self.dataset = pd.DataFrame() + self.dropped_recent_series = True + self.get_time_difference_current() + def create_time_series_dataset(self): if not self.classification: self.time_varying_unknown_reals = [ @@ -106,48 +233,6 @@ class Dataset(object): ) return ts_dataset - def check_gap(self): - max_gap = self.dataset["time"].diff().abs().max() - logging.info(f"Max time gap in series {max_gap}") - print(f"Max time gap in series {max_gap}") - series_freq = self.dataset["time"].diff().value_counts().index.values[0] - logging.info(f"Detected series with {series_freq} frequency") - print(f"Detected series with {series_freq} frequency") - # check series length - series = np.split( - self.dataset, - *np.where( - self.dataset["time"].diff().abs().fillna(0).astype(int) - >= np.abs(self.max_missing_values * series_freq) - ), - ) - logging.info(f"{len(series)} series found") - print(f"{len(series)} series found") - preprocessed_series = [] - for i, s in enumerate(series): - s = self.fill_na(s) - s = self.cut_nan_start(s) - s = self.add_obligatory_columns(s) - s = self.convert_formats(s) - s["split"] = "train" - if s.shape[0] > self.prediction_length * 2 + self.context_length: - s["series"] = i - preprocessed_series.append(s) - if i == len(series) - 1: - logging.info( - f"Fresh data rows: {s.shape[0]}, required fresh data rows: {self.prediction_length * 2 + self.context_length}" - ) - - logging.info(f"{len(preprocessed_series)} long enough series found") - print(f"{len(preprocessed_series)} long enough series found") - # logging.info(f"") - self.dataset = pd.concat(preprocessed_series) - if self.dataset["series"].max() != len(series) - 1: - self.dropped_recent_series = True - else: - self.dropped_recent_series = False - self.dataset.index = range(self.dataset.shape[0]) - def inherited_dataset(self, split1, split2): df1 = ( self.dataset[lambda x: x.split == split1] diff --git a/deployment/tft/test/model_predict_test.py b/deployment/tft/test/model_predict_test.py new file mode 100644 index 0000000000000000000000000000000000000000..bfecb97ddd0158021f119111a7d1c1d78003640a --- /dev/null +++ b/deployment/tft/test/model_predict_test.py @@ -0,0 +1,180 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_predict import predict +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = predict(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/tft/test/model_train_test.py b/deployment/tft/test/model_train_test.py new file mode 100644 index 0000000000000000000000000000000000000000..00f4ebd62dbe8ba04f7dcf0cf7a61158bd8559a2 --- /dev/null +++ b/deployment/tft/test/model_train_test.py @@ -0,0 +1,221 @@ +import sys + +sys.path.append(".") + +import pytest +from src.model_train import train +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :3 + ] + ] + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(1, 1001)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + print(df) + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :1000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = [ + int(x) + for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[ + :6000 + ] + ] + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(6000) + ] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +@pytest.fixture +def prediction_length(): + return 60 + + +@pytest.mark.parametrize( + "df,metric,prediction_length", + [ + ("df_1", metric, prediction_length), + ("df_2", metric, prediction_length), + ("df_3", metric, prediction_length), + ("df_4", metric, prediction_length), + ("df_5", metric, prediction_length), + ("df_6", metric, prediction_length), + ("df_7", metric, prediction_length), + ("df_8", metric, prediction_length), + ("df_9", metric, prediction_length), + ("df_10", metric, prediction_length), + ("df_11", metric, prediction_length), + ("df_12", metric, prediction_length), + ], + indirect=True, +) +def test_predict(df, metric, prediction_length): + df.to_csv("demo.csv") + output = train(metric, prediction_length) + print(output) + if output: + print("True") + assert True diff --git a/deployment/tft/test/preprocess_dataset_test.py b/deployment/tft/test/preprocess_dataset_test.py new file mode 100644 index 0000000000000000000000000000000000000000..7abb9e0d514d1b44c816e428150621e0a16c3977 --- /dev/null +++ b/deployment/tft/test/preprocess_dataset_test.py @@ -0,0 +1,214 @@ +import sys + +sys.path.append(".") + +import pytest +from src.preprocess_dataset import Dataset +import pandas as pd +import numpy as np +import random + + +@pytest.fixture +def df_1(): + df = pd.DataFrame({"ems_time": [], "metric_0": []}) + return df + + +@pytest.fixture +def df_2(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + df["metric_0"] = np.nan + return df + + +@pytest.fixture +def df_3(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.nan + return df + + +@pytest.fixture +def df_4(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_5(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 3)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = 1 + return df + + +@pytest.fixture +def df_6(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + return df + + +@pytest.fixture +def df_7(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None") + return df + + +@pytest.fixture +def df_8(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf) + return df + + +@pytest.fixture +def df_9(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + np.random.randint(0, df.shape[0] - 1, 990), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_10(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.inf + return df + + +@pytest.fixture +def df_11(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = np.random.rand(1000) + if i % 2 == 0: + df.loc[ + list(range(20, 300)), + f"metric_{i}", + ] = np.nan + return df + + +@pytest.fixture +def df_12(): + df = pd.DataFrame() + df["ems_time"] = np.array(range(0, 1000)) * 1e9 + for i in range(5): + df[f"metric_{i}"] = [ + np.nan if i % 2 == 0 else random.random() for i in range(1000) + ] + return df + + +@pytest.fixture +def df_13(): + df = pd.DataFrame() + df["ems_time"] = 1 + for i in range(5): + df[f"metric_{i}"] = [random.random() for i in range(1000)] + return df + + +@pytest.fixture +def df_14(): + df = pd.DataFrame() + df["ems_time"] = 10 + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(1000)] + return df + + +@pytest.fixture +def df_15(): + df = pd.DataFrame() + df["ems_time"] = [i * 30 * 1e5 for i in range(500)] + [ + i * 30 * 1e5 + 10000 for i in range(500) + ] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df_16(): + df = pd.DataFrame() + df["ems_time"] = [i for i in range(500)] + [i + 10000 for i in range(500)] + for i in range(5): + df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)] + return df + + +@pytest.fixture +def df(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def metric(): + return "metric_0" + + +class TestDataset: + @pytest.mark.parametrize( + "df,metric", + [ + ("df_1", metric), + ("df_2", metric), + ("df_3", metric), + ("df_4", metric), + ("df_5", metric), + ("df_6", metric), + ("df_7", metric), + ("df_8", metric), + ("df_9", metric), + ("df_10", metric), + ("df_11", metric), + ("df_12", metric), + ("df_13", metric), + ("df_14", metric), + ("df_15", metric), + ("df_16", metric), + ], + indirect=True, + ) + def test_init(self, df, metric): + preprocessed_dataset = Dataset(df, metric) + assert isinstance(preprocessed_dataset, Dataset) diff --git a/forecaster-cnn/.DS_Store b/forecaster-cnn/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..019fd405c5a9606650a9e54a682492254109c24f Binary files /dev/null and b/forecaster-cnn/.DS_Store differ diff --git a/forecaster-cnn/Dockerfile b/forecaster-cnn/Dockerfile index af2b49693c6751765429f56188b0b9ddd7a815d2..711682c2272e172268794fc06bf81a4e1ab87da1 100644 --- a/forecaster-cnn/Dockerfile +++ b/forecaster-cnn/Dockerfile @@ -5,7 +5,7 @@ RUN mkdir /app RUN mkdir -p /app/log ADD . /app -RUN pip install /app/datasetlib +RUN pip install /app/lib RUN pip install -r /app/amq_client/requirements.txt WORKDIR /app diff --git a/forecaster-cnn/__pycache__/main.cpython-36.pyc b/forecaster-cnn/__pycache__/main.cpython-36.pyc index 9ac341bdaa2f0da0a6f49a01ef61205cc338de1a..1f2986d120c235983a12815c1eff77d7ff86873a 100644 Binary files a/forecaster-cnn/__pycache__/main.cpython-36.pyc and b/forecaster-cnn/__pycache__/main.cpython-36.pyc differ diff --git a/forecaster-cnn/app.py b/forecaster-cnn/app.py index 30d717d2447818cf11b5bcc5df8fb2f10b10af75..ab5e8d8c8dd8af51e4025a106eddefdfb95242b4 100644 --- a/forecaster-cnn/app.py +++ b/forecaster-cnn/app.py @@ -1,4 +1,5 @@ -import os, json, time, stomp, pickle, logging +import os, json, time, stomp, pickle, logging +import pandas as pd from os import path from datetime import datetime from threading import Thread @@ -22,7 +23,7 @@ ml_model_path = os.environ.get("ML_MODEL_PATH","./models_trained") prediction_tolerance = os.environ.get("PREDICTION_TOLERANCE","85") forecasting_method_name = os.environ.get("FORECASTING_METHOD_NAME","cnn") #///////////////////////////////////////////////////////////////////////////////// -steps = 128 +steps = int(os.environ.get("BACKWARD_STEPS","64")) #///////////////////////////////////////////////////////////////////////////////// influxdb_hostname = os.environ.get("INFLUXDB_HOSTNAME","147.102.17.76") #persistent_storage_hostname influxdb_port = int(os.environ.get("INFLUXDB_PORT","8086")) @@ -33,7 +34,9 @@ influxdb_org = os.environ.get("INFLUXDB_ORG","morphemic") start_forecasting_queue = os.environ.get("START_FORECASTING","/topic/start_forecasting.cnn") metric_to_predict_queue = os.environ.get("METRIC_TO_PREDICT","/topic/metrics_to_predict") #////////////////////////////////////////////////////////////////////////////////// -_time_column_name = 'time' +_time_column_name = os.environ.get("TIME_COLUMN","time") +_column_to_remove_str = os.environ.get("COLUMNS_TO_REMOVE","ems_time,level,name,application") +_column_to_remove = _column_to_remove_str.split(",") _new_epoch = False #logging @@ -149,6 +152,7 @@ class Forecaster(Thread): self.publisher = publisher self.target = target self.application = application + self.tolerance_epoch_start = 1 self.features_dict = {} self.stop = False super(Forecaster,self).__init__() @@ -159,23 +163,35 @@ class Forecaster(Thread): def setStop(self): self.stop = True + def computeSleepingTime(self): + now = int(time.time()) + return (now - self.epoch_start) // self.prediction_horizon, (now - self.epoch_start) % self.prediction_horizon + def run(self): print("Forecaster started for target metric {0} ".format(self.target)) logging.info("Forecaster started for target metric {0} ".format(self.target)) while True: if int(time.time()) < self.epoch_start: + diff = self.epoch_start - int(time.time()) + print("Prediction starts in {0} sec".format(diff)) time.sleep(1) continue + if int(time.time()) > self.epoch_start + self.tolerance_epoch_start: + n, sleeping_time = self.computeSleepingTime() + print("{0} sec sleeping time before publishing".format(sleeping_time)) + time.sleep(sleeping_time) + self.epoch_start += self.prediction_horizon * n if self.stop: print("Forecaster stops after having receiving new epoch start") logging.info("Forecaster stops after having receiving new epoch start") break self.features = self.manager.getFeatureInput(self.application) if len(self.features) == 0: + print("Cannot proceed, number of feature is 0") time.sleep(self.prediction_horizon) self.epoch_start += self.prediction_horizon continue - predictor = Predictor(self.application, self.target, steps, self.features) + predictor = Predictor(self.application,_time_column_name, _column_to_remove, self.target, steps, self.features) response = predictor.predict() index = 1 for v, prob,interval in response: @@ -229,8 +245,29 @@ class ForecastingManager(): else: return None + def loadDataset(self, url_dataset): + try: + #return pd.read_csv(url_dataset, low_memory=False, on_bad_lines='skip') + return pd.read_csv(url_dataset, low_memory=False) + except Exception as e: + print("Could not load the dataset") + print(e) + return pd.DataFrame() + def getFeatureInput(self, application): - return [{'time':1602538628,'served_request':2110,'request_rate':426,'avgResponseTime':673.574009325832,'performance':0.626508734240462,'cpu_usage':31.6,'memory':71798784}] + #update dataset and return the last entry + response = self.prepareDataset(application) + if 'url' in response: + print("Dataset updated ...") + logging.info("Dataset updated ...") + data = self.loadDataset(response['url']) + return [data.to_dict('records')[len(data)-1]] #returning the last entry + print("Could not update datasets") + return [] + #test_url = "/home/jean-didier/Projects/morphemic-preprocessor/forecaster-cnn/datasets/historicValues.csv" + #data = self.loadDataset(test_url) + #return [data.to_dict('records')[len(data)-1]] #returning the last entry + def getModelFromMetric(self, metric): for key, model in self.applications.items(): @@ -241,11 +278,6 @@ class ForecastingManager(): def startForecasting(self, data): print("Start forecasting methods") logging.info("Start forecasting methods") - _json = None - metrics = None - epoch_start = None - number_of_forward_forecasting = None - prediction_horizon = None _json = json.loads(data) metrics = _json['metrics'] @@ -261,18 +293,21 @@ class ForecastingManager(): model.setPredictionHorizon(prediction_horizon) model.setEpochStart(epoch_start) self.trainModel(model) - - def simulateForcasting(self): - data = {"metrics":["avgResponseTime","memory"],"timestamp":1623242615043,"epoch_start":1623242815041,"number_of_forward_predictions":8,"prediction_horizon":30} + #data = {"metrics":["avgResponseTime","memory"],"timestamp":1623242615043,"epoch_start":1623242815041,"number_of_forward_predictions":8,"prediction_horizon":30} + data = {"metrics":["AvgResponseTime"],"timestamp":int(time.time())+20,"epoch_start":int(time.time())+20,"number_of_forward_predictions":8,"prediction_horizon":30} self.startForecasting(json.dumps(data)) def simulateMetricToPredict(self): + data = [ + {"refersTo": "default_application", "level":1, "metric": "AvgResponseTime", "publish_rate":3000} + ] + """ data = [ {"refersTo": "default_application", "level":1, "metric": "avgResponseTime", "publish_rate":3000}, {"refersTo": "default_application", "level":1, "metric": "memory", "publish_rate":3000} - ] + ]""" self.metricToPredict(json.dumps(data)) def metricToPredict(self, data): @@ -343,15 +378,23 @@ class ForecastingManager(): application = model.getApplication() response = self.prepareDataset(application) if not 'url' in response: - print("Cannot create dataset") - logging.info("Cannot create dataset") + print("Cannot create dataset, not data available") + logging.info("Cannot create dataset, not data available") return None + #test_url = "/home/jean-didier/Projects/morphemic-preprocessor/forecaster-cnn/datasets/historicValues.csv" model.setDatasetUrl(response['url']) - #model.setDatasetUrl("/home/jean-didier/Projects/morphemic/Morphemic_TimeSeries/datasets/ds.csv") - model.setDatasetCreationTime(time.time()) + #model.setDatasetUrl(test_url) + data = self.loadDataset(response['url']) + #data = self.loadDataset(test_url) + if len(data) <= steps: + print("Not enough data to train the model, the CNN requires at least {0} samples, the dataset has {1} rows".format(steps, len(data))) + logging.info("Not enough data to train the model, the CNN requires at least {0} samples, the dataset has {1} rows".format(steps, len(data))) + return False + #model.setDatasetUrl(test_url) + #model.setDatasetCreationTime(time.time()) #start training ml (application, url, metrics) metric = model.getMetric() - trainer = Train(application, metric, _time_column_name, model.getDatasetUrl(), model.getNumberOfForwardPredictions(), steps, model.getPredictionHorizon()) + trainer = Train(application, metric, _time_column_name, _column_to_remove, model.getDatasetUrl(), model.getNumberOfForwardPredictions(), steps, model.getPredictionHorizon()) model.setMLModelStatus('started') result = trainer.prepareTraining() if len(result) > 0: @@ -367,16 +410,19 @@ class ForecastingManager(): worker = Forecaster(self, model.getPredictionHorizon(), model.getEpochStart(), self.publisher, model.getMetric(), model.getApplication()) worker.start() self.workers.append(worker) + else: + model.setMLModelStatus('NotExist') def publishTrainingCompleted(self, model): data = model.getTrainingData() for tr in data: del tr['x_train'] del tr['y_train'] - message = {"metrics": ["cpu_usage"], "forecasting_method":"cnn","timestamp": int(time.time())} - #print(data) - self.publisher.setParameters(message, "training_models") - self.publisher.send() + message = {"metrics": [tr['target']], "forecasting_method":"cnn","timestamp": int(time.time())} + print("Training information -> ", message) + logging.info(message) + self.publisher.setParameters(message, "training_models") + self.publisher.send() def predict(self,application,model, target, features): predictor = Predictor(application, target, steps, features) @@ -405,7 +451,7 @@ class ForecastingManager(): #self.simulateMetricToPredict() #time.sleep(10) #self.simulateForcasting() - #time.sleep(100) + #time.sleep(2) #self.simulateForcasting() while True: for key, model in self.applications.items(): diff --git a/forecaster-cnn/datasetlib b/forecaster-cnn/datasetlib deleted file mode 160000 index c9c6d3c954b57f9dd3b5109514bd033da00c95db..0000000000000000000000000000000000000000 --- a/forecaster-cnn/datasetlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c9c6d3c954b57f9dd3b5109514bd033da00c95db diff --git a/forecaster-cnn/lib/CHANGES.txt b/forecaster-cnn/lib/CHANGES.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/forecaster-cnn/lib/Dataset_Maker.egg-info/PKG-INFO b/forecaster-cnn/lib/Dataset_Maker.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..bdce7448c8739cdb539ea2f34b4001ce34b41a65 --- /dev/null +++ b/forecaster-cnn/lib/Dataset_Maker.egg-info/PKG-INFO @@ -0,0 +1,79 @@ +Metadata-Version: 1.0 +Name: Dataset-Maker +Version: 0.0.1 +Summary: Python package for creating a dataset using InfluxDB data points +Home-page: http://git.dac.ds.unipi.gr/morphemic/datasetmaker +Author: Jean-Didier Totow +Author-email: totow@unipi.gr +License: LICENSE.txt +Description: 1. Generality + + Dataset maker is morphemic python library for + building dataset from data points registered into InfluxDB. + Dataset maker receives the name of an application, the start time + and the tolerance interval. More details are provided below. + + 2. InfluxDB format + + Data points in InfluxDB should have the following format for being used + correctly by the dataset maker: + + measurement : "application_name" #mandatory + timestamp : timestamp #optional + fields : dictionnary containing metric exposed by the given application + cpu_usage, memory_consumption, response_time, http_latency + tags : dictionnary of metrics related information + + The JSON describing the above information is the following: + + Ex.: + {"measurement": "application_name", + "timestamp": 155655476.453, + "fields": { + "cpu_usage": 40, + "memory_consumption": 67.9, + "response_time": 28, + "http_latency": 12 + }, + "tags": { + "core": 2 #cpu_usage of 40% is the usage of the cpu core number 2 + } + } + + If data points are presented as the above format, the dataset maker will output + a csv (application_name.csv) file with the following schema: + time, cpu_usage, memory_consumption, response_time, http_latency, core + + 3. Usage + + + Warming : make sure the above variables exist before importing dataset make library + + from morphemic.dataset import DatasetMaker + + data_maker = DatasetMaker(application, start, configs) + response = data_maker.make() + + application, string containing the application name + start, when to start building the dataset + Ex.: '10m' , build dataset containg data point stored the 10 last minute + Ex.: '3h', three hours + Ex.: '4d', four days + leave empty or set to None if you wish all data points stored in your InfluxDB + configs is dictionnary containg parameters + + { + "hostname": hostname or IP of InfluxDB + "port": port of InfluxDB + "username": InfluxDB username + "password": password of the above user + "dbname": database name + "path_dataset": path where the dataset will be saved + } + + the response contains + {'status': True,'url': url, 'application': application_name, 'features': features} + + or if an error occured + {'status': False,'message': "reason of the error"} +Platform: UNKNOWN diff --git a/forecaster-cnn/lib/Dataset_Maker.egg-info/SOURCES.txt b/forecaster-cnn/lib/Dataset_Maker.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..80dd9ca282142debe5336789f0ffa166c3f8e614 --- /dev/null +++ b/forecaster-cnn/lib/Dataset_Maker.egg-info/SOURCES.txt @@ -0,0 +1,9 @@ +README.txt +setup.py +Dataset_Maker.egg-info/PKG-INFO +Dataset_Maker.egg-info/SOURCES.txt +Dataset_Maker.egg-info/dependency_links.txt +Dataset_Maker.egg-info/requires.txt +Dataset_Maker.egg-info/top_level.txt +morphemic/__init__.py +morphemic/dataset/__init__.py \ No newline at end of file diff --git a/forecaster-cnn/lib/Dataset_Maker.egg-info/dependency_links.txt b/forecaster-cnn/lib/Dataset_Maker.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/forecaster-cnn/lib/Dataset_Maker.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/forecaster-cnn/lib/Dataset_Maker.egg-info/requires.txt b/forecaster-cnn/lib/Dataset_Maker.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..e20f7f050a243a245d049f39f136118021deedfd --- /dev/null +++ b/forecaster-cnn/lib/Dataset_Maker.egg-info/requires.txt @@ -0,0 +1,2 @@ +pandas +influxdb diff --git a/forecaster-cnn/lib/Dataset_Maker.egg-info/top_level.txt b/forecaster-cnn/lib/Dataset_Maker.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..047ceb8359adfa72a917b51af6e230a7b5c3e218 --- /dev/null +++ b/forecaster-cnn/lib/Dataset_Maker.egg-info/top_level.txt @@ -0,0 +1 @@ +morphemic diff --git a/forecaster-cnn/lib/LICENCE.txt b/forecaster-cnn/lib/LICENCE.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d70f41f07748e787bc3e8c0540add418f055ad4 --- /dev/null +++ b/forecaster-cnn/lib/LICENCE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2021 unipi.gr + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/forecaster-cnn/lib/README.txt b/forecaster-cnn/lib/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..941537098178b8bc0fe4824a56b34313502a7f52 --- /dev/null +++ b/forecaster-cnn/lib/README.txt @@ -0,0 +1,70 @@ +1. Generality + +Dataset maker is morphemic python library for +building dataset from data points registered into InfluxDB. +Dataset maker receives the name of an application, the start time +and the tolerance interval. More details are provided below. + +2. InfluxDB format + +Data points in InfluxDB should have the following format for being used +correctly by the dataset maker: + +measurement : "application_name" #mandatory +timestamp : timestamp #optional +fields : dictionnary containing metric exposed by the given application + cpu_usage, memory_consumption, response_time, http_latency +tags : dictionnary of metrics related information + +The JSON describing the above information is the following: + +Ex.: + {"measurement": "application_name", + "timestamp": 155655476.453, + "fields": { + "cpu_usage": 40, + "memory_consumption": 67.9, + "response_time": 28, + "http_latency": 12 + }, + "tags": { + "core": 2 #cpu_usage of 40% is the usage of the cpu core number 2 + } + } + +If data points are presented as the above format, the dataset maker will output +a csv (application_name.csv) file with the following schema: +time, cpu_usage, memory_consumption, response_time, http_latency, core + +3. Usage + + +Warming : make sure the above variables exist before importing dataset make library + +from morphemic.dataset import DatasetMaker + +data_maker = DatasetMaker(application, start, configs) +response = data_maker.make() + +application, string containing the application name +start, when to start building the dataset +Ex.: '10m' , build dataset containg data point stored the 10 last minute +Ex.: '3h', three hours +Ex.: '4d', four days +leave empty or set to None if you wish all data points stored in your InfluxDB +configs is dictionnary containg parameters + +{ + "hostname": hostname or IP of InfluxDB + "port": port of InfluxDB + "username": InfluxDB username + "password": password of the above user + "dbname": database name + "path_dataset": path where the dataset will be saved +} + +the response contains +{'status': True,'url': url, 'application': application_name, 'features': features} + +or if an error occured +{'status': False,'message': "reason of the error"} \ No newline at end of file diff --git a/forecaster-cnn/lib/datasetmaker.egg-info/PKG-INFO b/forecaster-cnn/lib/datasetmaker.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..384bb8a358d1d67494d892b812245da9eec5376e --- /dev/null +++ b/forecaster-cnn/lib/datasetmaker.egg-info/PKG-INFO @@ -0,0 +1,79 @@ +Metadata-Version: 1.0 +Name: datasetmaker +Version: 0.0.1 +Summary: Python package for creating a dataset using InfluxDB data points +Home-page: http://git.dac.ds.unipi.gr/morphemic/datasetmaker +Author: Jean-Didier Totow +Author-email: totow@unipi.gr +License: LICENSE.txt +Description: 1. Generality + + Dataset maker is morphemic python library for + building dataset from data points registered into InfluxDB. + Dataset maker receives the name of an application, the start time + and the tolerance interval. More details are provided below. + + 2. InfluxDB format + + Data points in InfluxDB should have the following format for being used + correctly by the dataset maker: + + measurement : "application_name" #mandatory + timestamp : timestamp #optional + fields : dictionnary containing metric exposed by the given application + cpu_usage, memory_consumption, response_time, http_latency + tags : dictionnary of metrics related information + + The JSON describing the above information is the following: + + Ex.: + {"measurement": "application_name", + "timestamp": 155655476.453, + "fields": { + "cpu_usage": 40, + "memory_consumption": 67.9, + "response_time": 28, + "http_latency": 12 + }, + "tags": { + "core": 2 #cpu_usage of 40% is the usage of the cpu core number 2 + } + } + + If data points are presented as the above format, the dataset maker will output + a csv (application_name.csv) file with the following schema: + time, cpu_usage, memory_consumption, response_time, http_latency, core + + 3. Usage + + + Warming : make sure the above variables exist before importing dataset make library + + from morphemic.dataset import DatasetMaker + + data_maker = DatasetMaker(application, start, configs) + response = data_maker.make() + + application, string containing the application name + start, when to start building the dataset + Ex.: '10m' , build dataset containg data point stored the 10 last minute + Ex.: '3h', three hours + Ex.: '4d', four days + leave empty or set to None if you wish all data points stored in your InfluxDB + configs is dictionnary containg parameters + + { + "hostname": hostname or IP of InfluxDB + "port": port of InfluxDB + "username": InfluxDB username + "password": password of the above user + "dbname": database name + "path_dataset": path where the dataset will be saved + } + + the response contains + {'status': True,'url': url, 'application': application_name, 'features': features} + + or if an error occured + {'status': False,'message': "reason of the error"} +Platform: UNKNOWN diff --git a/forecaster-cnn/lib/datasetmaker.egg-info/SOURCES.txt b/forecaster-cnn/lib/datasetmaker.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..3d8cbf44478cca30eab5b651db9f18ab1b9348fb --- /dev/null +++ b/forecaster-cnn/lib/datasetmaker.egg-info/SOURCES.txt @@ -0,0 +1,9 @@ +README.txt +setup.py +datasetmaker.egg-info/PKG-INFO +datasetmaker.egg-info/SOURCES.txt +datasetmaker.egg-info/dependency_links.txt +datasetmaker.egg-info/requires.txt +datasetmaker.egg-info/top_level.txt +morphemic/__init__.py +morphemic/dataset/__init__.py \ No newline at end of file diff --git a/forecaster-cnn/lib/datasetmaker.egg-info/dependency_links.txt b/forecaster-cnn/lib/datasetmaker.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/forecaster-cnn/lib/datasetmaker.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/forecaster-cnn/lib/datasetmaker.egg-info/requires.txt b/forecaster-cnn/lib/datasetmaker.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..e20f7f050a243a245d049f39f136118021deedfd --- /dev/null +++ b/forecaster-cnn/lib/datasetmaker.egg-info/requires.txt @@ -0,0 +1,2 @@ +pandas +influxdb diff --git a/forecaster-cnn/lib/datasetmaker.egg-info/top_level.txt b/forecaster-cnn/lib/datasetmaker.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..047ceb8359adfa72a917b51af6e230a7b5c3e218 --- /dev/null +++ b/forecaster-cnn/lib/datasetmaker.egg-info/top_level.txt @@ -0,0 +1 @@ +morphemic diff --git a/forecaster-cnn/lib/morphemic/__init__.py b/forecaster-cnn/lib/morphemic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-36.pyc b/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4a41f6352cbab9846bba6c984fe6bbaef058d5e Binary files /dev/null and b/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-36.pyc differ diff --git a/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-37.pyc b/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f27e8c483244bafadd7420e9bb4b52f19d60d5eb Binary files /dev/null and b/forecaster-cnn/lib/morphemic/__pycache__/__init__.cpython-37.pyc differ diff --git a/forecaster-cnn/lib/morphemic/dataset/__init__.py b/forecaster-cnn/lib/morphemic/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..db2c96f242c1efdde2a86de5515b6455ce02b0f7 --- /dev/null +++ b/forecaster-cnn/lib/morphemic/dataset/__init__.py @@ -0,0 +1,154 @@ +import os, json, time +from influxdb import InfluxDBClient +import pandas as pd +from datetime import datetime + +url_path_dataset = None + +class Row(): + def __init__(self, features,metricsname): + self.features = features + if "time" in self.features: + time_str = self.features["time"] + _obj = datetime.strptime(time_str,'%Y-%m-%dT%H:%M:%S.%fZ') + self.features["time"] = int(_obj.timestamp()) + if 'application' in metricsname: + metricsname.remove('application') + for field_name in metricsname: + if not field_name in self.features: + self.features[field_name] = None + + def getTime(self): + if "time" in self.features: + return self.features["time"] + if "timestamp" in self.features: + return self.features["timestamp"] + return None + + def makeCsvRow(self): + if "application" in self.features: + del self.features["application"] + result = '' + for key, _value in self.features.items(): + result += "{0},".format(_value) + return result[:-1] + "\n" + +class Dataset(): + def __init__(self): + self.rows = {} + self.size = 0 + def addRow(self,row): + self.rows[row.getTime()] = row + self.size +=1 + def reset(self): + self.rows = {} + self.size = 0 + print("Dataset reset") + def getSize(self): + return self.size + def sortRows(self): + return sorted(list(self.rows.values()), key=lambda x: x.getTime(), reverse=True) + def getRows(self): + return list(self.rows.values()) + def getRow(self,_time, tolerance): + for i in range(tolerance): + if int(_time + i) in self.rows: + return self.rows[int(_time+i)] + return None + def save(self,metricnames,application_name): + if "application" in metricnames: + metricnames.remove("application") + dataset_content = '' + for metric in metricnames: + dataset_content += "{0},".format(metric) + dataset_content = dataset_content[:-1] + "\n" + for row in list(self.rows.values()): + dataset_content += row.makeCsvRow() + _file = open(url_path_dataset + "{0}.csv".format(application_name),'w') + _file.write(dataset_content) + _file.close() + return url_path_dataset + "{0}.csv".format(application_name) + +class DatasetMaker(): + def __init__(self, application, start, configs): + self.application = application + self.start_filter = start + self.influxdb = InfluxDBClient(host=configs['hostname'], port=configs['port'], username=configs['username'], password=configs['password'], database=configs['dbname']) + self.dataset = Dataset() + self.tolerance = 5 + global url_path_dataset + url_path_dataset = configs['path_dataset'] + if url_path_dataset[-1] != "/": + url_path_dataset += "/" + + def getIndex(self, columns, name): + return columns.index(name) + + def makeRow(self,columns, values): + row = {} + index = 0 + for column in columns: + row[column] = values[index] + index +=1 + return row + + def prepareResultSet(self, result_set): + result = [] + columns = result_set["series"][0]["columns"] + series_values = result_set["series"][0]["values"] + index = 0 + for _values in series_values: + row = self.makeRow(columns,_values) + result.append(row) + return result + + def make(self): + try: + self.influxdb.ping() + except Exception as e: + print("Could not establish connexion with InfluxDB, please verify connexion parameters") + print(e) + return {"message": "Could not establish connexion with InfluxDB, please verify connexion parameters"} + if self.getData() == None: + return {"message":"No data found"} + + metricnames, _data = self.getData() + for _row in _data: + row = Row(_row,metricnames) + self.dataset.addRow(row) + + print("Rows construction completed") + print("{0} rows found".format(self.dataset.getSize())) + #self.dataset.sortRows() + url = self.dataset.save(metricnames,self.application) + features = self.getFeatures(url) + if features == None: + return {'status': False, 'message': 'An error occured while building dataset'} + return {'status': True,'url': url, 'application': self.application, 'features': features} + + def getFeatures(self, url): + try: + df = pd.read_csv(url) + return df.columns.to_list() + except Exception as e: + print("Cannot extract data feature list") + return None + + def extractMeasurement(self, _json): + return _json["series"][0]["columns"] + + def getData(self): + query = None + try: + if self.start_filter != None and self.start_filter != "": + query = "SELECT * FROM " + self.application +" WHERE time > now() - "+ self.start_filter + else: + query = "SELECT * FROM " + self.application + result_set = self.influxdb.query(query=query) + series = self.extractMeasurement(result_set.raw) + #self.influxdb.close() #closing connexion + return [series, self.prepareResultSet(result_set.raw)] + except Exception as e: + print("Could not collect query data points") + print(e) + return None diff --git a/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-36.pyc b/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9894da3b51ac629a70ae88ffcbbca133e8be4732 Binary files /dev/null and b/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-36.pyc differ diff --git a/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-37.pyc b/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd046025c6568187cf364fe9f4aabeebc1785b0b Binary files /dev/null and b/forecaster-cnn/lib/morphemic/dataset/__pycache__/__init__.cpython-37.pyc differ diff --git a/forecaster-cnn/lib/setup.py b/forecaster-cnn/lib/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..7e0ad6cc5b4a81ee0492517222212539681a1176 --- /dev/null +++ b/forecaster-cnn/lib/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup + +setup( + name = 'datasetmaker', + version = '0.0.1', + author = 'Jean-Didier Totow', + author_email = 'totow@unipi.gr', + packages = ['morphemic', 'morphemic.dataset'], + scripts = [], + url='http://git.dac.ds.unipi.gr/morphemic/datasetmaker', + license='LICENSE.txt', + description='Python package for creating a dataset using InfluxDB data points', + long_description=open('README.txt').read(), + install_requires=[ + "pandas", + "influxdb", + ], +) diff --git a/forecaster-cnn/main.py b/forecaster-cnn/main.py index 854d4191c11c4ceb44912aa8e515614d5112b059..75dfcf84c879a6f5c09d2ec1100e02445f02ab85 100755 --- a/forecaster-cnn/main.py +++ b/forecaster-cnn/main.py @@ -1,14 +1,15 @@ from pre_processing.preprocessing import load_data, percent_missing, datetime_conversion from pre_processing.preprocessing import important_data, resample, resample_median, missing_data_handling, resample_quantile from pre_processing.Data_transformation import reshape_data_single_lag, series_to_supervised, \ - prediction_and_score_for_CNN + prediction_and_score_for_CNN, missing_values_imputer from models.ML_models import LSTM_model, CNN_model, CNN_model_multi_steps from plots.plots import plot_train_test_loss from pre_processing.Data_transformation import predictions_and_scores, Min_max_scal, Min_max_scal_inverse -from pre_processing.Data_transformation import split_sequences, split_sequences_multi_steps, prediction_interval +from pre_processing.Data_transformation import split_sequences, split_sequences_multi_steps, prediction_interval, split_sequences_univariate #import matplotlib.pyplot as plt import pandas as pd import numpy as np +from numpy import hstack import os, time, pickle, json, psutil from os import path from tensorflow import keras @@ -19,65 +20,20 @@ ml_model_path = os.environ.get("ML_MODEL_PATH","./models_trained") ml_model = os.environ.get("ML_MODEL_PATH","./models") #/////////////////////////////////////////////////////////////////////////////// -#metrics = ['performance','request_rate', 'cpu_usage', 'memory','served_request'] -""" -metrics = ['cpu_usage', 'memory', 'request_rate',] - -data = load_data() - -data = data.round(decimals=2) - -data = missing_data_handling(data, rolling_mean=True) - -percent_missing(data) - -data = datetime_conversion(data, 'time') -print(data) -data = important_data(data, metrics) -print(data) -data = resample(data) -print(data) -data = Min_max_scal(data) -print(data) -#data = series_to_supervised(data, 24, 1) -#print(data) -X_train, y_train, X_test,y_test = split_sequences(data, n_steps=3) -print(X_train.shape) -print(X_test.shape) -print(y_train.shape) -print(y_test.shape) -# summarize the data -#for i in range(len(X_train)): -# print(X_train[i], y_train[i]) - -train_X, train_y, test_X, test_y, val_X, val_y = reshape_data_single_lag(data, 0.6, 0.2, 0.2 ) - -model = LSTM_model(train_X, train_y, test_X, test_y) - -model.summary() - -plot_train_test_loss(model) - -predictions_and_scores(model, test_X, test_y) - -model = CNN_model(n_steps=3, n_features=2, X=X_train, y=y_train, val_x=X_test, val_y=y_test) -plot_train_test_loss(model) -prediction_and_score_for_CNN(n_steps = 3,n_features=2, x_input=X_test, model=model,test_y=y_test) -model.summary() -""" - class Predictor(): - def __init__(self, application, target, steps, features): + def __init__(self, application, _time_column, _column_to_remove, target, steps, features): self.application = application self.target = target self.steps = steps + self.time_column = _time_column + self.column_to_remove = _column_to_remove self.feature_dict = features self.applications_model = None self.loadModel() def loadDataset(self, url): try: - return pd.read_csv(url, low_memory=False, error_bad_lines=False) + return missing_values_imputer(pd.read_csv(url, low_memory=False)) except Exception as e: print("Could not load the dataset") print(e) @@ -103,11 +59,11 @@ class Predictor(): #data preparation data = self.loadDataset(model_metadata["dataset_url"]) #if data.empty: - # return {'status': False, 'message': 'dataset empty', 'data': None} - for _dict in self.feature_dict: - data = data.append(_dict, ignore_index=True) + #return {'status': False, 'message': 'dataset empty', 'data': None} + #for _dict in self.feature_dict: + # data = data.append(_dict, ignore_index=True) #data['memory'] = data['memory']/1000000 - data = data.drop(columns=[self.target, 'time']) + data = data.drop(columns=[self.time_column]) #data = data.round(decimals=2) #data = missing_data_handling(data, rolling_mean=True) #percent_missing(data) @@ -115,12 +71,17 @@ class Predictor(): #important_features.remove(self.target) #data = important_data(data, important_features) important_feature = list(self.feature_dict[0].keys()) - important_feature.remove(self.target) - important_feature.remove('time') + #important_feature.remove(self.target) + important_feature.remove(self.time_column) + for col in self.column_to_remove: + if col in important_feature: + important_feature.remove(col) + + data = important_data(data, important_feature) #print(important_feature) - #data, scaler = Min_max_scal(data) + data, scaler = Min_max_scal(data) #print(data) - data = data.values + #data = data.values new_sample = data[-self.steps:] #new_sample = np.array(self.feature_dict) new_sample = new_sample.reshape((1, self.steps, len(important_feature))) @@ -131,7 +92,7 @@ class Predictor(): predictor = keras.models.load_model(path) #[lower_bound, y_predict, upper_bound] = prediction_interval(predictor, x_train, y_train, new_sample, alpha=0.05) y_predict = predictor.predict(new_sample, verbose=0) - + y_predict = Min_max_scal_inverse(scaler, y_predict) y_predict = y_predict[0].astype('float') returned_data = [] for v in y_predict: @@ -161,7 +122,7 @@ class MorphemicModel(): self.features = None self.steps = None self.epoch_start = None - self.number_of_forward_predictions = None + self.number_of_forward_predictions = 8 def getLowestPredictionProbability(self): return self.lowest_prediction_probability @@ -261,11 +222,12 @@ class Model(): self.dataset_characteristics = properties class Train(): - def __init__(self, application, metric, _time_column_name, url_dataset, number_of_forward_forecasting, steps, prediction_horizon): + def __init__(self, application, metric, _time_column_name, _column_to_remove, url_dataset, number_of_forward_forecasting, steps, prediction_horizon): self.application = application self.metric = metric self.features = None self.time_column_name = _time_column_name + self.column_to_remove = _column_to_remove self.applications_model = {} self.url_dataset = url_dataset self.number_of_foreward_forecating = number_of_forward_forecasting @@ -287,7 +249,7 @@ class Train(): def loadDataset(self): try: - return pd.read_csv(self.url_dataset, low_memory=False, error_bad_lines=False) + return missing_values_imputer(pd.read_csv(self.url_dataset, low_memory=False)) except Exception as e: print("Could not load the dataset") print(e) @@ -315,6 +277,7 @@ class Train(): model.setUrlDataset(self.url_dataset) key = None if not response["status"]: + print(response['message']) model.setStatus("Failed") else: key = self.makeKey(self.application, self.metric) @@ -331,8 +294,10 @@ class Train(): def train(self, target): print("Target metric = {0}".format(target)) - print("Sampling rate : {0}".format(self.prediction_horizon)) + print("Sampling rate : {0} Seconde".format(self.prediction_horizon)) data = self.loadDataset() + if len(str(data[self.time_column_name][0])) == 19: + data[self.time_column_name] = data[self.time_column_name]/1000000000 #data['memory'] = data['memory']/1000000 if len(data) == 0: return {"status": False, "message": "An error occured while loading the dataset", "data": None} @@ -341,28 +306,38 @@ class Train(): if not target in self.features: return {"status": False, "message": "target not in features list", "data": None} + data[target+"_target"] = data[target] #we duplicate the targeted column if not self.time_column_name in self.features: return {"status": False, "message": "time field ({0}) not found in dataset".format(self.time_column_name), "data": None} if not self.metric in self.features: - return {"status": False, "message": "Metric field ({0}) not found in dataset".format(metric), "data": None} + return {"status": False, "message": "Metric field ({0}) not found in dataset".format(self.metric), "data": None} - self.features.remove(target) - self.features.append(target) + #self.features.remove(target) + self.features.append(target+"_target") self.features.remove(self.time_column_name) + for col in self.column_to_remove: + if col in self.features: + self.features.remove(col) ########### _start = time.time() data = data.round(decimals=2) data = missing_data_handling(data, rolling_mean=True) - percent_missing(data) + missing_data = percent_missing(data) + print("---Missing resume---") + print(missing_data) + print("---End resume---") data = datetime_conversion(data, self.time_column_name) data = important_data(data, self.features) sampling_rate = '{0}S'.format(self.prediction_horizon) - data = resample_quantile(data, sampling_rate) - print(data) + data = resample(data, sampling_rate) + if len(data) * 0.33 < self.steps: + return {"status": False, "message": "No enough data after sampling", "data": None} #this will be removed data, scaler = Min_max_scal(data) + data = data[~np.isnan(data).any(axis=1)] #X_train, y_train, X_test,y_test = split_sequences(data, n_steps=steps) + #X_train, y_train, X_test,y_test = split_sequences_univariate(data, n_steps=self.number_of_foreward_forecating) X_train, y_train, X_test,y_test = split_sequences_multi_steps(data, n_steps_in=self.steps, n_steps_out=self.number_of_foreward_forecating) model = CNN_model_multi_steps(n_steps=self.steps, n_features=len(self.features)-1, X=X_train, y=y_train, val_x=X_test, val_y=y_test, n_steps_out=self.number_of_foreward_forecating) prediction_and_score_for_CNN(n_steps = self.steps,n_features=len(self.features)-1, x_input=X_test, model=model,test_y=y_test) diff --git a/forecaster-cnn/models/avgResponseTime/saved_model.pb b/forecaster-cnn/models/avgResponseTime/saved_model.pb deleted file mode 100644 index 72292e79a892ff42bdc7648551a7ec2afabfc98b..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/avgResponseTime/saved_model.pb and /dev/null differ diff --git a/forecaster-cnn/models/avgResponseTime/variables/variables.data-00000-of-00001 b/forecaster-cnn/models/avgResponseTime/variables/variables.data-00000-of-00001 deleted file mode 100644 index 96543be823f00f882b007dcdfa8b2e1d98ad9e35..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/avgResponseTime/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/forecaster-cnn/models/avgResponseTime/variables/variables.index b/forecaster-cnn/models/avgResponseTime/variables/variables.index deleted file mode 100644 index 14a2ef27af561d6e0f013a5880138ac44d13c22c..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/avgResponseTime/variables/variables.index and /dev/null differ diff --git a/forecaster-cnn/models/memory/saved_model.pb b/forecaster-cnn/models/memory/saved_model.pb deleted file mode 100644 index b5b58eb6c99ab107c5f559ff9ea782038a392aee..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/memory/saved_model.pb and /dev/null differ diff --git a/forecaster-cnn/models/memory/variables/variables.data-00000-of-00001 b/forecaster-cnn/models/memory/variables/variables.data-00000-of-00001 deleted file mode 100644 index 0ada2ca978ab93cffb2f2cb8aeb87051e9c8becf..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/memory/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/forecaster-cnn/models/memory/variables/variables.index b/forecaster-cnn/models/memory/variables/variables.index deleted file mode 100644 index de127bd4486e22cf45af795327b4c08538dd7987..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models/memory/variables/variables.index and /dev/null differ diff --git a/forecaster-cnn/models_trained/.DS_Store b/forecaster-cnn/models_trained/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 Binary files /dev/null and b/forecaster-cnn/models_trained/.DS_Store differ diff --git a/forecaster-cnn/models_trained/models.obj b/forecaster-cnn/models_trained/models.obj deleted file mode 100644 index d00daf8a7ec63fb380f60954123eb939e213593b..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models_trained/models.obj and /dev/null differ diff --git a/forecaster-cnn/models_trained/morphemic_models.obj b/forecaster-cnn/models_trained/morphemic_models.obj deleted file mode 100644 index 54e85a186fe2d4b9be92f2beb2a42f777cf1e4d2..0000000000000000000000000000000000000000 Binary files a/forecaster-cnn/models_trained/morphemic_models.obj and /dev/null differ diff --git a/forecaster-cnn/pre_processing/Data_transformation.py b/forecaster-cnn/pre_processing/Data_transformation.py index c726201e2673073a336f9074a24a4d1fc44172c4..42c343d8213ea35a69ca83e9a634cde65afd5493 100755 --- a/forecaster-cnn/pre_processing/Data_transformation.py +++ b/forecaster-cnn/pre_processing/Data_transformation.py @@ -5,6 +5,7 @@ import numpy as np # convert series to supervised learning from sklearn.model_selection import train_test_split from sklearn.preprocessing import MinMaxScaler +from sklearn.impute import SimpleImputer from numpy import concatenate from math import sqrt @@ -165,6 +166,7 @@ def split_sequences(sequences, n_steps): return array(X_train), array(y_train), array(X_test), array(y_test) #multivariate, multisteps +#an other change here def split_sequences_multi_steps(sequences, n_steps_in, n_steps_out): X, y = list(), list() for i in range(len(sequences)): @@ -181,6 +183,21 @@ def split_sequences_multi_steps(sequences, n_steps_in, n_steps_out): X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=0.33, random_state=42) return array(X_train), array(y_train), array(X_test), array(y_test) +def split_sequences_univariate(sequences, n_steps): + X, y = list(), list() + for i in range(len(sequences)): + # find the end of this pattern + end_ix = i + n_steps + # check if we are beyond the dataset + if end_ix > len(sequences): + break + # gather input and output parts of the pattern + seq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1] + X.append(seq_x) + y.append(seq_y) + X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=0.33, random_state=42) + return array(X_train), array(y_train), array(X_test), array(y_test) + def prediction_and_score_for_CNN(n_steps,n_features, x_input, model,test_y): #x_input = x_input.reshape((1, n_steps, n_features)) yhat = model.predict(x_input, verbose=2) @@ -230,4 +247,21 @@ def prediction_interval(model, X_train, y_train, x0, alpha: float = 0.05): qs = [100 * alpha / 2, 100 * (1 - alpha / 2)] percentiles = np.percentile(C, q = qs) - return percentiles[0], model.predict(x0), percentiles[1] \ No newline at end of file + return percentiles[0], model.predict(x0), percentiles[1] + +def missing_values_imputer(data): + # data = self.load_data() + + # TODO ---> make a configuration column + # TODO ---> use the missing values imputation based on configuration + + if data.isna().any().any() == False: + pass + elif data.isna().any().any() == True: + names = data.columns + # example + imp = SimpleImputer(missing_values=np.nan, strategy='mean') + imp.fit(data.values) + data = imp.transform(data.values) + data = pd.DataFrame(data, columns=names) + return data \ No newline at end of file diff --git a/forecaster-cnn/pre_processing/__pycache__/Data_transformation.cpython-36.pyc b/forecaster-cnn/pre_processing/__pycache__/Data_transformation.cpython-36.pyc index ad02dcf2fc8501dad29b16226f58385d346af23a..dd01f7a3730984fde45fc1c1b63c65627e6b711e 100644 Binary files a/forecaster-cnn/pre_processing/__pycache__/Data_transformation.cpython-36.pyc and b/forecaster-cnn/pre_processing/__pycache__/Data_transformation.cpython-36.pyc differ diff --git a/forecaster-cnn/pre_processing/__pycache__/preprocessing.cpython-36.pyc b/forecaster-cnn/pre_processing/__pycache__/preprocessing.cpython-36.pyc index 72045f2723286314b1f4161d2786e933ce0b5e17..e22c0d1ec223f31a58def017904a390b73ac0361 100644 Binary files a/forecaster-cnn/pre_processing/__pycache__/preprocessing.cpython-36.pyc and b/forecaster-cnn/pre_processing/__pycache__/preprocessing.cpython-36.pyc differ diff --git a/forecaster-cnn/pre_processing/preprocessing.py b/forecaster-cnn/pre_processing/preprocessing.py index db85629eae269b9f7fffed2418e9ff5c6a9fd63e..f67e23ec896b6a076f43cb2b71961b7c1c15cbec 100755 --- a/forecaster-cnn/pre_processing/preprocessing.py +++ b/forecaster-cnn/pre_processing/preprocessing.py @@ -27,7 +27,7 @@ def percent_missing(data): missing_value_df = missing_value_df.drop(columns=['index']) missing_value_df.sort_values('percent_missing', inplace=True, ascending=False) print(missing_value_df) #TODO dont know for sure if it has to return something or just to print - #return missing_value_df #if needed we can use return here + return missing_value_df #if needed we can use return here @@ -141,8 +141,6 @@ def missing_data_handling(data ,drop_all_nan = False, fill_with_mean = False, return data - - def datetime_conversion(data, column_name): data[column_name] = pd.to_datetime(data[column_name], unit='s') data = data.set_index(column_name) @@ -156,7 +154,7 @@ def important_data(data, list_of_important_features): #resampling 10 seconde -def resample(data, rate='30S'): +def resample(data, rate): resampled_data= data.resample(rate).mean() #TODO maybe the dot in data_. will cause problems return resampled_data diff --git a/morphemic-forecasting-eshybrid/Dockerfile b/morphemic-forecasting-eshybrid/Dockerfile index 7c211cb99eeadb1a5b3e8db377f0b732051888ea..9f595d29a6d63340fe4815a5e0772444e82725d6 100644 --- a/morphemic-forecasting-eshybrid/Dockerfile +++ b/morphemic-forecasting-eshybrid/Dockerfile @@ -1,26 +1,36 @@ FROM ubuntu:20.04 + +ADD https://bootstrap.pypa.io/get-pip.py /tmp/get-pip.py + RUN apt-get update && apt-get install -y \ - wget \ - python3 \ - python3-pip \ + wget curl \ + software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt install -y python3.9 python3.9-distutils \ + && python3.9 /tmp/get-pip.py \ && rm -rf /var/lib/apt/lists/* + COPY . /app ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc1.5/morphemic-preprocessor-morphemic-rc1.5.tar.gz /var/lib/morphemic/ - RUN cd /var/lib/morphemic/ \ && tar -zxf morphemic-preprocessor-morphemic-rc1.5.tar.gz \ && rm -rf /app/messaging \ && cp -R /var/lib/morphemic/morphemic-preprocessor-morphemic-rc1.5/amq-message-python-library /app/messaging \ && rm -rf /var/lib/morphemic +COPY docker-entrypoint.sh /app WORKDIR /app + RUN pip3 install -r requirements.txt \ - && pip3 install -r messaging/requirements.txt + && pip3 install -r messaging/requirements.txt \ + && chmod +x /app/docker-entrypoint.sh + +ENTRYPOINT ["/app/docker-entrypoint.sh"] -CMD ["python3","main.py"] \ No newline at end of file +CMD ["python3.9", "main.py"] diff --git a/morphemic-forecasting-eshybrid/docker-entrypoint.sh b/morphemic-forecasting-eshybrid/docker-entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..897c61a52a797f723d1757ea24a3ff066f65fed2 --- /dev/null +++ b/morphemic-forecasting-eshybrid/docker-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh + + +mkdir -p /logs +touch /logs/eshybrid.log + +"$@" > /logs/eshybrid.log 2>&1 + diff --git a/morphemic-forecasting-eshybrid/forecasting/eshybrid.py b/morphemic-forecasting-eshybrid/forecasting/eshybrid.py index 5b94d3936a91bbe8570636a9357f898a1936ec00..bf4084a4b078382a7c96c07152ce7e719eb6a684 100644 --- a/morphemic-forecasting-eshybrid/forecasting/eshybrid.py +++ b/morphemic-forecasting-eshybrid/forecasting/eshybrid.py @@ -21,7 +21,9 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen config['messaging']['username'], config['messaging']['password'], host=config['messaging']['host'], - port=config['messaging']['port'] + port=config['messaging']['port'], + timeout=6000, + keepalive=True ) self.model = morphemic.model.Model(self) self.application = config['persistence']['application'] @@ -60,10 +62,6 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen self.run() pass - def cleanup(self): - logging.debug("Cleaning Up...") - scheduler=False - pass def signal_handler(self, signum, frame): logging.debug("SIGHUP") @@ -72,7 +70,6 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen def stop(self): logging.debug("Stopping...") self._run = False - self.cleanup() def start(self): logging.debug("Starting ESHybrid") @@ -85,36 +82,31 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen if self.scheduler: self.scheduler.check(self) time.sleep(1) + self.connector.disconnect() + def on_schedule(self, times): for m in self.metrics: - # predictions = self.model.predict(self.application, m, times) - # if not predictions: - # continue + + predictions = self.model.predict( + self.application, + m, + times + ) for t in times: logging.debug("Sending prediction for time %s(%s) " % (datetime.datetime.fromtimestamp(t), t)) self.connector.send_to_topic( "intermediate_prediction.eshybrid.%s" % m, { - "metricValue": 12.43, + "metricValue": predictions[t], "timestamp": int(time.time()), "probability": 0.98, "confidence_interval": [float(8),float(15)], "predictionTime":t, } ) - # for p in predictions[m['metric']]: - - def on_train(self): - self.connector.send_to_topic("training_models", - { - "metrics": self.metrics, - "forecasting_method": self.id, - "timestamp": int(time.time() * 1000) - }) - def _train_model(self): @@ -122,6 +114,14 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen data = self.dataset.getData() self.model.train(self.metrics, data) + def on_train(self): + self.connector.send_to_topic("training_models", + { + "metrics": self.metrics, + "forecasting_method": self.id, + "timestamp": int(time.time() * 1000) + }) + def on_metrics_to_predict(self,res): logging.debug("[2] Metrics to predics %s " % res) @@ -166,14 +166,14 @@ class ESHybrid(morphemic.handler.ModelHandler,messaging.listener.MorphemicListen horizon= res[messaging.events.StartForecasting.PREDICTION_HORIZON] ) - def on_error(self, headers, body): logging.error("Headers %s",headers) logging.error(" %s",body) def on_disconnected(self): print('disconnected') - self.reconnect() + if self._run: + self.reconnect() def on(self, headers, res): diff --git a/morphemic-forecasting-eshybrid/main.py b/morphemic-forecasting-eshybrid/main.py index dd1e582cadef1a8053d0da91cefe37b85bba583c..545230ee9a0b08c84ebb87d37aa5dea02d83e41f 100644 --- a/morphemic-forecasting-eshybrid/main.py +++ b/morphemic-forecasting-eshybrid/main.py @@ -4,25 +4,25 @@ import os import argparse from forecasting import eshybrid -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) +def main(): + parser = argparse.ArgumentParser(description='Run eshybrid forecaster') + parser.add_argument('--config', help='Config file to run, default sync.cfg') -parser = argparse.ArgumentParser(description='Run eshybrid forecaster') -parser.add_argument('--config', help='Config file to run, default sync.cfg') + args = parser.parse_args() -args = parser.parse_args() + config_file = "%s/%s" % (os.getcwd(), args.config or "sync.cfg") + print("Config file %s ", config_file) + config = configparser.RawConfigParser() + config.read(config_file) + e = eshybrid.ESHybrid(dict(config)) + try: + e.start() + except KeyboardInterrupt: + e.stop() -config_file = "%s/%s" % (os.getcwd(), args.config or "sync.cfg") -print("Config file %s ", config_file) -config = configparser.RawConfigParser() -config.read(config_file) -config_dict = dict(config) -e = eshybrid.ESHybrid(config) +if __name__ == '__main__': + main() -try: - e.start() -except KeyboardInterrupt: - e.stop() diff --git a/morphemic-persistent-storage/database/Dockerfile b/morphemic-persistent-storage/database/Dockerfile index 5fd5001e55dcf174ee85f39fd007d186c9cf3d8a..cd9459633010e79ccc738ebc1751310eafcd532c 100644 --- a/morphemic-persistent-storage/database/Dockerfile +++ b/morphemic-persistent-storage/database/Dockerfile @@ -7,47 +7,6 @@ FROM python:3.7-slim as api-loader COPY inputapi/requirements.txt . RUN pip install --user -r requirements.txt -# --- Stage 2: Combined InfluxDB + Python API Image -FROM python:3.7-alpine3.12 - -# :: InfluxDB standard setup -RUN echo 'hosts: files dns' >> /etc/nsswitch.conf -RUN apk add --no-cache tzdata bash ca-certificates && \ - update-ca-certificates - -ENV INFLUXDB_VERSION 1.8.4 -RUN set -ex && \ - mkdir ~/.gnupg; \ - echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf; \ - apk add --no-cache --virtual .build-deps wget gnupg tar && \ - for key in \ - 05CE15085FC09D18E99EFB22684A14CF2582E0C5 ; \ - do \ - gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --keyserver pgp.mit.edu --recv-keys "$key" || \ - gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \ - done && \ - wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc && \ - wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ - gpg --batch --verify influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ - mkdir -p /usr/src && \ - tar -C /usr/src -xzf influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ - rm -f /usr/src/influxdb-*/influxdb.conf && \ - chmod +x /usr/src/influxdb-*/* && \ - cp -a /usr/src/influxdb-*/* /usr/bin/ && \ - gpgconf --kill all && \ - rm -rf *.tar.gz* /usr/src /root/.gnupg && \ - apk del .build-deps -COPY influxdb.conf /etc/influxdb/influxdb.conf - -EXPOSE 8086 - -VOLUME /var/lib/influxdb - -COPY entrypoint.sh /entrypoint.sh -COPY init-influxdb.sh /init-influxdb.sh - - # :: Python API setup # Copy compiled dependencies from # the standard user pip directory @@ -56,10 +15,10 @@ COPY --from=api-loader /root/.local /root/.local ENV PATH=/root/.local:$PATH # Copy Python API. -RUN mkdir inputapi -RUN mkdir -p inputapi/log -COPY ./inputapi/src ./inputapi/ +RUN mkdir /app +RUN mkdir -p /app/log +COPY ./inputapi/src /app/ +WORKDIR /app # Execute both in entrypoint.sh. -ENTRYPOINT ["/entrypoint.sh"] -CMD ["influxd"] +CMD ["python","-u","app.py"] diff --git a/morphemic-persistent-storage/database/Dockerfile.original b/morphemic-persistent-storage/database/Dockerfile.original new file mode 100644 index 0000000000000000000000000000000000000000..5fd5001e55dcf174ee85f39fd007d186c9cf3d8a --- /dev/null +++ b/morphemic-persistent-storage/database/Dockerfile.original @@ -0,0 +1,65 @@ +# --- Stage 1: API Dependency Loader + +# :: Initial dependency loading image. +FROM python:3.7-slim as api-loader + +# Get package dependencies. +COPY inputapi/requirements.txt . +RUN pip install --user -r requirements.txt + +# --- Stage 2: Combined InfluxDB + Python API Image +FROM python:3.7-alpine3.12 + +# :: InfluxDB standard setup +RUN echo 'hosts: files dns' >> /etc/nsswitch.conf +RUN apk add --no-cache tzdata bash ca-certificates && \ + update-ca-certificates + +ENV INFLUXDB_VERSION 1.8.4 +RUN set -ex && \ + mkdir ~/.gnupg; \ + echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf; \ + apk add --no-cache --virtual .build-deps wget gnupg tar && \ + for key in \ + 05CE15085FC09D18E99EFB22684A14CF2582E0C5 ; \ + do \ + gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --keyserver pgp.mit.edu --recv-keys "$key" || \ + gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \ + done && \ + wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc && \ + wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ + gpg --batch --verify influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ + mkdir -p /usr/src && \ + tar -C /usr/src -xzf influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \ + rm -f /usr/src/influxdb-*/influxdb.conf && \ + chmod +x /usr/src/influxdb-*/* && \ + cp -a /usr/src/influxdb-*/* /usr/bin/ && \ + gpgconf --kill all && \ + rm -rf *.tar.gz* /usr/src /root/.gnupg && \ + apk del .build-deps +COPY influxdb.conf /etc/influxdb/influxdb.conf + +EXPOSE 8086 + +VOLUME /var/lib/influxdb + +COPY entrypoint.sh /entrypoint.sh +COPY init-influxdb.sh /init-influxdb.sh + + +# :: Python API setup +# Copy compiled dependencies from +# the standard user pip directory +# and update PATH. +COPY --from=api-loader /root/.local /root/.local +ENV PATH=/root/.local:$PATH + +# Copy Python API. +RUN mkdir inputapi +RUN mkdir -p inputapi/log +COPY ./inputapi/src ./inputapi/ + +# Execute both in entrypoint.sh. +ENTRYPOINT ["/entrypoint.sh"] +CMD ["influxd"] diff --git a/morphemic-persistent-storage/database/inputapi/src/activemqlistermanager.py b/morphemic-persistent-storage/database/inputapi/src/activemqlistermanager.py index a90c677ca0b574ea382c7034d6dec8aea0df3d5f..edb29a4bfabf5dd62ed27a6056a4b05f62804654 100644 --- a/morphemic-persistent-storage/database/inputapi/src/activemqlistermanager.py +++ b/morphemic-persistent-storage/database/inputapi/src/activemqlistermanager.py @@ -53,6 +53,7 @@ class Worker(Thread): self.hostname, self.port, self.topic ) ) + _sleeping_time = 5 while True: if self.normal_stop: break @@ -74,7 +75,9 @@ class Worker(Thread): print("Could not connect to ActiveMQ broker") self.status = "error" print(e) - time.sleep(5) + print("Sleep before reconnection {0}sec.".format(_sleeping_time)) + time.sleep(_sleeping_time) + _sleeping_time = _sleeping_time * 2 print("End process") self.status = "stopped" diff --git a/morphemic-persistent-storage/database/inputapi/src/amq_client/MorphemicConnection.py b/morphemic-persistent-storage/database/inputapi/src/amq_client/MorphemicConnection.py index 9fd61b18c35ed6f26bd3197c53d5209aaa77ef41..9ffcbf9bffc48e89cd3b2755f497ba7e481f9b53 100644 --- a/morphemic-persistent-storage/database/inputapi/src/amq_client/MorphemicConnection.py +++ b/morphemic-persistent-storage/database/inputapi/src/amq_client/MorphemicConnection.py @@ -48,10 +48,8 @@ class Connection: def connect(self, wait=True): - if not self.conn: return - self.conn.connect(self.username, self.password, wait=wait) def disconnect(self): diff --git a/morphemic-persistent-storage/database/inputapi/src/app.py b/morphemic-persistent-storage/database/inputapi/src/app.py index 2ccfde653573ae1438c753123fcddd61119bf8f2..af0c9f62f95f831ba56b79f698b0546098c25893 100644 --- a/morphemic-persistent-storage/database/inputapi/src/app.py +++ b/morphemic-persistent-storage/database/inputapi/src/app.py @@ -9,7 +9,7 @@ logging.basicConfig(filename=logname,filemode='a',format='%(asctime)s,%(msecs)d url_api_ems = os.environ.get("URL_API_EMS", "localhost:8080") -influxdb_url = os.environ.get("INFLUXDB_URL", "http://localhost:8086") +influxdb_url = os.environ.get("INFLUXDB_URL", "http://influxdb:8086") metric_name_field_name = os.environ.get("METRIC_NAME_FIELD_NAME", "metricName") metric_name_field_value = os.environ.get("METRIC_NAME_FIELD_VALUE", "metricValue") @@ -20,7 +20,7 @@ metric_name_field_label = os.environ.get("METRIC_NAME_FIELD_LABEL", "labels") max_waiting_time = int(os.environ.get("MAX_WAITING_TIME", "20")) # minutes max_point_list = int(os.environ.get("MAX_POINT_LIST", "1000")) -influxdb_hostname = os.environ.get("INFLUXDB_HOSTNAME", "localhost") +influxdb_hostname = os.environ.get("INFLUXDB_HOSTNAME", "influxdb") influxdb_port = int(os.environ.get("INFLUXDB_PORT", "8086")) influxdb_username = os.environ.get("INFLUXDB_USERNAME", "morphemic") influxdb_password = os.environ.get("INFLUXDB_PASSWORD", "password") @@ -51,7 +51,7 @@ class Publisher(Thread): def connect(self): try: - self.conn = Connection(username=self.username, password=self.password, host=self.hostname,port=self.port, debug=False) + self.conn = Connection(username=self.username, password=self.password, host=self.host,port=self.port, debug=False) self.conn.connect() print("Publisher is connected to ActiveMQ") logging.info("Publisher is connected to ActiveMQ") @@ -408,7 +408,7 @@ class Ingestor(Thread): database=self.database, ) databases = self.influxdb.get_list_database() - print(databases) + #print(databases) db_exist = False for db in databases: if db["name"] == self.database: @@ -449,6 +449,8 @@ class Ingestor(Thread): else: application = fields[metric_name_field_application] timestamp = int(fields[metric_name_field_timestamp]/1000) #time in sec + #for an unknown error, i'll reduce the timestamp to 5 minutes for debugging purpose + #timestamp -= 60*5 metric = topic[topic.rindex('/')+1:] value = fields[metric_name_field_value] backet = self.backer_manager.getBacketBasedOnTime(application, timestamp) @@ -460,7 +462,7 @@ class Ingestor(Thread): # if tolerance != None: # backet.setTolerance(tolerance) else: - backet = Backet(application, 2, timestamp) + backet = Backet(application, 1, timestamp) backet.addLabels({"application": application, "level": fields["level"]}) backet.insert(metric, value, timestamp) self.backer_manager.addBacket(application, backet) @@ -469,6 +471,7 @@ class Ingestor(Thread): if backet != None: metrics = backet.getBacketSeries() timestamp = backet.getTimestamp() + metrics["ems_time"] = timestamp labels = backet.getLabels() self.insert2({"metrics": metrics, "labels": labels, "timestamp": timestamp}) @@ -625,6 +628,24 @@ class InputApi: def getSubscriberSize(self): return len(self.subscriptions.keys()) + def converterEMSToJSON(self, data): + if type(data) != type(""): + return None + try: + data = data.replace("{","").replace("}","") + data_splitted = data.split(",") + result = {} + for _d in data_splitted: + v, k = _d[_d.index("=")+1:], _d[:_d.index("=")] + k, v = k.strip(), v.strip() + if k == "level" or k == "timestamp": + result[k] = int(float(v)) + else: + result[k] = float(v) + return json.dumps(result) + except: + return None + def getData(self, data, topic): try: _json = json.loads(data) @@ -633,9 +654,11 @@ class InputApi: self.handleRequest(_json) return True except Exception as e: - print("Non JSON content received") - logging.warning("Non JSON content received") - return None + data = self.converterEMSToJSON(data) + if data == None: + print("Non JSON content received") + logging.warning("Non JSON content received") + return None self.ingestor.addToList(data, topic) self.data_points += 1 if time.time() - self.last_evaluation > self.evaluation_interval: diff --git a/morphemic-persistent-storage/docker-compose.yaml b/morphemic-persistent-storage/docker-compose.yaml index f01e8ab94d97c9a166a3e33c0067a3c476b99e14..c392f5e2c88e606b05f0680356dd42aac65d59aa 100644 --- a/morphemic-persistent-storage/docker-compose.yaml +++ b/morphemic-persistent-storage/docker-compose.yaml @@ -1,20 +1,26 @@ version: '2' services: - database: - image: jdtotow/persistent_storage - container_name: database + persistent_storage: + image: gitlab.ow2.org:4567/melodic/morphemic-preprocessor/persistent_storage:latest + container_name: persistent_storage restart: always env_file: - "./database/.env" volumes: - - "./database/data:/var/lib/influxdb" + - "./log:/app/log/" ports: - 8086:8086 environment: - "ACTIVEMQ_PORT=61610" - - "ACTIVEMQ_TOPIC=AAAA" - "ACTIVEMQ_HOST=147.102.17.76" + - "ACTIVEMQ_USERNAME=aaa" + - "ACTIVEMQ_PASSWORD=111" + - "INFLUXDB_HOSTNAME=ui-influxdb" + - "INFLUXDB_PORT=8086" + - "INFLUXDB_USERNAME=morphemic" + - "INFLUXDB_PASSWORD=password" + publisher: image: jdtotow/publisher container_name: publisher @@ -23,45 +29,5 @@ services: - "ACTIVEMQ_HOST=activemq" #- "APPLICATION_NAME=custom_name" - activemq: - image: jdtotow/activemq - container_name: activemq - ports: - # mqtt - - "1883:1883" - # amqp - - "5672:5672" - # ui - - "8161:8161" - # stomp - - "61613:61613" - # ws - - "61614:61614" - # jms - - "61616:61616" - # jms prometheus agent - - "8080:8080" - #volumes: ["activemq-data:/opt/activemq/conf", "activemq-data:/data/activemq", "activemq-data:/var/log/activemq"] - environment: - ACTIVEMQ_REMOVE_DEFAULT_ACCOUNT: "true" - ACTIVEMQ_ADMIN_LOGIN: aaa - ACTIVEMQ_ADMIN_PASSWORD: "111" - ACTIVEMQ_WRITE_LOGIN: aaa - ACTIVEMQ_WRITE_PASSWORD: "111" - ACTIVEMQ_READ_LOGIN: aaa - ACTIVEMQ_READ_PASSWORD: "111" - ACTIVEMQ_JMX_LOGIN: aaa - ACTIVEMQ_JMX_PASSWORD: "111" - ACTIVEMQ_STATIC_TOPICS: static-topic-1;static-topic-2 - ACTIVEMQ_STATIC_QUEUES: static-queue-1;static-queue-2 - ACTIVEMQ_ENABLED_SCHEDULER: "true" - ACTIVEMQ_MIN_MEMORY: 512 - ACTIVEMQ_MAX_MEMORY: 2048 - #prometheus: - # image: prom/prometheus - # ports: - # - 9090:9090 - # container_name: prometheus - # volumes: - # - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + diff --git a/morphemic-persistent-storage/example/ems/converter.py b/morphemic-persistent-storage/example/ems/converter.py new file mode 100644 index 0000000000000000000000000000000000000000..b241b906068ee08952d6520a81fe1020b2c16bbe --- /dev/null +++ b/morphemic-persistent-storage/example/ems/converter.py @@ -0,0 +1,22 @@ +import os, json, time + +_in = "{metricValue=0.0, level=3.0, timestamp=1.627629862279E12}" + +def converterEMSToJSON(data): + if type(data) != type(""): + return None + data = data.replace("{","").replace("}","") + data_splitted = data.split(",") + result = {} + for _d in data_splitted: + v, k = _d[_d.index("=")+1:], _d[:_d.index("=")] + k, v = k.strip(), v.strip() + if k == "level" or k == "timestamp": + result[k] = int(float(v)) + else: + result[k] = float(v) + return result + + +r = converterEMSToJSON(_in) +print(r) diff --git a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/communication/listeners/PredictionListener.java b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/communication/listeners/PredictionListener.java index 91dc1efe6a7ace533c3e9bd14cc0952dc37f11a0..ffd1e90c906ad9cb3f6d410b2645de135c1db525 100644 --- a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/communication/listeners/PredictionListener.java +++ b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/communication/listeners/PredictionListener.java @@ -29,8 +29,9 @@ public class PredictionListener extends ActiveMQListener { log.debug("Listener of topic {}: Converting event payload to PredictionFromForecasterMessage instance...", topicName); PredictionFromForecasterMessage predictionFromForecasterMessage = gson.fromJson(payload, PredictionFromForecasterMessage.class); log.info("Listener of topic {}: PredictionFromForecasterMessage instance: {}", topicName, predictionFromForecasterMessage.toString()); - boolean valueUpdated = predictionRegistry.processPrediction(new Prediction(predictionFromForecasterMessage)); - if (valueUpdated) { + int valueResult = predictionRegistry.processPrediction(new Prediction(predictionFromForecasterMessage)); + if ((valueResult == PredictionRegistry.VALUE_ADDED) || + (valueResult == PredictionRegistry.VALUE_UPDATED)) { metricHandler.launchMethodsPoolingIfNecessary(predictionFromForecasterMessage.getPredictionTime()); } } catch (JMSException e) { diff --git a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/NoAccessWithoutUpdatePredictionRegistry.java b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/NoAccessWithoutUpdatePredictionRegistry.java deleted file mode 100644 index 517e53655067bb69ef3274f3933c1022ee730c1d..0000000000000000000000000000000000000000 --- a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/NoAccessWithoutUpdatePredictionRegistry.java +++ /dev/null @@ -1,60 +0,0 @@ -package eu.morphemic.prediction_orchestrator.registries; - -import eu.morphemic.prediction_orchestrator.model.Prediction; -import eu.morphemic.prediction_orchestrator.properties.ForecastingConfiguration; -import lombok.extern.slf4j.Slf4j; - -import javax.jms.JMSException; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class NoAccessWithoutUpdatePredictionRegistry extends PredictionRegistry { - public final static Prediction PREDICTION_ALREADY_ACCESSED = null; - - private ConcurrentHashMap availableToAccess; - - public NoAccessWithoutUpdatePredictionRegistry(ForecastingConfiguration forecastingConfiguration, String registryName) { - super(forecastingConfiguration, registryName); - } - - @Override - void removeBufferRecord(long keyToRemove) { - this.predictions.remove(keyToRemove); - this.availableToAccess.remove(keyToRemove); - } - - @Override - public synchronized void reconfigureBuffer(ForecastingConfiguration forecastingConfiguration) { - super.reconfigureBuffer(forecastingConfiguration); - this.availableToAccess = new ConcurrentHashMap<>(); - } - - /** - * We need to synchronise this block as we need to makes sure that we block next access just after checking if we - * can get the element ("if" condition and updating the "availableToAccessMap" operations need to be bounded into atomic block) - */ - @Override - public Prediction getCopyPrediction(long predictionTime) { - synchronized (this) { - if (availableToAccess.getOrDefault(predictionTime, false)) { - availableToAccess.put(predictionTime, false); - return super.getCopyPrediction(predictionTime); - } else { - log.info("Registry {}: Prediction on predictionTime: {} has already been accessed. Wait for its update", registryName, - formatPredictionTime(predictionTime)); - return PREDICTION_ALREADY_ACCESSED; - } - } - } - - @Override - boolean updateBufferRecord(Prediction prediction) throws JMSException { - log.debug("Processing pooled prediction of registry {} for {}", registryName, - formatPredictionTime(prediction.getPredictionTime())); - boolean valueChanged = super.updateBufferRecord(prediction); - if (valueChanged || !availableToAccess.containsKey(prediction.getPredictionTime())) { - this.availableToAccess.put(prediction.getPredictionTime(), true); - } - return valueChanged; - } -} diff --git a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistry.java b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistry.java index 5b4e6fec5e5d4bd37b9f874ccfdd81f5e9b149da..fe776e071883824a22da8b9d8b3dd32e5b4977dd 100644 --- a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistry.java +++ b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistry.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import javax.jms.JMSException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -29,6 +30,10 @@ import java.util.concurrent.atomic.AtomicReference; @Slf4j public class PredictionRegistry { + public static int VALUE_ADDED = 2; + public static int VALUE_UPDATED = 1; + public static int VALUE_REJECTED = 0; + String registryName; private AtomicReference forecastingConfiguration; @@ -69,7 +74,7 @@ public class PredictionRegistry { earliestPredictionTime.get() + predictionHorizon * (forecastingConfiguration.getForward_prediction_number() - 1)); } - public synchronized boolean processPrediction(Prediction prediction) throws JMSException { + public synchronized int processPrediction(Prediction prediction) throws JMSException { if (prediction != null) { long predictionTime = prediction.getPredictionTime(); if (predictionTime < earliestPredictionTime.get()) { @@ -84,20 +89,24 @@ public class PredictionRegistry { return updateBufferRecord(prediction); } } - return false; + return VALUE_REJECTED; } public boolean containsPrediction(long predictionTime) { return predictions.containsKey(predictionTime); } - boolean updateBufferRecord(Prediction prediction) throws JMSException { - AtomicBoolean valueUpdated = new AtomicBoolean(false); + int updateBufferRecord(Prediction prediction) throws JMSException { + AtomicInteger valueResult = new AtomicInteger(VALUE_REJECTED); predictions.compute( prediction.getPredictionTime(), (k, v) -> { if ((v == null) || (v.getTimestamp() < prediction.getTimestamp())) { - valueUpdated.set(true); + if (v == null) { + valueResult.set(VALUE_ADDED); + } else { + valueResult.set(VALUE_UPDATED); + } log.debug("Updating value for {} inside registry: {} on predictionTime: {} and timestamp {}", prediction.getMetricValue(), registryName, prediction.getPredictionTime(), prediction.getTimestamp()); return prediction; @@ -109,7 +118,7 @@ public class PredictionRegistry { if (latestPredictionTime.get() < prediction.getPredictionTime()) { moveBuffer(prediction.getPredictionTime()); } - return valueUpdated.get(); + return valueResult.get(); } void removeBufferRecord(long keyToRemove) { diff --git a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/Coordinator.java b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/Coordinator.java index 1d5c8369b76f3a1a7c05b982d462709a7d8b14a1..418d19b6aa9efd34424032d690ab9e6e10922582 100644 --- a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/Coordinator.java +++ b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/Coordinator.java @@ -7,7 +7,7 @@ import eu.morphemic.prediction_orchestrator.log_utils.PredictionTimeFormatter; import eu.morphemic.prediction_orchestrator.model.Prediction; import eu.morphemic.prediction_orchestrator.properties.ForecastingConfiguration; import eu.morphemic.prediction_orchestrator.properties.Properties; -import eu.morphemic.prediction_orchestrator.registries.NoAccessWithoutUpdatePredictionRegistry; +import eu.morphemic.prediction_orchestrator.registries.PredictionRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -44,9 +44,9 @@ public class Coordinator { * and before getting them. But this is not a problem as we want to publish as much as possible assuming we had a full metric vector * at any point. If the buffer would move in the meantime we just wouldn't publish this specific value */ - //Communication 4 and 1 - synchronized void publishIfPooledValuesProvidedForAllMetrics(long predictionTime) { - + //Communication 5 and 1 + synchronized void publishIfPooledValuesProvidedForAllMetrics(long predictionTime, + int lastValueUpdate, String callingMetricHandler) { //metricName -> predictionAvailability HashMap isPooledPredictionAvailable = new HashMap<>(); metricHandlers.forEach((metricName, metricHandler) -> @@ -57,12 +57,18 @@ public class Coordinator { if (entireVectorIsReady) { log.info("Starting publishing pooled predictions for metrics for {} ", PredictionTimeFormatter.rawDateFormat(predictionTime)); - metricHandlers.forEach((metricName, metricHandler) -> { - Prediction prediction = metricHandler.getPooledPrediction(predictionTime); - if (prediction != NoAccessWithoutUpdatePredictionRegistry.PREDICTION_ALREADY_ACCESSED) { + //We send the entire vector only once, when all values are present for the first time + if (lastValueUpdate == PredictionRegistry.VALUE_ADDED) { + metricHandlers.forEach((metricName, metricHandler) -> { + Prediction prediction = metricHandler.getPooledPrediction(predictionTime); communicationService.publishPooledPrediction(prediction, metricName); - } - }); + }); + } else { + //we send only updated value as at this point the entire vector has been send earlier + Prediction prediction = metricHandlers.get(callingMetricHandler) + .getPooledPrediction(predictionTime); + communicationService.publishPooledPrediction(prediction, callingMetricHandler); + } } } diff --git a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/MetricHandler.java b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/MetricHandler.java index 32b5a35c481ffa7c010263e9a720c09940e02a26..cb4703d713552106bf11082871c2df72e2b40ad3 100644 --- a/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/MetricHandler.java +++ b/prediction_orchestrator/src/main/java/eu/morphemic/prediction_orchestrator/service/MetricHandler.java @@ -3,7 +3,6 @@ package eu.morphemic.prediction_orchestrator.service; import eu.morphemic.prediction_orchestrator.communication.CommunicationService; import eu.morphemic.prediction_orchestrator.log_utils.PredictionTimeFormatter; import eu.morphemic.prediction_orchestrator.properties.Properties; -import eu.morphemic.prediction_orchestrator.registries.NoAccessWithoutUpdatePredictionRegistry; import eu.morphemic.prediction_orchestrator.model.Prediction; import eu.morphemic.prediction_orchestrator.registries.PredictionRegistry; import eu.morphemic.prediction_orchestrator.service.pooling_strategy.PoolingStrategy; @@ -30,7 +29,7 @@ public class MetricHandler { //methodName -> MethodHandler private HashMap methodHandlers = new HashMap<>(); - private NoAccessWithoutUpdatePredictionRegistry pooledPredictionsRegistry; + private PredictionRegistry pooledPredictionsRegistry; private PoolingStrategy poolingStrategy; private CommunicationService communicationService; @@ -40,7 +39,7 @@ public class MetricHandler { this.metricName = metricName; this.poolingStrategy = PoolingStrategyFactory.getPoolingStrategy(properties); this.coordinator = coordinator; - this.pooledPredictionsRegistry = new NoAccessWithoutUpdatePredictionRegistry( + this.pooledPredictionsRegistry = new PredictionRegistry( forecastingConfiguration, PredictionRegistry.getPooledPredictionsRegistryName(metricName) ); @@ -57,9 +56,10 @@ public class MetricHandler { if (newPooledPrediction != PoolingStrategy.POOLED_PREDICTION_NOT_CREATED) { log.info("Pooling function has created pooled prediction for metric {} and predictionTime {}", metricName, PredictionTimeFormatter.rawDateFormat(predictionTime)); - boolean valueChanged = pooledPredictionsRegistry.processPrediction(newPooledPrediction); - if (valueChanged) { - coordinator.publishIfPooledValuesProvidedForAllMetrics(predictionTime); + int valueResult = pooledPredictionsRegistry.processPrediction(newPooledPrediction); + if ((valueResult == PredictionRegistry.VALUE_UPDATED) || + (valueResult == PredictionRegistry.VALUE_ADDED)) { + coordinator.publishIfPooledValuesProvidedForAllMetrics(predictionTime, valueResult, metricName); } } else { log.debug("Pooling function has not created pooled prediction for metric {} and predictionTime {}", diff --git a/prediction_orchestrator/src/main/resources/eu.morphemic.predictionorchestrator.properties b/prediction_orchestrator/src/main/resources/eu.morphemic.predictionorchestrator.properties index 0c1ecc30704d7c0789e8f705c65218c480284603..14676ebc12db0223f4185dda5b8ea1048fd48072 100644 --- a/prediction_orchestrator/src/main/resources/eu.morphemic.predictionorchestrator.properties +++ b/prediction_orchestrator/src/main/resources/eu.morphemic.predictionorchestrator.properties @@ -1,14 +1,51 @@ -forecasting_configuration.initial_prediction_horizon=120000 +#forecasting_configuration are properties that are sent further to The forecasters +#They explain pattern for predictionTimes, predictions's count and what should be the predictionTime +#of the first message send + +#global pattern is: predictionTime = epochStart + k*initial_prediction_horizon where k is integer + +#unit: seconds +#This property indicates what is the time difference between following predictionTimes +# both inside the forecasters and the PO +# Depending on the forecaster it may also mean how often the Forecaster would publish predictions +# to the PO +forecasting_configuration.initial_prediction_horizon=120 + +#This property indicates how many forward prediction the forecaster should send +#at the moment of publishing predictions forecasting_configuration.initial_forward_prediction_number=8 -forecasting_configuration.starting_forecasting_delay=50000 +#This property indicates what is epochStart and what predictionTime should have the first message send by the forecaster +#It also explains time before the forecasters start publishing +#EpochStart = currentTime + starting_forecasting_delay +# predictionTime of first predictionSend = EpochStart + initial_prediction_horizon +forecasting_configuration.starting_forecasting_delay=200 +#In the most common scenario: +#PO counts epochStart based on starting_forecasting_delay and send the above properties to the Forecaster +#At moment epochStart the forecaster would publish initial_forward_prediction_number predictions +#with predictionTimes equal = epochStart + (1,2,3 .. initial_forward_prediction_number) * initial_prediction_horizon +# At moment (epochStart + 1*initial_prediction_horizon) the Forecaster would send initial_forward_prediction_number predictions +#with predictionTimes equal = epochStart + (2,3 .. (initial_forward_prediction_number+1)) * initial_prediction_horizon + + +#This property indicates method for checking how many forecasters' data is needed to be abe to pool/ensemble +# predictions and publish them to EMS +# It currntly may equal PERCENTAGE_FORECASTERS_COUNT_NEEDED or STATIC_FORECASTERS_COUNT_NEEDED +# With the first option threshold property is in percents +#With the second it is a static count +#For example with threshold threshold=2 at least 2 synchronised forecasters are needed to be able to merge predictions and send them forward pooling.poolingStrategy=STATIC_FORECASTERS_COUNT_NEEDED pooling.threshold=2 jms.client.broker_properties_configuration_file_location=${MELODIC_CONFIG_DIR}/eu.melodic.event.brokerclient.properties -startingMethodsList=nbeats,es_hybrid,arima,tsetlin_machines,exponential_smoothing,lstm,gluon_machines,prophet +#List of connected forecaster that PO expects to be working. PO would not coummuncate with the forecasters not on that list +startingMethodsList=es_hybrid,cnn,prophet,tft,nbeats,arima,tsetlin_machines,exponential_smoothing,lstm,gluon_machines -logging.config=file:${MELODIC_CONFIG_DIR}/logback-conf/logback-spring.xml +#this is only for logging, have no influence on working flow of The PO logging.zone_id=Europe/Warsaw + +#Details of restarting connection to amq upon failure +activemq.restartinterval=10000 +activemq.restartcount=20 diff --git a/prediction_orchestrator/src/test/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistryTest.java b/prediction_orchestrator/src/test/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistryTest.java index ec33e9322f8334fa6d633f54ca3955e10f463fa4..768529a9d2e95390b3fa832d03714c4d4f6405bb 100644 --- a/prediction_orchestrator/src/test/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistryTest.java +++ b/prediction_orchestrator/src/test/java/eu/morphemic/prediction_orchestrator/registries/PredictionRegistryTest.java @@ -98,58 +98,6 @@ class PredictionRegistryTest { predictionRegistry, first_epoch_starting_forecast, prediction_horizon); } - @Test - void multipleReadersBlockingRegistryTest() throws InterruptedException { - int numberOfForwardPrediction = 10; - long first_epoch_starting_forecast = 100000000; - int prediction_horizon = 1000; - ForecastingConfiguration forecastingConfiguration = new ForecastingConfiguration( - first_epoch_starting_forecast, - prediction_horizon, - numberOfForwardPrediction - ); - //we need to publish more than the buffer can handle - ForecastingConfiguration forecastingConfigurationPublisher = new ForecastingConfiguration( - first_epoch_starting_forecast, - prediction_horizon, - 2 * numberOfForwardPrediction - ); - PredictionRegistry predictionRegistry = new NoAccessWithoutUpdatePredictionRegistry( - forecastingConfiguration, - "populateMovingTest" - ); - - List publishingThreads = populateTestBed(forecastingConfigurationPublisher, predictionRegistry); - waitForThreads(publishingThreads); - //At this moment we should have a registry with boundaries: [10,19] - - - ConcurrentHashMap> results = new ConcurrentHashMap<>(); - - List readingThreads = new LinkedList<>(); - for (int threadNo = 0; threadNo < THREAD_COUNT; threadNo++) { - Thread reading_thread = new Thread(() -> { - for (int j = 10; j < 20; j++) { - long predictionTime = (j * prediction_horizon) + first_epoch_starting_forecast; - Prediction prediction = - predictionRegistry.getCopyPrediction(predictionTime); - results.computeIfAbsent(predictionTime, k -> new LinkedList<>()).add(prediction); - } - }); - readingThreads.add(reading_thread); - reading_thread.start(); - } - waitForThreads(publishingThreads); - waitForThreads(readingThreads); - - // We will be checking if no element has been pulled twice - for (List pulledPredictions : results.values()) { - List withoutNullsList = pulledPredictions.stream().filter(Objects::nonNull).collect(Collectors.toList());; - log.info(withoutNullsList.toString()); - assert( (new HashSet<>(withoutNullsList)).size() == withoutNullsList.size()); - } - } - /** * This procedure launches pool of publishing threads and wait until they are finished */ diff --git a/scheduling-abstraction-layer/build.gradle b/scheduling-abstraction-layer/build.gradle index 217ead9ef1396dbf0fdf4610e3933728bb94b297..8cb8c29762a47c3a6f4ab49bc3dbbec1d7650af1 100644 --- a/scheduling-abstraction-layer/build.gradle +++ b/scheduling-abstraction-layer/build.gradle @@ -3,7 +3,7 @@ plugins { } group 'org.activeeon' -version '3.6-SNAPSHOT' +version '3.7-SNAPSHOT' sourceCompatibility = 1.8 diff --git a/scheduling-abstraction-layer/pom.xml b/scheduling-abstraction-layer/pom.xml index ac768a9a27f18d1ad94e34ee349dfef8586e0572..d98a0e9cb5bae773852f83a72335e8dd8575048e 100644 --- a/scheduling-abstraction-layer/pom.xml +++ b/scheduling-abstraction-layer/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.activeeon scheduling-abstraction-layer - 3.6-SNAPSHOT + 3.7-SNAPSHOT org.projectlombok diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java index e635841c2e62d2a91acf8ccee7e853de855e6abe..dab5e633f23c612a3a9eaec83b4b30730fe201ec 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java @@ -10,10 +10,8 @@ import org.json.JSONObject; import java.io.IOException; import java.math.BigDecimal; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; +import java.util.*; + import java.util.stream.Collectors; @Slf4j @@ -257,26 +255,47 @@ public class NodeCandidateUtils { //TODO: (Optimization) An images per region map structure could be the best here. // It can reduce the getNodeCandidates calls to PA. + List entries = new LinkedList<>(); + List openstackOsList = Arrays.asList("Ubuntu","Fedora","Centos","Debian"); consolidatedImages.forEach(image -> { String region = image.optString("location"); String imageReq; + String os = (String) ((JSONObject) image.get("operatingSystem")).get("family"); + os = os.substring(0, 1).toUpperCase() + os.substring(1); + String pair =os+":"+region; switch (paCloud.getCloudProviderName()) { case "aws-ec2": imageReq = "Linux"; break; case "openstack": - imageReq = "linux"; + imageReq = os; break; default: throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudProviderName() + " is not handled yet."); } - JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(), - region, imageReq); - nodeCandidates.forEach(nc -> { - JSONObject nodeCandidate = (JSONObject) nc; - EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud)); - EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud)); - }); + + if (paCloud.getCloudProviderName().equals("openstack")){ + if((!entries.contains(pair)) && openstackOsList.contains(os)) { + entries.add(pair); + JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(), + region, imageReq); + nodeCandidates.forEach(nc -> { + JSONObject nodeCandidate = (JSONObject) nc; + EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud)); + EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud)); + }); + + } + } + else { + JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(), + region, imageReq); + nodeCandidates.forEach(nc -> { + JSONObject nodeCandidate = (JSONObject) nc; + EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud)); + EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud)); + }); + } }); });