From a59675b0767a5f677f96b9e7562c263606dd9f74 Mon Sep 17 00:00:00 2001 From: Jean-Didier Date: Mon, 12 Jun 2023 10:43:10 +0300 Subject: [PATCH] wp1 changes --- .DS_Store | Bin 14340 -> 12292 bytes deployment/.DS_Store | Bin 0 -> 6148 bytes forecaster-cnn/app.py | 4 ++ morphemic-performance-model/.DS_Store | Bin 6148 -> 6148 bytes morphemic-performance-model/ml_code/.DS_Store | Bin 8196 -> 8196 bytes .../ml_code/src/activemq_logic.py | 39 ++++++++--- .../ml_code/src/ml_module.py | 65 ++++++++++++++++-- morphemic-persistent-storage/.DS_Store | Bin 6148 -> 6148 bytes .../database/.DS_Store | Bin 6148 -> 6148 bytes .../database/inputapi/.DS_Store | Bin 6148 -> 6148 bytes .../database/inputapi/src/app.py | 45 +++++++++--- polymorphic_solver/.DS_Store | Bin 8196 -> 8196 bytes polymorphic_solver/src/app.py | 25 ++++--- polymorphic_solver/src/morphemic.py | 22 +++--- polymorphic_solver/src/xmi_camel_utilities.py | 27 +++++++- 15 files changed, 179 insertions(+), 48 deletions(-) create mode 100644 deployment/.DS_Store diff --git a/.DS_Store b/.DS_Store index a653d74f7eb32c941dbcc397d7d9f883d465fa6c..5e890ad35c6b5f743e49bb779a9c327701905640 100644 GIT binary patch delta 343 zcmZoEXh~3DU|?W$DortDV9)?EIe-{M3-ADmb_NCo?uiQejCvCToF`vYi`w{Bj-OQw zD5Eu5KtOczb2X{WvFbsL!UpC#3PzR&wK@vbhNk8gItu2Nllg>ACQf#q%p)*=vZY2D zE+v!s1uQo&5R76JF~nlXW2XDIE(Wy0n;%0|qS_lufIGcd3MH2{GIH;`~e k4vx)&jPIEz^PBkcFhW9$0Z1E6&M=v{`G@i~CZKjx0E)j&0RR91 delta 242 zcmZokXem%&U|?W$DortDU@!nOIe-{M3-ADmb_NCoo{0+jj5-qooF_L6m`~;x;+Y(# zpud@wk&T~K2&hD6vVef-ldXgrCQB>sVMG>@-Mm3KigEHXHIL0fqC2@ZD=-Q$ zF-mUcQOIYSyhB-b@=WEu>=q_E3Py&;lh>&^Y@VxP#5{3A>}Fnp7Df~a5ysDKlNA`{ yBw2xW0)YWHkZ?tI=4L_0_so;|b$of4Ail5wshgakGja1T)vJs^Qz!EZ+ynr55I6n+ diff --git a/deployment/.DS_Store b/deployment/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c5cb3da5ea5c18db7d687b32a84e131dca9bf084 GIT binary patch literal 6148 zcmeHK&u`N(6n<{IEnx!n01_OKEO9NPMViXQB`e*51D6HC0Z>UufrvDol5{;(m2!vw zg){#M{|hJhp6y9Z3rJiLVt&%|=O6F0>(8|v6OkCrlL1keh&(uB=L(Xa7`L-;SnfZAPJw?<0p7bi6j4Db&1il9@{w>1KlrXq|8$?Glv9axMlof; zVr|DWcM+w=ClSK)iL2&<@?m!qh7!KJ54W${Hmil3#^#iV=vk;aTioIXE0X=*i*H;cDf3H}2efcsluz&2q&P zfC(I&+OAr>fKSLY1UL_iJX85Q)HBDYQ#!!{jp+&S01J|ANc0U+>xY+(Jje+;Z~+CKXWto0JIb delta 32 ncmZoMXffDumPy#cL`T8M(70Adq1w>I*iuKq+;sB;CKXWtn`a2* diff --git a/morphemic-performance-model/ml_code/.DS_Store b/morphemic-performance-model/ml_code/.DS_Store index 487298ce764505b146979b3686f3b1e6c2607ff4..2b8e773b752f0d52d96f8f2b472704fa8dbba0d6 100644 GIT binary patch delta 32 ncmZp1XmQxEMo8GeTt~sk(x6sHq1w>Y+(Jje+;a1NAy-}ionHw9 delta 32 ncmZp1XmQxEMo8GgL`T8M(70Adq1w>I*iuKq+;sDPAy-}ioiYgf diff --git a/morphemic-performance-model/ml_code/src/activemq_logic.py b/morphemic-performance-model/ml_code/src/activemq_logic.py index 8a8c8349..6f4d1b8e 100644 --- a/morphemic-performance-model/ml_code/src/activemq_logic.py +++ b/morphemic-performance-model/ml_code/src/activemq_logic.py @@ -1,10 +1,8 @@ -from email.mime import application -import os, time, subprocess, json -from shutil import ExecError +import os, time, json, math from threading import Thread from datetime import datetime from activemq.MorphemicConnection import Connection -import stomp +from typing import List, Dict #from initial_ML_module import Predictor from morphemic.dataset import DatasetMaker from ml_module import MultiComponentsManager @@ -132,6 +130,9 @@ class MessagingManager(Thread): def multiComponentsTrainHandler(self, data): #{"application": app_name, models:[{"target": target, "components_config": [{"name": comp_name, "variant": variant, "hardware": hardware}]}]} + print("==============Train request===============") + print(data) + print("===========================================") _json = None try: _json = json.loads(data) @@ -145,6 +146,9 @@ class MessagingManager(Thread): return None app_name = _json["application"] + self.manager.setFormulas(_json["formulas"]) + self.manager.setUtilityMetricName(_json["utility_metric_name"]) + configs = {'hostname': influxdb_hostname, 'port': influxdb_port, 'username': influxdb_username, @@ -200,8 +204,15 @@ class MessagingManager(Thread): _file.close() self.multiComponentsTrainHandler(data) + def completeVariables(self, variables: Dict[str,str]): + _variables = [] + for component in self.current_deployment_config: + variables.update({"component": component["component"]}) + _variables.append(variables) + return _variables + def multiComponentsPredictHandler(self, data): - print("-------------------Data------------------") + print("-------------------Prediction requests------------------") print(data) print("-----------------------------------------") _json = None @@ -225,6 +236,8 @@ class MessagingManager(Thread): return None sender_id = _json["sender_id"] variables = _json["variables"] + if type(variables) == type({}): + variables = self.completeVariables(variables) variables = self.completeMissingVariables(variables) if not variables: @@ -268,14 +281,24 @@ class MessagingManager(Thread): if not self.current_deployment_config: if not self.loadDeploymentConfigFile(): return variables + _final = [] for component in variables: + _component = {} + _component.update(component) if not "component" in component: return None if not "_variant" in component: - component["_variant"] = self.getComponentConfig(component["component"],"variant") + _component["_variant"] = self.getComponentConfig(component["component"],"variant") if not "_hardware" in component: - component["_hardware"] = self.getComponentConfig(component["component"],"hardware") - return variables + _component["_hardware"] = self.getComponentConfig(component["component"],"hardware") + if not "_cores" in component: + _component["_cores"] = self.getComponentConfig(component["component"],"cores") + if not "_memory" in component: + _component["_memory"] = self.getComponentConfig(component["component"],"memory") + if not "_instances" in component: + _component["_instances"] = self.getComponentConfig(component["component"],"instances") + _final.append(_component) + return _final def run(self): print("Messaging manager started ...") diff --git a/morphemic-performance-model/ml_code/src/ml_module.py b/morphemic-performance-model/ml_code/src/ml_module.py index 8ecb9075..e6e702ad 100644 --- a/morphemic-performance-model/ml_code/src/ml_module.py +++ b/morphemic-performance-model/ml_code/src/ml_module.py @@ -1,4 +1,4 @@ -import os, time, pickle, hashlib +import os, time, pickle, hashlib, math import pandas as pd import numpy as np from sklearn.impute import SimpleImputer @@ -8,7 +8,6 @@ from tensorflow import keras from tensorflow.keras.layers import LSTM, Dropout, Flatten, Dense, Conv1D from tensorflow.python.keras.layers import MaxPooling1D from sklearn.model_selection import train_test_split -from math import sqrt from sklearn.metrics import mean_squared_error, r2_score from morphemic.dataset import DatasetMaker @@ -29,6 +28,8 @@ columns_to_remove = columns_to_remove.split(",") _VARIANTS_HARDWARE = ["VM","DOCKER","SERVERLESS","HPC","EDGE","CPU","GPU","FPGA","NPU","TPU"] +functions = ['acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh', 'ceil', 'comb', 'copysign', 'cos', 'cosh', 'degrees', 'dist', 'e', 'erf', 'erfc', 'exp', 'expm1', 'fabs', 'factorial', 'floor', 'fmod', 'frexp', 'fsum', 'gamma', 'gcd', 'hypot', 'inf', 'isclose', 'isfinite', 'isinf', 'isnan', 'isqrt', 'lcm', 'ldexp', 'lgamma', 'log', 'log10', 'log1p', 'log2', 'modf', 'nan', 'nextafter', 'perm', 'pi', 'pow', 'prod', 'radians', 'remainder', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'tau', 'trunc', 'ulp'] + def get_index(name): if name in _VARIANTS_HARDWARE: return _VARIANTS_HARDWARE.index(name) @@ -49,7 +50,7 @@ def CNN_model(n_features, X, y): def prediction_and_score_for_CNN(model, x_input,test_y): #x_input = x_input.reshape((1, n_steps, n_features)) yhat = model.predict(x_input, verbose=2) - rmse = sqrt(mean_squared_error(test_y, yhat)) + rmse = math.sqrt(mean_squared_error(test_y, yhat)) r2score = r2_score(test_y, yhat) return rmse, r2score @@ -255,6 +256,8 @@ class MultiComponentsManager(): self.models = {} self.ml_loaded_models = {} self.ready_to_train = False + self.formulas = None + self.utility_metric_name = None self.loadModels() def addModel(self, components_config, target, url_file): @@ -271,8 +274,32 @@ class MultiComponentsManager(): self.saveModels() self.checkTrainStatus() + def setFormulas(self, formulas): + self.formulas = formulas + + def getFormulas(self): + return self.formulas + + def setUtilityMetricName(self, name): + print("utility metric name = ", name) + self.utility_metric_name = name + + def getUtilityMetricName(self) -> str: + return self.utility_metric_name + + def replaceFunction(self, _string: str): + for function in functions: + _exp = "{0}(".format(function) + if _exp in _string: + _string = _string.replace(_exp,"math.{0}(".format(function)) + return _string + def addPredictData(self, sender_id, variables, target): #variables : [{"component": name, "variant": var, "hardware": hardware, "_memory": mem, "_cores": core, "_instances": instances}] + print("target is ", target) + if target == "utility": + target = self.getUtilityMetricName() + print("target is ", target) components_config = [] for component in variables: mandatory_field = ["component", "_variant", "_hardware"] @@ -302,7 +329,7 @@ class MultiComponentsManager(): features_list = [] for component in variables: for field in list(component.keys()): - if not field in static_fields: + if not field in static_fields and not field in features_list: features_list.append(field) model = MultiComponentsModel() @@ -311,8 +338,31 @@ class MultiComponentsManager(): model.makeIdentifier() id = model.getIdentifier() if not id in self.models: - print("Model with id = {0} not found".format(id)) - return {"status": False, "message": "Model not found", "sender_id": sender_id} + print("Model with id = {0} not found, the formula will be used".format(id)) + if target in self.formulas: + _dict = {} + _dict.update(features_dict) + _dict.update({"math":math}) + for _ in range(len(list(self.formulas.keys()))): + for t, formula in self.formulas.items(): + if t == target: + continue + try: + formula = self.replaceFunction(formula) + result = eval(formula, _dict) + _dict[t] = result + except: + pass + print('Formula for = {0} found'.format(target)) + print('Formula = ', self.formulas[target]) + formula = self.replaceFunction(self.formulas[target]) + try: + result = eval(formula, _dict) + return {"status": True, "message": "", "result": {target: result}} + except Exception as e: + return {"status": False, "message": str(e), "result": None} + else: + return {"status": False, "message": "Model not found", "sender_id": sender_id} model = self.models[id] if not model.getLastTraining(): train_result = self.train(model) @@ -344,7 +394,7 @@ class MultiComponentsManager(): else: ml_model = self.ml_loaded_models[id] prediction = float(ml_model.predict(new_sample.values)[0][0]) - return {"status": True, "message": "", "results":{"prediction": prediction},"sender_id": sender_id} + return {"status": True, "message": "", "results":{target: prediction},"sender_id": sender_id} def readyToTrain(self): self.ready_to_train = True @@ -389,6 +439,7 @@ class MultiComponentsManager(): target = model.getTarget() if not target in list(data.columns): return {"status": False, "message": 'Target field not found in the dataset'} + data = model.filterRows(data) if data.empty: return {"status": False, "message": "An error occured, dataframe is empty after filtering"} diff --git a/morphemic-persistent-storage/.DS_Store b/morphemic-persistent-storage/.DS_Store index 0617f43e32b18a641eb10087291c440261bba41c..38432110c6b646bcd47d9632a5aa149660a1d553 100644 GIT binary patch delta 34 pcmZoMXffCzz$$8JV6LNJWNA>Vqfl*VYHp#UU~Xx;S(Wv^5CD~y2$}!@ delta 34 pcmZoMXffCzz$$8NVWFd7WNuQcqfl*VY+$6LU}kKzS(Wv^5CE0h2#x>% diff --git a/morphemic-persistent-storage/database/.DS_Store b/morphemic-persistent-storage/database/.DS_Store index ea777f464545b9ae8971c1d1f15939bcf5d77700..c187e3dba7e2a4756977f73f78f6a8a44449bf76 100644 GIT binary patch delta 34 pcmZoMXffEZgH_bfz+6Yc$kL!zN1@u#)Z9Wx!Q9ex^I2ANApoiF2|@q> delta 34 pcmZoMXffEZgH_bn!a_&E$lRn>N1@u#*uY3f!OYlb^I2ANApoi}2{r%# diff --git a/morphemic-persistent-storage/database/inputapi/.DS_Store b/morphemic-persistent-storage/database/inputapi/.DS_Store index 34656b34614ea8fa495da593ce87e09ba998c208..e7640f4f37dfeeb9be3adfe5c341761935651dfb 100644 GIT binary patch delta 34 pcmZoMXffE3z$|KLV6LNJWNA>Vqfl*VYHp#UU~Xx;xs