diff --git a/forecasting_gluonts/docker_image/Dockerfile b/forecasting_gluonts/docker_image/Dockerfile index e6f9d42f3215fa43098e3591a57ac5e40667d434..e4d2aa0e4f1f9ffead01d8d3727a85b9ce795602 100644 --- a/forecasting_gluonts/docker_image/Dockerfile +++ b/forecasting_gluonts/docker_image/Dockerfile @@ -1,50 +1,50 @@ -FROM python:3.8.10 -WORKDIR /morphemic_project -RUN apt-get clean && apt-get update -y -qq -RUN apt-get install -y --no-install-recommends apt-utils -RUN python -m pip install --upgrade pip -RUN apt-get install -y \ - build-essential \ - cmake \ - pkg-config \ - wget \ - swig \ - git \ - curl \ - unzip \ - libaio1 \ - nano \ - freetds-dev \ - unixodbc \ - unixodbc-dev \ - libjpeg-dev \ - libtiff5-dev \ - libpng-dev \ - libgtk2.0-dev \ - libavcodec-dev \ - libavformat-dev \ - libswscale-dev \ - libv4l-dev \ - libatlas-base-dev \ - gfortran \ - libhdf5-dev \ - libtbb2 \ - libtbb-dev \ - libgl1-mesa-glx - -COPY forecasting_gluonts/docker_image/requirements.txt . -RUN pip3 install --no-cache-dir -r requirements.txt - -ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.0/morphemic-preprocessor-morphemic-rc2.0.tar.gz /var/lib/morphemic/ - -COPY forecasting_gluonts/ ./ - -RUN cd /var/lib/morphemic/ \ - && tar -zxf morphemic-preprocessor-morphemic-rc2.0.tar.gz \ - && cd morphemic-preprocessor-morphemic-rc2.0 \ - && cd morphemic-datasetmaker && python3 setup.py install \ - && cd ../.. \ - && rm -rf /var/lib/morphemic \ - && mkdir -p /logs && mkdir /morphemic_project/models - - CMD python /morphemic_project/main.py > /logs/gluonts.log 2>&1 +FROM python:3.8.10 +WORKDIR /morphemic_project +RUN apt-get clean && apt-get update -y -qq +RUN apt-get install -y --no-install-recommends apt-utils +RUN python -m pip install --upgrade pip +RUN apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + wget \ + swig \ + git \ + curl \ + unzip \ + libaio1 \ + nano \ + freetds-dev \ + unixodbc \ + unixodbc-dev \ + libjpeg-dev \ + libtiff5-dev \ + libpng-dev \ + libgtk2.0-dev \ + libavcodec-dev \ + libavformat-dev \ + libswscale-dev \ + libv4l-dev \ + libatlas-base-dev \ + gfortran \ + libhdf5-dev \ + libtbb2 \ + libtbb-dev \ + libgl1-mesa-glx + +COPY forecasting_gluonts/docker_image/requirements.txt . +RUN pip3 install --no-cache-dir -r requirements.txt + +ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.0/morphemic-preprocessor-morphemic-rc2.0.tar.gz /var/lib/morphemic/ + +COPY forecasting_gluonts/ ./ + +RUN cd /var/lib/morphemic/ \ + && tar -zxf morphemic-preprocessor-morphemic-rc2.0.tar.gz \ + && cd morphemic-preprocessor-morphemic-rc2.0 \ + && cd morphemic-datasetmaker && python3 setup.py install \ + && cd ../.. \ + && rm -rf /var/lib/morphemic \ + && mkdir -p /logs && mkdir /morphemic_project/models + + CMD python /morphemic_project/main.py > /logs/gluonts.log 2>&1 diff --git a/forecasting_prophet/docker_image/Dockerfile b/forecasting_prophet/docker_image/Dockerfile index 81def71e24249ca0b6e0fcfaa462191e09c2f8d3..3b31be20a8912c3a90e0863fc25bf4f1e9894a19 100644 --- a/forecasting_prophet/docker_image/Dockerfile +++ b/forecasting_prophet/docker_image/Dockerfile @@ -1,87 +1,87 @@ -FROM python:3.8.10 -WORKDIR /morphemic_project -#ENV DEBIAN_FRONTEND noninteractive -RUN apt-get clean && apt-get update -y -qq -RUN apt-get install -y --no-install-recommends apt-utils -RUN python -m pip install --upgrade pip -# Install generic packages -# ------------------------ -RUN apt-get install -y \ - build-essential \ - cmake \ - pkg-config \ - wget \ - swig \ - git \ - curl \ - unzip \ - libaio1 \ - nano \ - freetds-dev \ - unixodbc \ - unixodbc-dev \ - libjpeg-dev \ - libtiff5-dev \ - libpng-dev \ - libgtk2.0-dev \ - libavcodec-dev \ - libavformat-dev \ - libswscale-dev \ - libv4l-dev \ - libatlas-base-dev \ - gfortran \ - libhdf5-dev \ - libtbb2 \ - libtbb-dev \ - libgl1-mesa-glx -# Install ML libraries -# ------------------------ -RUN pip install tensorflow keras -RUN pip install scikit-learn -RUN pip install pandas -RUN pip install matplotlib -RUN pip install numpy -RUN pip install plotly -RUN pip install scipy -RUN pip install convertdate -RUN pip install lunarcalendar -RUN pip install holidays -RUN pip install tqdm -# Install prophet -# ------------------------ -RUN pip install pystan==2.19.1.1 -RUN pip install fbprophet -#Install gluonts -#------------------------- -RUN pip install mxnet -RUN pip install gluonts -#Install ActiveMQ integration libraries -#------------------------- -RUN pip install stomp.py -RUN pip install PyYAML -RUN pip install Timeloop -RUN pip install python-slugify -#Install DatasetMaker integration libraries -#------------------------- -RUN pip install filelock -RUN pip install influxDB -# Force the installation of the version 0.10.8.1 of py4j -# This is due to the double connection from docker issue with the version 0.10.9 -# ------------------------ -RUN python -m pip install py4j==0.10.8.1 -RUN rm -rf ~/.cache/pip -#Clone the prophet forecaster code -#------------------------- -ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.0/morphemic-preprocessor-morphemic-rc2.0.tar.gz /var/lib/morphemic/ - -COPY forecasting_prophet/ ./ - -RUN cd /var/lib/morphemic/ \ - && tar -zxf morphemic-preprocessor-morphemic-rc2.0.tar.gz \ - && cd morphemic-preprocessor-morphemic-rc2.0 \ - && cd morphemic-datasetmaker && python3 setup.py install \ - && cd ../.. \ - && rm -rf /var/lib/morphemic \ - && mkdir -p /logs && mkdir /morphemic_project/models - -CMD python /morphemic_project/main.py > /logs/prophet.log 2>&1 +FROM python:3.8.10 +WORKDIR /morphemic_project +#ENV DEBIAN_FRONTEND noninteractive +RUN apt-get clean && apt-get update -y -qq +RUN apt-get install -y --no-install-recommends apt-utils +RUN python -m pip install --upgrade pip +# Install generic packages +# ------------------------ +RUN apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + wget \ + swig \ + git \ + curl \ + unzip \ + libaio1 \ + nano \ + freetds-dev \ + unixodbc \ + unixodbc-dev \ + libjpeg-dev \ + libtiff5-dev \ + libpng-dev \ + libgtk2.0-dev \ + libavcodec-dev \ + libavformat-dev \ + libswscale-dev \ + libv4l-dev \ + libatlas-base-dev \ + gfortran \ + libhdf5-dev \ + libtbb2 \ + libtbb-dev \ + libgl1-mesa-glx +# Install ML libraries +# ------------------------ +RUN pip install tensorflow keras +RUN pip install scikit-learn +RUN pip install pandas +RUN pip install matplotlib +RUN pip install numpy +RUN pip install plotly +RUN pip install scipy +RUN pip install convertdate +RUN pip install lunarcalendar +RUN pip install holidays +RUN pip install tqdm +# Install prophet +# ------------------------ +RUN pip install pystan==2.19.1.1 +RUN pip install fbprophet +#Install gluonts +#------------------------- +RUN pip install mxnet +RUN pip install gluonts +#Install ActiveMQ integration libraries +#------------------------- +RUN pip install stomp.py +RUN pip install PyYAML +RUN pip install Timeloop +RUN pip install python-slugify +#Install DatasetMaker integration libraries +#------------------------- +RUN pip install filelock +RUN pip install influxDB +# Force the installation of the version 0.10.8.1 of py4j +# This is due to the double connection from docker issue with the version 0.10.9 +# ------------------------ +RUN python -m pip install py4j==0.10.8.1 +RUN rm -rf ~/.cache/pip +#Clone the prophet forecaster code +#------------------------- +ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.0/morphemic-preprocessor-morphemic-rc2.0.tar.gz /var/lib/morphemic/ + +COPY forecasting_prophet/ ./ + +RUN cd /var/lib/morphemic/ \ + && tar -zxf morphemic-preprocessor-morphemic-rc2.0.tar.gz \ + && cd morphemic-preprocessor-morphemic-rc2.0 \ + && cd morphemic-datasetmaker && python3 setup.py install \ + && cd ../.. \ + && rm -rf /var/lib/morphemic \ + && mkdir -p /logs && mkdir /morphemic_project/models + +CMD python /morphemic_project/main.py > /logs/prophet.log 2>&1 diff --git a/morphemic-slo-severity-calculator/pom.xml b/morphemic-slo-severity-calculator/pom.xml index bfe193bdbf2cd451f6369c4f4083cf698c1f2889..56feb6467740defe13d8fba2b55c466646b44b9a 100644 --- a/morphemic-slo-severity-calculator/pom.xml +++ b/morphemic-slo-severity-calculator/pom.xml @@ -129,7 +129,6 @@ 4.13.1 test - diff --git a/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java b/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java index df80346dfeb5f8799373df5800c8f432bd9ea15e..634733150ccde84a0f001516daa943a07da0a86f 100644 --- a/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java +++ b/morphemic-slo-severity-calculator/src/main/java/metric_retrieval/AttributeSubscription.java @@ -102,6 +102,14 @@ public class AttributeSubscription { PREDICTION_EXISTS.setValue(true); PREDICTION_EXISTS.notifyAll(); } + }else{ + if(adaptation_times.contains(targeted_prediction_time)){ + Logger.getAnonymousLogger().log(info_logging_level, "Could not add the new targeted prediction time " + targeted_prediction_time + " from topic " + topic + " as it is already present"); + }else if (!adaptation_times_pending_processing.contains(targeted_prediction_time)){ + Logger.getAnonymousLogger().log(info_logging_level, "Could not add the new targeted prediction time " + targeted_prediction_time + " from topic " + topic + " as it is already pending processing"); + }else if (targeted_prediction_time * 1000 - time_horizon_seconds * 1000L - (Clock.systemUTC()).millis()<=0) { + Logger.getAnonymousLogger().log(info_logging_level, "Could not add the new targeted prediction time " + targeted_prediction_time + " from topic " + topic + " as it would expire in " + (targeted_prediction_time * 1000 - System.currentTimeMillis()) + " milliseconds and the prediction horizon is " + time_horizon_seconds * 1000L + " milliseconds"); + } } ADAPTATION_TIMES_MODIFY.setValue(true); ADAPTATION_TIMES_MODIFY.notifyAll(); diff --git a/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java b/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java index 433fd604e106509af27d94d76c8d0952e2e81091..158fb6ce2018ed967b8c9dda6ff65892ebcce114 100644 --- a/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java +++ b/morphemic-slo-severity-calculator/src/main/java/runtime/Main.java @@ -584,7 +584,9 @@ public class Main { can_modify_slo_rules.setValue(false); for (SLORule slo_rule : slo_rules) { for (SLOSubRule subrule : slo_rule.getSlo_subrules()) { - getPredicted_monitoring_attributes().get(subrule.getId()).remove(processed_adaptation_time); + if (getPredicted_monitoring_attributes().containsKey(subrule.getId())) { + getPredicted_monitoring_attributes().get(subrule.getId()).remove(processed_adaptation_time); + } } } can_modify_slo_rules.setValue(true); @@ -598,18 +600,24 @@ public class Main { * @param rule_severity The severity of the rule which has been determined * @return The probability of the rule being violated. The minimum value of this probability is 0, and increases as the severity increases */ - private static double determine_slo_violation_probability(double rule_severity) { + public static double determine_slo_violation_probability(double rule_severity) { if (slo_violation_determination_method.equals("all-metrics")) { //39.64 is the mean severity value when examining all integer severity values for roc x probability x confidence_interval x delta_value in (-100,100)x(0,100)x(0,100)x(-100,100) - if (rule_severity >= 39.64) { - return Math.min((50 + 100*(rule_severity - 39.64) / 120.72)/100,1); // in case we desire the probability to start from 50% - // return Math.min((100*(rule_severity - 39.64) / 60.36)/100,1); // in case we desire the probability to start from 0% + if (rule_severity >= 40) { + return Math.min((50 + 50*(rule_severity - 40) / 60)/100,1); // in case we desire the probability to start from 50% + // return Math.min((100*(rule_severity - 40) / 60)/100,1); // in case we desire the probability to start from 0% } else { return 0; } }else if (slo_violation_determination_method.equals("prconf-delta")){ - Logger.getAnonymousLogger().log(warning_logging_level,"The calculation of probability for the prconf-delta method needs to be implemented"); - return 0; + //Logger.getAnonymousLogger().log(warning_logging_level,"The calculation of probability for the prconf-delta method needs to be implemented"); + //return 0; + if (rule_severity >= 6.52){ + return Math.min((50+50*(rule_severity-6.52)/93.48)/100,1); + }else{ + return 0; + } + }else{ Logger.getAnonymousLogger().log(warning_logging_level,"Unknown severity calculation method"); return 0; diff --git a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java index 6804590c3e34edd537c5bcd744a1f734c32ba611..04391830f4c8dc6b5bea491cf0b5e9f0b9b5b6c2 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java +++ b/morphemic-slo-severity-calculator/src/main/java/utilities/SLOViolationCalculator.java @@ -51,6 +51,12 @@ public class SLOViolationCalculator { double severity_sum = (predictionAttribute.getDelta()*predictionAttribute.getProbability_confidence()*(100-predictionAttribute.getNormalizedConfidenceIntervalWidth()/100))/(100*100*100); //dividing by 10000 to normalize; Logger.getAnonymousLogger().log(info_logging_level,"The prconf-delta attribute severity for " + predictionAttribute.getName() + " based on a (prconf,delta,confidence_interval) triplet of (" + predictionAttribute.getProbability_confidence() + "," + predictionAttribute.getDelta() +","+predictionAttribute.getConfidence_interval_width()+ ") is " + severity_sum); + + if (severity_sum<0){ + Logger.getAnonymousLogger().log(info_logging_level,"A NaN severity value may be produced due to the root of a negative severity sum - returning zero instead for severity sum"); + severity_sum = 0; + } + return severity_sum; } } diff --git a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java index 3a80f8bb04e8c996a0822a9c556713a90c134c6a..636b2edc8e0184fb2f5aa3bed1410f30d0df1b29 100644 --- a/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java +++ b/morphemic-slo-severity-calculator/src/main/java/utility_beans/PredictedMonitoringAttribute.java @@ -82,6 +82,7 @@ public class PredictedMonitoringAttribute { //this.previous_delta = 100*Math.abs(current_value-threshold)/(threshold-getMonitoring_attributes_statistics().get(name).getLower_bound()); } + this.delta = Math.min(Math.max(this.delta,-100),100); this.probability_confidence = probability_confidence; this.confidence_interval_width = confidence_interval_width; //actual_metric_values = get_last_n_actual_values(Constants.elements_considered_in_prediction, MonitoringAttribute.get_monitoring_attributes_values_map().get(name),true);