Commit c27a2592 authored by Jean-Didier's avatar Jean-Didier
Browse files

polymorphic model support

parent b4f459f8
......@@ -43,3 +43,10 @@ stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:activemq_listener]
command=/app/start_activemq_listener.sh
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
......@@ -17,6 +17,7 @@ features_4 = ["EstimatedRemainingTimeContext","SimulationLeftNumber","Simulation
#post_data = {'url_file': url_file, 'application': 'application-1','target':'performance','features': features}
#post_data_2 = {'url_file': "", 'application': 'demo','target':'response_time','features': features_2, "variant": "vm"}
#post_data_3 = {'url_file': url_file_3, 'application': 'application-3','target':'performance','features': features_3}
# "default_application"
post_data_4 = {'url_file': url_file_4, 'application': 'genome','target':'EstimatedRemainingTimeContext','features': features_4,'variant':'vm'}
#print("Get model")
#response = requests.post(url+"/api/v1/model", data='{"application":"application-2"}', headers={'Content-Type':'application/json'})
......@@ -34,4 +35,29 @@ print(response)
#features = {'cpu_usage': 31, "memory": 4500.23, 'latency': 2.1, 'level': 1}
#post_data = {'application': 'fcr','target':'response_time','features':features}
#response = requests.post(url+"/api/v1/model/predict", data=json.dumps(post_data),headers={'Content-Type':'application/json'}).text
#print(response)
\ No newline at end of file
#print(response)
# utility : input (configs) - Mem , Cpu, Number vm provided to an application
# utility : metrics (formula) - datasets
# ??
# utility : output (utility value)
"""
Training
topic : performance_module_train
data: [
{'application': 'genome','target':'EstimatedRemainingTimeContext','features': features_4,'variant':'vm', 'sender_id':'solver_id'},
{'application': 'genome','target':'SimulationLeftNumber','features': features_4,'variant':'vm'},
]
Predict
topic : performance_module_predict
data : {'application': 'fcr','target':'response_time','features':features, 'variant': 'vm', 'sender_id':'solver_id'}
Prediction response
topic : performance_module_predict_feedback
data : {'application': 'app_name', 'target': 'targeted_field', 'variant': 'vm', 'value':'value_predicted (float)', 'sender_id': 'solver-id'}
Feedback
topic : performance_module_train_feedback
data : {'application': 'application_name', 'target': 'SimulationLeftNumber', 'variant': 'vm', 'sender_id':'solver_id'}
"""
\ No newline at end of file
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import service_pb2 as service__pb2
class PredictStub(object):
# missing associated documentation comment in .proto file
pass
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.PredictPerformance = channel.unary_unary(
'/proto.Predict/PredictPerformance',
request_serializer=service__pb2.PredictRequest.SerializeToString,
response_deserializer=service__pb2.PredictReply.FromString,
)
self.getModel = channel.unary_unary(
'/proto.Predict/getModel',
request_serializer=service__pb2.ModelRequest.SerializeToString,
response_deserializer=service__pb2.ModelReply.FromString,
)
self.trainModel = channel.unary_unary(
'/proto.Predict/trainModel',
request_serializer=service__pb2.TrainRequest.SerializeToString,
response_deserializer=service__pb2.TrainReply.FromString,
)
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.PredictPerformance = channel.unary_unary(
'/proto.Predict/PredictPerformance',
request_serializer=service__pb2.PredictRequest.SerializeToString,
response_deserializer=service__pb2.PredictReply.FromString,
)
self.getModel = channel.unary_unary(
'/proto.Predict/getModel',
request_serializer=service__pb2.ModelRequest.SerializeToString,
response_deserializer=service__pb2.ModelReply.FromString,
)
self.trainModel = channel.unary_unary(
'/proto.Predict/trainModel',
request_serializer=service__pb2.TrainRequest.SerializeToString,
response_deserializer=service__pb2.TrainReply.FromString,
)
class PredictServicer(object):
# missing associated documentation comment in .proto file
pass
def PredictPerformance(self, request, context):
# missing associated documentation comment in .proto file
pass
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def getModel(self, request, context):
# missing associated documentation comment in .proto file
pass
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def trainModel(self, request, context):
# missing associated documentation comment in .proto file
pass
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
"""Missing associated documentation comment in .proto file."""
def PredictPerformance(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def getModel(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def trainModel(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_PredictServicer_to_server(servicer, server):
rpc_method_handlers = {
'PredictPerformance': grpc.unary_unary_rpc_method_handler(
servicer.PredictPerformance,
request_deserializer=service__pb2.PredictRequest.FromString,
response_serializer=service__pb2.PredictReply.SerializeToString,
),
'getModel': grpc.unary_unary_rpc_method_handler(
servicer.getModel,
request_deserializer=service__pb2.ModelRequest.FromString,
response_serializer=service__pb2.ModelReply.SerializeToString,
),
'trainModel': grpc.unary_unary_rpc_method_handler(
servicer.trainModel,
request_deserializer=service__pb2.TrainRequest.FromString,
response_serializer=service__pb2.TrainReply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'proto.Predict', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
rpc_method_handlers = {
'PredictPerformance': grpc.unary_unary_rpc_method_handler(
servicer.PredictPerformance,
request_deserializer=service__pb2.PredictRequest.FromString,
response_serializer=service__pb2.PredictReply.SerializeToString,
),
'getModel': grpc.unary_unary_rpc_method_handler(
servicer.getModel,
request_deserializer=service__pb2.ModelRequest.FromString,
response_serializer=service__pb2.ModelReply.SerializeToString,
),
'trainModel': grpc.unary_unary_rpc_method_handler(
servicer.trainModel,
request_deserializer=service__pb2.TrainRequest.FromString,
response_serializer=service__pb2.TrainReply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'proto.Predict', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Predict(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def PredictPerformance(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/proto.Predict/PredictPerformance',
service__pb2.PredictRequest.SerializeToString,
service__pb2.PredictReply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def getModel(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/proto.Predict/getModel',
service__pb2.ModelRequest.SerializeToString,
service__pb2.ModelReply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def trainModel(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/proto.Predict/trainModel',
service__pb2.TrainRequest.SerializeToString,
service__pb2.TrainReply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
class Metric(enumerate):
"""
[0] (current/detected) Metrics & SLOs Events Format:
This event is aggregated by EMS and it is persisted in InfluxDB. Moreover,
Prediction Orchestrator will subscribe and receive the current metrics in order to
evaluate the forecasting methods, according to the defined KPIs (e.g., MAPE)
* Topic: [metric_name]
> (e.g. MaxCPULoad)
{
"metricValue": 12.34,
"level": 1,
"timestamp": 143532341251,
"refersTo": "MySQL_12345",
"cloud": "AWS-Dublin",
"provider": "AWS"
}
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
TIMESTAMP = "timestamp"
METRIC_VALUE = "metricValue"
REFERS_TO = "refersTo"
CLOUD = "cloud"
PROVIDER = "provider"
class PredictionMetric(enumerate):
"""
[1] Predicted Metrics & SLOs Events Format
This event is produced by the Prediction Orchestrator and reflects the final predicted value for a metric.
- Topic: prediction.[metric_name]
> (e.g. prediction.MaxCPULoad)
{
"metricValue": 12.34,
"level": 1,
"timestamp": 143532341251,
"probability": 0.98,
"confidence_interval " : [8,15]
"predictionTime": 143532342,
"refersTo": "MySQL_12345",
"cloud": "AWS-Dublin",
"provider": "AWS"
}
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
_match = "prediction."
METRICVALUE= "metricValue"
'''Predicted metric value'''
LEVEL= "level"
'''Level of VM where prediction occurred or refers'''
TIMESTAMP= "timestamp"
'''Prediction creation date/time from epoch'''
PROBABILITY= "probability"
'''Probability of the predicted metric value (range 0..1)'''
CONFIDENCE_INTERVAL= "confidence_interval"
'''the probability-confidence interval for the prediction'''
PREDICTION_TIME= "predictionTime"
'''This refers to time point in the imminent future (that is relative to the time
that is needed for reconfiguration) for which the predicted value is considered
valid/accurate (in UNIX Epoch)'''
REFERSTO= "refersTo"
'''The id of the application or component or (VM) host for which the prediction refers to'''
CLOUD= "cloud"
'''Cloud provider of the VM (with location)'''
PROVIDER= "provider"
'''Cloud provider name'''
class MetricsToPredict(enumerate):
"""
[2] Translator – to – Forecasting Methods/Prediction Orchestrator Events Format
This event is produced by the translator, to:
imform Dataset Maker which metrics should subscribe to in order to aggregate the appropriate tanning dataset in the time-series DB.
instruct each of the Forecasting methods to predict the values of one or more monitoring metrics
inform the Prediction Orchestrator for the metrics which will be forecasted
* Topic: metrics_to_predict
*Note:* This event could be communicated through Mule
[
{
"metric": "MaxCPULoad",
"level": 3,
"publish_rate": 60000,
},
{
"metric": "MinCPULoad",
"level": 3,
"publish_rate": 50000,
}
]
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
_match = "metrics_to_predict"
METRIC = "metric"
'''name of the metric to predict'''
LEVEL = "level"
'''Level of monitoring topology where this metric may be produced/found'''
PUBLISH_RATE = "publish_rate"
'''expected rate for datapoints regarding the specific metric (according to CAMEL)'''
class TraningModels(enumerate):
"""
[3] Forecasting Methods – to – Prediction Orchestrator Events Format
This event is produced by each of the Forecasting methods, to inform the
Prediction Orchestrator that the method has (re-)trained its model for one or more metrics.
* Topic: training_models
{
"metrics": ["MaxCPULoad","MinCPULoad"]",
"forecasting_method": "ESHybrid",
"timestamp": 143532341251,
}
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
_match = "training_models"
METRICS = "metrics"
'''metrics for which a certain forecasting method has successfully trained or re-trained its model'''
FORECASTING_METHOD = "forecasting_method"
'''the method that is currently re-training its models'''
TIMESTAMP = "timestamp"
'''date/time of model(s) (re-)training'''
class IntermediatePrediction(enumerate):
"""
[4] Forecasting Methods – to – Prediction Orchestrator Events Format
This event is produced by each of the Forecasting methods, and is used by the Prediction Orchestrator to determine the final prediction value for the particular metric.
* Topic: intermediate_prediction.[forecasting_method].[metric_name]
* (e.g. intermediate_prediction.ESHybrid.MaxCPULoad)
* We note that any component will be able to subscribe to topics like:
* intermediate_prediction.*.MaxCPULoad → gets MaxCPULoad predictions produced by all forecasting methods or
* intermediate_prediction.ESHybrid.* → gets all metrics predictions from ESHybrid method
* We consider that each forecasting method publishes a static (but configurable) number m of predicted values (under the same timestamp) for time points into the future. These time points into the future are relevant to the reconfiguration time that it is needed (and can also be updated).
* For example if we configure m=5 predictions into the future and the reconfiguration time needed is TR=10 minutes, then at t0 a forecasting method publishes 5 events with the same timestamp and prediction times t0+10, t0+20, t0+30, t0+40, t0+50.
{
"metricValue": 12.34,
"level": 3,
"timestamp": 143532341251,
"probability": 0.98,
"confidence_interval " : [8,15]
"predictionTime": 143532342,
"refersTo": "MySQL_12345",
"cloud": "AWS-Dublin",
"provider": "AWS"
}
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
_match="intermediate_prediction."
METRICVALUE = "metricValue"
'''Predicted metric value (more than one such events will be produced for different time points into the future – this can be valuable to the Prediction Orchestrator in certain situations e.g., forecasting method is unreachable for a time period)'''
LEVEL = "level"
'''Level of VM where prediction occurred or refers'''
TIMESTAMP = "timestamp"
'''Prediction creation date/time from epoch'''
PROBABILITY = "probability"
'''Probability of the predicted metric value (range 0..1)'''
CONFIDENCE_INTERVAL = "confidence_interval"
'''the probability-confidence interval for the prediction'''
PREDICTION_TIME = "predictionTime"
'''This refers to time point in the imminent future (that is relative to the time that is needed for reconfiguration) for which the predicted value is considered valid/accurate (in UNIX Epoch)'''
REFERS_TO = "refersTo"
'''The id of the application or component or (VM) host for which the prediction refers to'''
CLOUD = "cloud"
'''Cloud provider of the VM (with location)'''
PROVIDER = "provider"
'''Cloud provider name'''
class Prediction(enumerate):
"""
[5] Prediction Orchestrator – to – Severity-based SLO Violation Detector Events Format
This event is used by the Prediction Orchestrator to inform the SLO Violation Detector about the current values of a metric, which can possibly lead to an SLO Violation detection.
* Topic: prediction.[metric_name]
* (e.g. prediction.MaxCPULoad)
{
"metricValue": 12.34,
"level": 1,
"timestamp": 143532341251,
"probability": 0.98,
"confidence_interval " : [8,15]
"predictionTime": 143532342,
"refersTo": "MySQL_12345",
"cloud": "AWS-Dublin",
"provider": "AWS"
}
https://confluence.7bulls.eu/display/MOR/Forecasting+Mechanism+Sub-components+Communication
"""
_match = "prediction."
METRICVALUE = "metricValue"
'''Predicted metric value'''
LEVEL = "level"
'''Level of VM where prediction occurred or refers'''
TIMESTAMP = "timestamp"
'''Prediction creation date/time from epoch'''
PROBABILITY = "probability"
'''Probability of the predicted metric value (range 0..1)'''
CONFIDENCE_INTERVAL = "confidence_interval"
'''the probability-confidence interval for the prediction'''
PREDICTIONTIME = "predictionTime"
'''This refers to time point in the imminent future (that is relative to the time that is needed for reconfiguration) for which the predicted value is considered valid/accurate (in UNIX Epoch)'''
REFERSTO = "refersTo"
'''The id of the application or component or (VM) host for which the prediction refers to'''
CLOUD = "cloud"
'''Cloud provider of the VM (with location)'''
PROVIDER = "provider"
'''Cloud provider name'''