Skip to content

GitLab

Commit 3d67c58c authored by Jean-Didier Totow's avatar Jean-Didier Totow
Browse files

auto builder

parent 9e3caa20
Hide whitespace changes
Inline Side-by-side
labels.morphemic 0 → 100644
View file @ 3d67c58c
[{"configs": "docker"}, {"languages": ["python", "go", "java"]}]
\ No newline at end of file
morphemic-performance-model/README.md
View file @ 3d67c58c
......@@ -5,3 +5,5 @@ This is an experimental repository in which we try to predict the performance of
![](images/Morphemic-Page-2.png)
The performance model image can be built using the script /ml_code/build-performance-model.sh
\ No newline at end of file
morphemic-performance-model/ml_code/build-performance-model.sh
View file @ 3d67c58c
#!/bin/bash
python3 -m grpc_tools.protoc -I./src/protos --python_out=. --grpc_python_out=. ./src/protos/service.proto
sudo docker build -t jdtotow/performance_model -f ./deployment/Dockerfile .
sudo docker push jdtotow/performance_model
cd ../evaluator
sudo docker build -t jdtotow/evaluator .
sudo docker push jdtotow/evaluator
\ No newline at end of file
sudo docker push jdtotow/performance_model
\ No newline at end of file
morphemic-performance-model/ml_code/deployment/Dockerfile
View file @ 3d67c58c
FROM tiangolo/uwsgi-nginx-flask:python3.8
RUN python3 -m pip install --upgrade pip
RUN apt update &&\
apt upgrade -y && \
apt install -y swig
FROM python:3.7
RUN pip install --upgrade pip
COPY ./src/requirements.txt /
RUN pip install -r /requirements.txt
ENV NGINX_WORKER_PROCESSES auto
ENV NGINX_WORKER_CONNECTIONS 1024
ENV NGINX_WORKER_OPEN_FILES 1024
ENV LISTEN_PORT 8766
ADD ./datasetlib /app/datasetlib
RUN pip install /app/datasetlib
EXPOSE 8766:8766
EXPOSE 8767:8767
ENV UWSGI_CHEAPER 1
EXPOSE 8766 8767
RUN apt-get update
RUN mkdir -p /var/performance_model
RUN mkdir -p /var/performance_model/datasets &&\
mkdir -p /var/performance_model/models &&\
mkdir -p /var/performance_model/train &&\
mkdir -p /var/performance_model/logs &&\
mkdir -p /var/performance_model/db
RUN mkdir -p /run/pid
RUN mkdir -p /var/log/supervisor
COPY ./src /app
COPY ./deployment/uwsgi.ini /app
#COPY ./src/db/prediction.db /var/performance_model/db
RUN chown www-data:www-data /app/db/prediction.db
RUN chmod a+rw /app/db/ /app/db/*
RUN chown -R www-data:www-data /var/performance_model
\ No newline at end of file
RUN apt-get install -y supervisor
ADD ./src /app/
COPY ./deployment/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
WORKDIR /app
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
\ No newline at end of file
morphemic-performance-model/ml_code/deployment/Dockerfile-3 0 → 100644
View file @ 3d67c58c
FROM tiangolo/uwsgi-nginx-flask:python3.8
RUN python3 -m pip install --upgrade pip
RUN apt update &&\
apt upgrade -y && \
apt install -y swig
COPY ./src/requirements.txt /
RUN pip install -r /requirements.txt
ENV NGINX_WORKER_PROCESSES auto
ENV NGINX_WORKER_CONNECTIONS 1024
ENV NGINX_WORKER_OPEN_FILES 1024
ENV LISTEN_PORT 8766
ADD ./datasetlib /app/datasetlib
RUN pip install /app/datasetlib
EXPOSE 8766 8767
ENV UWSGI_CHEAPER 1
RUN mkdir -p /var/performance_model
RUN mkdir -p /var/performance_model/datasets &&\
mkdir -p /var/performance_model/models &&\
mkdir -p /var/performance_model/train &&\
mkdir -p /var/performance_model/logs &&\
mkdir -p /var/performance_model/db
COPY ./src /app
COPY ./deployment/uwsgi.ini /app
#COPY ./src/db/prediction.db /var/performance_model/db
RUN chown www-data:www-data /app/db/prediction.db
RUN chmod a+rw /app/db/ /app/db/*
RUN chown -R www-data:www-data /var/performance_model
morphemic-performance-model/ml_code/deployment/supervisord.conf
View file @ 3d67c58c
......@@ -16,15 +16,29 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///run/supervisor.sock
[program:nginx]
command=/usr/sbin/nginx -g "daemon off;" -c /etc/nginx/nginx.conf
[program:evaluator]
command=/app/start_evaluator.sh
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:uwsgi]
command=/usr/local/bin/uwsgi --ini /etc/uwsgi/apps-enabled/uwsgi.ini
[program:rpc_server]
command=/app/start_rpc.sh
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:db_checker]
command=/app/start_dbchecker.sh
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:manager]
command=/app/start_manager.sh
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
......
morphemic-performance-model/ml_code/example/simple-2.py
View file @ 3d67c58c
......@@ -6,28 +6,32 @@ url = "http://localhost:8766"
url_file = '/home/jean-didier/Projects/morphemic/performance-model/ml_code/example/dataset.csv'
url_file_3 = '/home/jean-didier/Projects/morphemic/performance-model/ml_code/example/all.csv'
url_file_4 = '/var/performance_model/datasets/all.csv'
#features = ['time','served_request','request_rate','response_time','performance','cpu_usage','memory']
#features_3 = ['number','served_request','request_rate','number_instances','response_time','performance','cpu_usage','cpu_alloc','memory','memory_alloc']
features_2 = ['cpu_usage','memory','level','response_time','latency']
#features_2 = ['cpu_usage','memory','level','response_time','latency']
features_4 = ["EstimatedRemainingTimeContext","SimulationLeftNumber","SimulationElapsedTime","NotFinishedOnTime","MinimumCoresContext","NotFinished","WillFinishTooSoonContext","NotFinishedOnTimeContext","MinimumCores","ETPercentile","RemainingSimulationTimeMetric","TotalCores"]
#post_data = {'url_file': url_file, 'application': 'application-1','target':'performance','features': features}
post_data_2 = {'url_file': "", 'application': 'fcr','target':'response_time','features': features_2}
#post_data_2 = {'url_file': "", 'application': 'demo','target':'response_time','features': features_2, "variant": "vm"}
#post_data_3 = {'url_file': url_file_3, 'application': 'application-3','target':'performance','features': features_3}
post_data_4 = {'url_file': url_file_4, 'application': 'genome','target':'EstimatedRemainingTimeContext','features': features_4,'variant':'vm'}
#print("Get model")
#response = requests.post(url+"/api/v1/model", data='{"application":"application-2"}', headers={'Content-Type':'application/json'})
#print(response.text)
#response = requests.post(url+"/api/v1/model/train", data=json.dumps(post_data_2),headers={'Content-Type':'application/json'}).text
#print("Training phase")
#print(response)
response = requests.post(url+"/api/v1/model/train", data=json.dumps(post_data_4),headers={'Content-Type':'application/json'}).text
print("Training phase")
print(response)
#time.sleep(5)
#prediction request
#features = {'cpu_alloc': 1 ,'memory_alloc': 64,'number_instances':4, "memory": 51086677.3333}
features = {'cpu_usage': 31, "memory": 4500.23, 'latency': 2.1, 'level': 1}
post_data = {'application': 'fcr','target':'response_time','features':features}
response = requests.post(url+"/api/v1/model/predict", data=json.dumps(post_data),headers={'Content-Type':'application/json'}).text
print(response)
\ No newline at end of file
#features = {'cpu_usage': 31, "memory": 4500.23, 'latency': 2.1, 'level': 1}
#post_data = {'application': 'fcr','target':'response_time','features':features}
#response = requests.post(url+"/api/v1/model/predict", data=json.dumps(post_data),headers={'Content-Type':'application/json'}).text
#print(response)
\ No newline at end of file
morphemic-performance-model/ml_code/service_pb2.py 0 → 100644
View file @ 3d67c58c
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: service.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='service.proto',
package='proto',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n\rservice.proto\x12\x05proto\"}\n\x03\x41ny\x12\x13\n\tint_value\x18\x01 \x01(\x03H\x00\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x16\n\x0cstring_value\x18\x03 \x01(\tH\x00\x12*\n\nlist_value\x18\x04 \x01(\x0b\x32\x14.proto.ListOfStringsH\x00\x42\x06\n\x04kind\".\n\x04Pair\x12\x0b\n\x03key\x18\x01 \x02(\t\x12\x19\n\x05value\x18\x02 \x02(\x0b\x32\n.proto.Any\"v\n\nDictionary\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.proto.Dictionary.FieldsEntry\x1a\x39\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.proto.Any:\x02\x38\x01\"k\n\x0ePredictRequest\x12\x13\n\x0b\x61pplication\x18\x01 \x02(\t\x12#\n\x08\x66\x65\x61tures\x18\x02 \x02(\x0b\x32\x11.proto.Dictionary\x12\x0e\n\x06target\x18\x03 \x02(\t\x12\x0f\n\x07variant\x18\x04 \x02(\t\"r\n\x0cPredictReply\x12\x0e\n\x06status\x18\x01 \x02(\x08\x12\"\n\x07results\x18\x02 \x02(\x0b\x32\x11.proto.Dictionary\x12\x1d\n\x02ml\x18\x03 \x02(\x0b\x32\x11.proto.Dictionary\x12\x0f\n\x07message\x18\x04 \x02(\t\"3\n\x0cModelRequest\x12\x13\n\x0b\x61pplication\x18\x01 \x02(\t\x12\x0e\n\x06target\x18\x02 \x02(\t\"0\n\nModelReply\x12\"\n\x07results\x18\x01 \x02(\x0b\x32\x11.proto.Dictionary\"8\n\tListValue\x12+\n\rlistOfStrings\x18\x01 \x03(\x0b\x32\x14.proto.ListOfStrings\" \n\rListOfStrings\x12\x0f\n\x07strings\x18\x01 \x03(\t\"~\n\x0cTrainRequest\x12\x13\n\x0b\x61pplication\x18\x01 \x02(\t\x12\x10\n\x08url_file\x18\x02 \x02(\t\x12&\n\x08\x66\x65\x61tures\x18\x03 \x02(\x0b\x32\x14.proto.ListOfStrings\x12\x0e\n\x06target\x18\x04 \x02(\t\x12\x0f\n\x07variant\x18\x05 \x02(\t\"B\n\nTrainReply\x12\x0e\n\x06status\x18\x01 \x02(\x08\x12\x0f\n\x07message\x18\x02 \x02(\t\x12\x13\n\x0b\x61pplication\x18\x03 \x02(\t2\xbb\x01\n\x07Predict\x12\x42\n\x12PredictPerformance\x12\x15.proto.PredictRequest\x1a\x13.proto.PredictReply\"\x00\x12\x34\n\x08getModel\x12\x13.proto.ModelRequest\x1a\x11.proto.ModelReply\"\x00\x12\x36\n\ntrainModel\x12\x13.proto.TrainRequest\x1a\x11.proto.TrainReply\"\x00')
)
_ANY = _descriptor.Descriptor(
name='Any',
full_name='proto.Any',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='int_value', full_name='proto.Any.int_value', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='float_value', full_name='proto.Any.float_value', index=1,
number=2, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='string_value', full_name='proto.Any.string_value', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='list_value', full_name='proto.Any.list_value', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='kind', full_name='proto.Any.kind',
index=0, containing_type=None, fields=[]),
],
serialized_start=24,
serialized_end=149,
)
_PAIR = _descriptor.Descriptor(
name='Pair',
full_name='proto.Pair',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='proto.Pair.key', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='proto.Pair.value', index=1,
number=2, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=151,
serialized_end=197,
)
_DICTIONARY_FIELDSENTRY = _descriptor.Descriptor(
name='FieldsEntry',
full_name='proto.Dictionary.FieldsEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='proto.Dictionary.FieldsEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='proto.Dictionary.FieldsEntry.value', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=_b('8\001'),
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=260,
serialized_end=317,
)
_DICTIONARY = _descriptor.Descriptor(
name='Dictionary',
full_name='proto.Dictionary',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='fields', full_name='proto.Dictionary.fields', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_DICTIONARY_FIELDSENTRY, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=199,
serialized_end=317,
)
_PREDICTREQUEST = _descriptor.Descriptor(
name='PredictRequest',
full_name='proto.PredictRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='application', full_name='proto.PredictRequest.application', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='features', full_name='proto.PredictRequest.features', index=1,
number=2, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='target', full_name='proto.PredictRequest.target', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='variant', full_name='proto.PredictRequest.variant', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=319,
serialized_end=426,
)
_PREDICTREPLY = _descriptor.Descriptor(
name='PredictReply',
full_name='proto.PredictReply',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='status', full_name='proto.PredictReply.status', index=0,
number=1, type=8, cpp_type=7, label=2,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='results', full_name='proto.PredictReply.results', index=1,
number=2, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='ml', full_name='proto.PredictReply.ml', index=2,
number=3, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='message', full_name='proto.PredictReply.message', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=428,
serialized_end=542,
)
_MODELREQUEST = _descriptor.Descriptor(
name='ModelRequest',
full_name='proto.ModelRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='application', full_name='proto.ModelRequest.application', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='target', full_name='proto.ModelRequest.target', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=544,
serialized_end=595,
)
_MODELREPLY = _descriptor.Descriptor(
name='ModelReply',
full_name='proto.ModelReply',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='results', full_name='proto.ModelReply.results', index=0,
number=1, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=597,
serialized_end=645,
)
_LISTVALUE = _descriptor.Descriptor(
name='ListValue',
full_name='proto.ListValue',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='listOfStrings', full_name='proto.ListValue.listOfStrings', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=647,
serialized_end=703,
)
_LISTOFSTRINGS = _descriptor.Descriptor(
name='ListOfStrings',
full_name='proto.ListOfStrings',
filename=None,