Commit 643fa49e authored by Sebastian Miller's avatar Sebastian Miller
Browse files

enhanced requests training

parent 99234821
......@@ -42,14 +42,20 @@ MQ_TOPIC = properties["components"]["stomp"]["destination"]
ROUTE = "http://" + HOST_IP + ":" + PORT + properties['api-endpoint']
METRIC = "MinCPULoad"
METRIC1 = "MinCPULoad"
METRIC2 = "MaxCPULoad"
TRAIN_CPU_MSG = \
'{ \
"metrics": [ \
{ \
"metric": "' + METRIC + '", \
"metric": "' + METRIC1 + '", \
"level":3, \
"publish_rate":50000 \
"publish_rate":30000 \
}, \
{ \
"metric": "' + METRIC2 + '", \
"level":3, \
"publish_rate":30000 \
} \
], \
"models": ["cnn","tft","nbeats"] \
......@@ -57,8 +63,7 @@ TRAIN_CPU_MSG = \
class TestRequests:
max_forecast = 200
train_timeout = 300 # max time in seconds given for the NN to complete its training
metric = METRIC
train_timeout = 500 # max time in seconds given for the NN to complete its training
conn = {}
if GITLAB_ACTIVEMQ_HOST_IP is None:
......@@ -67,10 +72,10 @@ class TestRequests:
conn = stomp.Connection([(GITLAB_ACTIVEMQ_HOST_IP, STOMP_PORT)])
conn.connect(USERNAME, PASSWORD, wait=True)
def create_json(self):
def create_json(self, metric):
json={
"method": "RL",
"metric": self.metric,
"metric": metric,
"predictionTime": 0,
"predictionToEnsemble": {
"tft": random.randrange(0, self.max_forecast),
......@@ -80,10 +85,23 @@ class TestRequests:
}
return json
def create_json_NaN(self):
def create_json_single_NaN(self, metric):
json={
"method": "RL",
"metric": self.metric,
"metric": metric,
"predictionTime": 0,
"predictionToEnsemble": {
"tft": random.randrange(0, self.max_forecast),
"cnn": random.randrange(0, self.max_forecast),
"nbeats": None
}
}
return json
def create_json_NaN(self, metric):
json={
"method": "RL",
"metric": metric,
"predictionTime": 0,
"predictionToEnsemble": {
"tft": None,
......@@ -93,10 +111,10 @@ class TestRequests:
}
return json
def create_json_wrong(self):
def create_json_wrong(self, metric):
json={
"method": "RL",
"metric": self.metric,
"metric": metric,
"predictionTime": 0,
"predictionToEnsemble": {
"tft": 10,
......@@ -106,54 +124,72 @@ class TestRequests:
}
return json
def post_request(self, NaN : bool = False, wrong : bool = False):
def post_request(
self,
metric,
singleNaN : bool = False,
NaN : bool = False,
wrong : bool = False
):
if wrong:
r = requests.post(ROUTE, json=self.create_json_wrong())
r = requests.post(ROUTE, json=self.create_json_wrong(metric))
elif singleNaN:
r = requests.post(ROUTE, json=self.create_json_single_NaN(metric))
elif NaN:
r = requests.post(ROUTE, json=self.create_json_NaN())
r = requests.post(ROUTE, json=self.create_json_NaN(metric))
else:
r = requests.post(ROUTE, json=self.create_json())
r = requests.post(ROUTE, json=self.create_json(metric))
return r
def send_train_request(self):
self.conn.send(MQ_TOPIC, TRAIN_CPU_MSG)
#TODO @pytest.mark.parametrize()
def test_response_untrained(self):
r = self.post_request()
print(f"Status Code: {r.status_code}, Response: {r.json()}")
print(r.json())
assert r.status_code == NO_MODEL_CODE
# sends a request to train a new model and waits until
# it has finished training
def ensure_trained(self):
self.send_train_request()
code = self.post_request().status_code
def ensure_trained(self, metric):
code = self.post_request(metric).status_code
i = 0
while code == NO_MODEL_CODE:
r = self.post_request()
r = self.post_request(metric)
code = r.status_code
print(i, r.json())
sleep(1.0) # sleep for 1.5 sec
sleep(1.0) # sleep for 1.0 sec
assert(i < self.train_timeout)
i = i+1
assert code == RESPONSE_CODE
# sends a request to train two new models and waits until
# they have finished training
def order_and_ensure_training(self):
self.send_train_request()
self.ensure_trained(METRIC1)
self.ensure_trained(METRIC2)
#TODO @pytest.mark.parametrize()
def test_many_requests(self):
self.ensure_trained()
self.order_and_ensure_training()
# we have received a response code with the first ensembled prediction,
# indicating that the training has ended
for i in range (100):
r = self.post_request()
metric = random.choice([METRIC1, METRIC2])
r = self.post_request(metric)
print(i, r.json())
assert r.status_code == RESPONSE_CODE
for i in range (100):
r = self.post_request(wrong = True) # NaN values in forecaster predictions
metric = random.choice([METRIC1, METRIC2])
r = self.post_request(metric, singleNaN = True) # single NaN value in forecaster predictions
print(i, r.json())
assert r.status_code == NO_MODEL_CODE
for i in range (100):
metric = random.choice([METRIC1, METRIC2])
r = self.post_request(metric, wrong = True) # inexistent forecaster name
print(i, r.json())
assert r.status_code == NO_MODEL_CODE
for i in range (100):
r = self.post_request(NaN = True) # NaN values in forecaster predictions
metric = random.choice([METRIC1, METRIC2])
r = self.post_request(metric, NaN = True) # NaN values in forecaster predictions
print(i, r.json())
assert r.status_code == RESPONSE_CODE
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment