Commit 098f33ab authored by Sebastian Miller's avatar Sebastian Miller
Browse files

Documented properties; yet another approach to requests test CI

parent bbc3c3d9
......@@ -53,9 +53,7 @@ build:ensembler:
integration-tests:ensembler:
stage: integration-tests
image:
name: docker/compose:1.29.2
entrypoint: [""]
image: $DOCKER_DIND_IMAGE
#image: python:3.8-slim-buster
only:
- integration
......@@ -79,8 +77,9 @@ integration-tests:ensembler:
# /ensembler/config/"] # Volume mounting workaround
script:
- docker load -i image/ensembler.tar # Load the artifact image
- apt-get update -y
- apt-get install -y python3.8
- apk add --no-cache python3 py3-pip
- pip3 install --no-cache-dir docker-compose
- docker-compose -v
- cd $SYSTEM_SETUP_DIR
#- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
#- docker pull $CONTAINER_BASE_TAG:$CI_COMMIT_SHA
......@@ -89,8 +88,8 @@ integration-tests:ensembler:
- chmod u+x ./fill_db.sh
- ./fill_db.sh '127.0.0.1' 'sample_data_separate_runs' # Upload test data to InfluxDB
- cd $CI_PROJECT_DIR/$ENSEMBLER_DIR
- pip install --no-cache-dir --upgrade pip
- pip install --no-cache-dir poetry
- pip3 install --no-cache-dir --upgrade pip
- pip3 install --no-cache-dir poetry
- poetry install
- ENSEMBLER_HOST_IP='127.0.0.1' poetry run pytest tests/test_requests.py # Run the test
- docker-compose stop # Stop the test environment
......
---
# Codes and messages associated with various errors.
errors:
no-adequate-model-code: 512
no-adequate-model-msg: "None of the currently trained models is suitable for ensembling such predictions."
# Params for connecting to InfluxDB and ActiveMQ
components:
influxdb:
port: 8086
......@@ -13,42 +15,49 @@ components:
username: admin
password: admin
destination: /topic/start_ensembler
id: ensemble_settings
reconnect_attempts_max: -1
reconnect_sleep_initial: 10
id: ensemble_settings # Internal id of the topic subscription. Unimportant
reconnect_attempts_max: -1 # -1 means that stomp.py will attempt to reconnect indefinitely
reconnect_sleep_initial: 10 # Time between consecutive reconnect attempts
# Model training hyperparameters
hyperparameters:
n_steps: 100
batch_size: 25
n_epochs: 30
gamma: 0.9
learning_rate: 0.003
dataset_traversals: 6
window_size: 10
episode_length: 100
mode: "continuous"
extend_predictions_by: 0.0
reward_diff_factor: 0.05
extended_state: False
n_steps: 100 # S-B3 n_steps PPO hyperparam
batch_size: 25 # S-B3 batch_size PPO hyperparam
n_epochs: 30 # S-B3 n_epochs PPO hyperparam
gamma: 0.9 # S-B3 gamma PPO hyperparam
learning_rate: 0.003 # S-B3 learning_rate PPO hyperparam
dataset_traversals: 6 # How many times should the whole dataset be stepped through during training
window_size: 10 # Length of the past predictions window (the state passed to the model)
episode_length: 100 # Length of an episode - steps between artificial state resets
mode: "continuous" # Discrete or continuous (enhanced) reward
extend_predictions_by: 0.0 # Mocked forecasters prediction value extension (0.0 means no mocked forecasters)
reward_diff_factor: 0.05 # Factor for the continuous reward
extended_state: False # Whether or not the extended state should be used
# WARNING: the `extended_state` and `extend_predictions_by` params are INCOMPATIBLE with the component.
# Setting them to something other than the default values breaks the component.
# Parameters of the data used for training
training-data-params:
training_set_size: 500
continual_training_set_size: 300
training_set_size_eps: 0.1
run_break_len_factor: 20
max_prediction_value: 100.0
first_prediction_column: 2
grand_truth_column: 1
fresh_data_ratio: 0.5
minimum_cohesive_run_len: 50
training_set_size: 500 # Target training data set size (how many data points should be downloaded from InfluxDB)
continual_training_set_size: 300 # Target continual training data set size
training_set_size_eps: 0.1 # How much more data should be downloaded from InfluxDB (to account for data trimming)
run_break_len_factor: 20 # If the difference between the timestamps of two consecutive downloaded data points
# is greater than 20 * publish_rate, then the data is split into separate runs.
max_prediction_value: 100.0 # Arbitrary value - used only for generating the initial random state
first_prediction_column: 2 # First column in downloaded data that contains predictions
grand_truth_column: 1 # The column in downloaded data that contains the real values
fresh_data_ratio: 0.5 # The proportion: fresh_data_points_number/continual_training_set_size
# (How much fresh data should be used in a single continual training)
minimum_cohesive_run_len: 50 # Minimum number of data points of a single cohesive run
real_values_db_name: melodic_ui
predictions_db_name: morphemic
misc:
training_thread_pool_size: 20
training_thread_pool_size: 20 # The number of threads involved in training.
logging:
filename: ensembler.out
level: DEBUG
filename: ensembler.out # Logs filename
level: DEBUG # Logging level
date-format: "%Y-%m-%d %H:%M:%S"
format: "%(asctime)s.%(msecs)03d %(levelname)s - %(threadName)s - %(module)s - %(funcName)s: %(message)s"
# Gunicorn port (Ensembler component port)
PORT=5000
\ No newline at end of file
---
# Codes and messages associated with various errors.
errors:
no-adequate-model-code: 512
no-adequate-model-msg: "None of the currently trained models is suitable for ensembling such predictions."
# Params for connecting to InfluxDB and ActiveMQ
components:
influxdb:
port: 8086
......@@ -13,42 +15,49 @@ components:
username: admin
password: admin
destination: /topic/start_ensembler
id: ensemble_settings
reconnect_attempts_max: -1
reconnect_sleep_initial: 10
id: ensemble_settings # Internal id of the topic subscription. Unimportant
reconnect_attempts_max: -1 # -1 means that stomp.py will attempt to reconnect indefinitely
reconnect_sleep_initial: 10 # Time between consecutive reconnect attempts
# Model training hyperparameters
hyperparameters:
n_steps: 100
batch_size: 25
n_epochs: 30
gamma: 0.9
learning_rate: 0.003
dataset_traversals: 6
window_size: 10
episode_length: 100
mode: "continuous"
extend_predictions_by: 0.0
reward_diff_factor: 0.05
extended_state: False
n_steps: 100 # S-B3 n_steps PPO hyperparam
batch_size: 25 # S-B3 batch_size PPO hyperparam
n_epochs: 30 # S-B3 n_epochs PPO hyperparam
gamma: 0.9 # S-B3 gamma PPO hyperparam
learning_rate: 0.003 # S-B3 learning_rate PPO hyperparam
dataset_traversals: 6 # How many times should the whole dataset be stepped through during training
window_size: 10 # Length of the past predictions window (the state passed to the model)
episode_length: 100 # Length of an episode - steps between artificial state resets
mode: "continuous" # Discrete or continuous (enhanced) reward
extend_predictions_by: 0.0 # Mocked forecasters prediction value extension (0.0 means no mocked forecasters)
reward_diff_factor: 0.05 # Factor for the continuous reward
extended_state: False # Whether or not the extended state should be used
# WARNING: the `extended_state` and `extend_predictions_by` params are INCOMPATIBLE with the component.
# Setting them to something other than the default values breaks the component.
# Parameters of the data used for training
training-data-params:
training_set_size: 500
continual_training_set_size: 300
training_set_size_eps: 0.1
run_break_len_factor: 20
max_prediction_value: 100.0
first_prediction_column: 2
grand_truth_column: 1
fresh_data_ratio: 0.5
minimum_cohesive_run_len: 50
training_set_size: 500 # Target training data set size (how many data points should be downloaded from InfluxDB)
continual_training_set_size: 300 # Target continual training data set size
training_set_size_eps: 0.1 # How much more data should be downloaded from InfluxDB (to account for data trimming)
run_break_len_factor: 20 # If the difference between the timestamps of two consecutive downloaded data points
# is greater than 20 * publish_rate, then the data is split into separate runs.
max_prediction_value: 100.0 # Arbitrary value - used only for generating the initial random state
first_prediction_column: 2 # First column in downloaded data that contains predictions
grand_truth_column: 1 # The column in downloaded data that contains the real values
fresh_data_ratio: 0.5 # The proportion: fresh_data_points_number/continual_training_set_size
# (How much fresh data should be used in a single continual training)
minimum_cohesive_run_len: 50 # Minimum number of data points of a single cohesive run
real_values_db_name: melodic_ui
predictions_db_name: morphemic
misc:
training_thread_pool_size: 20
training_thread_pool_size: 20 # The number of threads involved in training.
logging:
filename: ensembler.out
level: DEBUG
filename: ensembler.out # Logs filename
level: DEBUG # Logging level
date-format: "%Y-%m-%d %H:%M:%S"
format: "%(asctime)s.%(msecs)03d %(levelname)s - %(threadName)s - %(module)s - %(funcName)s: %(message)s"
# Gunicorn port (Ensembler component port)
PORT=5000
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment