Commit 8b503d6d authored by mriedl's avatar mriedl
Browse files

Merge with 2.0

parents 4b5c9b67 92f6f0b5
Pipeline #21761 passed with stages
in 41 minutes and 3 seconds
import sys
sys.path.append(".")
import pytest
from nbeats.preprocess_dataset import Dataset
import pandas as pd
import numpy as np
import random
@pytest.fixture
def df_1():
df = pd.DataFrame({"ems_time": [], "metric_0": []})
return df
@pytest.fixture
def df_2():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
df["metric_0"] = np.nan
return df
@pytest.fixture
def df_3():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.nan
return df
@pytest.fixture
def df_4():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 3)) * 1e9
for i in range(5):
df[f"metric_{i}"] = 1
return df
@pytest.fixture
def df_5():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 3)) * 1e9
for i in range(5):
df[f"metric_{i}"] = 1
return df
@pytest.fixture
def df_6():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
return df
@pytest.fixture
def df_7():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
np.random.randint(0, df.shape[0] - 1, 990),
f"metric_{i}",
] = np.nan
df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None")
return df
@pytest.fixture
def df_8():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
np.random.randint(0, df.shape[0] - 1, 990),
f"metric_{i}",
] = np.nan
df[f"metric_{i}"] = df[f"metric_{i}"].fillna(np.inf)
return df
@pytest.fixture
def df_9():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
np.random.randint(0, df.shape[0] - 1, 990),
f"metric_{i}",
] = np.nan
return df
@pytest.fixture
def df_10():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
list(range(20, 300)),
f"metric_{i}",
] = np.inf
return df
@pytest.fixture
def df_11():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
if i % 2 == 0:
df.loc[
list(range(20, 300)),
f"metric_{i}",
] = np.nan
return df
@pytest.fixture
def df_12():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = [
np.nan if i % 2 == 0 else random.random() for i in range(1000)
]
return df
@pytest.fixture
def df_13():
df = pd.DataFrame()
df["ems_time"] = 1
for i in range(5):
df[f"metric_{i}"] = [random.random() for i in range(1000)]
return df
@pytest.fixture
def df_14():
df = pd.DataFrame()
df["ems_time"] = 10
for i in range(5):
df[f"metric_{i}"] = [np.nan for i in range(1000)]
return df
@pytest.fixture
def df_15():
df = pd.DataFrame()
df["ems_time"] = [i * 30 * 1e5 for i in range(500)] + [
i * 30 * 1e5 + 10000 for i in range(500)
]
for i in range(5):
df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)]
return df
@pytest.fixture
def df_16():
df = pd.DataFrame()
df["ems_time"] = [i for i in range(500)] + [i + 10000 for i in range(500)]
for i in range(5):
df[f"metric_{i}"] = [np.nan for i in range(500)] + [2 for i in range(500)]
return df
@pytest.fixture
def df(request):
return request.getfixturevalue(request.param)
@pytest.fixture
def metric():
return "metric_0"
class TestDataset:
@pytest.mark.parametrize(
"df,metric",
[
("df_1", metric),
("df_2", metric),
("df_3", metric),
("df_4", metric),
("df_5", metric),
("df_6", metric),
("df_7", metric),
("df_8", metric),
("df_9", metric),
("df_10", metric),
("df_11", metric),
("df_12", metric),
("df_13", metric),
("df_14", metric),
("df_15", metric),
("df_16", metric),
],
indirect=True,
)
def test_init(self, df, metric):
preprocessed_dataset = Dataset(df, metric)
assert isinstance(preprocessed_dataset, Dataset)
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
\ No newline at end of file
......@@ -2,14 +2,23 @@ FROM python:3.8-slim-buster
# Install Python dependencies.
WORKDIR /wd
# COPY deployment/tft/requirements.txt .
COPY deployment/tft/requirements.txt .
RUN pip3 install --no-cache-dir -r requirements.txt && mkdir models
COPY deployment/tft/poetry.lock deployment/tft/pyproject.toml /wd/
# Install and setup poetry
RUN pip install -U pip \
&& apt-get update \
&& apt install -y curl netcat \
&& curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
ENV PATH="${PATH}:/root/.poetry/bin"
RUN poetry config virtualenvs.create false \
&& poetry install --no-interaction --no-ansi --no-dev
ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.5/morphemic-preprocessor-morphemic-rc2.5.tar.gz /var/lib/morphemic/
# Copy the rest of the codebase into the image
COPY deployment/tft/ ./
COPY deployment/tft ./
RUN cd /var/lib/morphemic/ \
&& tar -zxf morphemic-preprocessor-morphemic-rc2.5.tar.gz \
......@@ -18,9 +27,7 @@ RUN cd /var/lib/morphemic/ \
&& cd ../.. \
&& cp -R /var/lib/morphemic/morphemic-preprocessor-morphemic-rc2.5/amq-message-python-library /wd/amq_message_python_library \
&& rm -rf /var/lib/morphemic \
&& mkdir -p /wd/logs
CMD ["python3", "main.py"]
&& mkdir -p /wd/logs \
&& mkdir -p /wd/models
CMD ["python3", "main.py"]
\ No newline at end of file
docker run -t --env-file=env --network=host stomp_app
This diff is collapsed.
......@@ -2,15 +2,15 @@ import time
import os
import stomp
import threading
from src.model_predict import predict
from tft.model_predict import predict
from amq_message_python_library import *
from src.influxdb_predictions import InfluxdbPredictionsSender
from tft.influxdb_predictions import InfluxdbPredictionsSender
import json
import sys
import pandas as pd
import logging
from datetime import datetime
from src.dataset_maker import CSVData
from tft.dataset_maker import CSVData
from pytz import timezone
import pytz
from datetime import datetime
......
[tool.poetry]
name = "tft"
version = "0.1.0"
description = ""
authors = ["Anna Warno <awarno@7bulls.com>"]
[tool.poetry.dependencies]
python = "^3.8"
"stomp.py" = "^8.0.1"
pandas = "1.1.3"
pytorch-lightning = "1.2.7"
pytorch-forecasting = "0.8.4"
filelock = "3.0.12"
influxdb = "^5.3.1"
python-slugify = "^6.1.2"
torchmetrics = "0.5.0"
setproctitle = "^1.2.3"
protobuf = "3.20.1"
[tool.poetry.dev-dependencies]
pytest = "^5.2"
isort = "^5.10.1"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
stomp.py
pandas==1.1.3
pytorch-lightning==1.2.7
pytorch-forecasting==0.8.4
filelock==3.0.12
influxdb
python-slugify
torchmetrics==0.5.0
setproctitle
......@@ -3,9 +3,9 @@ import sys
import json
import logging
import time
from src.model_train import train
from tft.model_train import train
from amq_message_python_library import *
from src.dataset_maker import CSVData
from tft.dataset_maker import CSVData
import pytz
import time
from pytz import timezone
......
#!/bin/sh
python3 morphemic-datasetmaker/setup.py install
# rm -r morphemic-datasetmaker
# python3 main.py
# mv amq-message-python-library amq_message_python_library
import sys
sys.path.append(".")
import pytest
from src.model_train import train
import pandas as pd
import numpy as np
import random
@pytest.fixture
def df_1():
df = pd.DataFrame({"ems_time": [], "metric_0": []})
return df
@pytest.fixture
def df_2():
df = pd.DataFrame()
df["ems_time"] = [
int(x)
for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[
:1000
]
]
df["metric_0"] = np.nan
return df
@pytest.fixture
def df_3():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 1000)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.nan
return df
@pytest.fixture
def df_4():
df = pd.DataFrame()
df["ems_time"] = [
int(x)
for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[
:3
]
]
for i in range(5):
df[f"metric_{i}"] = 1
return df
@pytest.fixture
def df_5():
df = pd.DataFrame()
df["ems_time"] = np.array(range(0, 3)) * 1e9
for i in range(5):
df[f"metric_{i}"] = 1
return df
@pytest.fixture
def df_6():
df = pd.DataFrame()
df["ems_time"] = np.array(range(1, 1001)) * 1e9
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
return df
@pytest.fixture
def df_7():
df = pd.DataFrame()
df["ems_time"] = [
int(x)
for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[
:1000
]
]
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
np.random.randint(0, df.shape[0] - 1, 990),
f"metric_{i}",
] = np.nan
df[f"metric_{i}"] = df[f"metric_{i}"].fillna("None")
print(df)
return df
@pytest.fixture
def df_8():
df = pd.DataFrame()
df["ems_time"] = [
int(x)
for x in pd.date_range(start="2016-01-01", end="2020-12-31", freq="10S").values[
:1000
]
]
for i in range(5):
df[f"metric_{i}"] = np.random.rand(1000)
df.loc[
np.random.randint(0, df.shape[0] - 1, 990),
f"metric_{i}",
] =