Commit 778df4a7 authored by Marta Różańska's avatar Marta Różańska
Browse files

Merge branch 'tft-nbeats-2.0' into 'morphemic-rc2.0'

tft + nbeats with poetry

See merge request !308
parents a708a4e7 d7b0bb65
Pipeline #21387 passed with stages
in 47 minutes and 17 seconds
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
\ No newline at end of file
......@@ -2,14 +2,23 @@ FROM python:3.8-slim-buster
# Install Python dependencies.
WORKDIR /wd
# COPY deployment/nbeats/requirements.txt .
COPY deployment/nbeats/requirements.txt .
RUN pip3 install --no-cache-dir -r requirements.txt && mkdir models
COPY deployment/nbeats/poetry.lock deployment/nbeats/pyproject.toml /wd/
# Install and setup poetry
RUN pip install -U pip \
&& apt-get update \
&& apt install -y curl netcat \
&& curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
ENV PATH="${PATH}:/root/.poetry/bin"
RUN poetry config virtualenvs.create false \
&& poetry install --no-interaction --no-ansi --no-dev
ADD https://gitlab.ow2.org/melodic/morphemic-preprocessor/-/archive/morphemic-rc2.0/morphemic-preprocessor-morphemic-rc2.0.tar.gz /var/lib/morphemic/
# Copy the rest of the codebase into the image
COPY deployment/nbeats/ ./
COPY deployment/nbeats ./
RUN cd /var/lib/morphemic/ \
&& tar -zxf morphemic-preprocessor-morphemic-rc2.0.tar.gz \
......@@ -18,9 +27,7 @@ RUN cd /var/lib/morphemic/ \
&& cd ../.. \
&& cp -R /var/lib/morphemic/morphemic-preprocessor-morphemic-rc2.0/amq-message-python-library /wd/amq_message_python_library \
&& rm -rf /var/lib/morphemic \
&& mkdir -p /wd/logs
CMD ["python3", "main.py"]
&& mkdir -p /wd/logs \
&& mkdir -p /wd/models
CMD ["python3", "main.py"]
\ No newline at end of file
docker run -t --env-file=env --network=host stomp_app
......@@ -10,7 +10,6 @@ from pytz import timezone
import time
import setproctitle
# from src.log import logger
AMQ_USER = os.environ.get("AMQ_USER", "admin")
AMQ_PASSWORD = os.environ.get("AMQ_PASSWORD", "admin")
......
......@@ -6,7 +6,7 @@ import time
import os
import torch
from filelock import FileLock
from src.preprocess_dataset import Dataset
from nbeats.preprocess_dataset import Dataset
from pytorch_forecasting import NBeats
import scipy.stats as st
import logging
......
......@@ -9,7 +9,7 @@ import logging
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
from pytorch_forecasting.metrics import QuantileLoss, MAE, RMSE, CrossEntropy
from pytorch_forecasting import NBeats
from src.preprocess_dataset import Dataset
from nbeats.preprocess_dataset import Dataset
import pickle
import pytz
from datetime import datetime
......@@ -127,7 +127,7 @@ def train(target_column, prediction_length, yaml_file="model.yaml", publish_rate
trainer.fit(
model,
train_dataloader=train_dataloader,
train_dataloaders=train_dataloader,
val_dataloaders=val_dataloader,
)
if os.path.isfile(lockfile):
......
......@@ -235,7 +235,7 @@ class Dataset(object):
max_prediction_length=self.prediction_length,
min_prediction_length=self.prediction_length,
add_relative_time_idx=False,
allow_missings=False,
# allow_missings=False,
)
return ts_dataset
......
This diff is collapsed.
......@@ -2,15 +2,15 @@ import time
import os
import stomp
import threading
from src.model_predict import predict
from nbeats.model_predict import predict
from amq_message_python_library import *
from src.influxdb_predictions import InfluxdbPredictionsSender
from nbeats.influxdb_predictions import InfluxdbPredictionsSender
import json
import sys
import pandas as pd
import logging
from datetime import datetime
from src.dataset_maker import CSVData
from nbeats.dataset_maker import CSVData
from pytz import timezone
import pytz
from datetime import datetime
......
[tool.poetry]
name = "nbeats"
version = "0.1.0"
description = ""
authors = ["Anna Warno <awarno@7bulls.com>"]
[tool.poetry.dependencies]
python = "3.8.13"
"stomp.py" = "^8.0.1"
filelock = "3.0.12"
influxdb = "^5.3.1"
python-slugify = "^6.1.2"
torchmetrics = "0.5.0"
setproctitle = "^1.2.3"
protobuf = "3.20"
pytorch-forecasting = "^0.10.2"
[tool.poetry.dev-dependencies]
pytest = "^5.2"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
stomp.py
pandas==1.1.3
pytorch-lightning==1.2.7
pytorch-forecasting==0.8.4
absl-py==1.0.0; python_version >= "3.6"
aiohttp==3.8.1; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
aiosignal==1.2.0; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
alembic==1.7.7; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
async-timeout==4.0.2; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
attrs==21.4.0; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
autopage==0.5.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
cachetools==5.2.0; python_version >= "3.7" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
certifi==2022.5.18.1; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
charset-normalizer==2.0.12; python_full_version >= "3.6.1" and python_version > "3.6" and python_full_version < "4.0.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
cliff==3.10.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
cmaes==0.8.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
cmd2==2.4.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
colorama==0.4.4; python_full_version >= "3.6.1" and python_version >= "3.6" and platform_system == "Windows" and python_full_version < "4.0.0" and sys_platform == "win32"
colorlog==6.6.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
cycler==0.11.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
docopt==0.6.2; python_version >= "3.6" and python_version < "4.0"
filelock==3.0.12
influxdb
python-slugify
torchmetrics==0.5.0
setproctitle
fonttools==4.33.3; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
frozenlist==1.3.0; python_version >= "3.7" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
fsspec==0.8.7; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version > "3.6"
future==0.18.2; python_full_version >= "3.6.1" and python_version >= "3.6" and python_full_version < "4.0.0"
google-auth-oauthlib==0.4.6; python_version >= "3.6"
google-auth==2.6.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
greenlet==1.1.2; python_version >= "3.6" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
grpcio==1.46.3; python_version >= "3.6"
idna==3.3; python_full_version >= "3.6.1" and python_version > "3.6" and python_full_version < "4.0.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
importlib-metadata==4.11.4; python_version < "3.9" and python_version >= "3.7" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
importlib-resources==5.7.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version < "3.9"
influxdb==5.3.1
joblib==1.1.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
kiwisolver==1.4.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
mako==1.2.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
markdown==3.3.7; python_version >= "3.6"
markupsafe==2.1.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
matplotlib==3.5.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
msgpack==1.0.3
multidict==6.0.2; python_version >= "3.7" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
numpy==1.22.4; python_version >= "3.8" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
oauthlib==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
optuna==2.10.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
packaging==21.3; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
pandas==1.1.3; python_full_version >= "3.6.1"
patsy==0.5.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
pbr==5.9.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
pillow==9.1.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
prettytable==3.3.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
protobuf==3.20.0; python_version >= "3.7"
pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
pyasn1==0.4.8; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
pyparsing==3.0.9; python_full_version >= "3.6.8" and python_full_version < "4.0.0" and python_version >= "3.7"
pyperclip==1.8.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
pyreadline3==3.4.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6" and sys_platform == "win32"
python-dateutil==2.8.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
python-slugify==6.1.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
pytorch-forecasting==0.8.4; python_full_version >= "3.6.1" and python_full_version < "4.0.0"
pytorch-lightning==1.2.7; python_version >= "3.6"
pytz==2022.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0"
pyyaml==6.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
requests-oauthlib==1.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
requests==2.27.1; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
rsa==4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
scikit-learn==0.24.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
scipy==1.6.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
setproctitle==1.2.3; python_version >= "3.6"
setuptools-scm==6.4.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
six==1.16.0; python_full_version >= "3.6.1" and python_version >= "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and python_full_version < "4.0.0"
sqlalchemy==1.4.36; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
statsmodels==0.13.2; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
stevedore==3.5.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
stomp.py==8.0.1; python_version >= "3.6" and python_version < "4.0"
tensorboard-data-server==0.6.1; python_version >= "3.6"
tensorboard-plugin-wit==1.8.1; python_version >= "3.6"
tensorboard==2.9.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
text-unidecode==1.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
threadpoolctl==3.1.0; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.6"
tomli==2.0.1; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
torch==1.11.0; python_full_version >= "3.7.0" and python_version >= "3.6" and python_full_version < "4.0.0"
torchmetrics==0.5.0; python_version >= "3.6"
tqdm==4.64.0; python_full_version >= "3.6.1" and python_version >= "3.6" and python_full_version < "4.0.0"
typing-extensions==4.2.0; python_full_version >= "3.7.0" and python_version >= "3.7" and python_full_version < "4.0.0"
urllib3==1.26.9; python_full_version >= "3.6.1" and python_version < "4" and python_version > "3.6" and python_full_version < "4.0.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
wcwidth==0.2.5; python_full_version >= "3.6.1" and python_full_version < "4.0.0" and python_version >= "3.7"
werkzeug==2.1.2; python_version >= "3.7"
yarl==1.7.2; python_version > "3.6" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
zipp==3.8.0; python_version < "3.9" and python_version >= "3.7" and python_full_version >= "3.6.1" and python_full_version < "4.0.0"
......@@ -3,9 +3,9 @@ import sys
import json
import logging
import time
from src.model_train import train
from nbeats.model_train import train
from amq_message_python_library import *
from src.dataset_maker import CSVData
from nbeats.dataset_maker import CSVData
import pytz
import time
from pytz import timezone
......
#!/bin/sh
python3 morphemic-datasetmaker/setup.py install
# rm -r morphemic-datasetmaker
# python3 main.py
# mv amq-message-python-library amq_message_python_library
......@@ -3,7 +3,7 @@ import sys
sys.path.append(".")
import pytest
from src.model_predict import predict
from nbeats.model_predict import predict
import pandas as pd
import numpy as np
import random
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment