From 695118bcb94f667286180a904eb301ee39b04089 Mon Sep 17 00:00:00 2001
From: szysad <szysad108@gmail.com>
Date: Wed, 19 May 2021 13:19:44 +0200
Subject: [PATCH 1/5] added heuristics to set hparams of tunable objective. Now
 AssembledModel is ensemble model, added more tests of AssembledModel

---
 SupervisedGym/SupervisedGym/__main__.py       |  45 +++++-
 .../model_assembler/model_assembler.py        |  50 ++++++-
 .../SupervisedGym/models/assembled_model.py   | 107 ++++++++++-----
 .../SupervisedGym/modeltuner/__init__.py      |   4 +-
 .../{utils/data => modeltuner}/datamodule.py  |   2 +-
 .../SupervisedGym/modeltuner/modeltuner.py    |  42 +++---
 .../SupervisedGym/modeltuner/objective.py     |  68 +++++----
 .../SupervisedGym/modeltuner/utils.py         |  72 ++++++++++
 SupervisedGym/SupervisedGym/utils/__init__.py |  14 ++
 .../SupervisedGym/utils/data/__init__.py      |   2 -
 .../SupervisedGym/utils/domain_rounder.py     |   6 +-
 .../SupervisedGym/utils/problem_analyzer.py   |   8 ++
 .../tests/models/test_assmebled_model.py      | 129 +++++++++++++++++-
 .../tests/modeltuner/test_model_tuner.py      |  20 ++-
 14 files changed, 453 insertions(+), 116 deletions(-)
 rename SupervisedGym/SupervisedGym/{utils/data => modeltuner}/datamodule.py (94%)
 create mode 100644 SupervisedGym/SupervisedGym/modeltuner/utils.py
 create mode 100644 SupervisedGym/SupervisedGym/utils/__init__.py

diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py
index c5fb358..fa49b05 100644
--- a/SupervisedGym/SupervisedGym/__main__.py
+++ b/SupervisedGym/SupervisedGym/__main__.py
@@ -1,5 +1,7 @@
+from SupervisedGym.utils import domain_rounder
+from SupervisedGym.modeltuner.modeltuner import TrainedModelStats
 import torch
-
+import pickle
 from SupervisedGym.model_assembler.model_assembler import (
     ModelAssebler as GenomAssembler,
 )
@@ -7,14 +9,43 @@ from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable
 from SupervisedGym.models.assembled_model import AssembledModel
 
 
-SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/13-05-genom/rrn-11h-scaling"
+SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/test-model"
 TIMEOUT = 60 * 1  # 1 minutes
 
 if __name__ == "__main__":
-    assebler = GenomAssembler(
-        raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss()
-    )  # placeholder
+    # assebler = GenomAssembler(
+    #    raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss()
+    # )  # placeholder
+
+    # model = assebler.assemble_model(
+    #    timeout=TIMEOUT,
+    #    n_trials=5,
+    #    assembly_size=3,
+    #    avaible_model_types=[FeedForwardTunable]
+    # )
 
-    model = assebler.assemble_model(TIMEOUT, [FeedForwardTunable])
-    #with open(SAVE_PATH, "wb") as f:
+    # with open(SAVE_PATH, "wb") as f:
     #    f.write(model.parse_model())
+
+    rnnpaths = [
+        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-9h-gpu",
+        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-w-early-stopping-gpu",
+        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-gpu",
+    ]
+
+    rrnmodels = []
+    rounder = None
+
+    for p in rnnpaths:
+        data = None
+        with open(p, "rb") as f:
+            data = pickle.loads(f.read())
+        model = StackedRNNTunable.load_model(data["tunable_model"])
+        rounder = data["domain_rounder"]
+        rrnmodels.append(model)
+
+    stats = [TrainedModelStats(model, 1) for model in rrnmodels]
+
+    assemble = AssembledModel(domain_rounder=rounder, model_stats=stats)
+    with open(SAVE_PATH, "wb") as f:
+        f.write(assemble.parse_model())
diff --git a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
index 1e88ab6..bcb8e8d 100644
--- a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
+++ b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
@@ -8,7 +8,7 @@ from ..models.base import IBaseTunableModel
 from ..models import avaible_model_types
 from ..models.assembled_model import AssembledModel
 from ..utils.problem_analyzer import DummyGenomAnalyzer as ProblemAnalyzer
-from ..modeltuner import ModelTuner
+from ..modeltuner import ModelTuner, TrainedModelStats
 
 
 class ModelAssebler:
@@ -35,26 +35,62 @@ class ModelAssebler:
             raw_data, cp_problem
         )
 
+    def __val_loss_to_weight(self, loss: float) -> float:
+        """
+        transforms validation loss of model
+        to its weight in ensembled model
+        """
+        return 1 / (loss + 0.33)
+
     def assemble_model(
         self,
         timeout: int,
         n_trials: Optional[int] = None,
-        avaible_model_types: List[Type[IBaseTunableModel]] = avaible_model_types
+        assembly_size: int = 5,
+        avaible_model_types: List[Type[IBaseTunableModel]] = avaible_model_types,
     ) -> AssembledModel:
         """
         based on data and cp_problem
         builds AssembledModel
         """
 
+        model_stats: List[TrainedModelStats] = []
+
+        def __receive_stat(new_stat: TrainedModelStats):
+            """updates model stats with new model if has val_loss
+            small enough"""
+            nonlocal model_stats
+
+            if len(model_stats) < assembly_size:
+                model_stats.append(new_stat)
+
+            max_idx = 0
+            max_loss = float("-inf")
+            for i, stat in enumerate(model_stats):
+                if stat.val_loss > max_loss:
+                    max_loss = stat.val_loss
+                    max_idx = i
+
+            if new_stat.val_loss < max_loss:
+                model_stats[max_idx] = new_stat
+
         tuner = ModelTuner(
             val_loss=self.val_loss,
             tsdata_static_config=self.data_static_conf,
-            fast_dev_run=False
+            fast_dev_run=False,
         )
 
-        best_model = tuner.tune_model(
+        tuner.tune(
             timeout=timeout,
-            tunable_model_type=avaible_model_types[0],
-            n_trials=n_trials
+            tunable_model_type=avaible_model_types[
+                0
+            ],  # CURRENTLY FIXED FIRST MODEL TYPE
+            n_trials=n_trials,
+            model_callback=__receive_stat,
+        )
+        return AssembledModel(
+            model_stats=map(
+                lambda m, val_loss: (m, self.__val_loss_to_weight(val_loss))
+            ),
+            domain_rounder=self.dom_rounder,
         )
-        return AssembledModel(tunable_model=best_model, domain_rounder=self.dom_rounder)
diff --git a/SupervisedGym/SupervisedGym/models/assembled_model.py b/SupervisedGym/SupervisedGym/models/assembled_model.py
index 38bc312..cbc5412 100644
--- a/SupervisedGym/SupervisedGym/models/assembled_model.py
+++ b/SupervisedGym/SupervisedGym/models/assembled_model.py
@@ -1,11 +1,11 @@
 from __future__ import annotations
+from SupervisedGym.models.base.IBaseTunableModel import IBaseTunableModel
 import torch
 import pickle
-from typing import Dict, Union, List
+from typing import Dict, Any, Union, List, Type, Tuple
 from itertools import count
 
-from ..utils.domain_rounder import DomainRounder
-from .base import IBaseTunableModel
+from ..utils import DomainRounder
 from .base.parsable_model import IParsableModel
 
 
@@ -14,15 +14,22 @@ class AssembledModel(IParsableModel):
     input_idx: Dict[str, int]
 
     def __init__(
-        self, tunable_model: IBaseTunableModel, domain_rounder: DomainRounder
+        self,
+        models: List[Tuple[IBaseTunableModel, float]],
+        domain_rounder: DomainRounder,
     ) -> AssembledModel:
 
-        self.tunable_model = tunable_model
+        self.models = models
         self.domain_rounder = domain_rounder
-        self.data_adapter = tunable_model.data_adapter
-        self.input_idx = dict(
-            zip(tunable_model.data_adapter.input_columns, count(start=0, step=1))
+        self._input_columns = models[0][0].data_adapter.input_columns
+        self._target_columns = models[0][0].data_adapter.target_columns
+        self._input_sequence_len = max(
+            map(lambda s: s[0].data_adapter.input_sequence_len, models)
         )
+        self.input_idx = dict(zip(self._input_columns, count(start=0, step=1)))
+        # enter eval mode for all models
+        for m in self.models:
+            m[0].eval()
 
     def __validate_input(self, x: Dict[str, List[Union[float, str, int]]]):
         """Checks if input data for prediction is correct
@@ -34,7 +41,7 @@ class AssembledModel(IParsableModel):
         Raises:
             ValueError: [description]
         """
-        for var_name in self.data_adapter.input_columns:
+        for var_name in self._input_columns:
             if var_name not in x:
                 raise ValueError(
                     f"sequence of {var_name} is required to make"
@@ -42,12 +49,45 @@ class AssembledModel(IParsableModel):
                 )
 
         for var_name, seq in x.items():
-            if len(seq) != self.data_adapter.input_sequence_len:
+            if len(seq) != self._input_sequence_len:
                 raise ValueError(
                     f"sequence of {var_name} data has length {len(seq)},"
-                    " expected {self.data_adapter.input_sequence_len}."
+                    f" expected {self._input_sequence_len}."
                 )
 
+    def __assembly_forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        performes forward pass using all assemby models
+        and then morging their preiction using weighted
+        average based on their val_loss
+        """
+        # currently all weights are equal
+        w_sum = 0
+        out = torch.zeros(len(self._target_columns))
+        for model, w in self.models:
+            seq_len = model.data_adapter.input_sequence_len
+            xx = x[:, -seq_len:, :]
+            xx = model.data_adapter.transform_input(xx)
+            raw_out = model.forward(xx)
+            transformed_out = model.data_adapter.transform_target(
+                raw_out, inverse=True
+            ).squeeze()
+            out += transformed_out * w
+            w_sum += w
+        return out / w_sum
+
+    @property
+    def input_columns(self) -> List[str]:
+        return self._input_columns
+
+    @property
+    def target_columns(self) -> List[str]:
+        return self._target_columns
+
+    @property
+    def input_sequence_len(self) -> int:
+        return self._input_sequence_len
+
     def predict(self, x: Dict[str, List[Union[float, str, int]]]) -> Dict[float, int]:
         """
         generates model prediction based
@@ -61,44 +101,39 @@ class AssembledModel(IParsableModel):
                 latest. Each value list must have lenght
                 equal to input_sequence_length
         """
-        self.tunable_model.eval()
         self.__validate_input(x)
-        seq_len = self.data_adapter.input_sequence_len
         # we need to pass 3d tensor with first dimention
         # equal to batch_size which we set to 1
         tensor_in = torch.cat(
             tuple(
-                torch.Tensor(x[col]).view(seq_len, 1)
-                for col in self.data_adapter.input_columns
+                torch.Tensor(x[col]).view(self._input_sequence_len, 1)
+                for col in self._input_columns
             ),
             dim=1,
         ).unsqueeze(dim=0)
-        tensor_out = self.tunable_model.forward(
-            self.data_adapter.transform_input(tensor_in)
-        )
-        self.tunable_model.train(True)
-
-        # batch size equals to 1 so squeeze
-        prediction = self.data_adapter.transform_target(tensor_out).squeeze()
-        pred_dict = {
-            col: v.item()
-            for col, v in zip(self.data_adapter.target_columns, prediction)
-        }
+
+        prediction = self.__assembly_forward(tensor_in)
+        pred_dict = {col: v.item() for col, v in zip(self._target_columns, prediction)}
         return self.domain_rounder.round(pred_dict)
 
     @classmethod
     def load_model(cls, encoding: bytes) -> AssembledModel:
         data = pickle.loads(encoding)
-        tunable_type = data["tunable_type"]
-        return AssembledModel(
-            tunable_model=tunable_type.load_model(data["tunable_model"]),
-            domain_rounder=data["domain_rounder"],
-        )
+        models: List[Tuple[IBaseTunableModel, float]] = []
+        for state_dict in data["model_data"]:
+            model_type: Type[IBaseTunableModel] = state_dict["type"]
+            models.append(
+                (model_type.load_model(state_dict["encoding"]), state_dict["weight"])
+            )
+        return AssembledModel(models=models, domain_rounder=data["domain_rounder"])
 
     def parse_model(self) -> bytes:
-        data = {
-            "domain_rounder": self.domain_rounder,
-            "tunable_model": self.tunable_model.parse_model(),
-            "tunable_type": type(self.tunable_model),
-        }
+        model_data: List[Dict[str, Any]] = []
+        for model, weight in self.models:
+            model_data.append(
+                {"encoding": model.parse_model(), "weight": weight, "type": type(model)}
+            )
+
+        data = {"domain_rounder": self.domain_rounder, "model_data": model_data}
+
         return pickle.dumps(data)
diff --git a/SupervisedGym/SupervisedGym/modeltuner/__init__.py b/SupervisedGym/SupervisedGym/modeltuner/__init__.py
index e14248e..3a59cb3 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/__init__.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/__init__.py
@@ -1,4 +1,4 @@
-from .modeltuner import ModelTuner
+from .modeltuner import ModelTuner, TrainedModelStats
 
 
-__all__ = [ModelTuner]
+__all__ = [ModelTuner, TrainedModelStats]
diff --git a/SupervisedGym/SupervisedGym/utils/data/datamodule.py b/SupervisedGym/SupervisedGym/modeltuner/datamodule.py
similarity index 94%
rename from SupervisedGym/SupervisedGym/utils/data/datamodule.py
rename to SupervisedGym/SupervisedGym/modeltuner/datamodule.py
index d083ba3..e6f2da0 100644
--- a/SupervisedGym/SupervisedGym/utils/data/datamodule.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/datamodule.py
@@ -1,7 +1,7 @@
 from pytorch_lightning import LightningDataModule
 from torch.utils.data import DataLoader
 
-from .TimeSeriesDataSet.timeseries_data import TimeSeriesData
+from SupervisedGym.utils.data import TimeSeriesData
 
 
 class DataModule(LightningDataModule):
diff --git a/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py b/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py
index 400f6e2..1d8fe7e 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py
@@ -1,10 +1,15 @@
 import optuna
 import torch
-from typing import Union, Optional, Type
+from typing import Callable, Optional, Type, NamedTuple
 from ..models.base import IBaseTunableModel
-from ..utils.callbacks.optuna import NewBestTrialCallbackTrigger
 from ..utils.data import TimeSeriesDataStaticConfig
 from .objective import Objective
+from .utils import generate_objective_hparams
+
+
+class TrainedModelStats(NamedTuple):
+    model: IBaseTunableModel
+    val_loss: float
 
 
 class ModelTuner:
@@ -21,18 +26,16 @@ class ModelTuner:
     ):
         self.tsdata_static_config = tsdata_static_config
         self.val_loss = val_loss
-        self._best_model: Union[IBaseTunableModel, None] = None
+        self._best_model: Optional[IBaseTunableModel] = None
         self.fast_dev_run = fast_dev_run
 
-    def _save_curr_model(self, objective: Objective):
-        self._best_model = objective.current_model()
-
-    def tune_model(
+    def tune(
         self,
         timeout: int,
         tunable_model_type: Type[IBaseTunableModel],
+        model_callback: Callable[[TrainedModelStats], None],
         n_trials: Optional[int] = None,
-    ) -> IBaseTunableModel:
+    ) -> None:
         """
         Performs network tuning
         """
@@ -42,8 +45,11 @@ class ModelTuner:
         objective = Objective(
             tsdata_static_config=self.tsdata_static_config,
             tunable_model_type=tunable_model_type,
-            val_loss=self.val_loss,
+            val_func=self.val_loss,
             fast_dev_run=self.fast_dev_run,
+            objective_hyper_params=generate_objective_hparams(
+                conf=self.tsdata_static_config, val_loss=self.val_loss
+            ),
         )
 
         # enqueue trial with suggested params
@@ -54,18 +60,10 @@ class ModelTuner:
             n_trials=n_trials,
             timeout=timeout,
             callbacks=[
-                NewBestTrialCallbackTrigger(lambda: self._save_curr_model(objective))
+                lambda _, ft: model_callback(
+                    TrainedModelStats(
+                        model=objective.current_model(), val_loss=ft.value
+                    )
+                )
             ],
         )
-
-        print("Number of finished trials: {}".format(len(study.trials)))
-        print("Best trial:")
-        trial = study.best_trial
-
-        print("Value: {}".format(trial.value))
-
-        print("Params:")
-        for key, value in trial.params.items():
-            print("{}: {}".format(key, value))
-
-        return self._best_model
diff --git a/SupervisedGym/SupervisedGym/modeltuner/objective.py b/SupervisedGym/SupervisedGym/modeltuner/objective.py
index a3ffd75..de3c0e7 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/objective.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/objective.py
@@ -1,7 +1,7 @@
 import pytorch_lightning as pl
 import torch
 import optuna
-from typing import Dict, Any, Union, Optional, Type
+from typing import Dict, Any, Optional, Type, TypedDict, Tuple
 from sklearn.base import TransformerMixin
 from sklearn import preprocessing
 from optuna.integration import PyTorchLightningPruningCallback
@@ -10,27 +10,37 @@ from pytorch_lightning.callbacks import EarlyStopping
 from ..utils.data import (
     TimeSeriesDataStaticConfig,
     TimeSeriesDataFitableConfig,
-    DataModule,
     DataAdapter,
     make_time_series_data,
 )
 from ..models.base import IBaseTunableModel
+from .datamodule import DataModule
 
 
-VALIDATE_EVERY_N_ITEMS = 10 * (10 ** 3)
-EARLY_STOPPING_PATIENCE = 3
-MAX_EPOCHS = 100
+class ObjectiveHParams(TypedDict):
+    """
+    dict containing Objective hyper
+    parameters which are hyperparameters
+    but such that concerns piepline
+    as whole and not single model
+    """
+
+    max_epochs: int
+    min_epochs: int
+    es_patience: int
+    es_divergence_threshold: float
+    val_check_interval: float
 
 
 class Objective:
-    # TODO make val_check_interval dynamic
     """objective for tuner"""
 
     def __init__(
         self,
         tsdata_static_config: TimeSeriesDataStaticConfig,
         tunable_model_type: Type[IBaseTunableModel],
-        val_loss: torch.nn.Module,
+        val_func: torch.nn.Module,
+        objective_hyper_params: ObjectiveHParams,
         fast_dev_run: bool = False,
     ):
         """
@@ -45,7 +55,7 @@ class Objective:
             tunableModel (type):
                 subtype of BaseTunableModel which hyperparameters will be searched
                 and this model will be trained
-            val_loss (torch.nn.Module):
+            val_func (torch.nn.Module):
                 loss function used to evaluate validation error, the better model should have
                 lower validation loss value. This might not be the same as loss function used
                 in training. But val_loss is used in model pruning.
@@ -60,14 +70,17 @@ class Objective:
 
         self.tsdata_static_config = tsdata_static_config
         self.tunable_model_type = tunable_model_type
-        self.val_loss = val_loss
-        self.model: Union[IBaseTunableModel, None] = None
+        self.val_func = val_func
+        self.model: Optional[IBaseTunableModel] = None
         self.fast_dev_run = fast_dev_run
+        self.ohparams = objective_hyper_params
 
     def current_model(self) -> IBaseTunableModel:
         """
-        returns model connected to his trainer
+        returns currently trained model with its validation loss
         """
+        if self.model is None:
+            raise Exception("There is no trained model in objective")
         return self.model
 
     def suggested_params(self) -> Dict[str, Any]:
@@ -86,8 +99,6 @@ class Objective:
         batch_size_pow = trial.suggest_int("batch_size_pow", 5, 12)
         batch_size = 2 ** batch_size_pow
 
-        # generate x_transform
-        # TODO add more scalers
         input_scaler_name = trial.suggest_categorical(
             "x_scaler_name",
             ["MinMaxScaler", "MaxAbsScaler", "StandardScaler", "RobustScaler", "None"],
@@ -101,7 +112,7 @@ class Objective:
         # generate y_transform
         # TODO add more scales
         target_scaler_name = trial.suggest_categorical(
-            "y_scaler_name", ["MinMaxScaler", "MaxAbsScaler", "None"]
+            "y_scaler_name", ["MinMaxScaler", "MaxAbsScaler", "RobustScaler", "None"]
         )
         target_scaler_type: Optional[Type[TransformerMixin]] = (
             getattr(preprocessing, target_scaler_name)
@@ -125,31 +136,28 @@ class Objective:
 
         # generate tunable model
         self.model = self.tunable_model_type.get_trialed_model(
-            trial=trial, data_adapter=adapter, val_loss=self.val_loss
+            trial=trial, data_adapter=adapter, val_loss=self.val_func
         )
 
         callbacks = [
-            PyTorchLightningPruningCallback(trial, monitor="val_loss")
-            ]
-
-        if trial.number < 50:
-            callbacks.append(
-                EarlyStopping(
-                    monitor="val_loss",
-                    mode="min",
-                    patience=EARLY_STOPPING_PATIENCE,  
-                    divergence_threshold=1000.0,  # TODO MAKE IT DYNAMIC
-                )
-            )
+            PyTorchLightningPruningCallback(trial, monitor="val_loss"),
+            EarlyStopping(
+                monitor="val_loss",
+                mode="min",
+                patience=self.ohparams["es_patience"],
+                divergence_threshold=self.ohparams["es_divergence_threshold"],
+            ),
+        ]
 
         trainer = pl.Trainer(
             logger=False,
-            val_check_interval=0.5,  # TODO make it dynamic
+            val_check_interval=self.ohparams["val_check_interval"],
             fast_dev_run=self.fast_dev_run,
             checkpoint_callback=False,
-            max_epochs=MAX_EPOCHS,
+            max_epochs=self.ohparams["max_epochs"],
+            min_epochs=self.ohparams["min_epochs"],
             gradient_clip_val=gradient_clip_val,
-            callbacks=callbacks
+            callbacks=callbacks,
         )
 
         trainer.fit(model=self.model, datamodule=datamodule)
diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py
new file mode 100644
index 0000000..a0172b6
--- /dev/null
+++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py
@@ -0,0 +1,72 @@
+import torch
+from math import log
+from torch.utils.data import Dataset, DataLoader, Sampler
+from typing import Literal, Optional
+from SupervisedGym.utils.data import (
+    TimeSeriesDataStaticConfig,
+    TimeSeriesDataFitableConfig,
+    make_time_series_data,
+)
+from .objective import ObjectiveHParams
+
+
+def naive_model_loss(
+    val_func: torch.nn.Module,
+    dataset: Dataset,
+    mode_type: Literal["mean"] = "mean",
+    sampler: Optional[Sampler] = None,
+) -> float:
+    loader = DataLoader(dataset=dataset, sampler=sampler, batch_size=1)
+    pred: torch.Tensor
+    if mode_type == "mean":
+        # calculate average values of targets
+        # using moving average
+        cma = None
+        for i, (_, y) in enumerate(loader, start=1):
+            if i == 1:
+                cma = torch.Tensor(y)
+                continue
+            cma = (y + i * cma) / (i + 1)
+        pred = cma
+
+        # calculate avg loss of average prediction
+        # using moving average
+        loss = 0
+        for i, (_, y) in enumerate(loader, start=1):
+            loss = (val_func(pred, y) + i * loss) / (i + 1)
+        return loss
+
+    else:
+        raise Exception(f"type {mode_type} is not handled")
+
+
+def generate_objective_hparams(
+    conf: TimeSeriesDataStaticConfig, val_loss: torch.nn.Module
+) -> ObjectiveHParams:
+    """
+    based on TimeSeriesDataStaticConfig and problem validation
+    function generates ObjectiveHParams
+    """
+    train_items = int(
+        (conf["data"].size / 100) * conf["train_val_test_percentage"].train
+    )
+    min_epochs = max(1, int(log(train_items, 100)))
+
+    dummy_conf = TimeSeriesDataFitableConfig(input_sequence_len=1)
+    tmp_tsdataset = make_time_series_data(static_conf=conf, fitable_conf=dummy_conf)
+    es_divergence_threshold = naive_model_loss(
+        val_func=val_loss,
+        dataset=tmp_tsdataset.generate_train_dataset(),
+        sampler=tmp_tsdataset.generate_uniform_target_sampler(),
+    )
+    del tmp_tsdataset
+    val_check_interval = 2.5e5 / train_items
+    if val_check_interval > 1:
+        val_check_interval = int(val_check_interval)
+    return ObjectiveHParams(
+        val_check_interval=val_check_interval,
+        max_epochs=min_epochs * 50,
+        min_epochs=min_epochs,
+        es_divergence_threshold=es_divergence_threshold,
+        es_patience=min_epochs * 3,
+    )
diff --git a/SupervisedGym/SupervisedGym/utils/__init__.py b/SupervisedGym/SupervisedGym/utils/__init__.py
new file mode 100644
index 0000000..9df67c9
--- /dev/null
+++ b/SupervisedGym/SupervisedGym/utils/__init__.py
@@ -0,0 +1,14 @@
+from . import data
+from .domain_rounder import DomainRounder
+from .problem_analyzer import ProblemAnalyzer, DummyGenomAnalyzer
+from . import callbacks
+
+
+__all__ = [
+    data,
+    DomainRounder,
+    ProblemAnalyzer,
+    DomainRounder,
+    callbacks,
+    DummyGenomAnalyzer,
+]
diff --git a/SupervisedGym/SupervisedGym/utils/data/__init__.py b/SupervisedGym/SupervisedGym/utils/data/__init__.py
index 7efe594..be11e5e 100644
--- a/SupervisedGym/SupervisedGym/utils/data/__init__.py
+++ b/SupervisedGym/SupervisedGym/utils/data/__init__.py
@@ -7,7 +7,6 @@ from .TimeSeriesDataSet.timeseries_data_configs import (
 )
 from .TimeSeriesDataSet.tsdataset import TsDataset
 from .dataadapter import DataAdapter
-from .datamodule import DataModule
 
 
 __all__ = [
@@ -19,5 +18,4 @@ __all__ = [
     TimeSeriesDataStaticConfig,
     make_time_series_data,
     DataAdapter,
-    DataModule,
 ]
diff --git a/SupervisedGym/SupervisedGym/utils/domain_rounder.py b/SupervisedGym/SupervisedGym/utils/domain_rounder.py
index e8f41f0..e3e245d 100644
--- a/SupervisedGym/SupervisedGym/utils/domain_rounder.py
+++ b/SupervisedGym/SupervisedGym/utils/domain_rounder.py
@@ -1,5 +1,5 @@
 import numpy as np
-from typing import Dict, Union, List
+from typing import Dict, List
 
 
 class DomainRounder:
@@ -15,12 +15,12 @@ class DomainRounder:
 
     domain: Dict[str, np.array]
 
-    def __init__(self, domain: Dict[str, List[Union[int, float]]]):
+    def __init__(self, domain: Dict[str, List[int]]):
         self.domain = {
             col: np.sort(np.array(var_dom)) for col, var_dom in domain.items()
         }
 
-    def round(self, x: Dict[str, Union[float, int]]) -> Dict[str, Union[int, float]]:
+    def round(self, x: Dict[str, int]) -> Dict[str, int]:
         rez = {}
         for col, y in x.items():
             closest_idx = 0
diff --git a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
index cf5603c..f62d5be 100644
--- a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
+++ b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
@@ -5,6 +5,14 @@ from .domain_rounder import DomainRounder
 from .data import TimeSeriesDataStaticConfig, Train_val_test_perc
 
 
+class BaseProblemAnalyzerExcepion(Exception):
+    """
+    base class for ProblemAnalyzer exceptions
+    """
+
+    pass
+
+
 class ProblemAnalyzer:
     """
     based on cp-problem file generates:
diff --git a/SupervisedGym/tests/models/test_assmebled_model.py b/SupervisedGym/tests/models/test_assmebled_model.py
index 94cb565..6a1db64 100644
--- a/SupervisedGym/tests/models/test_assmebled_model.py
+++ b/SupervisedGym/tests/models/test_assmebled_model.py
@@ -1,7 +1,10 @@
+from SupervisedGym import models
+from SupervisedGym.modeltuner.modeltuner import TrainedModelStats
 import pandas as pd
 import numpy as np
 import pytest
 from SupervisedGym.models.assembled_model import AssembledModel
+from SupervisedGym.utils import domain_rounder
 from SupervisedGym.utils.data import (
     DataAdapter,
     TimeSeriesDataFitableConfig,
@@ -49,7 +52,7 @@ def test_simple_case():
     adapter = DataAdapter.create_from_timeseries_data(data)
     rounder = DomainRounder({"OUT1": [1], "OUT2": [2]})
     for model in DummyModelFactory.all_dummys(adapter):
-        assembled_model = AssembledModel(tunable_model=model, domain_rounder=rounder)
+        assembled_model = AssembledModel(models=[(model, 1)], domain_rounder=rounder)
 
         assert (
             assembled_model.predict(
@@ -96,7 +99,7 @@ def test_handles_invalid_input():
     data = make_time_series_data(static_conf, fitable_conf)
     adapter = DataAdapter.create_from_timeseries_data(data)
     model = AssembledModel(
-        tunable_model=DummyModelFactory.get_feed_forward_dummy(adapter),
+        models=[(DummyModelFactory.get_feed_forward_dummy(adapter), 1)],
         domain_rounder=DomainRounder({"OUT1": [69], "OUT2": [420]}),
     )
 
@@ -177,7 +180,7 @@ def test_load_parse():
         }
     )
     for model in DummyModelFactory.all_dummys(adapter):
-        assembled = AssembledModel(tunable_model=model, domain_rounder=rounder)
+        assembled = AssembledModel(models=[(model, 1)], domain_rounder=rounder)
         loaded_model = AssembledModel.load_model(assembled.parse_model())
         for _ in range(TRIES):
             pred_in = {
@@ -186,3 +189,123 @@ def test_load_parse():
                 "IN3": np.random.randn(SEQ_LEN).tolist(),
             }
             assert assembled.predict(pred_in) == loaded_model.predict(pred_in)
+
+
+def test_multi_forward():
+    """
+    tests if assemby model made of
+    same models makes same prediction as
+    only single model
+    """
+
+    TRIES = 5
+    SIZE = 50
+    SEQ_LEN = 7
+    data = pd.DataFrame(
+        {
+            "IN1": np.random.rand(SIZE),
+            "IN2": np.random.rand(SIZE),
+            "IN3": np.random.rand(SIZE),
+            "OUT1": np.zeros(SIZE),
+            "OUT2": np.zeros(SIZE),
+            "GR_ID": np.arange(SIZE) // 20,
+        }
+    )
+    static_conf = TimeSeriesDataStaticConfig(
+        data=data,
+        group_id="GR_ID",
+        input_continuous=["IN1", "IN2"],
+        input_categorical=["IN3"],
+        target_continuous=["OUT1"],
+        target_categorical=["OUT2"],
+        prediction_horizon=3,
+        input_predictions=["IN2"],
+        train_val_test_percentage=Train_val_test_perc(70, 15, 15),
+        shuffle_groups=True,
+    )
+    fitable_conf = TimeSeriesDataFitableConfig(input_sequence_len=SEQ_LEN)
+    data = make_time_series_data(static_conf, fitable_conf)
+    adapter = DataAdapter.create_from_timeseries_data(data)
+    rounder = DomainRounder(
+        {
+            "OUT1": [1e-1 * i for i in range(100)],
+            "OUT2": [1e-1 * i for i in range(100)],
+        }
+    )
+    singe_model = DummyModelFactory.get_feed_forward_dummy(adapter)
+    multi_assemby = AssembledModel(
+        models=[(model, 1) for model in [singe_model] * 5], domain_rounder=rounder
+    )
+    single_assembly = AssembledModel(models=[(singe_model, 1)], domain_rounder=rounder)
+
+    for _ in range(TRIES):
+        pred_in = {
+            "IN1": np.random.randn(SEQ_LEN).tolist(),
+            "IN2": np.random.randn(SEQ_LEN).tolist(),
+            "IN3": np.random.randn(SEQ_LEN).tolist(),
+        }
+        assert multi_assemby.predict(pred_in) == single_assembly.predict(pred_in)
+
+
+def test_ensemble_weights():
+    """
+    tests if assembly model behaves
+    correctly with diffrent weights
+    """
+    SIZE = 50
+    SEQ_LEN = 7
+    data = pd.DataFrame(
+        {
+            "IN1": np.random.rand(SIZE),
+            "IN2": np.random.rand(SIZE),
+            "IN3": np.random.rand(SIZE),
+            "OUT1": np.zeros(SIZE),
+            "OUT2": np.zeros(SIZE),
+            "GR_ID": np.arange(SIZE) // 20,
+        }
+    )
+    static_conf = TimeSeriesDataStaticConfig(
+        data=data,
+        group_id="GR_ID",
+        input_continuous=["IN1", "IN2"],
+        input_categorical=["IN3"],
+        target_continuous=["OUT1"],
+        target_categorical=["OUT2"],
+        prediction_horizon=3,
+        input_predictions=["IN2"],
+        train_val_test_percentage=Train_val_test_perc(70, 15, 15),
+        shuffle_groups=True,
+    )
+    fitable_conf = TimeSeriesDataFitableConfig(input_sequence_len=SEQ_LEN)
+    data = make_time_series_data(static_conf, fitable_conf)
+    adapter = DataAdapter.create_from_timeseries_data(data)
+    rounder = DomainRounder(
+        {
+            "OUT1": [1e-1 * i for i in range(100)],
+            "OUT2": [1e-1 * i for i in range(100)],
+        }
+    )
+
+    ffmodel = DummyModelFactory.get_feed_forward_dummy(adapter)
+    rrnmodel = DummyModelFactory.get_stacked_rnn_dummy(adapter)
+    pred_in = {
+        "IN1": np.random.randn(SEQ_LEN).tolist(),
+        "IN2": np.random.randn(SEQ_LEN).tolist(),
+        "IN3": np.random.randn(SEQ_LEN).tolist(),
+    }
+
+    assert AssembledModel(
+        models=[(ffmodel, 3), (rrnmodel, 0)], domain_rounder=rounder
+    ).predict(pred_in) == AssembledModel(
+        models=[(ffmodel, 1)], domain_rounder=rounder
+    ).predict(
+        pred_in
+    )
+
+    assert AssembledModel(
+        models=[(ffmodel, 0), (rrnmodel, 5)], domain_rounder=rounder
+    ).predict(pred_in) == AssembledModel(
+        models=[(rrnmodel, 0.1)], domain_rounder=rounder
+    ).predict(
+        pred_in
+    )
diff --git a/SupervisedGym/tests/modeltuner/test_model_tuner.py b/SupervisedGym/tests/modeltuner/test_model_tuner.py
index 080a6f3..81546d4 100644
--- a/SupervisedGym/tests/modeltuner/test_model_tuner.py
+++ b/SupervisedGym/tests/modeltuner/test_model_tuner.py
@@ -1,7 +1,9 @@
+from SupervisedGym.modeltuner.modeltuner import TrainedModelStats
 import pytest
 import numpy as np
 import torch.nn as nn
 import pandas as pd
+from typing import Optional
 
 from SupervisedGym.modeltuner import ModelTuner
 from SupervisedGym.utils.data import (
@@ -45,7 +47,19 @@ def test_fast_dev_run():
     with pytest.raises(Exception):
         tuner.best_model
 
-    best_model = tuner.tune_model(
-        timeout=10 ** 3, n_trials=1, tunable_model_type=StackedRNNTunable
+    trained_model_stat: Optional[TrainedModelStats] = None
+
+    def set_tms(tms: TrainedModelStats):
+        nonlocal trained_model_stat
+        trained_model_stat = tms
+
+    tuner.tune(
+        timeout=10 ** 3,
+        n_trials=1,
+        tunable_model_type=StackedRNNTunable,
+        model_callback=set_tms,
     )
-    assert isinstance(best_model, IBaseTunableModel)
+
+    assert isinstance(trained_model_stat, TrainedModelStats)
+    assert isinstance(trained_model_stat.model, IBaseTunableModel)
+    assert isinstance(trained_model_stat.val_loss, float)
-- 
GitLab


From 98187f67e4a4ec2ccaf8914c02679e52c1c7aeef Mon Sep 17 00:00:00 2001
From: szysad <szysad108@gmail.com>
Date: Wed, 19 May 2021 15:31:52 +0200
Subject: [PATCH 2/5] added TunableTransformer to tests, formated code

---
 SupervisedGym/SupervisedGym/__main__.py       | 53 ++++-----------
 .../models/Transformer/Transformer.py         | 31 ++++-----
 .../SupervisedGym/models/__init__.py          |  7 +-
 .../SupervisedGym/utils/problem_analyzer.py   | 66 ++++++++++---------
 SupervisedGym/tests/models/dummy_factory.py   | 24 ++++++-
 SupervisedGym/tests/utils/fcrCP.py            |  2 +-
 SupervisedGym/tests/utils/genomCP.py          |  2 +-
 .../tests/utils/test_problem_analyzer.py      | 58 ++++++++++------
 8 files changed, 133 insertions(+), 110 deletions(-)

diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py
index 303ad9d..eb027ab 100644
--- a/SupervisedGym/SupervisedGym/__main__.py
+++ b/SupervisedGym/SupervisedGym/__main__.py
@@ -1,51 +1,24 @@
-from SupervisedGym.utils import domain_rounder
-from SupervisedGym.modeltuner.modeltuner import TrainedModelStats
 import torch
-import pickle
 from SupervisedGym.model_assembler.model_assembler import (
     ModelAssebler as GenomAssembler,
 )
-from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable, TransformerTunable
-from SupervisedGym.models.assembled_model import AssembledModel
+from SupervisedGym.models import FeedForwardTunable
 
 
-SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/test-model"
-TIMEOUT = 60 * 1  # 1 minutes
+SAVE_PATH = "<model save path>"
+TIMEOUT = 60 * 60 * 10  # 10 hours minutes
 
 if __name__ == "__main__":
-    # assebler = GenomAssembler(
-    #    raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss()
-    # )  # placeholder
+    assebler = GenomAssembler(
+        raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss()
+    )  # placeholder
 
-    # model = assebler.assemble_model(
-    #    timeout=TIMEOUT,
-    #    n_trials=5,
-    #    assembly_size=3,
-    #    avaible_model_types=[FeedForwardTunable]
-    # )
+    model = assebler.assemble_model(
+        timeout=TIMEOUT,
+        n_trials=5,
+        assembly_size=5,
+        avaible_model_types=[FeedForwardTunable],
+    )
 
-    # with open(SAVE_PATH, "wb") as f:
-    #    f.write(model.parse_model())
-
-    rnnpaths = [
-        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-9h-gpu",
-        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-w-early-stopping-gpu",
-        "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-gpu",
-    ]
-
-    rrnmodels = []
-    rounder = None
-
-    for p in rnnpaths:
-        data = None
-        with open(p, "rb") as f:
-            data = pickle.loads(f.read())
-        model = StackedRNNTunable.load_model(data["tunable_model"])
-        rounder = data["domain_rounder"]
-        rrnmodels.append(model)
-
-    stats = [TrainedModelStats(model, 1) for model in rrnmodels]
-
-    assemble = AssembledModel(domain_rounder=rounder, model_stats=stats)
     with open(SAVE_PATH, "wb") as f:
-        f.write(assemble.parse_model())
+        f.write(model.parse_model())
diff --git a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py
index cb18080..bd268b2 100644
--- a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py
+++ b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py
@@ -3,7 +3,6 @@ import torch
 import math
 
 
-
 class PositionalEncoding(nn.Module):
     """
     Inject some information about the relative or absolute position of the tokens
@@ -20,19 +19,21 @@ class PositionalEncoding(nn.Module):
         >>> pos_encoder = PositionalEncoding(d_model)
     """
 
-    def __init__(self, d_model: int, max_len: int=5000):
+    def __init__(self, d_model: int, max_len: int = 5000):
         super(PositionalEncoding, self).__init__()
 
         pe = torch.zeros(max_len, d_model)
         position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
-        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+        div_term = torch.exp(
+            torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
+        )
         pe[:, 0::2] = torch.sin(position * div_term)
-        if d_model%2 != 0:
-            pe[:, 1::2] = torch.cos(position * div_term)[:,0:-1]
+        if d_model % 2 != 0:
+            pe[:, 1::2] = torch.cos(position * div_term)[:, 0:-1]
         else:
             pe[:, 1::2] = torch.cos(position * div_term)
         pe = pe.unsqueeze(0).transpose(0, 1)
-        self.register_buffer('pe', pe)
+        self.register_buffer("pe", pe)
 
     def forward(self, x):
         """
@@ -47,7 +48,7 @@ class PositionalEncoding(nn.Module):
             >>> output = pos_encoder(x)
         """
 
-        x = x + self.pe[:x.size(0), :]
+        x = x + self.pe[: x.size(0), :]
         return x
 
 
@@ -109,16 +110,16 @@ class Transformer(nn.Module):
 
         self.primary_embedding = nn.Conv1d(input_size, features, 1)
         self.positionalEncoder = PositionalEncoding(features)
-        transformerEncoderLayers = nn.TransformerEncoderLayer(features,
-                                                            heads_number,
-                                                            dim_feedforward=feed_forward_size,
-                                                            dropout=dropout)
-        self.transformerEncoder = nn.TransformerEncoder(transformerEncoderLayers, self.num_layers);
-        
+        transformerEncoderLayers = nn.TransformerEncoderLayer(
+            features, heads_number, dim_feedforward=feed_forward_size, dropout=dropout
+        )
+        self.transformerEncoder = nn.TransformerEncoder(
+            transformerEncoderLayers, self.num_layers
+        )
+
         self.fc1 = nn.Linear(features * seq_len, pre_output)
         self.fc2 = nn.Linear(pre_output, output_size)
 
-
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         """
         Performs pass throught network
@@ -155,5 +156,5 @@ class Transformer(nn.Module):
         out = self.fc1(out)
         out = self.activation(out)
         out = self.fc2(out)
-        
+
         return out
diff --git a/SupervisedGym/SupervisedGym/models/__init__.py b/SupervisedGym/SupervisedGym/models/__init__.py
index ce749dc..740b84f 100644
--- a/SupervisedGym/SupervisedGym/models/__init__.py
+++ b/SupervisedGym/SupervisedGym/models/__init__.py
@@ -11,4 +11,9 @@ avaible_model_types: List[Type[IBaseTunableModel]] = [
     TransformerTunable,
 ]
 
-__all__ = [FeedForwardTunable, StackedRNNTunable, TransformerTunable, avaible_model_types]
+__all__ = [
+    FeedForwardTunable,
+    StackedRNNTunable,
+    TransformerTunable,
+    avaible_model_types,
+]
diff --git a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
index c55dad7..96bb8cd 100644
--- a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
+++ b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py
@@ -7,14 +7,6 @@ from .data import TimeSeriesDataStaticConfig, Train_val_test_perc
 import re
 
 
-class BaseProblemAnalyzerExcepion(Exception):
-    """
-    base class for ProblemAnalyzer exceptions
-    """
-
-    pass
-
-
 class ProblemAnalyzer:
     """
     based on cp-problem file generates:
@@ -26,7 +18,9 @@ class ProblemAnalyzer:
 
     @staticmethod
     def generate_domain_rounder(cp_problem: ET.ElementTree) -> DomainRounder:
-        return DomainRounder(ProblemAnalyzer._get_domains_from_constraint_problem(cp_problem))
+        return DomainRounder(
+            ProblemAnalyzer._get_domains_from_constraint_problem(cp_problem)
+        )
 
     @staticmethod
     def generate_tsdata_static_conf(
@@ -34,52 +28,60 @@ class ProblemAnalyzer:
     ) -> TimeSeriesDataStaticConfig:
         raise NotImplementedError()
 
-    def _get_domains_from_constraint_problem(cp_problem: ET.ElementTree) -> Dict[str, List[int]]:
+    def _get_domains_from_constraint_problem(
+        cp_problem: ET.ElementTree,
+    ) -> Dict[str, List[int]]:
         """
         Returns a dictionary containing domains of variables.
         """
 
         result = {}
-        type_re = re.compile(r'^({.*})?type$')
+        type_re = re.compile(r"^({.*})?type$")
 
-        for variable_node in cp_problem.findall('cpVariables'):
+        for variable_node in cp_problem.findall("cpVariables"):
 
-            variable_name = variable_node.attrib.get('id')
+            variable_name = variable_node.attrib.get("id")
 
-            domain_node = variable_node.find('domain')
+            domain_node = variable_node.find("domain")
             variable_domain = []
 
-            if variable_name == None:
-                raise Exception(f"No 'id' attribute in 'cpVariables' tag.")
-            if domain_node == None:
-                raise Exception(f"Variable {variable_name} has no 'domain' child in CP.")
+            if variable_name is None:
+                raise Exception("No 'id' attribute in 'cpVariables' tag.")
+            if domain_node is None:
+                raise Exception(
+                    f"Variable {variable_name} has no 'domain' child in CP."
+                )
 
             domain_type = "No type attribute."
 
             for key, value in domain_node.attrib.items():
-                if type_re.match(key) != None:
+                if not type_re.match(key) is None:
                     domain_type = value
                     break
 
-            if domain_type == 'cp:RangeDomain':
+            if domain_type == "cp:RangeDomain":
 
-                from_node = domain_node.find('from')
-                to_node = domain_node.find('to')
-                if from_node == None or to_node == None:
-                    raise Exception(f"Wrong domain fromat of {variable_name} variable."
-                                    "Expected 'from' and 'to' nodes")
+                from_node = domain_node.find("from")
+                to_node = domain_node.find("to")
+                if from_node is None or to_node is None:
+                    raise Exception(
+                        f"Wrong domain fromat of {variable_name} variable."
+                        "Expected 'from' and 'to' nodes"
+                    )
 
-                start_value = int(from_node.attrib.get('value', "0"))
-                end_value = int(to_node.attrib.get('value', "0")) + 1
+                start_value = int(from_node.attrib.get("value", "0"))
+                end_value = int(to_node.attrib.get("value", "0")) + 1
                 variable_domain = range(start_value, end_value)
 
-            elif domain_type == 'cp:NumericListDomain':
+            elif domain_type == "cp:NumericListDomain":
 
-                for value in domain_node.findall('values'):
-                    variable_domain.append(int(value.attrib.get('value', "0")))
+                for value in domain_node.findall("values"):
+                    variable_domain.append(int(value.attrib.get("value", "0")))
 
             else:
-                raise Exception(f"Unknown type of domain of {variable_name} variable: {domain_type}.")
+                raise Exception(
+                    f"Unknown type of domain of {variable_name} variable: {domain_type}."
+                )
 
             result[variable_name] = variable_domain
 
@@ -122,7 +124,7 @@ class DummyGenomAnalyzer(ProblemAnalyzer):
     ) -> TimeSeriesDataStaticConfig:
         return TimeSeriesDataStaticConfig(
             data=pd.read_csv(
-                r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\trening\alpha-star-solver\FCRtraining\Genom\genom-newest-at-05-28.csv"
+                "/home/szysad/mimuw/3rok/ZPP/training-data/genom-data/newest/genom-newest-at-05-28.csv"
             ),
             group_id="experimentId",
             input_continuous=[
diff --git a/SupervisedGym/tests/models/dummy_factory.py b/SupervisedGym/tests/models/dummy_factory.py
index abde9fb..0a77c7e 100644
--- a/SupervisedGym/tests/models/dummy_factory.py
+++ b/SupervisedGym/tests/models/dummy_factory.py
@@ -4,7 +4,11 @@ from typing import List
 
 from SupervisedGym.utils.data import DataAdapter
 from SupervisedGym.models.base import IBaseTunableModel
-from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable
+from SupervisedGym.models import (
+    FeedForwardTunable,
+    StackedRNNTunable,
+    TransformerTunable,
+)
 
 
 class DummyModelFactory:
@@ -40,9 +44,27 @@ class DummyModelFactory:
             val_loss=torch.nn.L1Loss(),
         )
 
+    @staticmethod
+    def get_transformer_dummy(data_adapter: DataAdapter) -> TransformerTunable:
+        return TransformerTunable(
+            data_adapter=data_adapter,
+            features=3,
+            heads_number=1,
+            layers_number=1,
+            feed_forward_size=3,
+            pre_output=3,
+            dropout=0,
+            activation=torch.nn.ReLU(),
+            loss=torch.nn.L1Loss(),
+            lr=1,
+            val_loss=torch.nn.L1Loss(),
+            optimizer_type=torch.optim.SGD,
+        )
+
     @staticmethod
     def all_dummys(data_adapter: DataAdapter) -> List[IBaseTunableModel]:
         return [
             DummyModelFactory.get_feed_forward_dummy(data_adapter),
             DummyModelFactory.get_stacked_rnn_dummy(data_adapter),
+            DummyModelFactory.get_transformer_dummy(data_adapter),
         ]
diff --git a/SupervisedGym/tests/utils/fcrCP.py b/SupervisedGym/tests/utils/fcrCP.py
index ff31fb8..6ba70c3 100644
--- a/SupervisedGym/tests/utils/fcrCP.py
+++ b/SupervisedGym/tests/utils/fcrCP.py
@@ -405,4 +405,4 @@ fcrCP = r"""<?xml version="1.0" encoding="ASCII"?>
     <value xsi:type="types:IntegerValueUpperware" value="1"/>
   </cpMetrics>
 </cp:ConstraintProblem>
-"""
\ No newline at end of file
+"""
diff --git a/SupervisedGym/tests/utils/genomCP.py b/SupervisedGym/tests/utils/genomCP.py
index 8943142..f832314 100644
--- a/SupervisedGym/tests/utils/genomCP.py
+++ b/SupervisedGym/tests/utils/genomCP.py
@@ -211,4 +211,4 @@ genomCP = r"""<?xml version="1.0" encoding="ASCII"?>
     <value xsi:type="types:IntegerValueUpperware" value="1"/>
   </cpMetrics>
 </cp:ConstraintProblem>
-"""
\ No newline at end of file
+"""
diff --git a/SupervisedGym/tests/utils/test_problem_analyzer.py b/SupervisedGym/tests/utils/test_problem_analyzer.py
index 2cf2787..76cc9e0 100644
--- a/SupervisedGym/tests/utils/test_problem_analyzer.py
+++ b/SupervisedGym/tests/utils/test_problem_analyzer.py
@@ -13,7 +13,7 @@ def test_genom_domain():
     """
 
     xml_file = ET.fromstring(genomCP)
-    assert(ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == {
+    assert ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == {
         "WorkerCardinality": range(1, 31),
         "provider_ComponentSparkWorker": [0],
         "WorkerCores": [2, 4, 8, 16],
@@ -31,7 +31,8 @@ def test_genom_domain():
             65536,
             70041,
         ],
-    })
+    }
+
 
 @pytest.mark.ProblemAnalyzer
 def test_FCR_domain():
@@ -41,20 +42,39 @@ def test_FCR_domain():
     """
 
     xml_file = ET.fromstring(fcrCP)
-    assert(ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == {
-        "AppCardinality": range(1, 41),                                       
-        "provider_Component_App": [0],                                                
-        "AppCores": [2, 4, 8, 16, 32, 36, 40, 48],                      
-        "AppStorage": [0, 10, 420, 1024, 2097152],                        
-        "LBCardinality": range(1, 2),                                        
-        "provider_Component_LB": [0],                                                
-        "LBRam": [1024, 1740, 2048, 3750, 3840, 4096, 7168, 7680,
-        8192, 15360, 15616, 16384, 17510, 22528, 23552,
-        30720, 31232, 32768, 35020, 62464, 70041],          
-        "LBStorage": [0, 10, 420],                                       
-        "DBCardinality": range(1, 2),                                        
-        "provider_Component_DB": [0],                                                
-        "DBCores": [1, 2, 4, 8],                                       
-        "DBStorage": [0, 10, 420],                                        
-    })
-
+    assert ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == {
+        "AppCardinality": range(1, 41),
+        "provider_Component_App": [0],
+        "AppCores": [2, 4, 8, 16, 32, 36, 40, 48],
+        "AppStorage": [0, 10, 420, 1024, 2097152],
+        "LBCardinality": range(1, 2),
+        "provider_Component_LB": [0],
+        "LBRam": [
+            1024,
+            1740,
+            2048,
+            3750,
+            3840,
+            4096,
+            7168,
+            7680,
+            8192,
+            15360,
+            15616,
+            16384,
+            17510,
+            22528,
+            23552,
+            30720,
+            31232,
+            32768,
+            35020,
+            62464,
+            70041,
+        ],
+        "LBStorage": [0, 10, 420],
+        "DBCardinality": range(1, 2),
+        "provider_Component_DB": [0],
+        "DBCores": [1, 2, 4, 8],
+        "DBStorage": [0, 10, 420],
+    }
-- 
GitLab


From 3aa286d12cb1bae9348284136648a78691e63396 Mon Sep 17 00:00:00 2001
From: szymon <szysad108@gmail.com>
Date: Wed, 19 May 2021 16:17:24 +0200
Subject: [PATCH 3/5] fixed some small errors

---
 .../SupervisedGym/model_assembler/model_assembler.py      | 4 +---
 SupervisedGym/SupervisedGym/modeltuner/objective.py       | 2 ++
 SupervisedGym/SupervisedGym/utils/data/dataadapter.py     | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
index bcb8e8d..bacffd3 100644
--- a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
+++ b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py
@@ -89,8 +89,6 @@ class ModelAssebler:
             model_callback=__receive_stat,
         )
         return AssembledModel(
-            model_stats=map(
-                lambda m, val_loss: (m, self.__val_loss_to_weight(val_loss))
-            ),
+            models=[(stat.model, self.__val_loss_to_weight(stat.val_loss)) for stat in model_stats],
             domain_rounder=self.dom_rounder,
         )
diff --git a/SupervisedGym/SupervisedGym/modeltuner/objective.py b/SupervisedGym/SupervisedGym/modeltuner/objective.py
index de3c0e7..0f35d33 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/objective.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/objective.py
@@ -151,6 +151,7 @@ class Objective:
 
         trainer = pl.Trainer(
             logger=False,
+            gpus=-1,
             val_check_interval=self.ohparams["val_check_interval"],
             fast_dev_run=self.fast_dev_run,
             checkpoint_callback=False,
@@ -158,6 +159,7 @@ class Objective:
             min_epochs=self.ohparams["min_epochs"],
             gradient_clip_val=gradient_clip_val,
             callbacks=callbacks,
+            progress_bar_refresh_rate=0
         )
 
         trainer.fit(model=self.model, datamodule=datamodule)
diff --git a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py
index b36964e..a25b7da 100644
--- a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py
+++ b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py
@@ -70,10 +70,10 @@ class DataAdapter:
         with torch.no_grad():
             if inverse:
                 return torch.from_numpy(
-                    self.__input_transformer.inverse_transform(x.numpy())
+                    self.__input_transformer.inverse_transform(x.cpu().numpy())
                 )
             else:
-                return torch.from_numpy(self.__input_transformer.transform(x.numpy()))
+                return torch.from_numpy(self.__input_transformer.transform(x.cpu().numpy()))
 
     def transform_target(self, y: torch.Tensor, inverse: bool = False) -> torch.Tensor:
         """
@@ -84,7 +84,7 @@ class DataAdapter:
         with torch.no_grad():
             if inverse:
                 return torch.from_numpy(
-                    self.__target_transformer.inverse_transform(y.numpy())
+                    self.__target_transformer.inverse_transform(y.cpu().numpy())
                 )
             else:
-                return torch.from_numpy(self.__target_transformer.transform(y.numpy()))
+                return torch.from_numpy(self.__target_transformer.transform(y.cpu().numpy()))
-- 
GitLab


From 83ffea7645ded21f950c2ccc68949ac3fd3a04ef Mon Sep 17 00:00:00 2001
From: szysad <szysad108@gmail.com>
Date: Wed, 19 May 2021 16:38:57 +0200
Subject: [PATCH 4/5] added docstring, fixed divergence_threhold heuristic

---
 SupervisedGym/SupervisedGym/modeltuner/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py
index a0172b6..fffcc0a 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/utils.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py
@@ -16,6 +16,9 @@ def naive_model_loss(
     mode_type: Literal["mean"] = "mean",
     sampler: Optional[Sampler] = None,
 ) -> float:
+    '''
+        calculates naive prediction loss on train dataset
+    '''
     loader = DataLoader(dataset=dataset, sampler=sampler, batch_size=1)
     pred: torch.Tensor
     if mode_type == "mean":
@@ -67,6 +70,6 @@ def generate_objective_hparams(
         val_check_interval=val_check_interval,
         max_epochs=min_epochs * 50,
         min_epochs=min_epochs,
-        es_divergence_threshold=es_divergence_threshold,
+        es_divergence_threshold=es_divergence_threshold * 10,
         es_patience=min_epochs * 3,
     )
-- 
GitLab


From 77b0b419ffc722cae8496f573047381fa1637d71 Mon Sep 17 00:00:00 2001
From: szysad <szysad108@gmail.com>
Date: Wed, 19 May 2021 16:39:48 +0200
Subject: [PATCH 5/5] typo fix

---
 SupervisedGym/SupervisedGym/modeltuner/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py
index fffcc0a..003dac7 100644
--- a/SupervisedGym/SupervisedGym/modeltuner/utils.py
+++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py
@@ -70,6 +70,6 @@ def generate_objective_hparams(
         val_check_interval=val_check_interval,
         max_epochs=min_epochs * 50,
         min_epochs=min_epochs,
-        es_divergence_threshold=es_divergence_threshold * 10,
+        es_divergence_threshold=es_divergence_threshold * 100,
         es_patience=min_epochs * 3,
     )
-- 
GitLab