From 695118bcb94f667286180a904eb301ee39b04089 Mon Sep 17 00:00:00 2001 From: szysad <szysad108@gmail.com> Date: Wed, 19 May 2021 13:19:44 +0200 Subject: [PATCH 1/5] added heuristics to set hparams of tunable objective. Now AssembledModel is ensemble model, added more tests of AssembledModel --- SupervisedGym/SupervisedGym/__main__.py | 45 +++++- .../model_assembler/model_assembler.py | 50 ++++++- .../SupervisedGym/models/assembled_model.py | 107 ++++++++++----- .../SupervisedGym/modeltuner/__init__.py | 4 +- .../{utils/data => modeltuner}/datamodule.py | 2 +- .../SupervisedGym/modeltuner/modeltuner.py | 42 +++--- .../SupervisedGym/modeltuner/objective.py | 68 +++++---- .../SupervisedGym/modeltuner/utils.py | 72 ++++++++++ SupervisedGym/SupervisedGym/utils/__init__.py | 14 ++ .../SupervisedGym/utils/data/__init__.py | 2 - .../SupervisedGym/utils/domain_rounder.py | 6 +- .../SupervisedGym/utils/problem_analyzer.py | 8 ++ .../tests/models/test_assmebled_model.py | 129 +++++++++++++++++- .../tests/modeltuner/test_model_tuner.py | 20 ++- 14 files changed, 453 insertions(+), 116 deletions(-) rename SupervisedGym/SupervisedGym/{utils/data => modeltuner}/datamodule.py (94%) create mode 100644 SupervisedGym/SupervisedGym/modeltuner/utils.py create mode 100644 SupervisedGym/SupervisedGym/utils/__init__.py diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py index c5fb358..fa49b05 100644 --- a/SupervisedGym/SupervisedGym/__main__.py +++ b/SupervisedGym/SupervisedGym/__main__.py @@ -1,5 +1,7 @@ +from SupervisedGym.utils import domain_rounder +from SupervisedGym.modeltuner.modeltuner import TrainedModelStats import torch - +import pickle from SupervisedGym.model_assembler.model_assembler import ( ModelAssebler as GenomAssembler, ) @@ -7,14 +9,43 @@ from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable from SupervisedGym.models.assembled_model import AssembledModel -SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/13-05-genom/rrn-11h-scaling" +SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/test-model" TIMEOUT = 60 * 1 # 1 minutes if __name__ == "__main__": - assebler = GenomAssembler( - raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss() - ) # placeholder + # assebler = GenomAssembler( + # raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss() + # ) # placeholder + + # model = assebler.assemble_model( + # timeout=TIMEOUT, + # n_trials=5, + # assembly_size=3, + # avaible_model_types=[FeedForwardTunable] + # ) - model = assebler.assemble_model(TIMEOUT, [FeedForwardTunable]) - #with open(SAVE_PATH, "wb") as f: + # with open(SAVE_PATH, "wb") as f: # f.write(model.parse_model()) + + rnnpaths = [ + "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-9h-gpu", + "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-w-early-stopping-gpu", + "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-gpu", + ] + + rrnmodels = [] + rounder = None + + for p in rnnpaths: + data = None + with open(p, "rb") as f: + data = pickle.loads(f.read()) + model = StackedRNNTunable.load_model(data["tunable_model"]) + rounder = data["domain_rounder"] + rrnmodels.append(model) + + stats = [TrainedModelStats(model, 1) for model in rrnmodels] + + assemble = AssembledModel(domain_rounder=rounder, model_stats=stats) + with open(SAVE_PATH, "wb") as f: + f.write(assemble.parse_model()) diff --git a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py index 1e88ab6..bcb8e8d 100644 --- a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py +++ b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py @@ -8,7 +8,7 @@ from ..models.base import IBaseTunableModel from ..models import avaible_model_types from ..models.assembled_model import AssembledModel from ..utils.problem_analyzer import DummyGenomAnalyzer as ProblemAnalyzer -from ..modeltuner import ModelTuner +from ..modeltuner import ModelTuner, TrainedModelStats class ModelAssebler: @@ -35,26 +35,62 @@ class ModelAssebler: raw_data, cp_problem ) + def __val_loss_to_weight(self, loss: float) -> float: + """ + transforms validation loss of model + to its weight in ensembled model + """ + return 1 / (loss + 0.33) + def assemble_model( self, timeout: int, n_trials: Optional[int] = None, - avaible_model_types: List[Type[IBaseTunableModel]] = avaible_model_types + assembly_size: int = 5, + avaible_model_types: List[Type[IBaseTunableModel]] = avaible_model_types, ) -> AssembledModel: """ based on data and cp_problem builds AssembledModel """ + model_stats: List[TrainedModelStats] = [] + + def __receive_stat(new_stat: TrainedModelStats): + """updates model stats with new model if has val_loss + small enough""" + nonlocal model_stats + + if len(model_stats) < assembly_size: + model_stats.append(new_stat) + + max_idx = 0 + max_loss = float("-inf") + for i, stat in enumerate(model_stats): + if stat.val_loss > max_loss: + max_loss = stat.val_loss + max_idx = i + + if new_stat.val_loss < max_loss: + model_stats[max_idx] = new_stat + tuner = ModelTuner( val_loss=self.val_loss, tsdata_static_config=self.data_static_conf, - fast_dev_run=False + fast_dev_run=False, ) - best_model = tuner.tune_model( + tuner.tune( timeout=timeout, - tunable_model_type=avaible_model_types[0], - n_trials=n_trials + tunable_model_type=avaible_model_types[ + 0 + ], # CURRENTLY FIXED FIRST MODEL TYPE + n_trials=n_trials, + model_callback=__receive_stat, + ) + return AssembledModel( + model_stats=map( + lambda m, val_loss: (m, self.__val_loss_to_weight(val_loss)) + ), + domain_rounder=self.dom_rounder, ) - return AssembledModel(tunable_model=best_model, domain_rounder=self.dom_rounder) diff --git a/SupervisedGym/SupervisedGym/models/assembled_model.py b/SupervisedGym/SupervisedGym/models/assembled_model.py index 38bc312..cbc5412 100644 --- a/SupervisedGym/SupervisedGym/models/assembled_model.py +++ b/SupervisedGym/SupervisedGym/models/assembled_model.py @@ -1,11 +1,11 @@ from __future__ import annotations +from SupervisedGym.models.base.IBaseTunableModel import IBaseTunableModel import torch import pickle -from typing import Dict, Union, List +from typing import Dict, Any, Union, List, Type, Tuple from itertools import count -from ..utils.domain_rounder import DomainRounder -from .base import IBaseTunableModel +from ..utils import DomainRounder from .base.parsable_model import IParsableModel @@ -14,15 +14,22 @@ class AssembledModel(IParsableModel): input_idx: Dict[str, int] def __init__( - self, tunable_model: IBaseTunableModel, domain_rounder: DomainRounder + self, + models: List[Tuple[IBaseTunableModel, float]], + domain_rounder: DomainRounder, ) -> AssembledModel: - self.tunable_model = tunable_model + self.models = models self.domain_rounder = domain_rounder - self.data_adapter = tunable_model.data_adapter - self.input_idx = dict( - zip(tunable_model.data_adapter.input_columns, count(start=0, step=1)) + self._input_columns = models[0][0].data_adapter.input_columns + self._target_columns = models[0][0].data_adapter.target_columns + self._input_sequence_len = max( + map(lambda s: s[0].data_adapter.input_sequence_len, models) ) + self.input_idx = dict(zip(self._input_columns, count(start=0, step=1))) + # enter eval mode for all models + for m in self.models: + m[0].eval() def __validate_input(self, x: Dict[str, List[Union[float, str, int]]]): """Checks if input data for prediction is correct @@ -34,7 +41,7 @@ class AssembledModel(IParsableModel): Raises: ValueError: [description] """ - for var_name in self.data_adapter.input_columns: + for var_name in self._input_columns: if var_name not in x: raise ValueError( f"sequence of {var_name} is required to make" @@ -42,12 +49,45 @@ class AssembledModel(IParsableModel): ) for var_name, seq in x.items(): - if len(seq) != self.data_adapter.input_sequence_len: + if len(seq) != self._input_sequence_len: raise ValueError( f"sequence of {var_name} data has length {len(seq)}," - " expected {self.data_adapter.input_sequence_len}." + f" expected {self._input_sequence_len}." ) + def __assembly_forward(self, x: torch.Tensor) -> torch.Tensor: + """ + performes forward pass using all assemby models + and then morging their preiction using weighted + average based on their val_loss + """ + # currently all weights are equal + w_sum = 0 + out = torch.zeros(len(self._target_columns)) + for model, w in self.models: + seq_len = model.data_adapter.input_sequence_len + xx = x[:, -seq_len:, :] + xx = model.data_adapter.transform_input(xx) + raw_out = model.forward(xx) + transformed_out = model.data_adapter.transform_target( + raw_out, inverse=True + ).squeeze() + out += transformed_out * w + w_sum += w + return out / w_sum + + @property + def input_columns(self) -> List[str]: + return self._input_columns + + @property + def target_columns(self) -> List[str]: + return self._target_columns + + @property + def input_sequence_len(self) -> int: + return self._input_sequence_len + def predict(self, x: Dict[str, List[Union[float, str, int]]]) -> Dict[float, int]: """ generates model prediction based @@ -61,44 +101,39 @@ class AssembledModel(IParsableModel): latest. Each value list must have lenght equal to input_sequence_length """ - self.tunable_model.eval() self.__validate_input(x) - seq_len = self.data_adapter.input_sequence_len # we need to pass 3d tensor with first dimention # equal to batch_size which we set to 1 tensor_in = torch.cat( tuple( - torch.Tensor(x[col]).view(seq_len, 1) - for col in self.data_adapter.input_columns + torch.Tensor(x[col]).view(self._input_sequence_len, 1) + for col in self._input_columns ), dim=1, ).unsqueeze(dim=0) - tensor_out = self.tunable_model.forward( - self.data_adapter.transform_input(tensor_in) - ) - self.tunable_model.train(True) - - # batch size equals to 1 so squeeze - prediction = self.data_adapter.transform_target(tensor_out).squeeze() - pred_dict = { - col: v.item() - for col, v in zip(self.data_adapter.target_columns, prediction) - } + + prediction = self.__assembly_forward(tensor_in) + pred_dict = {col: v.item() for col, v in zip(self._target_columns, prediction)} return self.domain_rounder.round(pred_dict) @classmethod def load_model(cls, encoding: bytes) -> AssembledModel: data = pickle.loads(encoding) - tunable_type = data["tunable_type"] - return AssembledModel( - tunable_model=tunable_type.load_model(data["tunable_model"]), - domain_rounder=data["domain_rounder"], - ) + models: List[Tuple[IBaseTunableModel, float]] = [] + for state_dict in data["model_data"]: + model_type: Type[IBaseTunableModel] = state_dict["type"] + models.append( + (model_type.load_model(state_dict["encoding"]), state_dict["weight"]) + ) + return AssembledModel(models=models, domain_rounder=data["domain_rounder"]) def parse_model(self) -> bytes: - data = { - "domain_rounder": self.domain_rounder, - "tunable_model": self.tunable_model.parse_model(), - "tunable_type": type(self.tunable_model), - } + model_data: List[Dict[str, Any]] = [] + for model, weight in self.models: + model_data.append( + {"encoding": model.parse_model(), "weight": weight, "type": type(model)} + ) + + data = {"domain_rounder": self.domain_rounder, "model_data": model_data} + return pickle.dumps(data) diff --git a/SupervisedGym/SupervisedGym/modeltuner/__init__.py b/SupervisedGym/SupervisedGym/modeltuner/__init__.py index e14248e..3a59cb3 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/__init__.py +++ b/SupervisedGym/SupervisedGym/modeltuner/__init__.py @@ -1,4 +1,4 @@ -from .modeltuner import ModelTuner +from .modeltuner import ModelTuner, TrainedModelStats -__all__ = [ModelTuner] +__all__ = [ModelTuner, TrainedModelStats] diff --git a/SupervisedGym/SupervisedGym/utils/data/datamodule.py b/SupervisedGym/SupervisedGym/modeltuner/datamodule.py similarity index 94% rename from SupervisedGym/SupervisedGym/utils/data/datamodule.py rename to SupervisedGym/SupervisedGym/modeltuner/datamodule.py index d083ba3..e6f2da0 100644 --- a/SupervisedGym/SupervisedGym/utils/data/datamodule.py +++ b/SupervisedGym/SupervisedGym/modeltuner/datamodule.py @@ -1,7 +1,7 @@ from pytorch_lightning import LightningDataModule from torch.utils.data import DataLoader -from .TimeSeriesDataSet.timeseries_data import TimeSeriesData +from SupervisedGym.utils.data import TimeSeriesData class DataModule(LightningDataModule): diff --git a/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py b/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py index 400f6e2..1d8fe7e 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py +++ b/SupervisedGym/SupervisedGym/modeltuner/modeltuner.py @@ -1,10 +1,15 @@ import optuna import torch -from typing import Union, Optional, Type +from typing import Callable, Optional, Type, NamedTuple from ..models.base import IBaseTunableModel -from ..utils.callbacks.optuna import NewBestTrialCallbackTrigger from ..utils.data import TimeSeriesDataStaticConfig from .objective import Objective +from .utils import generate_objective_hparams + + +class TrainedModelStats(NamedTuple): + model: IBaseTunableModel + val_loss: float class ModelTuner: @@ -21,18 +26,16 @@ class ModelTuner: ): self.tsdata_static_config = tsdata_static_config self.val_loss = val_loss - self._best_model: Union[IBaseTunableModel, None] = None + self._best_model: Optional[IBaseTunableModel] = None self.fast_dev_run = fast_dev_run - def _save_curr_model(self, objective: Objective): - self._best_model = objective.current_model() - - def tune_model( + def tune( self, timeout: int, tunable_model_type: Type[IBaseTunableModel], + model_callback: Callable[[TrainedModelStats], None], n_trials: Optional[int] = None, - ) -> IBaseTunableModel: + ) -> None: """ Performs network tuning """ @@ -42,8 +45,11 @@ class ModelTuner: objective = Objective( tsdata_static_config=self.tsdata_static_config, tunable_model_type=tunable_model_type, - val_loss=self.val_loss, + val_func=self.val_loss, fast_dev_run=self.fast_dev_run, + objective_hyper_params=generate_objective_hparams( + conf=self.tsdata_static_config, val_loss=self.val_loss + ), ) # enqueue trial with suggested params @@ -54,18 +60,10 @@ class ModelTuner: n_trials=n_trials, timeout=timeout, callbacks=[ - NewBestTrialCallbackTrigger(lambda: self._save_curr_model(objective)) + lambda _, ft: model_callback( + TrainedModelStats( + model=objective.current_model(), val_loss=ft.value + ) + ) ], ) - - print("Number of finished trials: {}".format(len(study.trials))) - print("Best trial:") - trial = study.best_trial - - print("Value: {}".format(trial.value)) - - print("Params:") - for key, value in trial.params.items(): - print("{}: {}".format(key, value)) - - return self._best_model diff --git a/SupervisedGym/SupervisedGym/modeltuner/objective.py b/SupervisedGym/SupervisedGym/modeltuner/objective.py index a3ffd75..de3c0e7 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/objective.py +++ b/SupervisedGym/SupervisedGym/modeltuner/objective.py @@ -1,7 +1,7 @@ import pytorch_lightning as pl import torch import optuna -from typing import Dict, Any, Union, Optional, Type +from typing import Dict, Any, Optional, Type, TypedDict, Tuple from sklearn.base import TransformerMixin from sklearn import preprocessing from optuna.integration import PyTorchLightningPruningCallback @@ -10,27 +10,37 @@ from pytorch_lightning.callbacks import EarlyStopping from ..utils.data import ( TimeSeriesDataStaticConfig, TimeSeriesDataFitableConfig, - DataModule, DataAdapter, make_time_series_data, ) from ..models.base import IBaseTunableModel +from .datamodule import DataModule -VALIDATE_EVERY_N_ITEMS = 10 * (10 ** 3) -EARLY_STOPPING_PATIENCE = 3 -MAX_EPOCHS = 100 +class ObjectiveHParams(TypedDict): + """ + dict containing Objective hyper + parameters which are hyperparameters + but such that concerns piepline + as whole and not single model + """ + + max_epochs: int + min_epochs: int + es_patience: int + es_divergence_threshold: float + val_check_interval: float class Objective: - # TODO make val_check_interval dynamic """objective for tuner""" def __init__( self, tsdata_static_config: TimeSeriesDataStaticConfig, tunable_model_type: Type[IBaseTunableModel], - val_loss: torch.nn.Module, + val_func: torch.nn.Module, + objective_hyper_params: ObjectiveHParams, fast_dev_run: bool = False, ): """ @@ -45,7 +55,7 @@ class Objective: tunableModel (type): subtype of BaseTunableModel which hyperparameters will be searched and this model will be trained - val_loss (torch.nn.Module): + val_func (torch.nn.Module): loss function used to evaluate validation error, the better model should have lower validation loss value. This might not be the same as loss function used in training. But val_loss is used in model pruning. @@ -60,14 +70,17 @@ class Objective: self.tsdata_static_config = tsdata_static_config self.tunable_model_type = tunable_model_type - self.val_loss = val_loss - self.model: Union[IBaseTunableModel, None] = None + self.val_func = val_func + self.model: Optional[IBaseTunableModel] = None self.fast_dev_run = fast_dev_run + self.ohparams = objective_hyper_params def current_model(self) -> IBaseTunableModel: """ - returns model connected to his trainer + returns currently trained model with its validation loss """ + if self.model is None: + raise Exception("There is no trained model in objective") return self.model def suggested_params(self) -> Dict[str, Any]: @@ -86,8 +99,6 @@ class Objective: batch_size_pow = trial.suggest_int("batch_size_pow", 5, 12) batch_size = 2 ** batch_size_pow - # generate x_transform - # TODO add more scalers input_scaler_name = trial.suggest_categorical( "x_scaler_name", ["MinMaxScaler", "MaxAbsScaler", "StandardScaler", "RobustScaler", "None"], @@ -101,7 +112,7 @@ class Objective: # generate y_transform # TODO add more scales target_scaler_name = trial.suggest_categorical( - "y_scaler_name", ["MinMaxScaler", "MaxAbsScaler", "None"] + "y_scaler_name", ["MinMaxScaler", "MaxAbsScaler", "RobustScaler", "None"] ) target_scaler_type: Optional[Type[TransformerMixin]] = ( getattr(preprocessing, target_scaler_name) @@ -125,31 +136,28 @@ class Objective: # generate tunable model self.model = self.tunable_model_type.get_trialed_model( - trial=trial, data_adapter=adapter, val_loss=self.val_loss + trial=trial, data_adapter=adapter, val_loss=self.val_func ) callbacks = [ - PyTorchLightningPruningCallback(trial, monitor="val_loss") - ] - - if trial.number < 50: - callbacks.append( - EarlyStopping( - monitor="val_loss", - mode="min", - patience=EARLY_STOPPING_PATIENCE, - divergence_threshold=1000.0, # TODO MAKE IT DYNAMIC - ) - ) + PyTorchLightningPruningCallback(trial, monitor="val_loss"), + EarlyStopping( + monitor="val_loss", + mode="min", + patience=self.ohparams["es_patience"], + divergence_threshold=self.ohparams["es_divergence_threshold"], + ), + ] trainer = pl.Trainer( logger=False, - val_check_interval=0.5, # TODO make it dynamic + val_check_interval=self.ohparams["val_check_interval"], fast_dev_run=self.fast_dev_run, checkpoint_callback=False, - max_epochs=MAX_EPOCHS, + max_epochs=self.ohparams["max_epochs"], + min_epochs=self.ohparams["min_epochs"], gradient_clip_val=gradient_clip_val, - callbacks=callbacks + callbacks=callbacks, ) trainer.fit(model=self.model, datamodule=datamodule) diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py new file mode 100644 index 0000000..a0172b6 --- /dev/null +++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py @@ -0,0 +1,72 @@ +import torch +from math import log +from torch.utils.data import Dataset, DataLoader, Sampler +from typing import Literal, Optional +from SupervisedGym.utils.data import ( + TimeSeriesDataStaticConfig, + TimeSeriesDataFitableConfig, + make_time_series_data, +) +from .objective import ObjectiveHParams + + +def naive_model_loss( + val_func: torch.nn.Module, + dataset: Dataset, + mode_type: Literal["mean"] = "mean", + sampler: Optional[Sampler] = None, +) -> float: + loader = DataLoader(dataset=dataset, sampler=sampler, batch_size=1) + pred: torch.Tensor + if mode_type == "mean": + # calculate average values of targets + # using moving average + cma = None + for i, (_, y) in enumerate(loader, start=1): + if i == 1: + cma = torch.Tensor(y) + continue + cma = (y + i * cma) / (i + 1) + pred = cma + + # calculate avg loss of average prediction + # using moving average + loss = 0 + for i, (_, y) in enumerate(loader, start=1): + loss = (val_func(pred, y) + i * loss) / (i + 1) + return loss + + else: + raise Exception(f"type {mode_type} is not handled") + + +def generate_objective_hparams( + conf: TimeSeriesDataStaticConfig, val_loss: torch.nn.Module +) -> ObjectiveHParams: + """ + based on TimeSeriesDataStaticConfig and problem validation + function generates ObjectiveHParams + """ + train_items = int( + (conf["data"].size / 100) * conf["train_val_test_percentage"].train + ) + min_epochs = max(1, int(log(train_items, 100))) + + dummy_conf = TimeSeriesDataFitableConfig(input_sequence_len=1) + tmp_tsdataset = make_time_series_data(static_conf=conf, fitable_conf=dummy_conf) + es_divergence_threshold = naive_model_loss( + val_func=val_loss, + dataset=tmp_tsdataset.generate_train_dataset(), + sampler=tmp_tsdataset.generate_uniform_target_sampler(), + ) + del tmp_tsdataset + val_check_interval = 2.5e5 / train_items + if val_check_interval > 1: + val_check_interval = int(val_check_interval) + return ObjectiveHParams( + val_check_interval=val_check_interval, + max_epochs=min_epochs * 50, + min_epochs=min_epochs, + es_divergence_threshold=es_divergence_threshold, + es_patience=min_epochs * 3, + ) diff --git a/SupervisedGym/SupervisedGym/utils/__init__.py b/SupervisedGym/SupervisedGym/utils/__init__.py new file mode 100644 index 0000000..9df67c9 --- /dev/null +++ b/SupervisedGym/SupervisedGym/utils/__init__.py @@ -0,0 +1,14 @@ +from . import data +from .domain_rounder import DomainRounder +from .problem_analyzer import ProblemAnalyzer, DummyGenomAnalyzer +from . import callbacks + + +__all__ = [ + data, + DomainRounder, + ProblemAnalyzer, + DomainRounder, + callbacks, + DummyGenomAnalyzer, +] diff --git a/SupervisedGym/SupervisedGym/utils/data/__init__.py b/SupervisedGym/SupervisedGym/utils/data/__init__.py index 7efe594..be11e5e 100644 --- a/SupervisedGym/SupervisedGym/utils/data/__init__.py +++ b/SupervisedGym/SupervisedGym/utils/data/__init__.py @@ -7,7 +7,6 @@ from .TimeSeriesDataSet.timeseries_data_configs import ( ) from .TimeSeriesDataSet.tsdataset import TsDataset from .dataadapter import DataAdapter -from .datamodule import DataModule __all__ = [ @@ -19,5 +18,4 @@ __all__ = [ TimeSeriesDataStaticConfig, make_time_series_data, DataAdapter, - DataModule, ] diff --git a/SupervisedGym/SupervisedGym/utils/domain_rounder.py b/SupervisedGym/SupervisedGym/utils/domain_rounder.py index e8f41f0..e3e245d 100644 --- a/SupervisedGym/SupervisedGym/utils/domain_rounder.py +++ b/SupervisedGym/SupervisedGym/utils/domain_rounder.py @@ -1,5 +1,5 @@ import numpy as np -from typing import Dict, Union, List +from typing import Dict, List class DomainRounder: @@ -15,12 +15,12 @@ class DomainRounder: domain: Dict[str, np.array] - def __init__(self, domain: Dict[str, List[Union[int, float]]]): + def __init__(self, domain: Dict[str, List[int]]): self.domain = { col: np.sort(np.array(var_dom)) for col, var_dom in domain.items() } - def round(self, x: Dict[str, Union[float, int]]) -> Dict[str, Union[int, float]]: + def round(self, x: Dict[str, int]) -> Dict[str, int]: rez = {} for col, y in x.items(): closest_idx = 0 diff --git a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py index cf5603c..f62d5be 100644 --- a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py +++ b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py @@ -5,6 +5,14 @@ from .domain_rounder import DomainRounder from .data import TimeSeriesDataStaticConfig, Train_val_test_perc +class BaseProblemAnalyzerExcepion(Exception): + """ + base class for ProblemAnalyzer exceptions + """ + + pass + + class ProblemAnalyzer: """ based on cp-problem file generates: diff --git a/SupervisedGym/tests/models/test_assmebled_model.py b/SupervisedGym/tests/models/test_assmebled_model.py index 94cb565..6a1db64 100644 --- a/SupervisedGym/tests/models/test_assmebled_model.py +++ b/SupervisedGym/tests/models/test_assmebled_model.py @@ -1,7 +1,10 @@ +from SupervisedGym import models +from SupervisedGym.modeltuner.modeltuner import TrainedModelStats import pandas as pd import numpy as np import pytest from SupervisedGym.models.assembled_model import AssembledModel +from SupervisedGym.utils import domain_rounder from SupervisedGym.utils.data import ( DataAdapter, TimeSeriesDataFitableConfig, @@ -49,7 +52,7 @@ def test_simple_case(): adapter = DataAdapter.create_from_timeseries_data(data) rounder = DomainRounder({"OUT1": [1], "OUT2": [2]}) for model in DummyModelFactory.all_dummys(adapter): - assembled_model = AssembledModel(tunable_model=model, domain_rounder=rounder) + assembled_model = AssembledModel(models=[(model, 1)], domain_rounder=rounder) assert ( assembled_model.predict( @@ -96,7 +99,7 @@ def test_handles_invalid_input(): data = make_time_series_data(static_conf, fitable_conf) adapter = DataAdapter.create_from_timeseries_data(data) model = AssembledModel( - tunable_model=DummyModelFactory.get_feed_forward_dummy(adapter), + models=[(DummyModelFactory.get_feed_forward_dummy(adapter), 1)], domain_rounder=DomainRounder({"OUT1": [69], "OUT2": [420]}), ) @@ -177,7 +180,7 @@ def test_load_parse(): } ) for model in DummyModelFactory.all_dummys(adapter): - assembled = AssembledModel(tunable_model=model, domain_rounder=rounder) + assembled = AssembledModel(models=[(model, 1)], domain_rounder=rounder) loaded_model = AssembledModel.load_model(assembled.parse_model()) for _ in range(TRIES): pred_in = { @@ -186,3 +189,123 @@ def test_load_parse(): "IN3": np.random.randn(SEQ_LEN).tolist(), } assert assembled.predict(pred_in) == loaded_model.predict(pred_in) + + +def test_multi_forward(): + """ + tests if assemby model made of + same models makes same prediction as + only single model + """ + + TRIES = 5 + SIZE = 50 + SEQ_LEN = 7 + data = pd.DataFrame( + { + "IN1": np.random.rand(SIZE), + "IN2": np.random.rand(SIZE), + "IN3": np.random.rand(SIZE), + "OUT1": np.zeros(SIZE), + "OUT2": np.zeros(SIZE), + "GR_ID": np.arange(SIZE) // 20, + } + ) + static_conf = TimeSeriesDataStaticConfig( + data=data, + group_id="GR_ID", + input_continuous=["IN1", "IN2"], + input_categorical=["IN3"], + target_continuous=["OUT1"], + target_categorical=["OUT2"], + prediction_horizon=3, + input_predictions=["IN2"], + train_val_test_percentage=Train_val_test_perc(70, 15, 15), + shuffle_groups=True, + ) + fitable_conf = TimeSeriesDataFitableConfig(input_sequence_len=SEQ_LEN) + data = make_time_series_data(static_conf, fitable_conf) + adapter = DataAdapter.create_from_timeseries_data(data) + rounder = DomainRounder( + { + "OUT1": [1e-1 * i for i in range(100)], + "OUT2": [1e-1 * i for i in range(100)], + } + ) + singe_model = DummyModelFactory.get_feed_forward_dummy(adapter) + multi_assemby = AssembledModel( + models=[(model, 1) for model in [singe_model] * 5], domain_rounder=rounder + ) + single_assembly = AssembledModel(models=[(singe_model, 1)], domain_rounder=rounder) + + for _ in range(TRIES): + pred_in = { + "IN1": np.random.randn(SEQ_LEN).tolist(), + "IN2": np.random.randn(SEQ_LEN).tolist(), + "IN3": np.random.randn(SEQ_LEN).tolist(), + } + assert multi_assemby.predict(pred_in) == single_assembly.predict(pred_in) + + +def test_ensemble_weights(): + """ + tests if assembly model behaves + correctly with diffrent weights + """ + SIZE = 50 + SEQ_LEN = 7 + data = pd.DataFrame( + { + "IN1": np.random.rand(SIZE), + "IN2": np.random.rand(SIZE), + "IN3": np.random.rand(SIZE), + "OUT1": np.zeros(SIZE), + "OUT2": np.zeros(SIZE), + "GR_ID": np.arange(SIZE) // 20, + } + ) + static_conf = TimeSeriesDataStaticConfig( + data=data, + group_id="GR_ID", + input_continuous=["IN1", "IN2"], + input_categorical=["IN3"], + target_continuous=["OUT1"], + target_categorical=["OUT2"], + prediction_horizon=3, + input_predictions=["IN2"], + train_val_test_percentage=Train_val_test_perc(70, 15, 15), + shuffle_groups=True, + ) + fitable_conf = TimeSeriesDataFitableConfig(input_sequence_len=SEQ_LEN) + data = make_time_series_data(static_conf, fitable_conf) + adapter = DataAdapter.create_from_timeseries_data(data) + rounder = DomainRounder( + { + "OUT1": [1e-1 * i for i in range(100)], + "OUT2": [1e-1 * i for i in range(100)], + } + ) + + ffmodel = DummyModelFactory.get_feed_forward_dummy(adapter) + rrnmodel = DummyModelFactory.get_stacked_rnn_dummy(adapter) + pred_in = { + "IN1": np.random.randn(SEQ_LEN).tolist(), + "IN2": np.random.randn(SEQ_LEN).tolist(), + "IN3": np.random.randn(SEQ_LEN).tolist(), + } + + assert AssembledModel( + models=[(ffmodel, 3), (rrnmodel, 0)], domain_rounder=rounder + ).predict(pred_in) == AssembledModel( + models=[(ffmodel, 1)], domain_rounder=rounder + ).predict( + pred_in + ) + + assert AssembledModel( + models=[(ffmodel, 0), (rrnmodel, 5)], domain_rounder=rounder + ).predict(pred_in) == AssembledModel( + models=[(rrnmodel, 0.1)], domain_rounder=rounder + ).predict( + pred_in + ) diff --git a/SupervisedGym/tests/modeltuner/test_model_tuner.py b/SupervisedGym/tests/modeltuner/test_model_tuner.py index 080a6f3..81546d4 100644 --- a/SupervisedGym/tests/modeltuner/test_model_tuner.py +++ b/SupervisedGym/tests/modeltuner/test_model_tuner.py @@ -1,7 +1,9 @@ +from SupervisedGym.modeltuner.modeltuner import TrainedModelStats import pytest import numpy as np import torch.nn as nn import pandas as pd +from typing import Optional from SupervisedGym.modeltuner import ModelTuner from SupervisedGym.utils.data import ( @@ -45,7 +47,19 @@ def test_fast_dev_run(): with pytest.raises(Exception): tuner.best_model - best_model = tuner.tune_model( - timeout=10 ** 3, n_trials=1, tunable_model_type=StackedRNNTunable + trained_model_stat: Optional[TrainedModelStats] = None + + def set_tms(tms: TrainedModelStats): + nonlocal trained_model_stat + trained_model_stat = tms + + tuner.tune( + timeout=10 ** 3, + n_trials=1, + tunable_model_type=StackedRNNTunable, + model_callback=set_tms, ) - assert isinstance(best_model, IBaseTunableModel) + + assert isinstance(trained_model_stat, TrainedModelStats) + assert isinstance(trained_model_stat.model, IBaseTunableModel) + assert isinstance(trained_model_stat.val_loss, float) -- GitLab From 98187f67e4a4ec2ccaf8914c02679e52c1c7aeef Mon Sep 17 00:00:00 2001 From: szysad <szysad108@gmail.com> Date: Wed, 19 May 2021 15:31:52 +0200 Subject: [PATCH 2/5] added TunableTransformer to tests, formated code --- SupervisedGym/SupervisedGym/__main__.py | 53 ++++----------- .../models/Transformer/Transformer.py | 31 ++++----- .../SupervisedGym/models/__init__.py | 7 +- .../SupervisedGym/utils/problem_analyzer.py | 66 ++++++++++--------- SupervisedGym/tests/models/dummy_factory.py | 24 ++++++- SupervisedGym/tests/utils/fcrCP.py | 2 +- SupervisedGym/tests/utils/genomCP.py | 2 +- .../tests/utils/test_problem_analyzer.py | 58 ++++++++++------ 8 files changed, 133 insertions(+), 110 deletions(-) diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py index 303ad9d..eb027ab 100644 --- a/SupervisedGym/SupervisedGym/__main__.py +++ b/SupervisedGym/SupervisedGym/__main__.py @@ -1,51 +1,24 @@ -from SupervisedGym.utils import domain_rounder -from SupervisedGym.modeltuner.modeltuner import TrainedModelStats import torch -import pickle from SupervisedGym.model_assembler.model_assembler import ( ModelAssebler as GenomAssembler, ) -from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable, TransformerTunable -from SupervisedGym.models.assembled_model import AssembledModel +from SupervisedGym.models import FeedForwardTunable -SAVE_PATH = "/home/szysad/mimuw/3rok/ZPP/models/test-model" -TIMEOUT = 60 * 1 # 1 minutes +SAVE_PATH = "<model save path>" +TIMEOUT = 60 * 60 * 10 # 10 hours minutes if __name__ == "__main__": - # assebler = GenomAssembler( - # raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss() - # ) # placeholder + assebler = GenomAssembler( + raw_data=None, cp_problem=None, val_loss=torch.nn.L1Loss() + ) # placeholder - # model = assebler.assemble_model( - # timeout=TIMEOUT, - # n_trials=5, - # assembly_size=3, - # avaible_model_types=[FeedForwardTunable] - # ) + model = assebler.assemble_model( + timeout=TIMEOUT, + n_trials=5, + assembly_size=5, + avaible_model_types=[FeedForwardTunable], + ) - # with open(SAVE_PATH, "wb") as f: - # f.write(model.parse_model()) - - rnnpaths = [ - "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-9h-gpu", - "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-w-early-stopping-gpu", - "/home/szysad/mimuw/3rok/ZPP/models/15-05-genom/rrn-5h-gpu", - ] - - rrnmodels = [] - rounder = None - - for p in rnnpaths: - data = None - with open(p, "rb") as f: - data = pickle.loads(f.read()) - model = StackedRNNTunable.load_model(data["tunable_model"]) - rounder = data["domain_rounder"] - rrnmodels.append(model) - - stats = [TrainedModelStats(model, 1) for model in rrnmodels] - - assemble = AssembledModel(domain_rounder=rounder, model_stats=stats) with open(SAVE_PATH, "wb") as f: - f.write(assemble.parse_model()) + f.write(model.parse_model()) diff --git a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py index cb18080..bd268b2 100644 --- a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py +++ b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py @@ -3,7 +3,6 @@ import torch import math - class PositionalEncoding(nn.Module): """ Inject some information about the relative or absolute position of the tokens @@ -20,19 +19,21 @@ class PositionalEncoding(nn.Module): >>> pos_encoder = PositionalEncoding(d_model) """ - def __init__(self, d_model: int, max_len: int=5000): + def __init__(self, d_model: int, max_len: int = 5000): super(PositionalEncoding, self).__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) - div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + div_term = torch.exp( + torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model) + ) pe[:, 0::2] = torch.sin(position * div_term) - if d_model%2 != 0: - pe[:, 1::2] = torch.cos(position * div_term)[:,0:-1] + if d_model % 2 != 0: + pe[:, 1::2] = torch.cos(position * div_term)[:, 0:-1] else: pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) - self.register_buffer('pe', pe) + self.register_buffer("pe", pe) def forward(self, x): """ @@ -47,7 +48,7 @@ class PositionalEncoding(nn.Module): >>> output = pos_encoder(x) """ - x = x + self.pe[:x.size(0), :] + x = x + self.pe[: x.size(0), :] return x @@ -109,16 +110,16 @@ class Transformer(nn.Module): self.primary_embedding = nn.Conv1d(input_size, features, 1) self.positionalEncoder = PositionalEncoding(features) - transformerEncoderLayers = nn.TransformerEncoderLayer(features, - heads_number, - dim_feedforward=feed_forward_size, - dropout=dropout) - self.transformerEncoder = nn.TransformerEncoder(transformerEncoderLayers, self.num_layers); - + transformerEncoderLayers = nn.TransformerEncoderLayer( + features, heads_number, dim_feedforward=feed_forward_size, dropout=dropout + ) + self.transformerEncoder = nn.TransformerEncoder( + transformerEncoderLayers, self.num_layers + ) + self.fc1 = nn.Linear(features * seq_len, pre_output) self.fc2 = nn.Linear(pre_output, output_size) - def forward(self, x: torch.Tensor) -> torch.Tensor: """ Performs pass throught network @@ -155,5 +156,5 @@ class Transformer(nn.Module): out = self.fc1(out) out = self.activation(out) out = self.fc2(out) - + return out diff --git a/SupervisedGym/SupervisedGym/models/__init__.py b/SupervisedGym/SupervisedGym/models/__init__.py index ce749dc..740b84f 100644 --- a/SupervisedGym/SupervisedGym/models/__init__.py +++ b/SupervisedGym/SupervisedGym/models/__init__.py @@ -11,4 +11,9 @@ avaible_model_types: List[Type[IBaseTunableModel]] = [ TransformerTunable, ] -__all__ = [FeedForwardTunable, StackedRNNTunable, TransformerTunable, avaible_model_types] +__all__ = [ + FeedForwardTunable, + StackedRNNTunable, + TransformerTunable, + avaible_model_types, +] diff --git a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py index c55dad7..96bb8cd 100644 --- a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py +++ b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py @@ -7,14 +7,6 @@ from .data import TimeSeriesDataStaticConfig, Train_val_test_perc import re -class BaseProblemAnalyzerExcepion(Exception): - """ - base class for ProblemAnalyzer exceptions - """ - - pass - - class ProblemAnalyzer: """ based on cp-problem file generates: @@ -26,7 +18,9 @@ class ProblemAnalyzer: @staticmethod def generate_domain_rounder(cp_problem: ET.ElementTree) -> DomainRounder: - return DomainRounder(ProblemAnalyzer._get_domains_from_constraint_problem(cp_problem)) + return DomainRounder( + ProblemAnalyzer._get_domains_from_constraint_problem(cp_problem) + ) @staticmethod def generate_tsdata_static_conf( @@ -34,52 +28,60 @@ class ProblemAnalyzer: ) -> TimeSeriesDataStaticConfig: raise NotImplementedError() - def _get_domains_from_constraint_problem(cp_problem: ET.ElementTree) -> Dict[str, List[int]]: + def _get_domains_from_constraint_problem( + cp_problem: ET.ElementTree, + ) -> Dict[str, List[int]]: """ Returns a dictionary containing domains of variables. """ result = {} - type_re = re.compile(r'^({.*})?type$') + type_re = re.compile(r"^({.*})?type$") - for variable_node in cp_problem.findall('cpVariables'): + for variable_node in cp_problem.findall("cpVariables"): - variable_name = variable_node.attrib.get('id') + variable_name = variable_node.attrib.get("id") - domain_node = variable_node.find('domain') + domain_node = variable_node.find("domain") variable_domain = [] - if variable_name == None: - raise Exception(f"No 'id' attribute in 'cpVariables' tag.") - if domain_node == None: - raise Exception(f"Variable {variable_name} has no 'domain' child in CP.") + if variable_name is None: + raise Exception("No 'id' attribute in 'cpVariables' tag.") + if domain_node is None: + raise Exception( + f"Variable {variable_name} has no 'domain' child in CP." + ) domain_type = "No type attribute." for key, value in domain_node.attrib.items(): - if type_re.match(key) != None: + if not type_re.match(key) is None: domain_type = value break - if domain_type == 'cp:RangeDomain': + if domain_type == "cp:RangeDomain": - from_node = domain_node.find('from') - to_node = domain_node.find('to') - if from_node == None or to_node == None: - raise Exception(f"Wrong domain fromat of {variable_name} variable." - "Expected 'from' and 'to' nodes") + from_node = domain_node.find("from") + to_node = domain_node.find("to") + if from_node is None or to_node is None: + raise Exception( + f"Wrong domain fromat of {variable_name} variable." + "Expected 'from' and 'to' nodes" + ) - start_value = int(from_node.attrib.get('value', "0")) - end_value = int(to_node.attrib.get('value', "0")) + 1 + start_value = int(from_node.attrib.get("value", "0")) + end_value = int(to_node.attrib.get("value", "0")) + 1 variable_domain = range(start_value, end_value) - elif domain_type == 'cp:NumericListDomain': + elif domain_type == "cp:NumericListDomain": - for value in domain_node.findall('values'): - variable_domain.append(int(value.attrib.get('value', "0"))) + for value in domain_node.findall("values"): + variable_domain.append(int(value.attrib.get("value", "0"))) else: - raise Exception(f"Unknown type of domain of {variable_name} variable: {domain_type}.") + raise Exception( + f"Unknown type of domain of {variable_name} variable: {domain_type}." + ) result[variable_name] = variable_domain @@ -122,7 +124,7 @@ class DummyGenomAnalyzer(ProblemAnalyzer): ) -> TimeSeriesDataStaticConfig: return TimeSeriesDataStaticConfig( data=pd.read_csv( - r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\trening\alpha-star-solver\FCRtraining\Genom\genom-newest-at-05-28.csv" + "/home/szysad/mimuw/3rok/ZPP/training-data/genom-data/newest/genom-newest-at-05-28.csv" ), group_id="experimentId", input_continuous=[ diff --git a/SupervisedGym/tests/models/dummy_factory.py b/SupervisedGym/tests/models/dummy_factory.py index abde9fb..0a77c7e 100644 --- a/SupervisedGym/tests/models/dummy_factory.py +++ b/SupervisedGym/tests/models/dummy_factory.py @@ -4,7 +4,11 @@ from typing import List from SupervisedGym.utils.data import DataAdapter from SupervisedGym.models.base import IBaseTunableModel -from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable +from SupervisedGym.models import ( + FeedForwardTunable, + StackedRNNTunable, + TransformerTunable, +) class DummyModelFactory: @@ -40,9 +44,27 @@ class DummyModelFactory: val_loss=torch.nn.L1Loss(), ) + @staticmethod + def get_transformer_dummy(data_adapter: DataAdapter) -> TransformerTunable: + return TransformerTunable( + data_adapter=data_adapter, + features=3, + heads_number=1, + layers_number=1, + feed_forward_size=3, + pre_output=3, + dropout=0, + activation=torch.nn.ReLU(), + loss=torch.nn.L1Loss(), + lr=1, + val_loss=torch.nn.L1Loss(), + optimizer_type=torch.optim.SGD, + ) + @staticmethod def all_dummys(data_adapter: DataAdapter) -> List[IBaseTunableModel]: return [ DummyModelFactory.get_feed_forward_dummy(data_adapter), DummyModelFactory.get_stacked_rnn_dummy(data_adapter), + DummyModelFactory.get_transformer_dummy(data_adapter), ] diff --git a/SupervisedGym/tests/utils/fcrCP.py b/SupervisedGym/tests/utils/fcrCP.py index ff31fb8..6ba70c3 100644 --- a/SupervisedGym/tests/utils/fcrCP.py +++ b/SupervisedGym/tests/utils/fcrCP.py @@ -405,4 +405,4 @@ fcrCP = r"""<?xml version="1.0" encoding="ASCII"?> <value xsi:type="types:IntegerValueUpperware" value="1"/> </cpMetrics> </cp:ConstraintProblem> -""" \ No newline at end of file +""" diff --git a/SupervisedGym/tests/utils/genomCP.py b/SupervisedGym/tests/utils/genomCP.py index 8943142..f832314 100644 --- a/SupervisedGym/tests/utils/genomCP.py +++ b/SupervisedGym/tests/utils/genomCP.py @@ -211,4 +211,4 @@ genomCP = r"""<?xml version="1.0" encoding="ASCII"?> <value xsi:type="types:IntegerValueUpperware" value="1"/> </cpMetrics> </cp:ConstraintProblem> -""" \ No newline at end of file +""" diff --git a/SupervisedGym/tests/utils/test_problem_analyzer.py b/SupervisedGym/tests/utils/test_problem_analyzer.py index 2cf2787..76cc9e0 100644 --- a/SupervisedGym/tests/utils/test_problem_analyzer.py +++ b/SupervisedGym/tests/utils/test_problem_analyzer.py @@ -13,7 +13,7 @@ def test_genom_domain(): """ xml_file = ET.fromstring(genomCP) - assert(ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == { + assert ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == { "WorkerCardinality": range(1, 31), "provider_ComponentSparkWorker": [0], "WorkerCores": [2, 4, 8, 16], @@ -31,7 +31,8 @@ def test_genom_domain(): 65536, 70041, ], - }) + } + @pytest.mark.ProblemAnalyzer def test_FCR_domain(): @@ -41,20 +42,39 @@ def test_FCR_domain(): """ xml_file = ET.fromstring(fcrCP) - assert(ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == { - "AppCardinality": range(1, 41), - "provider_Component_App": [0], - "AppCores": [2, 4, 8, 16, 32, 36, 40, 48], - "AppStorage": [0, 10, 420, 1024, 2097152], - "LBCardinality": range(1, 2), - "provider_Component_LB": [0], - "LBRam": [1024, 1740, 2048, 3750, 3840, 4096, 7168, 7680, - 8192, 15360, 15616, 16384, 17510, 22528, 23552, - 30720, 31232, 32768, 35020, 62464, 70041], - "LBStorage": [0, 10, 420], - "DBCardinality": range(1, 2), - "provider_Component_DB": [0], - "DBCores": [1, 2, 4, 8], - "DBStorage": [0, 10, 420], - }) - + assert ProblemAnalyzer._get_domains_from_constraint_problem(xml_file) == { + "AppCardinality": range(1, 41), + "provider_Component_App": [0], + "AppCores": [2, 4, 8, 16, 32, 36, 40, 48], + "AppStorage": [0, 10, 420, 1024, 2097152], + "LBCardinality": range(1, 2), + "provider_Component_LB": [0], + "LBRam": [ + 1024, + 1740, + 2048, + 3750, + 3840, + 4096, + 7168, + 7680, + 8192, + 15360, + 15616, + 16384, + 17510, + 22528, + 23552, + 30720, + 31232, + 32768, + 35020, + 62464, + 70041, + ], + "LBStorage": [0, 10, 420], + "DBCardinality": range(1, 2), + "provider_Component_DB": [0], + "DBCores": [1, 2, 4, 8], + "DBStorage": [0, 10, 420], + } -- GitLab From 3aa286d12cb1bae9348284136648a78691e63396 Mon Sep 17 00:00:00 2001 From: szymon <szysad108@gmail.com> Date: Wed, 19 May 2021 16:17:24 +0200 Subject: [PATCH 3/5] fixed some small errors --- .../SupervisedGym/model_assembler/model_assembler.py | 4 +--- SupervisedGym/SupervisedGym/modeltuner/objective.py | 2 ++ SupervisedGym/SupervisedGym/utils/data/dataadapter.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py index bcb8e8d..bacffd3 100644 --- a/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py +++ b/SupervisedGym/SupervisedGym/model_assembler/model_assembler.py @@ -89,8 +89,6 @@ class ModelAssebler: model_callback=__receive_stat, ) return AssembledModel( - model_stats=map( - lambda m, val_loss: (m, self.__val_loss_to_weight(val_loss)) - ), + models=[(stat.model, self.__val_loss_to_weight(stat.val_loss)) for stat in model_stats], domain_rounder=self.dom_rounder, ) diff --git a/SupervisedGym/SupervisedGym/modeltuner/objective.py b/SupervisedGym/SupervisedGym/modeltuner/objective.py index de3c0e7..0f35d33 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/objective.py +++ b/SupervisedGym/SupervisedGym/modeltuner/objective.py @@ -151,6 +151,7 @@ class Objective: trainer = pl.Trainer( logger=False, + gpus=-1, val_check_interval=self.ohparams["val_check_interval"], fast_dev_run=self.fast_dev_run, checkpoint_callback=False, @@ -158,6 +159,7 @@ class Objective: min_epochs=self.ohparams["min_epochs"], gradient_clip_val=gradient_clip_val, callbacks=callbacks, + progress_bar_refresh_rate=0 ) trainer.fit(model=self.model, datamodule=datamodule) diff --git a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py index b36964e..a25b7da 100644 --- a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py +++ b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py @@ -70,10 +70,10 @@ class DataAdapter: with torch.no_grad(): if inverse: return torch.from_numpy( - self.__input_transformer.inverse_transform(x.numpy()) + self.__input_transformer.inverse_transform(x.cpu().numpy()) ) else: - return torch.from_numpy(self.__input_transformer.transform(x.numpy())) + return torch.from_numpy(self.__input_transformer.transform(x.cpu().numpy())) def transform_target(self, y: torch.Tensor, inverse: bool = False) -> torch.Tensor: """ @@ -84,7 +84,7 @@ class DataAdapter: with torch.no_grad(): if inverse: return torch.from_numpy( - self.__target_transformer.inverse_transform(y.numpy()) + self.__target_transformer.inverse_transform(y.cpu().numpy()) ) else: - return torch.from_numpy(self.__target_transformer.transform(y.numpy())) + return torch.from_numpy(self.__target_transformer.transform(y.cpu().numpy())) -- GitLab From 83ffea7645ded21f950c2ccc68949ac3fd3a04ef Mon Sep 17 00:00:00 2001 From: szysad <szysad108@gmail.com> Date: Wed, 19 May 2021 16:38:57 +0200 Subject: [PATCH 4/5] added docstring, fixed divergence_threhold heuristic --- SupervisedGym/SupervisedGym/modeltuner/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py index a0172b6..fffcc0a 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/utils.py +++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py @@ -16,6 +16,9 @@ def naive_model_loss( mode_type: Literal["mean"] = "mean", sampler: Optional[Sampler] = None, ) -> float: + ''' + calculates naive prediction loss on train dataset + ''' loader = DataLoader(dataset=dataset, sampler=sampler, batch_size=1) pred: torch.Tensor if mode_type == "mean": @@ -67,6 +70,6 @@ def generate_objective_hparams( val_check_interval=val_check_interval, max_epochs=min_epochs * 50, min_epochs=min_epochs, - es_divergence_threshold=es_divergence_threshold, + es_divergence_threshold=es_divergence_threshold * 10, es_patience=min_epochs * 3, ) -- GitLab From 77b0b419ffc722cae8496f573047381fa1637d71 Mon Sep 17 00:00:00 2001 From: szysad <szysad108@gmail.com> Date: Wed, 19 May 2021 16:39:48 +0200 Subject: [PATCH 5/5] typo fix --- SupervisedGym/SupervisedGym/modeltuner/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py index fffcc0a..003dac7 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/utils.py +++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py @@ -70,6 +70,6 @@ def generate_objective_hparams( val_check_interval=val_check_interval, max_epochs=min_epochs * 50, min_epochs=min_epochs, - es_divergence_threshold=es_divergence_threshold * 10, + es_divergence_threshold=es_divergence_threshold * 100, es_patience=min_epochs * 3, ) -- GitLab