From a6991b22d769c5f57452da4b8e53b3749abd3ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Szyma=C5=84ski?= Date: Wed, 19 May 2021 17:53:41 +0200 Subject: [PATCH 1/5] Running changes --- SupervisedGym/SupervisedGym/__main__.py | 9 +++++---- .../SupervisedGym/models/Transformer/Transformer.py | 3 +++ SupervisedGym/SupervisedGym/modeltuner/objective.py | 5 +++-- SupervisedGym/SupervisedGym/utils/data/dataadapter.py | 6 +++--- SupervisedGym/SupervisedGym/utils/data/datamodule.py | 6 +++--- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py index ee464cd..5996172 100644 --- a/SupervisedGym/SupervisedGym/__main__.py +++ b/SupervisedGym/SupervisedGym/__main__.py @@ -7,8 +7,9 @@ from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable, Transfor from SupervisedGym.models.assembled_model import AssembledModel -SAVE_PATH = r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\optuna\test_run" -TIMEOUT = 60 * 1 # 1 minute +SAVE_PATH = r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\optuna\10h" +# TIMEOUT = 60 * 60 * 10 # 10 hours +TIMEOUT = 60 * 10 # 10 minute if __name__ == "__main__": assebler = GenomAssembler( @@ -16,5 +17,5 @@ if __name__ == "__main__": ) # placeholder model = assebler.assemble_model(TIMEOUT, None, [TransformerTunable]) - #with open(SAVE_PATH, "wb") as f: - # f.write(model.parse_model()) + with open(SAVE_PATH, "wb") as f: + f.write(model.parse_model()) diff --git a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py index cb18080..c80bfcd 100644 --- a/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py +++ b/SupervisedGym/SupervisedGym/models/Transformer/Transformer.py @@ -97,6 +97,9 @@ class Transformer(nn.Module): ): super().__init__() + # features must be divisible by heads_number, so it is rounded up + features = ((features + heads_number - 1) // heads_number) * heads_number + self.input_size = input_size self.output_size = output_size self.seq_len = seq_len diff --git a/SupervisedGym/SupervisedGym/modeltuner/objective.py b/SupervisedGym/SupervisedGym/modeltuner/objective.py index a3ffd75..ccceceb 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/objective.py +++ b/SupervisedGym/SupervisedGym/modeltuner/objective.py @@ -83,7 +83,7 @@ class Objective: def __call__(self, trial: optuna.trial.Trial) -> float: # generate batch size pow - batch_size_pow = trial.suggest_int("batch_size_pow", 5, 12) + batch_size_pow = trial.suggest_int("batch_size_pow", 5, 8) batch_size = 2 ** batch_size_pow # generate x_transform @@ -149,7 +149,8 @@ class Objective: checkpoint_callback=False, max_epochs=MAX_EPOCHS, gradient_clip_val=gradient_clip_val, - callbacks=callbacks + callbacks=callbacks, + gpus=-1 ) trainer.fit(model=self.model, datamodule=datamodule) diff --git a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py index b36964e..bb71aaa 100644 --- a/SupervisedGym/SupervisedGym/utils/data/dataadapter.py +++ b/SupervisedGym/SupervisedGym/utils/data/dataadapter.py @@ -84,7 +84,7 @@ class DataAdapter: with torch.no_grad(): if inverse: return torch.from_numpy( - self.__target_transformer.inverse_transform(y.numpy()) - ) + self.__target_transformer.inverse_transform(y.cpu().numpy()) + ).to('cuda') else: - return torch.from_numpy(self.__target_transformer.transform(y.numpy())) + return torch.from_numpy(self.__target_transformer.transform(y.cpu().numpy())).cuda() diff --git a/SupervisedGym/SupervisedGym/utils/data/datamodule.py b/SupervisedGym/SupervisedGym/utils/data/datamodule.py index d083ba3..f05f32d 100644 --- a/SupervisedGym/SupervisedGym/utils/data/datamodule.py +++ b/SupervisedGym/SupervisedGym/utils/data/datamodule.py @@ -15,7 +15,7 @@ class DataModule(LightningDataModule): dataset=self.tsdata.generate_train_dataset(), batch_size=self.batch_size, sampler=self.tsdata.generate_uniform_target_sampler(), - num_workers=4 # , + num_workers=0 # , # pin_memory=True ) @@ -23,7 +23,7 @@ class DataModule(LightningDataModule): return DataLoader( dataset=self.tsdata.generate_validation_dataset(), batch_size=self.batch_size, - num_workers=4 # , + num_workers=0 # , # pin_memory=True ) @@ -31,6 +31,6 @@ class DataModule(LightningDataModule): return DataLoader( dataset=self.tsdata.generate_test_dataset(), batch_size=self.batch_size, - num_workers=4 # , + num_workers=0 # , # pin_memory=True ) -- GitLab From 6dd1da1d5d72275cc22670b6a03405c57113effd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Szyma=C5=84ski?= Date: Wed, 19 May 2021 18:00:13 +0200 Subject: [PATCH 2/5] Fixed path --- SupervisedGym/SupervisedGym/utils/problem_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py index 96bb8cd..3444335 100644 --- a/SupervisedGym/SupervisedGym/utils/problem_analyzer.py +++ b/SupervisedGym/SupervisedGym/utils/problem_analyzer.py @@ -124,7 +124,7 @@ class DummyGenomAnalyzer(ProblemAnalyzer): ) -> TimeSeriesDataStaticConfig: return TimeSeriesDataStaticConfig( data=pd.read_csv( - "/home/szysad/mimuw/3rok/ZPP/training-data/genom-data/newest/genom-newest-at-05-28.csv" + r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\trening\alpha-star-solver\FCRtraining\Genom\genom-newest-at-05-28.csv" ), group_id="experimentId", input_continuous=[ -- GitLab From dd3473b3417cd2c129c4f0f964292718f7a752b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Szyma=C5=84ski?= Date: Thu, 20 May 2021 10:08:36 +0200 Subject: [PATCH 3/5] Added patience tuning --- .../models/Transformer/TransformerTunable.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SupervisedGym/SupervisedGym/models/Transformer/TransformerTunable.py b/SupervisedGym/SupervisedGym/models/Transformer/TransformerTunable.py index f699dac..b9db829 100644 --- a/SupervisedGym/SupervisedGym/models/Transformer/TransformerTunable.py +++ b/SupervisedGym/SupervisedGym/models/Transformer/TransformerTunable.py @@ -39,6 +39,8 @@ class TransformerTunable(IBaseTunableModel): initial learning rate val_loss (nn.Module): loss function for validation + patience (int): + patience for lr scheduler """ def __init__( @@ -55,6 +57,7 @@ class TransformerTunable(IBaseTunableModel): optimizer_type: torch.optim.Optimizer, lr: float, val_loss: nn.Module, + patience: int, ): super().__init__() self.save_hyperparameters() @@ -74,6 +77,7 @@ class TransformerTunable(IBaseTunableModel): self.optimizer_type = optimizer_type self.lr = lr self.val_loss = val_loss + self.patience = patience self.net = Transformer( input_size=len(data_adapter.input_columns), @@ -121,7 +125,7 @@ class TransformerTunable(IBaseTunableModel): returns optimizer """ optimizer = self.optimizer_type(self.parameters(), lr=self.lr) - scheduler = ReduceLROnPlateau(optimizer, "min", patience=5) + scheduler = ReduceLROnPlateau(optimizer, "min", patience=self.patience) return { "optimizer": optimizer, "lr_scheduler": scheduler, @@ -154,6 +158,9 @@ class TransformerTunable(IBaseTunableModel): ) optimizer_type = getattr(torch.optim, optimizer_name) + # generate patience + patience = trial.suggest_categorical("lr_patience", [5, 20, 1000]) + return TransformerTunable( data_adapter=data_adapter, features=trial.suggest_int("features", 10, 100), @@ -167,6 +174,7 @@ class TransformerTunable(IBaseTunableModel): optimizer_type=optimizer_type, lr=trial.suggest_float("lr", 1e-5, 5e-2, log=True), val_loss=val_loss, + patience=patience, ) @staticmethod @@ -175,6 +183,7 @@ class TransformerTunable(IBaseTunableModel): "loss_name": "MSELoss", "activation_name": "ReLU", "optimizer_name": "Adam", + "lr_patience": 5, "features": 50, "heads_number": 2, "layers_number": 7, -- GitLab From ebc7caaa6e65db008d911be76c863877537ae697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Szyma=C5=84ski?= Date: Thu, 20 May 2021 08:16:11 +0000 Subject: [PATCH 4/5] Update __main__.py -- GitLab From 8a16f664ec8f10896d7888f4b04761715faf4634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Szyma=C5=84ski?= Date: Thu, 3 Jun 2021 18:12:42 +0200 Subject: [PATCH 5/5] Added training script and fixed val_check_interval --- SupervisedGym/SupervisedGym/__main__.py | 47 +++++++++++++------ .../SupervisedGym/modeltuner/utils.py | 2 +- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/SupervisedGym/SupervisedGym/__main__.py b/SupervisedGym/SupervisedGym/__main__.py index cd061ce..e03c5da 100644 --- a/SupervisedGym/SupervisedGym/__main__.py +++ b/SupervisedGym/SupervisedGym/__main__.py @@ -6,11 +6,11 @@ import torch import pandas as pd import xml.etree.ElementTree as ET from SupervisedGym.model_assembler import ModelAssembler -from SupervisedGym.models import FeedForwardTunable +from SupervisedGym.models import FeedForwardTunable, StackedRNNTunable, TransformerTunable -SAVE_PATH = "" -TIMEOUT = 60 * 60 * 10 # 10 hours minutes +# SAVE_PATH = "" +# TIMEOUT = 60 * 60 * 10 # 10 hours minutes DATA_CONF = DataConf( group_id="experimentId", pred_horizon=10, @@ -18,25 +18,44 @@ DATA_CONF = DataConf( shuffle=False, ) +DIRECTORY = r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\last1\genom" + if __name__ == "__main__": + assebler = ModelAssembler( raw_data=pd.read_csv( - "/home/szysad/mimuw/3rok/ZPP/training-data/FCR-data/set5/SD-01-29-2021.csv" + # "/home/szysad/mimuw/3rok/ZPP/training-data/FCR-data/set5/SD-01-29-2021.csv" + r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\trening\alpha-star-solver\FCRtraining\Genom\genom-newest-at-05-28.csv" ), cp_problem=ET.parse( - "/home/szysad/mimuw/3rok/ZPP/cp_solver/FCR-config/FCRclear-CP.xmi" + # "/home/szysad/mimuw/3rok/ZPP/cp_solver/FCR-config/FCRclear-CP.xmi" + r"C:\Users\wojci\OneDrive\Documents\Studia\ZPP\trening\alpha-star-solver\FCRtraining\Genom\GenomZPPSolverSmall-CP.xmi" ), val_loss=torch.nn.L1Loss(), data_conf=DATA_CONF, ) - model = assebler.assemble_model( - timeout=TIMEOUT, - n_trials=5, - ensemble_size=5, - avaible_model_types=[FeedForwardTunable], - fast_dev_run=True, - ) + for time in [60 * 60, 5 * 60 * 60]: + for networks, name in [ + ([FeedForwardTunable], "FF"), + ([StackedRNNTunable], "RNN"), + ([TransformerTunable], "Trans"), + ([FeedForwardTunable, StackedRNNTunable, TransformerTunable], "All") + ]: + for ensembling in [1, 5]: + + time_per_model = time // len(networks) + models = {} + for model_type in networks: + models[model_type] = {"timeout": time_per_model, "n_trials": None} + + model = assebler.assemble_model( + ensemble_size=ensembling, + models=models + ) + + file_name = "Time=" + str(time) + " Model=" + name + " Ensembling=" + str(ensembling) + file_path = DIRECTORY + "\\" + file_name - with open(SAVE_PATH, "wb") as f: - f.write(model.parse_model()) + with open(file_path, "wb") as f: + f.write(model.parse_model()) diff --git a/SupervisedGym/SupervisedGym/modeltuner/utils.py b/SupervisedGym/SupervisedGym/modeltuner/utils.py index 8b9fe62..ea76662 100644 --- a/SupervisedGym/SupervisedGym/modeltuner/utils.py +++ b/SupervisedGym/SupervisedGym/modeltuner/utils.py @@ -51,7 +51,7 @@ def generate_objective_hparams( ) min_epochs = max(1, int(log(train_items, 100))) return ObjectiveHParams( - val_check_interval=1, + val_check_interval=1.0, max_epochs=min_epochs * 50, min_epochs=min_epochs, es_divergence_threshold=None, -- GitLab