From de891ff5d067925e59adfb735153645bdb46b89e Mon Sep 17 00:00:00 2001 From: Kazuto Date: Wed, 9 Sep 2020 12:09:34 +0900 Subject: [PATCH] delete Make_models --- .gitignore | 3 +- LSTM/val_loss_time/Test_LSTM.csv | 2 - Make_models/configs/config_CNN_AE.json | 88 --- Make_models/configs/config_LSTM.json | 87 --- Make_models/dataloaders/Dataloader.py | 531 ------------------ Make_models/main.py | 61 -- Make_models/models/CNN_AE.py | 303 ---------- Make_models/models/LSTM.py | 219 -------- Make_models/trainers/Trainer.py | 505 ----------------- Make_models/utils/config.py | 25 - Make_models/utils/utils.py | 90 --- MulitiScaleCNNAE.py => Muliti-Scale_CNN-AE.py | 0 README.md | 14 +- 13 files changed, 8 insertions(+), 1920 deletions(-) delete mode 100644 LSTM/val_loss_time/Test_LSTM.csv delete mode 100644 Make_models/configs/config_CNN_AE.json delete mode 100644 Make_models/configs/config_LSTM.json delete mode 100644 Make_models/dataloaders/Dataloader.py delete mode 100644 Make_models/main.py delete mode 100644 Make_models/models/CNN_AE.py delete mode 100644 Make_models/models/LSTM.py delete mode 100644 Make_models/trainers/Trainer.py delete mode 100644 Make_models/utils/config.py delete mode 100644 Make_models/utils/utils.py rename MulitiScaleCNNAE.py => Muliti-Scale_CNN-AE.py (100%) diff --git a/.gitignore b/.gitignore index adbb97d..fcc8c02 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -data/ \ No newline at end of file +data/ +Make_models/ \ No newline at end of file diff --git a/LSTM/val_loss_time/Test_LSTM.csv b/LSTM/val_loss_time/Test_LSTM.csv deleted file mode 100644 index 6fc70a6..0000000 --- a/LSTM/val_loss_time/Test_LSTM.csv +++ /dev/null @@ -1,2 +0,0 @@ -val_loss,time -0.050651609897613525,32.40736722946167 diff --git a/Make_models/configs/config_CNN_AE.json b/Make_models/configs/config_CNN_AE.json deleted file mode 100644 index 38e4a9d..0000000 --- a/Make_models/configs/config_CNN_AE.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "GPU": { - "number": "3", - "multi_gpu": false, - "gpu_count": 1 - }, - "save": { - "val_time_save": false, - "history_save": true - }, - "data_loader": { - "num_of_ts": 500, - "path_to_present_dir": "../", - "x_num": 384, - "y_num": 192, - "phys_num": 3, - "kind_num": 80, - "aumont_kind": 100, - "with_shape": false, - "train": true, - "ratio_tr_te": 0.2, - "": "--------- generator condition ---------", - "generator": false - }, - "model": { - "name": "CNN-AE", - "multi_scale": true, - "optimizer": "adam", - "": "---- if you use SGD as optimizer ----", - "lr": 0.01, - "momentum": 0.0, - "decay": 0.0, - "nesterov": false, - "": "-------------------------------------", - "act": "relu", - "filsize": [ - 3, - 5, - 9 - ], - "layer_nm": [ - 1, - 4, - 2 - ], - "chanel_nm": [ - 16, - 8, - 4 - ], - "loss": "gdl_mse" - }, - "trainer": { - "": "--------- cross validation ---------", - "use_CV": false, - "n_splits": 4, - "": "------------------------------------", - "num_epochs": 100, - "batch_size": 50, - "shuffle": true, - "verbose": 1 - }, - "callbacks": { - "": "--------- model checkpoint ---------", - "modelcheckpoint": true, - "save_file": "/CNN_autoencoder/", - "model_name": "Test_CNN_AE", - "cp_monitor": "val_loss", - "cp_save_best_only": true, - "cp_verbose": 1, - "": "--------- early stopping ---------", - "earlystopping": false, - "es_monitor": "val_loss", - "es_patience": 40, - "es_verbose": 1, - "": "--------- tensor borad ---------", - "tensorboard": false, - "tensorboard_log_dir": "", - "tensorboard_write_graph": true, - "": "--------- reduce LR ---------", - "reduceLR": false, - "reduceLR_monitor": "val_loss", - "": "new_LR = LR * reduceLR_factor", - "reduceLR_factor": 0.2, - "reduceLR_patience": 10, - "reduceLR_verbose": 1 - } -} \ No newline at end of file diff --git a/Make_models/configs/config_LSTM.json b/Make_models/configs/config_LSTM.json deleted file mode 100644 index 466a87a..0000000 --- a/Make_models/configs/config_LSTM.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "GPU": { - "number": "3", - "multi_gpu": false, - "gpu_count": 1 - }, - "save": { - "val_time_save": true, - "history_save": true - }, - "data_loader": { - "dataset_name": "72_values_MS-BN-1_dataset.csv", - "generator": false, - "num_of_ts": 500, - "num_of_ts_for_data": 550, - "kind_num": 80, - "": "---------------------------------------------------------", - "": "num_of_ts + time_step * (maxlen - 1) < num_of_ts_for_data", - "": "---------------------------------------------------------", - "maxlen": 20, - "time_step": 1, - "data_size": 72, - "path_to_present_dir": "../", - "with_shape": true, - "train": true, - "ratio_tr_te": 0.2 - }, - "model": { - "name": "LSTM", - "layer_num": 3, - "unit_num": [ - 128, - 128, - 128 - ], - "return_sequence": [ - true, - true, - true - ], - "shape_input_layer": 1, - "optimizer": "adam", - "": "---- if you use SGD as optimizer ----", - "lr": 0.01, - "momentum": 0.0, - "decay": 0.0, - "nesterov": false, - "": "-------------------------------------", - "act": "tanh", - "loss": "mse" - }, - "trainer": { - "": "--------- cross validation ---------", - "use_CV": false, - "n_splits": 5, - "": "------------------------------------", - "num_epochs": 2, - "batch_size": 1000, - "shuffle": true, - "verbose": 1 - }, - "callbacks": { - "": "--------- model checkpoint ---------", - "modelcheckpoint": true, - "save_file": "/LSTM/", - "model_name": "Test_LSTM", - "cp_monitor": "val_loss", - "cp_save_best_only": true, - "cp_verbose": 1, - "": "--------- early stopping ---------", - "earlystopping": false, - "es_monitor": "val_loss", - "es_patience": 40, - "es_verbose": 1, - "": "--------- tensor borad ---------", - "tensorboard": false, - "tensorboard_log_dir": "", - "tensorboard_write_graph": true, - "": "--------- reduce LR ---------", - "reduceLR": true, - "reduceLR_monitor": "val_loss", - "": "new_LR = LR * reduceLR_factor", - "reduceLR_factor": 0.2, - "reduceLR_patience": 20, - "reduceLR_verbose": 1 - } -} \ No newline at end of file diff --git a/Make_models/dataloaders/Dataloader.py b/Make_models/dataloaders/Dataloader.py deleted file mode 100644 index 55ed5cb..0000000 --- a/Make_models/dataloaders/Dataloader.py +++ /dev/null @@ -1,531 +0,0 @@ -import numpy as np -import math -from tqdm import tqdm -import pandas as pd -import pickle -from sklearn.model_selection import train_test_split, KFold -from keras.utils import Sequence -import random - - -class Dataloader(): - def __init__(self, config): - self.config = config - self.num_of_ts = self.config.data_loader.num_of_ts - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.x_num = self.config.data_loader.x_num - self.y_num = self.config.data_loader.y_num - self.phys_num = self.config.data_loader.phys_num - self.kind_num = self.config.data_loader.kind_num - self.X = None - self.x_train = None - self.x_test = None - self.y_train = None - self.y_test = None - self.data_load() - - def data_load(self): - if self.config.data_loader.train: - self.X = np.zeros([self.kind_num, self.num_of_ts, - self.x_num, self.y_num, self.phys_num]) - - for i in tqdm(range(1, self.kind_num + 1)): - fnstr = self.path_to_present_dir + '/data/pickles/data_' + \ - '{0:03d}'.format(i)+'.pickle' - # Pickle load - - with open(fnstr, 'rb') as f: - obj = pickle.load(f) - self.X[i - 1, :, :, :, :self.phys_num] = obj[:self.num_of_ts] - - self.X = np.reshape( - self.X, - [-1, - self.X.shape[-3], - self.X.shape[-2], - self.X.shape[-1]] - ) - - else: - self.X = np.zeros([self.aumont_kind - self.kind_num, - self.num_of_ts, self.x_num, self.y_num, - self.phys_num]) - - for i in tqdm( - range(1, self.config.aumont_kind - self.kind_num + 1) - ): - fnstr = self.path_to_present_dir + '/data/pickles/Test_data/data_' \ - + '{0:03d}'.format(i)+'.pickle' - - # Pickle load - with open(fnstr, 'rb') as f: - obj = pickle.load(f) - self.X[i - 1, :, :, :, :self.phys_num] = \ - obj[:self.x_numnum_of_ts] - - def test_train_split(self): - x_train, x_test, y_train, y_test = \ - train_test_split(self.X, - self.X[:, :, :, :self.phys_num], - test_size=self.config.data_loader.ratio_tr_te, - random_state=None) - return x_train, x_test, y_train, y_test - - -class Dataloader_LSTM(): - def __init__(self, config): - self.config = config - self.num_of_ts = self.config.data_loader.num_of_ts - self.num_of_ts_for_data = self.config.data_loader.num_of_ts_for_data - self.number_of_shape = self.config.data_loader.kind_num - self.maxlen = self.config.data_loader.maxlen - self.time_step = self.config.data_loader.time_step - self.data_size = self.config.data_loader.data_size - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.path_data = self.path_to_present_dir + '/data/LSTM/Dataset/' + \ - self.config.data_loader.dataset_name - - assert self.num_of_ts + self.time_step * (self.maxlen - 1) < \ - self.num_of_ts_for_data, 'The data aumont is not enough.' - - self.data_load() - self.make_dataset() - - def data_load(self): - self.data_LSTM = pd.read_csv( - self.path_data, header=None, delim_whitespace=False - ) - self.data_LSTM = self.data_LSTM.values - - if self.config.data_loader.with_shape: - self.X_CNN = np.zeros([ - self.number_of_shape * self.num_of_ts, - 120, 120, 1 - ]) - for i in range(self.number_of_shape): - data_CNN = pd.read_csv( - self.path_to_present_dir + - '/data/LSTM/Flags/Flag' + - '{0:03d}'.format(i + 1) + '.csv', - header=None, - delim_whitespace=False - ) - data_CNN = data_CNN.values - self.X_CNN[ - i * self.num_of_ts: (i + 1) * self.num_of_ts, - :, :, 0 - ] = data_CNN - - def make_dataset(self): - self.X = np.zeros( - [ - self.number_of_shape * self.num_of_ts, - self.maxlen, - self.data_size - ] - ) - if self.config.model.return_sequence[-1]: - self.Y = np.zeros( - [ - self.number_of_shape * self.num_of_ts, - self.maxlen, - self.data_size - ] - ) - else: - self.Y = np.zeros( - [ - self.number_of_shape * self.num_of_ts, - self.data_size - ] - ) - for i in range(self.number_of_shape): - for j in range(self.num_of_ts): - self.X[i * self.num_of_ts + j] = \ - self.data_LSTM[ - i * self.num_of_ts_for_data + j: - i * self.num_of_ts_for_data + j + - self.time_step * self.maxlen: self.time_step - ] - if self.config.model.return_sequence[-1]: - self.Y[i * self.num_of_ts + j] = \ - self.data_LSTM[ - i * self.num_of_ts_for_data + j + 1: - i * self.num_of_ts_for_data + j + - self.time_step * self.maxlen + 1: self.time_step - ] - else: - self.Y[i * self.num_of_ts + j] = \ - self.data_LSTM[ - i * self.num_of_ts_for_data + j + - self.time_step * self.maxlen - ] - - def test_train_split(self): - if self.config.data_loader.with_shape: - X_CNN_train, X_CNN_test, X_train, X_test, y_train, y_test = \ - train_test_split(self.X_CNN, - self.X, - self.Y, - test_size=self.config.data_loader.ratio_tr_te, - random_state=None) - x_train = [X_CNN_train, X_train] - x_test = [X_CNN_test, X_test] - return x_train, x_test, y_train, y_test - - else: - x_train, x_test, y_train, y_test = \ - train_test_split(self.X, - self.Y, - test_size=self.config.data_loader.ratio_tr_te, - random_state=None) - return x_train, x_test, y_train, y_test - - -class Datagenerator_for_AE(Sequence): - def __init__(self, config): - self.config = config - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.batch_size = self.config.trainer.batch_size - self.x_num = self.config.data_loader.x_num - self.y_num = self.config.data_loader.y_num - self.phys_num = self.config.data_loader.phys_num - self.index = [] - self.devide_train_test() - self.length = math.ceil( - len(self.X) / self.n_split * (self.n_split - 1) / self.batch_size) - - def devide_train_test(self): - for i in range(self.config.data_loader.kind_num): - for j in range(self.config.data_loader.num_of_ts): - self.index.append([i + 1, j + 1]) - random.shuffle(self.index) - self.val_num = int(len(self.index) * - self.config.data_loader.ratio_tr_te) - self.val_index = self.index[-self.val_num:] - self.tra_index = self.index[:-self.val_num] - - def make_val_data(self): - val_data = np.zeros([len(self.val_index), self.x_num, - self.y_num, - self.phys_num]) - print('\nLoading the validation data.') - for i in tqdm(range(len(self.val_index))): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format( - self.val_index[i][0]) + '_' + '{0:04d}'.format(self.val_index[i][1]) - VAL_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - VAL_DATA = np.array(VAL_DATA) - val_data[i] = VAL_DATA.reshape( - self.x_num, - self.y_num, - self.phys_num) - return val_data, val_data - - def __getitem__(self, idx): - start_pos = self.batch_size * idx - end_pos = start_pos + self.batch_size - if end_pos > len(self.tra_index): - bs = len(self.tra_index) - start_pos - else: - bs = self.batch_size - tra_data = np.zeros([bs, self.x_num, self.y_num, self.phys_num]) - for i in range(bs): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format(self.tra_index[idx * self.batch_size + i][0]) + '_' + \ - '{0:04d}'.format(self.tra_index[idx * self.batch_size + i][1]) - TRA_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - TRA_DATA = np.array(TRA_DATA) - tra_data[i] = TRA_DATA.reshape(self.x_num, - self.y_num, - self.phys_num) - return tra_data, tra_data - - def __len__(self): - return self.length - - def on_epoch_end(self): - pass - - -class gen_for_AE_with_shape(Sequence): - def __init__(self, config): - self.config = config - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.batch_size = self.config.trainer.batch_size - self.x_num = self.config.data_loader.x_num - self.y_num = self.config.data_loader.y_num - self.phys_num = self.config.data_loader.phys_num - self.index = [] - self.devide_train_test() - self.length = math.ceil( - len(self.X) / self.n_split * (self.n_split - 1) / self.batch_size) - - def devide_train_test(self): - for i in range(self.config.data_loader.kind_num): - for j in range(self.config.data_loader.num_of_ts): - self.index.append([i + 1, j + 1]) - random.shuffle(self.index) - self.val_num = int(len(self.index) * - self.config.data_loader.ratio_tr_te) - self.val_index = self.index[-self.val_num:] - self.tra_index = self.index[:-self.val_num] - - def make_val_data(self): - val_data = np.zeros([len(self.val_index), - self.x_num, self.y_num, self.phys_num + 1]) - print('\nLoading the validation data.') - for i in tqdm(range(len(self.val_index))): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format( - self.val_index[i][0]) + '_' + '{0:04d}'.format(self.val_index[i][1]) - VAL_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - VAL_DATA = np.array(VAL_DATA) - val_data[i, :, :, :self.phys_num] = VAL_DATA.reshape( - self.x_num, - self.y_num, - self.phys_num) - tempfn = self.path_to_present_dir + \ - '/CNN_autoencoder/Flags_for_training/Flag' + \ - '{0:03d}'.format(self.val_index[i][0])+'.csv' - data = pd.read_csv(tempfn, header=None, - delim_whitespace=False) - data = data.values - val_data[i, :, :, self.phys_num] = data - return val_data, val_data[:, :, :, :self.phys_num] - - def __getitem__(self, idx): - start_pos = self.batch_size * idx - end_pos = start_pos + self.batch_size - if end_pos > len(self.tra_index): - bs = len(self.tra_index) - start_pos - else: - bs = self.batch_size - tra_data = np.zeros([bs, - self.x_num, self.y_num, self.phys_num + 1]) - for i in range(bs): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format(self.tra_index[idx * self.batch_size + i][0]) + '_' + \ - '{0:04d}'.format(self.tra_index[idx * self.batch_size + i][1]) - TRA_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - TRA_DATA = np.array(TRA_DATA) - tra_data[i, :, :, :self.phys_num] = TRA_DATA.reshape(self.x_num, - self.y_num, - self.phys_num) - tempfn = self.path_to_present_dir + \ - '/CNN_autoencoder/Flags_for_training/Flag' + \ - '{0:03d}'.format( - self.tra_index[idx * self.batch_size + i][0])+'.csv' - data = pd.read_csv(tempfn, header=None, - delim_whitespace=False) - data = data.values - tra_data[i, :, :, self.phys_num] = data - return tra_data, tra_data[:, :, :, :self.phys_num] - - def __len__(self): - return self.length - - def on_epoch_end(self): - pass - - -class Datagenerator_for_AE_with_CV(Sequence): - def __init__(self, config): - self.config = config - self.n_split = self.config.trainer.n_splits - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.batch_size = self.config.trainer.batch_size - self.x_num = self.config.data_loader.x_num - self.y_num = self.config.data_loader.y_num - self.phys_num = self.config.data_loader.phys_num - self.X = [] - for i in range(self.config.data_loader.kind_num): - for j in range(self.config.data_loader.num_of_ts): - self.X.append([i + 1, j + 1]) - self.X = np.array(self.X) - self.index_generator = self.devide_train_test() - self.length = math.ceil( - len(self.X) / self.n_split * (self.n_split - 1) / self.batch_size) - - def devide_train_test(self): - self.val_num = int(len(self.X) * self.config.data_loader.ratio_tr_te) - kf = KFold(n_splits=self.n_split, random_state=None, shuffle=True) - while True: - for train_index, test_index in kf.split(self.X): - self.tra_index, self.val_index = \ - self.X[train_index], self.X[test_index] - yield - - def make_val_data(self): - next(self.index_generator) - val_data = np.zeros([len(self.val_index), self.x_num, - self.y_num, - self.phys_num]) - print('\nLoading the validation data.') - for i in tqdm(range(len(self.val_index))): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format( - self.val_index[i][0]) + '_' + '{0:04d}'.format(self.val_index[i][1]) - VAL_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - VAL_DATA = np.array(VAL_DATA) - val_data[i] = VAL_DATA.reshape( - self.x_num, - self.y_num, - self.phys_num) - return val_data, val_data - - def __getitem__(self, idx): - start_pos = self.batch_size * idx - end_pos = start_pos + self.batch_size - if end_pos > len(self.tra_index): - bs = len(self.tra_index) - start_pos - else: - bs = self.batch_size - tra_data = np.zeros([bs, self.x_num, self.y_num, self.phys_num]) - for i in range(bs): - file_name = self.path_to_present_dir + '/Training_data/UVP_' + \ - '{0:03d}'.format(self.tra_index[idx * self.batch_size + i][0]) + '_' + \ - '{0:04d}'.format(self.tra_index[idx * self.batch_size + i][1]) - TRA_DATA = pd.read_csv( - file_name, header=None, delim_whitespace=False) - TRA_DATA = np.array(TRA_DATA) - tra_data[i] = TRA_DATA.reshape(self.x_num, - self.y_num, - self.phys_num) - return tra_data, tra_data - - def __len__(self): - return self.length - - def on_epoch_end(self): - pass - - -class gen_for_LSTM(Sequence): - def __init__(self, config): - self.config = config - self.num_of_ts = self.config.data_loader.num_of_ts - self.num_of_ts_for_data = self.config.data_loader.num_of_ts_for_data - self.number_of_shape = self.config.data_loader.kind_num - self.maxlen = self.config.data_loader.maxlen - self.time_step = self.config.data_loader.time_step - self.data_size = self.config.data_loader.data_size - self.val_ratio = self.config.data_loader.ratio_tr_te - self.tra_ratio = 1 - self.val_ratio - self.len_val = int(self.num_of_ts * self.val_ratio) - self.len_tra = int(self.num_of_ts * self.tra_ratio) - self.path_to_present_dir = self.config.data_loader.path_to_present_dir - self.path_data = self.path_to_present_dir + '/LSTM/Dataset/' + \ - self.config.data_loader.dataset_name - self.data_load() - self.make_dataset() - - def data_load(self): - self.data_LSTM = pd.read_csv( - self.path_data, header=None, delim_whitespace=False - ) - self.data_LSTM = self.data_LSTM.values - self.data_LSTM = np.reshape( - self.data_LSTM, ( - 80, - self.num_of_ts_for_data, - self.data_size - ) - ) - - if self.config.data_loader.with_shape: - self.data_CNN = np.zeros([ - self.number_of_shape, self.num_of_ts, - 120, 120, 1 - ]) - for i in range(self.number_of_shape): - DATA_CNN = pd.read_csv( - self.path_to_present_dir + - '/LSTM/Flags_for_training_LSTM/Flag' + - '{0:03d}'.format(i + 1) + '.csv', - header=None, - delim_whitespace=False - ) - DATA_CNN = DATA_CNN.values - self.data_CNN[i, :, :, :, 0] = DATA_CNN - - def make_dataset(self): - self.X_LSTM = [] - if self.config.data_loader.with_shape: - self.X_CNN = [] - self.Y = [] - self.length = [] - for i in range(self.len_tra): - LENGTH = int(random.uniform(1, self.maxlen)) - self.length.append(LENGTH) - input_LSTM = self.data_LSTM[ - :self.number_of_shape, i: i + self.time_step * LENGTH: self.time_step - ] - self.X_LSTM.append(input_LSTM) - if self.config.data_loader.with_shape: - input_CNN = self.data_CNN[:, i] - self.X_CNN.append(input_CNN) - if self.config.model.return_sequence[-1]: - output = self.data_LSTM[ - :self.number_of_shape, i + 1: i + self.time_step * LENGTH + 1: self.time_step - ] - else: - output = self.data_LSTM[ - :self.number_of_shape, i + self.time_step * LENGTH + 1 - ] - self.Y.append(output) - - self.X_LSTM_VAL = np.zeros([ - self.len_val * self.number_of_shape, self.maxlen, self.data_size - ]) - if self.config.data_loader.with_shape: - self.X_CNN_VAL = np.zeros([ - self.len_val * self.number_of_shape, 120, 120, 1 - ]) - if self.config.model.return_sequence[-1]: - self.Y_VAL = np.zeros([ - self.len_val * self.number_of_shape, self.maxlen, self.data_size - ]) - else: - self.Y_VAL = np.zeros([ - self.len_val * self.number_of_shape, 1, self.data_size - ]) - for i in range(self.len_tra, int(self.len_val + self.len_tra)): - for j in range(self.number_of_shape): - self.X_LSTM_VAL[j * self.number_of_shape + i] = \ - self.data_LSTM[j, i: i + self.time_step * - self.maxlen: self.time_step] - if self.config.data_loader.with_shape: - self.X_CNN_VAL[j * self.number_of_shape + i] = \ - self.data_CNN[j, i] - if self.config.model.return_sequence[-1]: - self.Y_VAL[j * self.number_of_shape + i] = \ - self.data_LSTM[j, - i + 1: i + self.time_step * self.maxlen + 1: self.time_step] - else: - self.Y_VAL[j * self.number_of_shape + i] = \ - self.data_LSTM[j, - i + self.time_step * self.maxlen + 1] - - def make_val_data(self): - if self.config.data_loader.with_shape: - return [self.X_CNN_VAL, self.X_LSTM_VAL], self.Y_VAL - else: - return self.X_LSTM_VAL, self.Y_VAL - - def __getitem__(self, idx): - if self.config.data_loader.with_shape: - return [self.X_CNN[idx], self.X_LSTM[idx]], self.Y[idx] - else: - return self.X_LSTM[idx], self.Y[idx] - - def __len__(self): - return self.len_tra - - def on_epoch_end(self): - pass diff --git a/Make_models/main.py b/Make_models/main.py deleted file mode 100644 index 935dab5..0000000 --- a/Make_models/main.py +++ /dev/null @@ -1,61 +0,0 @@ -from dataloaders.Dataloader import * -from models.CNN_AE import * -from models.LSTM import * -from utils.config import process_config -from utils.utils import get_args -from trainers.Trainer import * - - -def main(): - try: - args = get_args() - config = process_config(args.config) - except: - print("missing or invalid arguments") - exit(0) - - if config.GPU.multi_gpu: - config.trainer.batch_size = config.trainer.batch_size *\ - config.GPU.gpu_count - - print('Create the data generator.') - if config.model.name == 'CNN-AE': - if config.data_loader.generator: - if config.data_loader.with_shape: - data_loader = gen_for_AE_with_shape(config) - else: - if config.trainer.use_CV: - data_loader = Datagenerator_for_AE_with_CV(config) - else: - data_loader = Datagenerator_for_AE(config) - else: - data_loader = Dataloader(config) - elif config.model.name == 'LSTM': - if config.data_loader.generator: - data_loader = gen_for_LSTM(config) - else: - data_loader = Dataloader_LSTM(config) - - print('Create the model.') - if config.model.name == 'CNN-AE': - if config.model.multi_scale and not config.model.fc: - model = MS_CNN_AE(config) - elif config.model.multi_scale and config.model.fc: - model = MS_FC_CNN_AE(config) - elif config.model.name == 'LSTM': - if config.data_loader.with_shape: - model = LSTM_with_shape(config) - else: - model = simple_LSTM(config) - - if config.trainer.use_CV: - TRAINER = trainer_with_CV(model, data_loader, config) - else: - TRAINER = trainer(model, data_loader, config) - print('Trainer was created.\n') - - TRAINER.train() - - -if __name__ == '__main__': - main() diff --git a/Make_models/models/CNN_AE.py b/Make_models/models/CNN_AE.py deleted file mode 100644 index b760ded..0000000 --- a/Make_models/models/CNN_AE.py +++ /dev/null @@ -1,303 +0,0 @@ -from keras.layers import (Input, Conv2D, MaxPooling2D, UpSampling2D, - Add, BatchNormalization, Activation, Reshape, - Dense) -from keras.models import Model -from keras.optimizers import SGD -import tensorflow as tf -from keras.utils.training_utils import multi_gpu_model -import tensorflow as tf -from keras.backend import tensorflow_backend - -def conv_down_block(input_img, size, config): - for i in range(config.model.layer_nm[0]): - if i == 0: - x = Conv2D(config.model.chanel_nm[0], - (size, size), - padding='same')(input_img) - else: - x = Conv2D(config.model.chanel_nm[0], - (size, size), - padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - x = MaxPooling2D((2, 2), padding='same')(x) - - for i in range(config.model.layer_nm[1]): - x = Conv2D(config.model.chanel_nm[1], - (size, size), - padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - x = MaxPooling2D((2, 2), padding='same')(x) - - for i in range(config.model.layer_nm[2]): - x = Conv2D(config.model.chanel_nm[2], - (size, size), - padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - if i != config.model.layer_nm[2] - 1: - x = MaxPooling2D((2, 2), padding='same')(x) - return x - -def conv_up_block(encoded, size, config): - for i in range(config.model.layer_nm[-1] - 1): - if i == 0: - x = Conv2D(config.model.chanel_nm[-1], - (size, size), - padding='same')(encoded) - else: - x = Conv2D(config.model.chanel_nm[-1], - (size, size), - padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - x = UpSampling2D((2, 2))(x) # 12,6,4 - - for i in range(config.model.layer_nm[-2]): - x = Conv2D(config.model.chanel_nm[-2], - (size, size), - padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - x = UpSampling2D((2, 2))(x) # 24,12,8 - - for i in range(config.model.layer_nm[-3]): - x = Conv2D( - config.model.chanel_nm[-3], (size, size), padding='same')(x) - x = BatchNormalization()(x) - x = Activation(config.model.act)(x) - x = UpSampling2D((2, 2))(x) # 384,192,16 - x = Conv2D(config.data_loader.phys_num, (size, size), - activation='linear', padding='same')(x) - return x - -class MS_CNN_AE(): - def __init__(self, config): - self.config = config - - def conf_model(self): - input_img = Input( - shape=( - self.config.data_loader.x_num, - self.config.data_loader.y_num, - self.config.data_loader.phys_num) - ) - if self.config.data_loader.with_shape: - input_img = Input( - shape=( - self.config.data_loader.x_num, - self.config.data_loader.y_num, - self.config.data_loader.phys_num + 1) - ) - - filsize1 = self.config.model.filsize[0] - filsize2 = self.config.model.filsize[1] - filsize3 = self.config.model.filsize[2] - - conv1 = conv_down_block( - input_img, filsize1, self.config) - conv2 = conv_down_block( - input_img, filsize2, self.config) - conv3 = conv_down_block( - input_img, filsize3, self.config) - x = Add()([conv1, conv2, conv3]) - x = Conv2D( - self.config.model.chanel_nm[2], (3, 3), padding='same')(x) - x = BatchNormalization()(x) - encoded = Activation(self.config.model.act)(x) - x = Conv2D(self.config.model.chanel_nm[2], (3, 3), padding='same')( - encoded) - x = BatchNormalization()(x) - x = Activation(self.config.model.act)(x) - conv4 = conv_up_block(x, filsize1, self.config) - conv5 = conv_up_block(x, filsize2, self.config) - conv6 = conv_up_block(x, filsize3, self.config) - decoded = Add()([conv4, conv5, conv6]) - print('\n\nModel was created.') - print('\n----------------Model Configuration----------------\n') - print('Model : Multi-scale CNN\n') - print('Input shape of model : %d, %d, %d' - % (input_img.shape[-3], - input_img.shape[-2], - input_img.shape[-1])) - print('Shape of encoded data : %d, %d, %d' - % (encoded.shape[-3], - encoded.shape[-2], - encoded.shape[-1])) - print('Output shape of model : %d, %d, %d' - % (decoded.shape[-3], - decoded.shape[-2], - decoded.shape[-1])) - print('\nOptimizer : ' + self.config.model.optimizer) - print('Loss function : ' + self.config.model.loss) - print('Activation function : ' + self.config.model.act) - print('\n---------------------------------------------------\n') - - model = Model(input_img, decoded) - return model - - def compile_model(self, model): - if self.config.model.optimizer == 'SGD': - sgd = SGD( - lr=self.config.model.lr, - momentum=self.config.model.momentum, - decay=self.config.model.decay, - nesterov=self.config.model.nesterov - ) - model.compile( - optimizer=sgd, - loss=self.config.model.loss - ) - else: - model.compile( - optimizer=self.config.model.optimizer, - loss=self.config.model.loss - ) - return model - - def make_model(self): - if self.config.GPU.multi_gpu: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - with tf.device("/cpu:0"): - base_model = self.conf_model() - model = multi_gpu_model(base_model, gpus=self.config.GPU.gpu_count) - model = self.compile_model(model) - else: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - model = self.conf_model() - model = self.compile_model(model) - base_model = None - return model, base_model - -class MS_FC_CNN_AE(): - def __init__(self, config): - self.config = config - - def conf_model(self): - input_img = Input( - shape=( - self.config.data_loader.x_num, - self.config.data_loader.y_num, - self.config.data_loader.phys_num) - ) - if self.config.data_loader.with_shape: - input_img = Input( - shape=( - self.config.data_loader.x_num, - self.config.data_loader.y_num, - self.config.data_loader.phys_num + 1) - ) - - filsize1 = self.config.model.filsize[0] - filsize2 = self.config.model.filsize[1] - filsize3 = self.config.model.filsize[2] - - conv1 = conv_down_block( - input_img, filsize1, self.config) - conv2 = conv_down_block( - input_img, filsize2, self.config) - conv3 = conv_down_block( - input_img, filsize3, self.config) - x = Add()([conv1, conv2, conv3]) - x = Reshape([72])(x) - for i in range(self.config.model.fc_num): - x = Dense(self.config.model.uni_num[i])(x) - x = BatchNormalization()(x) - x = Activation(self.config.model.act)(x) - encoded = Dense(self.config.model.encoded_size)(x) - x = BatchNormalization()(encoded) - x = Activation(self.config.model.act)(x) - for i in reversed(range(self.config.model.fc_num)): - x = Dense(self.config.model.uni_num[i])(x) - x = BatchNormalization()(x) - x = Activation(self.config.model.act)(x) - x = Dense(72)(x) - x = BatchNormalization()(x) - x = Activation(self.config.model.act)(x) - x = Reshape([6, 3, 4])(x) - conv4 = conv_up_block(x, filsize1, self.config) - conv5 = conv_up_block(x, filsize2, self.config) - conv6 = conv_up_block(x, filsize3, self.config) - decoded = Add()([conv4, conv5, conv6]) - print('\n\nModel was created.') - print('\n----------------Model Configuration----------------\n') - print('Model : Multi-scale FC CNN\n') - print('Input shape of model : %d, %d, %d' - % (input_img.shape[-3], - input_img.shape[-2], - input_img.shape[-1])) - print('Shape of encoded data : %d' - % (encoded.shape[-1])) - print('Output shape of model : %d, %d, %d' - % (decoded.shape[-3], - decoded.shape[-2], - decoded.shape[-1])) - print('\nOptimizer : ' + self.config.model.optimizer) - print('Loss function : ' + self.config.model.loss) - print('Activation function : ' + self.config.model.act) - print('\n---------------------------------------------------\n') - - model = Model(input_img, decoded) - return model - - def compile_model(self, model): - if self.config.model.optimizer == 'SGD': - sgd = SGD( - lr=self.config.model.lr, - momentum=self.config.model.momentum, - decay=self.config.model.decay, - nesterov=self.config.model.nesterov - ) - model.compile( - optimizer=sgd, - loss=self.config.model.loss - ) - else: - model.compile( - optimizer=self.config.model.optimizer, - loss=self.config.model.loss - ) - return model - - def make_model(self): - if self.config.GPU.multi_gpu: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - with tf.device("/cpu:0"): - base_model = self.conf_model() - model = multi_gpu_model(base_model, gpus=self.config.GPU.gpu_count) - model = self.compile_model(model) - else: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - model = self.conf_model() - model = self.compile_model(model) - base_model = None - return model, base_model diff --git a/Make_models/models/LSTM.py b/Make_models/models/LSTM.py deleted file mode 100644 index 1c2ffd2..0000000 --- a/Make_models/models/LSTM.py +++ /dev/null @@ -1,219 +0,0 @@ -from keras.layers import (LSTM, Activation, Add, BatchNormalization, Conv2D, - Dense, Dropout, Input, Reshape) -from keras.models import Model -from keras.optimizers import SGD -import tensorflow as tf -from keras.utils.training_utils import multi_gpu_model -from keras.backend import tensorflow_backend - - -class simple_LSTM(): - def __init__(self, config): - self.config = config - - def conf_model(self): - input_img = Input(shape=(None, - self.config.data_loader.data_size)) - for i in range(self.config.model.layer_num): - if i == 0: - x = LSTM(self.config.model.unit_num[i], - activation=self.config.model.act, - return_sequences=self.config.model.return_sequence[i] - )(input_img) - # x = BatchNormalization()(x) - x = Dropout(0.15)(x) - elif i == self.config.model.layer_num - 1: - x = LSTM(self.config.model.unit_num[i], - activation=self.config.model.act, - return_sequences=self.config.model.return_sequence[i] - )(x) - x = Dense(self.config.data_loader.data_size)(x) - else: - x = LSTM(self.config.model.unit_num[i], - activation=self.config.model.act, - return_sequences=self.config.model.return_sequence[i] - )(x) - # x = BatchNormalization()(x) - x = Dropout(0.15)(x) - x = Activation("linear")(x) - model = Model(input_img, x) - print('\n\nModel was created.') - print('\n----------------Model Configuration----------------\n') - print('Model : LSTM\n') - print('Input shape of model : ', input_img.shape) - print('Output shape of model : ', x.shape) - print('Number of layers : ' + str(self.config.model.layer_num)) - print('Number of units : ', - ", ".join(repr(e) for e in self.config.model.unit_num)) - print('\nOptimizer : ' + self.config.model.optimizer) - print('Loss function : ' + self.config.model.loss) - print('Activation function : ' + self.config.model.act) - print('\n---------------------------------------------------\n') - - return model - - def compile_model(self, model): - if self.config.model.optimizer == 'SGD': - sgd = SGD( - lr=self.config.model.lr, - momentum=self.config.model.momentum, - decay=self.config.model.decay, - nesterov=self.config.model.nesterov - ) - model.compile( - optimizer=sgd, - loss=self.config.model.loss - ) - else: - model.compile( - optimizer=self.config.model.optimizer, - loss=self.config.model.loss - ) - return model - - def make_model(self): - if self.config.GPU.multi_gpu: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - with tf.device("/cpu:0"): - base_model = self.conf_model() - model = multi_gpu_model(base_model, gpus=self.config.GPU.gpu_count) - model = self.compile_model(model) - else: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - model = self.conf_model() - model = self.compile_model(model) - base_model = None - return model, base_model - - -class LSTM_with_shape(): - def __init__(self, config): - self.config = config - - def conf_model(self): - act = self.config.model.act - input_img_CNN = Input(shape=(120, 120, 1)) - y = Conv2D(8, (21, 21), padding='valid')(input_img_CNN) # 100,100,8 - y = BatchNormalization()(y) - y = Activation(act)(y) - y = Conv2D(8, (4, 4), padding='valid', strides=(2, 2))(y) # 49,49,8 - y = BatchNormalization()(y) - y = Activation(act)(y) - y = Conv2D(8, (3, 3), padding='valid', strides=(2, 2))(y) # 24,24,8 - y = BatchNormalization()(y) - y = Activation(act)(y) - y = Conv2D(1, (2, 2), padding='valid', strides=(2, 2))(y) # 12,12,8 - y = BatchNormalization()(y) - y = Activation(act)(y) - y = Reshape([144])(y) - y = Dense(128)(y) - y = BatchNormalization()(y) - y = Activation(act)(y) - - input_img = Input(shape=(None, - self.config.data_loader.data_size)) - for i in range(self.config.model.layer_num): - if i == 0: - x = LSTM(self.config.model.unit_num[i], - activation=act, - return_sequences=self.config.model.return_sequence[i] - )(input_img) - # x = BatchNormalization()(x) - x = Dropout(0.15)(x) - elif i == self.config.model.layer_num - 1: - x = LSTM(self.config.model.unit_num[i], - activation=act, - return_sequences=self.config.model.return_sequence[i] - )(x) - x = Dense(self.config.data_loader.data_size)(x) - else: - x = LSTM(self.config.model.unit_num[i], - activation=act, - return_sequences=self.config.model.return_sequence[i] - )(x) - # x = BatchNormalization()(x) - x = Dropout(0.15)(x) - if i == self.config.model.shape_input_layer - 1: - x = Add()([x, y]) - x = Activation("linear")(x) - model = Model([input_img_CNN, input_img], x) - - print('\n\nModel was created.') - print('\n----------------Model Configuration----------------\n') - print('Model : LSTM with shape input\n') - print('Input shape of model : ', - input_img_CNN.shape, input_img.shape) - print('Output shape of model : ', x.shape) - print('Number of layers : ' + str(self.config.model.layer_num)) - print('Number of units : ', - ", ".join(repr(e) for e in self.config.model.unit_num)) - print('Layer to be input shape : After ' + - str(self.config.model.shape_input_layer) + 'st layer') - print('\nOptimizer : ' + self.config.model.optimizer) - print('Loss function : ' + self.config.model.loss) - print('Activation function : ' + self.config.model.act) - print('\n---------------------------------------------------\n') - - return model - - def compile_model(self, model): - if self.config.model.optimizer == 'SGD': - sgd = SGD( - lr=self.config.model.lr, - momentum=self.config.model.momentum, - decay=self.config.model.decay, - nesterov=self.config.model.nesterov - ) - model.compile( - optimizer=sgd, - loss=self.config.model.loss - ) - else: - model.compile( - optimizer=self.config.model.optimizer, - loss=self.config.model.loss - ) - - return model - - def make_model(self): - if self.config.GPU.multi_gpu: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - with tf.device("/cpu:0"): - base_model = self.conf_model() - model = multi_gpu_model(base_model, gpus=self.config.GPU.gpu_count) - model = self.compile_model(model) - else: - config_gpu = tf.ConfigProto( - gpu_options=tf.GPUOptions( - allow_growth=True, - visible_device_list=self.config.GPU.number - ) - ) - session = tf.Session(config=config_gpu) - tensorflow_backend.set_session(session) - model = self.conf_model() - model = self.compile_model(model) - base_model = None - return model, base_model diff --git a/Make_models/trainers/Trainer.py b/Make_models/trainers/Trainer.py deleted file mode 100644 index f7db190..0000000 --- a/Make_models/trainers/Trainer.py +++ /dev/null @@ -1,505 +0,0 @@ -import os -import time -import pandas as pd -from keras.callbacks import ( - ModelCheckpoint, EarlyStopping, - ReduceLROnPlateau, TensorBoard -) -from keras import backend as K -from utils.utils import MultiGPUCheckpointCallback -from sklearn.model_selection import KFold -import numpy as np -import sys - -class trainer(): - def __init__(self, model, data, config): - self.model, self.base_model = model.make_model() - self.data = data - self.config = config - self.callbacks = [] - self.val_loss_time = { - 'val_loss': [], - 'time': [] - } - self.init_callbacks() - - def init_callbacks(self): - if self.config.callbacks.modelcheckpoint and \ - not self.config.GPU.multi_gpu: - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/', - exist_ok=True - ) - self.callbacks.append( - ModelCheckpoint( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '.hdf5', - monitor=self.config.callbacks.cp_monitor, - save_best_only=self.config.callbacks.cp_save_best_only, - verbose=self.config.callbacks.cp_verbose - ) - ) - if self.config.callbacks.modelcheckpoint and \ - self.config.GPU.multi_gpu: - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/', - exist_ok=True - ) - self.callbacks.append( - MultiGPUCheckpointCallback( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '.hdf5', - self.base_model, - monitor=self.config.callbacks.cp_monitor, - save_best_only=self.config.callbacks.cp_save_best_only, - verbose=self.config.callbacks.cp_verbose - ) - ) - if self.config.callbacks.earlystopping: - self.callbacks.append( - EarlyStopping( - monitor=self.config.callbacks.es_monitor, - patience=self.config.callbacks.es_patience, - verbose=self.config.callbacks.es_verbose - ) - ) - - if self.config.callbacks.tensorboard: - self.callbacks.append( - TensorBoard( - log_dir=self.config.callbacks.tensorboard_log_dir, - write_graph=self.config.callbacks.tensorboard_write_graph, - ) - ) - - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - self.callbacks.append( - ReduceLROnPlateau( - monitor=self.config.callbacks.reduceLR_monitor, - factor=self.config.callbacks.reduceLR_factor, - patience=self.config.callbacks.reduceLR_patience, - verbose=self.config.callbacks.reduceLR_verbose, - ) - ) - - def train(self): - if self.config.data_loader.generator: - self.x_test, self.y_test = \ - self.data.make_val_data() - - print('\n-----------------Training Condition----------------\n') - if self.config.GPU.multi_gpu: - print('This model will be trained using ' + str(self.config.GPU.gpu_count) + ' GPUs.') - print('This model will be trained by fit_generator.') - print('Number of epochs : ' + str(self.config.trainer.num_epochs)) - if type(self.x_test) is not list: - print('X test data : ', self.x_test.shape) - if type(self.y_test) is not list: - print('Y test data : ', self.y_test.shape) - if self.config.callbacks.modelcheckpoint: - print('Callbacks : Model Checkpoint') - if self.config.callbacks.earlystopping: - print(' : Early Stopping') - if self.config.callbacks.tensorboard: - print(' : Tensor Board') - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print(' : Reduce Learning Rate On Plateau') - if self.config.callbacks.earlystopping: - print('Patiance of ES : ' + str(self.config.callbacks.es_patience)) - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print('Patiance of RLRP : ' + str(self.config.callbacks.reduceLR_patience)) - print('\n---------------------------------------------------\n') - - print('Training is now begining.') - - start = time.time() - history = self.model.fit_generator( - self.data, - epochs=self.config.trainer.num_epochs, - verbose=self.config.trainer.verbose, - callbacks=self.callbacks, - validation_data=( - self.x_test, - self.y_test - ), - max_queue_size=15, - workers=10, - use_multiprocessing=True, - shuffle=self.config.trainer.shuffle, - ) - elapsed_time = time.time() - start - - else: - self.x_train, self.x_test, self.y_train, self.y_test = \ - self.data.test_train_split() - - print('\n-----------------Training Condition----------------\n') - if self.config.GPU.multi_gpu: - print('This model will be trained using ' + str(self.config.GPU.gpu_count) + ' GPUs.') - print('Number of epochs : ' + str(self.config.trainer.num_epochs)) - if type(self.x_train) is not list: - print('X training data : ', self.x_train.shape) - if type(self.y_train) is not list: - print('Y training data : ', self.y_train.shape) - if type(self.x_test) is not list: - print('X test data : ', self.x_test.shape) - if type(self.y_test) is not list: - print('Y test data : ', self.y_test.shape) - if self.config.callbacks.modelcheckpoint: - print('Callbacks : Model Checkpoint') - if self.config.callbacks.earlystopping: - print(' : Early Stopping') - if self.config.callbacks.tensorboard: - print(' : Tensor Board') - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print(' : Reduce Learning Rate On Plateau') - if self.config.callbacks.earlystopping: - print('Patiance of ES : ' + str(self.config.callbacks.es_patience)) - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print('Patiance of RLRP : ' + str(self.config.callbacks.reduceLR_patience)) - print('\n---------------------------------------------------\n') - - print('Training is now begining.') - - start = time.time() - history = self.model.fit( - self.x_train, - self.y_train, - epochs=self.config.trainer.num_epochs, - batch_size=self.config.trainer.batch_size, - shuffle=self.config.trainer.shuffle, - validation_data=( - self.x_test, - self.y_test - ), - callbacks=self.callbacks, - verbose=self.config.trainer.verbose - ) - elapsed_time = time.time() - start - self.val_loss_time['time'].append(elapsed_time) - self.val_loss_time['val_loss'].append( - min(history.history['val_loss']) - ) - df_results = pd.DataFrame(history.history) - df_results['epoch'] = history.epoch - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'History/', - exist_ok=True - ) - if self.config.save.history_save: - df_results.to_csv( - path_or_buf=self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'History/' + - self.config.callbacks.model_name + - '.csv', - index=False - ) - print('History was saved.') - - print("elapsed_time:{0}".format(elapsed_time) + "[sec]") - - K.clear_session() - print('The session was cleared.') - - df = pd.DataFrame(self.val_loss_time) - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'val_loss_time/', - exist_ok=True - ) - if self.config.save.val_time_save: - df.to_csv( - path_or_buf=self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'val_loss_time/' + - self.config.callbacks.model_name + - '.csv', - index=None - ) - print('Validation loss & time were saved.') - -class trainer_with_CV(): - def __init__(self, model, data, config): - self.model_maker = model - self.data = data - self.config = config - self.model = None - self.val_loss_time = { - 'val_loss': [], - 'time': [] - } - self.model_number = \ - ['Model_' + str(i + 1) for i in range(self.config.trainer.n_splits)] - - def make_callbacks(self): - self.callbacks = [] - if self.config.callbacks.modelcheckpoint and \ - not self.config.GPU.multi_gpu: - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '/', - exist_ok=True - ) - self.callbacks.append( - ModelCheckpoint( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '/' + - self.model_number[self.i] + - '.hdf5', - monitor=self.config.callbacks.cp_monitor, - save_best_only=self.config.callbacks.cp_save_best_only, - verbose=self.config.callbacks.cp_verbose - ) - ) - if self.config.callbacks.modelcheckpoint and \ - self.config.GPU.multi_gpu: - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '/', - exist_ok=True - ) - self.callbacks.append( - MultiGPUCheckpointCallback( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'Model/' + - self.config.callbacks.model_name + - '/' + - self.model_number[self.i] + - '.hdf5', - self.base_model, - monitor=self.config.callbacks.cp_monitor, - save_best_only=self.config.callbacks.cp_save_best_only, - verbose=self.config.callbacks.cp_verbose - ) - ) - - if self.config.callbacks.earlystopping: - self.callbacks.append( - EarlyStopping( - monitor=self.config.callbacks.es_monitor, - patience=self.config.callbacks.es_patience, - verbose=self.config.callbacks.es_verbose - ) - ) - - if self.config.callbacks.tensorboard: - self.callbacks.append( - TensorBoard( - log_dir=self.config.callbacks.tensorboard_log_dir, - write_graph=self.config.callbacks.tensorboard_write_graph, - ) - ) - - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - self.callbacks.append( - ReduceLROnPlateau( - monitor=self.config.callbacks.reduceLR_monitor, - factor=self.config.callbacks.reduceLR_factor, - patience=self.config.callbacks.reduceLR_patience, - verbose=self.config.callbacks.reduceLR_verbose, - ) - ) - - def train(self): - print('Training is now begining.') - print('The model will be trained using cross validation') - print('The number of total fold is ' + - str(self.config.trainer.n_splits)) - - kf = KFold(n_splits=self.config.trainer.n_splits, random_state=None, shuffle=True) - for self.i, (train_index, test_index) in enumerate(kf.split(self.data.X)): - if self.config.data_loader.generator: - self.x_test, self.y_test = \ - self.data.make_val_data() - - print('\n-----------------Training Condition----------------\n') - if self.config.GPU.multi_gpu: - print('This model will be trained using ' + str(self.config.GPU.gpu_count) + ' GPUs.') - print('This model will be trained by fit_generator.') - print('Number of epochs : ' + str(self.config.trainer.num_epochs)) - if self.config.model.name != 'LSTM' and not self.config.data_loader.with_shape: - print('X test data : ', self.x_test.shape) - print('Y test data : ', self.y_test.shape) - if self.config.callbacks.modelcheckpoint: - print('Callbacks : Model Checkpoint') - if self.config.callbacks.earlystopping: - print(' : Early Stopping') - if self.config.callbacks.tensorboard: - print(' : Tensor Board') - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print(' : Reduce Learning Rate On Plateau') - if self.config.callbacks.earlystopping: - print('Patiance of ES : ' + str(self.config.callbacks.es_patience)) - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print('Patiance of RLRP : ' + str(self.config.callbacks.reduceLR_patience)) - print('\nThe nomber of fold : %d' % (self.i + 1)) - print('\n---------------------------------------------------\n') - - print('Training is now begining.') - - else: - if self.config.model.name == 'CNN-AE': - self.x_train, self.x_test = \ - self.data.X[train_index], self.data.X[test_index] - self.y_train, self.y_test = \ - self.data.X[train_index, :, :, :self.config.data_loader.phys_num], \ - self.data.X[test_index, :, :, :self.config.data_loader.phys_num] - else: - if self.config.data_loader.with_shape: - self.x_train, self.x_test = \ - [self.data.X_CNN[train_index], self.data.X[train_index]], \ - [self.data.X_CNN[test_index], self.data.X[test_index]] - self.y_train, self.y_test = \ - self.data.Y[train_index], self.data.Y[test_index] - else: - self.x_train, self.x_test = \ - self.data.X[train_index], self.data.X[test_index] - self.y_train, self.y_test = \ - self.data.Y[train_index], self.data.Y[test_index] - - print('\n-----------------Training Condition----------------\n') - if self.config.GPU.multi_gpu: - print('This model will be trained using ' + str(self.config.GPU.gpu_count) + ' GPUs.') - print('Number of epochs : ' + str(self.config.trainer.num_epochs)) - if self.config.model.name != 'LSTM' and not self.config.data_loader.with_shape: - print('X training data : ', self.x_train.shape) - print('Y training data : ', self.y_train.shape) - print('X test data : ', self.x_test.shape) - print('Y test data : ', self.y_test.shape) - if self.config.callbacks.modelcheckpoint: - print('Callbacks : Model Checkpoint') - if self.config.callbacks.earlystopping: - print(' : Early Stopping') - if self.config.callbacks.tensorboard: - print(' : Tensor Board') - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print(' : Reduce Learning Rate On Plateau') - if self.config.callbacks.earlystopping: - print('Patiance of ES : ' + str(self.config.callbacks.es_patience)) - if self.config.model.optimizer == 'SGD' and \ - self.config.callbacks.reduceLR: - print('Patiance of RLRP : ' + str(self.config.callbacks.reduceLR_patience)) - print('\nThe nomber of fold : %d' % (self.i + 1)) - print('\n---------------------------------------------------\n') - - if self.model is not None: - del self.model - self.model, self.base_model = self.model_maker.make_model() - - self.make_callbacks() - - if self.config.data_loader.generator: - start = time.time() - history = self.model.fit_generator( - self.data, - epochs=self.config.trainer.num_epochs, - verbose=self.config.trainer.verbose, - callbacks=self.callbacks, - validation_data=( - self.x_test, - self.y_test - ), - max_queue_size=20, - workers=10, - use_multiprocessing=True, - shuffle=self.config.trainer.shuffle, - ) - elapsed_time = time.time() - start - - else: - start = time.time() - history = self.model.fit( - self.x_train, - self.y_train, - epochs=self.config.trainer.num_epochs, - batch_size=self.config.trainer.batch_size, - shuffle=self.config.trainer.shuffle, - validation_data=( - self.x_test, - self.y_test - ), - callbacks=self.callbacks, - verbose=self.config.trainer.verbose - ) - elapsed_time = time.time() - start - self.val_loss_time['time'].append(elapsed_time) - self.val_loss_time['val_loss'].append( - min(history.history['val_loss']) - ) - df_results = pd.DataFrame(history.history) - df_results['epoch'] = history.epoch - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'History/' + - self.config.callbacks.model_name + - '/', - exist_ok=True - ) - if self.config.save.history_save: - df_results.to_csv( - path_or_buf=self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'History/' + - self.config.callbacks.model_name + - '/' + - self.model_number[self.i] + - '.csv', - index=False - ) - print('History was saved.') - - print("elapsed_time:{0}".format(elapsed_time) + "[sec]") - - K.clear_session() - print('The session was cleared.') - - df = pd.DataFrame(self.val_loss_time, index=self.model_number) - os.makedirs( - self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'val_loss_time/', - exist_ok=True - ) - if self.config.save.val_time_save: - df.to_csv( - path_or_buf=self.config.data_loader.path_to_present_dir + - self.config.callbacks.save_file + - 'val_loss_time/' + - self.config.callbacks.model_name + - '.csv', - ) - print('Validation loss & time were saved.') diff --git a/Make_models/utils/config.py b/Make_models/utils/config.py deleted file mode 100644 index 287e098..0000000 --- a/Make_models/utils/config.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -from dotmap import DotMap - - -def get_config_from_json(json_file): - """ - Get the config from a json file - :param json_file: - :return: config(namespace) or config(dictionary) - """ - # parse the configurations from the config json file provided - with open(json_file, 'r') as config_file: - config_dict = json.load(config_file) - - # convert the dictionary to a namespace using bunch lib - config = DotMap(config_dict) - - return config, config_dict - - -def process_config(json_file): - config, _ = get_config_from_json(json_file) - # config.callb""" acks.tensorboard_log_dir = os.path.join("experiments", time.strftime("%Y-%m-%d/",time.localtime()), config.exp.name, "logs/") - # config.callb """acks.checkpoint_dir = os.path.join("experiments", time.strftime("%Y-%m-%d/",time.localtime()), config.exp.name, "checkpoints/") - return config diff --git a/Make_models/utils/utils.py b/Make_models/utils/utils.py deleted file mode 100644 index 3e821f1..0000000 --- a/Make_models/utils/utils.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -from keras.callbacks import Callback - -import numpy as np -import warnings - - -def get_args(): - argparser = argparse.ArgumentParser(description=__doc__) - argparser.add_argument( - '-c', '--config', - dest='config', - metavar='C', - default='None', - help='The Configuration file') - args = argparser.parse_args() - return args - - -class MultiGPUCheckpointCallback(Callback): - - def __init__(self, filepath, base_model, monitor='val_loss', verbose=0, - save_best_only=False, save_weights_only=False, - mode='auto', period=1): - super(MultiGPUCheckpointCallback, self).__init__() - self.base_model = base_model - self.monitor = monitor - self.verbose = verbose - self.filepath = filepath - self.save_best_only = save_best_only - self.save_weights_only = save_weights_only - self.period = period - self.epochs_since_last_save = 0 - - if mode not in ['auto', 'min', 'max']: - warnings.warn('ModelCheckpoint mode %s is unknown, ' - 'fallback to auto mode.' % (mode), - RuntimeWarning) - mode = 'auto' - - if mode == 'min': - self.monitor_op = np.less - self.best = np.Inf - elif mode == 'max': - self.monitor_op = np.greater - self.best = -np.Inf - else: - if 'acc' in self.monitor or self.monitor.startswith('fmeasure'): - self.monitor_op = np.greater - self.best = -np.Inf - else: - self.monitor_op = np.less - self.best = np.Inf - - def on_epoch_end(self, epoch, logs=None): - logs = logs or {} - self.epochs_since_last_save += 1 - if self.epochs_since_last_save >= self.period: - self.epochs_since_last_save = 0 - filepath = self.filepath.format(epoch=epoch + 1, **logs) - if self.save_best_only: - current = logs.get(self.monitor) - if current is None: - warnings.warn('Can save best model only with %s available, ' - 'skipping.' % (self.monitor), RuntimeWarning) - else: - if self.monitor_op(current, self.best): - if self.verbose > 0: - print('Epoch %05d: %s improved from %0.5f to %0.5f,' - ' saving model to %s' - % (epoch + 1, self.monitor, self.best, - current, filepath)) - self.best = current - if self.save_weights_only: - self.base_model.save_weights( - filepath, overwrite=True) - else: - self.base_model.save(filepath, overwrite=True) - else: - if self.verbose > 0: - print('Epoch %05d: %s did not improve' % - (epoch + 1, self.monitor)) - else: - if self.verbose > 0: - print('Epoch %05d: saving model to %s' % - (epoch + 1, filepath)) - if self.save_weights_only: - self.base_model.save_weights(filepath, overwrite=True) - else: - self.base_model.save(filepath, overwrite=True) diff --git a/MulitiScaleCNNAE.py b/Muliti-Scale_CNN-AE.py similarity index 100% rename from MulitiScaleCNNAE.py rename to Muliti-Scale_CNN-AE.py diff --git a/README.md b/README.md index 7020925..6c9794e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # Machine-learning-based reduced-order modeling for unsteady flows around bluff bodies of various shapes -This repository contains the source codes of "Machine-learning-based reduced-order modeling for unsteady flows around bluff bodies of various shapes" [Theor. Comput. Fluid Dyn. 34, 367-383 (2020).][thesis] (Preprint: [arXiv:2003.07548 [physics.flu-dyn]][airxiv]) +This repository contains the simple source codes of "Machine-learning-based reduced-order modeling for unsteady flows around bluff bodies of various shapes," [Theor. Comput. Fluid Dyn. 34, 367-383 (2020).][thesis] (Preprint: [arXiv:2003.07548 [physics.flu-dyn]][airxiv]) # Informations -Author: Kazuto Hasegawa ([Keio University][fukagatalab]) +Author: Kazuto Hasegawa ([Keio University][fukagatalab], Politecnico di Milano) This repository consists -1. Make_models (for creation of CNN and LSTM models with arbitrary options) -2. MultiScaleCNNAE.py (to create Multi-scale CNN-AE easily) -3. LSTM_with_shape.py (to create LSTM model easily) +1. Multi-Scale_CNN-AE.py (to create Multi-scale CNN-AE) +2. LSTM_with_shape.py (to create LSTM model) For citations, please use the reference below: > K. Hasegawa, K. Fukami, T. Murata, and K. Fukagata, @@ -25,15 +24,14 @@ The code is written for educational clarity and not for speed. * numpy * pandas * tqdm - * dotmap # Directory structure ML-ROM_Various_Shapes ── CNN_autoencoder/ ├─ Make_models/ ├─ data ─── pickles ─── data_001.pikle ~ data080.pickle │ │ └─ Test_data/data_001.pikle ~ data020.pickle - │ └─ LSTM ─── Dataset - │ └─ Flags + │ └─ LSTM ─── Dataset/ + │ └─ Flags/ ├─ .gitignore ├─ LSTM_with_shape.py ├─ MultiScaleCNNAE.py