From a6866d3020979e6dca34caac4b7ecaebb882d273 Mon Sep 17 00:00:00 2001 From: Arne Nix Date: Wed, 25 May 2022 10:11:22 +0200 Subject: [PATCH] New Singularity version + deleted tracking --- Singularity.v0.2.def | 2 + Singularity.v0.4.def | 114 +++++++++++++ nntransfer/analysis/plot.py | 14 +- nntransfer/configs/trainer/base.py | 4 +- nntransfer/trainer/trainer.py | 1 + nntransfer/trainer/utils/tracking.py | 244 --------------------------- 6 files changed, 128 insertions(+), 251 deletions(-) create mode 100644 Singularity.v0.4.def delete mode 100644 nntransfer/trainer/utils/tracking.py diff --git a/Singularity.v0.2.def b/Singularity.v0.2.def index 34ef05d..7d1c374 100644 --- a/Singularity.v0.2.def +++ b/Singularity.v0.2.def @@ -18,6 +18,8 @@ From: ghcr.io/sinzlab/pytorch-singularity:v3.9-torch1.10.2-dj0.12.7.def # dvipng \ + + python3.9 -m pip --no-cache-dir install \ ipykernel \ requests \ diff --git a/Singularity.v0.4.def b/Singularity.v0.4.def new file mode 100644 index 0000000..d5d616c --- /dev/null +++ b/Singularity.v0.4.def @@ -0,0 +1,114 @@ +BootStrap: docker +From: ubuntu:21.10 + +%labels + MAINTAINER Arne Nix +%files + ../bias_transfer /src/bias_transfer + ../nntransfer /src/nntransfer + ../neuralpredictors /src/neuralpredictors + ../pytorch_warmup /src/pytorch_warmup + ../nnfabrik /src/nnfabrik + ../ffcv /src/ffcv + +%environment + export PATH="/opt/conda/bin:${PATH}" + +%post + apt update && apt install -y software-properties-common + add-apt-repository universe + + apt update && apt install -y \ + build-essential \ + git \ + wget \ + vim \ + curl \ + zip \ + zlib1g-dev \ + unzip \ + pkg-config \ + libblas-dev \ + liblapack-dev \ + python3-tk \ + python3-wheel \ + graphviz \ + libhdf5-dev \ + texlive-latex-extra \ + texlive-fonts-recommended \ + texlive-base \ + dvipng + + + # apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* + # wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + # && mkdir /root/.conda \ + # && bash Miniconda3-latest-Linux-x86_64.sh -b \ + # && rm -f Miniconda3-latest-Linux-x86_64.sh + + curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda install -y python=${PYTHON_VERSION} conda-build pyyaml numpy ipython&& \ + /opt/conda/bin/conda clean -ya + + export PATH="/opt/conda/bin:${PATH}" + + # conda create -y -n ffcv python=3.9 ipython cupy pkg-config compilers libjpeg-turbo opencv pytorch torchvision cudatoolkit=11.3 numba -c pytorch -c conda-forge + conda create -y -n ffcv python=3.9 ipython cupy pkg-config compilers libjpeg-turbo opencv numba -c conda-forge + + echo "source activate" >> ~/.bashrc + echo "conda activate ffcv" >> ~/.bashrc + + conda run -n ffcv pip --no-cache-dir install \ + torch==1.10.2+cu113 \ + torchvision==0.11.3+cu113 \ + torchaudio==0.10.2+cu113 \ + -f https://download.pytorch.org/whl/cu113/torch_stable.html + + + conda run -n ffcv pip install --no-cache-dir jupyterlab_vim \ + checkout_code \ + requests \ + imageio \ + scikit-image \ + einops \ + vit-pytorch \ + blackcellmagic\ + pytest \ + pytest-cov \ + numpy \ + matplotlib \ + scipy \ + pandas \ + jupyter \ + scikit-learn \ + scikit-image \ + seaborn \ + graphviz \ + gpustat \ + h5py \ + gitpython \ + Pillow==8.0.1 \ + jupyterlab \ + ipykernel \ + opencv-python \ + datajoint==0.12.7 + + conda run -n ffcv pip install -e /src/bias_transfer + conda run -n ffcv pip install -e /src/nntransfer + conda run -n ffcv pip install -e /src/nnfabrik + conda run -n ffcv pip install -e /src/neuralpredictors + conda run -n ffcv pip install -e /src/pytorch_warmup + conda run -n ffcv pip install -e /src/ffcv + + conda run -n ffcv python -m ipykernel install --user --name=ffcv + +%startscript + exec "$@" + +%runscript + exec "$@" + + diff --git a/nntransfer/analysis/plot.py b/nntransfer/analysis/plot.py index 57e98c3..bb8a4e4 100644 --- a/nntransfer/analysis/plot.py +++ b/nntransfer/analysis/plot.py @@ -53,10 +53,10 @@ def plot_wrapper(*args, **kwargs): if "talk" in style or "beamer" in style: sns.set_context("talk") font = "sans-serif" - small_fontsize = 13 - xsmall_fontsize = 13 - normal_fontsize = 14 - large_fontsize = 17 + small_fontsize = 14 + xsmall_fontsize = 14 + normal_fontsize = 15 + large_fontsize = 16 else: sns.set_context("paper") font = "serif" @@ -106,7 +106,9 @@ def plot_wrapper(*args, **kwargs): for i in range(len(ax)): ax[i] = [ax[i]] - plt.grid(True, linestyle=":", linewidth=0.5) + for i in range(len(ax)): + for j in range(len(ax[i])): + ax[i][j].grid(True, linestyle=":", linewidth=0.8) # execute the actual plotting plot_function(*args, fig=fig, ax=ax, **kwargs) @@ -215,11 +217,13 @@ def set_size(style, ratio=None, fraction=1, subplots=(1, 1), gridspec_kw=None): def save_plot(fig, name, types=("pgf", "pdf", "png")): for file_type in types: + fig.patch.set_alpha(0.0) fig.savefig( name + "." + file_type, facecolor=fig.get_facecolor(), edgecolor=fig.get_edgecolor(), bbox_inches="tight", + transparent=True ) plt.close(fig) diff --git a/nntransfer/configs/trainer/base.py b/nntransfer/configs/trainer/base.py index a1d402e..289a96f 100644 --- a/nntransfer/configs/trainer/base.py +++ b/nntransfer/configs/trainer/base.py @@ -27,10 +27,10 @@ def __init__(self, **kwargs): "keep_selection": (), } self.keep_checkpoints: bool = False - self.patience: int = 10 + self.patience: int = None self.threshold: float = 0.0001 self.verbose: bool = False - self.lr_milestones: Tuple = (30, 60) + self.lr_milestones: Tuple = () self.min_lr: float = 0.00001 # lr scheduler min learning rate self.threshold_mode: str = "rel" self.train_cycler: str = "LongCycler" diff --git a/nntransfer/trainer/trainer.py b/nntransfer/trainer/trainer.py index c047566..9714d39 100644 --- a/nntransfer/trainer/trainer.py +++ b/nntransfer/trainer/trainer.py @@ -331,6 +331,7 @@ def train(self): test_result = self.test_final_model(epoch) copy_ensemble_param_to_buffer(self.model, self.config.ensemble_iteration) + print(self.model.state_dict().keys()) if self.config.switch_teacher: return ( diff --git a/nntransfer/trainer/utils/tracking.py b/nntransfer/trainer/utils/tracking.py deleted file mode 100644 index 8030ad2..0000000 --- a/nntransfer/trainer/utils/tracking.py +++ /dev/null @@ -1,244 +0,0 @@ -import collections -import copy -import time -from collections import defaultdict -from typing import Dict - -import numpy as np - - -def deep_update(d, u): - for k, v in u.items(): - if isinstance(v, collections.abc.Mapping): - d[k] = deep_update(d.get(k, {}), v) - else: - d[k] = v - return d - - -class Tracker: - def log_objective(self, obj): - raise NotImplementedError() - - def finalize(self, obj): - pass - - -class TimeObjectiveTracker(Tracker): - def __init__(self): - self.tracker = np.array([[time.time(), 0.0]]) - - def log_objective(self, obj): - new_track_point = np.array([[time.time(), obj]]) - self.tracker = np.concatenate((self.tracker, new_track_point), axis=0) - - def finalize(self): - self.tracker[:, 0] -= self.tracker[0, 0] - - -class MultipleObjectiveTracker(Tracker): - def __init__(self, **objectives): - self.objectives = objectives - self.log = defaultdict(list) - self.time = [] - - def log_objective(self, obj): - t = time.time() - self.time.append(t) - for name, objective in self.objectives.items(): - self.log[name].append(objective()) - - def finalize(self): - self.time = np.array(self.time) - self.time -= self.time[0] - for k, l in self.log.items(): - self.log[k] = np.array(l) - - -class AdvancedMultipleObjectiveTracker(Tracker): - def _initialize_log(self, objectives): - log = {} - for key, objective in objectives.items(): - if isinstance(objective, dict): - log[key] = self._initialize_log(objective) - elif not callable(objective): - log[key] = [] - return log - - def __init__(self, main_objective=(), save_each_epoch=True, **objectives): - """ - In principle, `objectives` is expected to be a dictionary of dictionaries - The hierarchy can in principle be arbitrary deep. - The only restriction is that the lowest level has to be a dictionary with values being - either a numerical value which will be interpreted as the intial value for this objective - (to be accumulated manually) or a callable (e.g. a function) that returns the objective value. - - :param objectives: e.g. {"dataset": {"objective1": o_fct1, "objective2": 0, "normalization": 0},...} - or {"dataset": {"task_key": {"objective1": o_fct1, "objective2": 0},...},...} - - """ - self.objectives = objectives - self.log = self._initialize_log(objectives) - self.time = [] - self.main_objective = main_objective - self.epoch = -1 - self.best_epoch = -1 - if not save_each_epoch: - self.save_each_epoch = True - self.start_epoch() # add first element to arrays - self.save_each_epoch = save_each_epoch - - def add_objectives(self, objectives, init_epoch=False): - deep_update(self.objectives, objectives) - new_log = self._initialize_log(objectives) - if init_epoch: - _save_each_epoch = self.save_each_epoch - self.save_each_epoch = True - self._initialize_epoch(new_log, objectives) - self.save_each_epoch = _save_each_epoch - deep_update(self.log, new_log) - - def _initialize_epoch(self, log, objectives): - for key, objective in objectives.items(): - if isinstance(objective, dict): - self._initialize_epoch(log[key], objective) - elif not callable(objective): - if self.save_each_epoch: - while len(log[key]) <= self.epoch: - log[key].append(objective) - else: - log[key][0] = objective - - def start_epoch(self): - t = time.time() - if self.save_each_epoch: - self.time.append(t) - self.epoch += 1 - self._initialize_epoch(self.log, self.objectives) - - def _log_objective_value(self, obj, log, keys=()): - if len(keys) > 1: - self._log_objective_value(obj, log[keys[0]], keys[1:]) - else: - log[keys[0]][self.epoch] += obj - - def _log_objective_callables(self, log, objectives): - for key, objective in objectives.items(): - if isinstance(objective, dict): - self._log_objective_callables(log[key], objective) - elif callable(objective): - log[key].append(objective()) - - def log_objective(self, obj, keys=()): - if keys: - self._log_objective_value(obj, self.log, keys) - else: - self._log_objective_callables(self.log, self.objectives) - - def _normalize_log(self, log): - if isinstance(log, dict): - n_log = {} - norm = None - for key, l in log.items(): - res = self._normalize_log(l) # to turn into arrays - if key == "normalization": - assert isinstance(res, np.ndarray) - norm = res - else: - n_log[key] = res - if norm is not None: - nonzero_start = (norm != 0).argmax(axis=0) - norm = norm[nonzero_start:] - for key, l in n_log.items(): - l = l[nonzero_start:] - if isinstance(l, np.ndarray): - n_log[key] = l / np.where(norm > 0, norm, np.ones_like(norm)) - return n_log - else: - return np.array(log) - - def _gather_log(self, log, keys, index=-1): - if len(keys) > 1: - return self._gather_log(log[keys[0]], keys[1:], index) - elif keys: - return self._gather_log(log[keys[0]], (), index) - elif isinstance(log, dict): - gathered = {} - for key, l in log.items(): - logs = self._gather_log(l, (), index) - for k, v in logs.items(): - gathered[key + " " + k] = v - return gathered - else: - return {"": "{:03.4f}".format(log[index])} - - def display_log(self, keys=(), tqdm_iterator=None): - # normalize (if possible) and turn into np.arrays: - n_log = self._normalize_log(self.log) - # flatten a subset of the dictionary: - current_log = self._gather_log(n_log, keys, index=-1) - if tqdm_iterator: - tqdm_iterator.set_postfix(**current_log) - else: - print(current_log) - - def _check_isfinite(self, log): - if isinstance(log, dict): - for k, l in log.items(): - if not self._check_isfinite(l): - return False - else: - return np.isfinite(log).any() - return True - - def check_isfinite(self): - return self._check_isfinite(self._normalize_log(self.log)) - - def _get_objective(self, log, keys): - if len(keys) > 1: - return self._get_objective(log[keys[0]], keys[1:]) - else: - return log[keys[0]] - - def get_objective(self, *keys): - return self._get_objective(self._normalize_log(self.log), keys) - - def get_current_objective(self, *keys): - return self._get_objective(self._normalize_log(self.log), keys)[-1] - - def get_current_main_objective(self, *keys): - return self.get_current_objective(*keys, *self.main_objective) - - def finalize(self): - self.time = np.array(self.time) - self.time -= self.time[0] - self.log = self._normalize_log(self.log) - - def state_dict(self): - """ - Serializes this instance to a Python dictionary. - - Returns: - :obj:`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, - """ - output = copy.deepcopy(self.__dict__) - return output - - def load_state_dict(self, tracker_dict: Dict): - self.main_objective = tracker_dict["main_objective"] - self.objectives = tracker_dict["objectives"] - self.log = tracker_dict["log"] - self.time = tracker_dict["time"] - self.epoch = tracker_dict["epoch"] - self.best_epoch = tracker_dict["best_epoch"] - - @classmethod - def from_dict(cls, tracker_dict: Dict) -> "AdvancedMultipleObjectiveTracker": - tracker = cls( - main_objective=tracker_dict["main_objective"], **tracker_dict["objectives"] - ) - tracker.log = tracker_dict["log"] - tracker.time = tracker_dict["time"] - tracker.epoch = tracker_dict["epoch"] - tracker.best_epoch = tracker_dict["best_epoch"] - return tracker