03_learner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# Contains code used/modified by fastai_minima author from fastai\n",
    "# Copyright 2019 the fast.ai team.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.test import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import pickle\n",
    "from contextlib import contextmanager\n",
    "from functools import partial\n",
    "\n",
    "from fastcore.basics import GetAttr, add_props, store_attr, detuplify, patch, Self\n",
    "from fastcore.foundation import L\n",
    "from fastcore.meta import delegates\n",
    "from fastcore.xtras import Path, join_path_file, ContextManagers\n",
    "\n",
    "from fastai_minima.callback.core import (\n",
    "    event, Callback, GatherPredsCallback,\n",
    "    CancelStepException,CancelFitException,\n",
    "    CancelEpochException,CancelTrainException,\n",
    "    CancelValidException,CancelBatchException,\n",
    "    TrainEvalCallback, GatherPredsCallback, \n",
    ")\n",
    "from fastai_minima.optimizer import Adam\n",
    "from fastai_minima.utils import (\n",
    "    defaults, trainable_params, noop, norm_bias_params, \n",
    "    find_device, tensor, rank_distrib, get_model, distrib_barrier, NoneType,\n",
    "    to_detach, find_bs\n",
    ")\n",
    "\n",
    "import torch, time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from builtins import NotImplementedError"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learner, Metrics, and Basic Callbacks\n",
    "\n",
    "> Basic class for handling the training loop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the actual fastai documentation, you should go to the [Learner](docs.fast.ai/Learner) documentation. These are minimal docs simply to bring in the source code and related tests to ensure that minimal functionality is met"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You probably want to jump directly to the definition of `Learner`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DataLoaders(GetAttr):\n",
    "    \"Basic wrapper around several `DataLoader`s.\"\n",
    "    _default='train'\n",
    "    def __init__(self, *loaders, path='.'):\n",
    "        self.loaders,self.path = list(loaders),Path(path)\n",
    "\n",
    "    def __getitem__(self, i): return self.loaders[i]\n",
    "    def new_empty(self):\n",
    "        loaders = [dl.new(dl.dataset.new_empty()) for dl in self.loaders]\n",
    "        return type(self)(*loaders, path=self.path, device=self.device)\n",
    "\n",
    "    def _set(i, self, v): self.loaders[i] = v\n",
    "    train   ,valid    = add_props(lambda i,x: x[i], _set)\n",
    "    train_ds,valid_ds = add_props(lambda i,x: x[i].dataset)\n",
    "    \n",
    "    def one_batch(self, ds_idx=0):\n",
    "        \"Grab one batch of data from the `DataLoader` at `ds_idx` in `self.loaders`\"\n",
    "        return next(iter(self.loaders[ds_idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utils function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#For tests\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "def synth_dbunch(a=2, b=3, bs=16, n_train=10, n_valid=2):\n",
    "    \"A simple dataset where `x` is random and `y = a*x + b` plus some noise.\"\n",
    "    def get_data(n):\n",
    "        x = torch.randn(int(bs*n))\n",
    "        return TensorDataset(x, a*x + b + 0.1*torch.randn(int(bs*n)))\n",
    "    train_ds = get_data(n_train)\n",
    "    valid_ds = get_data(n_valid)\n",
    "    train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True, num_workers=0)\n",
    "    valid_dl = DataLoader(valid_ds, batch_size=bs, num_workers=0)\n",
    "    return DataLoaders(train_dl, valid_dl)\n",
    "\n",
    "class RegModel(nn.Module):\n",
    "    \"A r\"\n",
    "    def __init__(self): \n",
    "        super().__init__()\n",
    "        self.a,self.b = nn.Parameter(torch.randn(1)),nn.Parameter(torch.randn(1))\n",
    "    def forward(self, x): return x*self.a + self.b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "defaults.lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def replacing_yield(o, attr, val):\n",
    "    \"Context manager to temporarily replace an attribute\"\n",
    "    old = getattr(o,attr)\n",
    "    try:     yield setattr(o,attr,val)\n",
    "    finally: setattr(o,attr,old)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _A:\n",
    "    def __init__(self, a): self.a = a\n",
    "    @contextmanager\n",
    "    def a_changed(self, v): return replacing_yield(self, 'a', v)\n",
    "\n",
    "a = _A(42)\n",
    "with a.a_changed(32):\n",
    "    test_eq(a.a, 32)\n",
    "test_eq(a.a, 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mk_metric(m):\n",
    "    \"Convert `m` to an `AvgMetric`, unless it's already a `Metric`\"\n",
    "    if isinstance(m,type): m = m()\n",
    "    return m if isinstance(m, Metric) else AvgMetric(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See the class `Metric` below for more information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def save_model(file, model, opt, with_opt=True, pickle_protocol=2):\n",
    "    \"Save `model` to `file` along with `opt` (if available, and if `with_opt`)\"\n",
    "    if rank_distrib(): return # don't save if child proc\n",
    "    if opt is None: with_opt=False\n",
    "    state = get_model(model).state_dict()\n",
    "    if with_opt: state = {'model': state, 'opt':opt.state_dict()}\n",
    "    torch.save(state, file, pickle_protocol=pickle_protocol)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` can be a `Path` object, a string or an opened file object. `pickle_protocol` is passed along to `torch.save`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def load_model(file, model, opt, with_opt=True, device=None, strict=True):\n",
    "    \"Load `model` from `file` along with `opt` (if available, and if `with_opt`)\"\n",
    "    distrib_barrier()\n",
    "    if isinstance(device, int): device = torch.device('cuda', device)\n",
    "    elif device is None: device = 'cpu'\n",
    "    state = torch.load(file, map_location=device)\n",
    "    hasopt = set(state)=={'model', 'opt'}\n",
    "    model_state = state['model'] if hasopt else state\n",
    "    get_model(model).load_state_dict(model_state, strict=strict)\n",
    "    if hasopt and with_opt:\n",
    "        try: opt.load_state_dict(state['opt'])\n",
    "        except:\n",
    "            if with_opt: warn(\"Could not load the optimizer state.\")\n",
    "    elif with_opt: warn(\"Saved filed doesn't contain an optimizer state.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` can be a `Path` object, a string or an opened file object. If a `device` is passed, the model is loaded on it, otherwise it's loaded on the CPU. \n",
    "\n",
    "If `strict` is `True`, the file must exactly contain weights for every parameter key in `model`, if `strict` is `False`, only the keys that are in the saved model are loaded in `model`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def _try_concat(o):\n",
    "    try:    return torch.cat(o)\n",
    "    except: return sum([L(o_[i,:] for i in range_of(o_)) for o_ in o], L())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "_before_epoch = [event.before_fit, event.before_epoch]\n",
    "_after_epoch  = [event.after_epoch, event.after_fit]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class _ConstantFunc():\n",
    "    \"Returns a function that returns `o`\"\n",
    "    def __init__(self, o): self.o = o\n",
    "    def __call__(self, *args, **kwargs): return self.o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learner -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "_loop = ['Start Fit', 'before_fit', 'Start Epoch Loop', 'before_epoch', 'Start Train', 'before_train',\n",
    "         'Start Batch Loop', 'before_batch', 'after_pred', 'after_loss', 'before_backward', 'before_step',\n",
    "         'after_step', 'after_cancel_batch', 'after_batch','End Batch Loop','End Train',\n",
    "         'after_cancel_train', 'after_train', 'Start Valid', 'before_validate','Start Batch Loop',\n",
    "         '**CBs same as train batch**', 'End Batch Loop', 'End Valid', 'after_cancel_validate',\n",
    "         'after_validate', 'End Epoch Loop', 'after_cancel_epoch', 'after_epoch', 'End Fit',\n",
    "         'after_cancel_fit', 'after_fit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class Learner(GetAttr):\n",
    "    _default='model'\n",
    "    def __init__(self, dls, model, loss_func=None, opt_func=Adam, lr=defaults.lr, splitter=trainable_params, cbs=None,\n",
    "                 metrics=None, path=None, model_dir='models', wd=None, wd_bn_bias=False, train_bn=True,\n",
    "                 moms=(0.95,0.85,0.95)):\n",
    "        \"Group together a `model`, some `dls` and a `loss_func` to handle training\"\n",
    "        path = Path(path) if path is not None else getattr(dls, 'path', Path('.'))\n",
    "        if loss_func is None:\n",
    "            loss_func = getattr(dls.train_ds, 'loss_func', None)\n",
    "            assert loss_func is not None, \"Could not infer loss function from the data, please pass a loss function.\"\n",
    "        self.dls,self.model = dls,model\n",
    "        store_attr(but='dls,model,cbs')\n",
    "        self.training,self.create_mbar,self.logger,self.opt,self.cbs = False,True,print,None,L()\n",
    "        self.add_cbs(L(defaults.callbacks)+L(cbs))\n",
    "        self(\"after_create\")\n",
    "\n",
    "    @property\n",
    "    def metrics(self): return self._metrics\n",
    "    @metrics.setter\n",
    "    def metrics(self,v): self._metrics = L(v).map(mk_metric)\n",
    "\n",
    "    def _grab_cbs(self, cb_cls): return L(cb for cb in self.cbs if isinstance(cb, cb_cls))\n",
    "\n",
    "    def add_cbs(self, cbs):\n",
    "        \"Add `cbs` to the list of `Callback` and register `self` as their learner\"\n",
    "        L(cbs).map(self.add_cb)\n",
    "        return self\n",
    "\n",
    "    def remove_cbs(self, cbs):\n",
    "        \"Remove `cbs` from the list of `Callback` and deregister `self` as their learner\"\n",
    "        L(cbs).map(self.remove_cb)\n",
    "        return self\n",
    "\n",
    "    def add_cb(self, cb):\n",
    "        \"Add `cb` to the list of `Callback` and register `self` as their learner\"\n",
    "        if isinstance(cb, type): cb = cb()\n",
    "        cb.learn = self\n",
    "        setattr(self, cb.name, cb)\n",
    "        self.cbs.append(cb)\n",
    "        return self\n",
    "\n",
    "    def remove_cb(self, cb):\n",
    "        \"Add `cb` from the list of `Callback` and deregister `self` as their learner\"\n",
    "        if isinstance(cb, type): self.remove_cbs(self._grab_cbs(cb))\n",
    "        else:\n",
    "            cb.learn = None\n",
    "            if hasattr(self, cb.name): delattr(self, cb.name)\n",
    "            if cb in self.cbs: self.cbs.remove(cb)\n",
    "        return self\n",
    "\n",
    "    @contextmanager\n",
    "    def added_cbs(self, cbs):\n",
    "        \"Context manage that temporarily adds `cbs`\"\n",
    "        self.add_cbs(cbs)\n",
    "        try: yield\n",
    "        finally: self.remove_cbs(cbs)\n",
    "\n",
    "    @contextmanager\n",
    "    def removed_cbs(self, cbs):\n",
    "        \"Context manage that temporarily removes `cbs`\"\n",
    "        self.remove_cbs(cbs)\n",
    "        try: yield self\n",
    "        finally: self.add_cbs(cbs)\n",
    "\n",
    "    def ordered_cbs(self, event): \n",
    "        \"List of `Callback`s, in order, for an `event` in the training loop\"\n",
    "        return [cb for cb in self.cbs.sorted('order') if hasattr(cb, event)]\n",
    "    def __call__(self, event_name): \n",
    "        \"Call `event_name` for all `Callback`s in `self.cbs`\"\n",
    "        L(event_name).map(self._call_one)\n",
    "\n",
    "    def _call_one(self, event_name):\n",
    "        if not hasattr(event, event_name): raise Exception(f'missing {event_name}')\n",
    "        for cb in self.cbs.sorted('order'): cb(event_name)\n",
    "\n",
    "    def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)\n",
    "    def create_opt(self):\n",
    "        \"Create an optimizer with default hyper-parameters\"\n",
    "        self.opt = self.opt_func(self.splitter(self.model), lr=self.lr)\n",
    "        if not self.wd_bn_bias:\n",
    "            for p in self._bn_bias_state(True ): p['do_wd'] = False\n",
    "        if self.train_bn:\n",
    "            for p in self._bn_bias_state(False): p['force_train'] = True\n",
    "\n",
    "    def _split(self, b):\n",
    "        i = getattr(self.dls, 'n_inp', 1 if len(b)==1 else len(b)-1)\n",
    "        self.xb,self.yb = b[:i],b[i:]\n",
    "\n",
    "    def _with_events(self, f, event_type, ex, final=noop):\n",
    "        try: self(f'before_{event_type}');  f()\n",
    "        except ex: self(f'after_cancel_{event_type}')\n",
    "        self(f'after_{event_type}');  final()\n",
    "\n",
    "    def all_batches(self):\n",
    "        \"Train or evaluate `self.model` on all the batches of `self.dl`\"\n",
    "        self.n_iter = len(self.dl)\n",
    "        for o in enumerate(self.dl): self.one_batch(*o)\n",
    "\n",
    "    def _do_one_batch(self):\n",
    "        self.pred = self.model(*self.xb)\n",
    "        self('after_pred')\n",
    "        if len(self.yb):\n",
    "            self.loss_grad = self.loss_func(self.pred, *self.yb)\n",
    "            self.loss = self.loss_grad.clone()\n",
    "        self('after_loss')\n",
    "        if not self.training or not len(self.yb): return\n",
    "        self('before_backward')\n",
    "        self.loss_grad.backward()\n",
    "        self._with_events(self.opt.step, 'step', CancelStepException)\n",
    "        self.opt.zero_grad()\n",
    "\n",
    "    def one_batch(self, i, b):\n",
    "        \"Train or evaluate `self.model` on batch `(xb,yb)`\"\n",
    "        self.iter = i\n",
    "        self._split(b)\n",
    "        self._with_events(self._do_one_batch, 'batch', CancelBatchException)\n",
    "\n",
    "    def _do_epoch_train(self):\n",
    "        self.dl = self.dls.train\n",
    "        self._with_events(self.all_batches, 'train', CancelTrainException)\n",
    "\n",
    "    def _do_epoch_validate(self, ds_idx=1, dl=None):\n",
    "        if dl is None: dl = self.dls[ds_idx]\n",
    "        self.dl = dl\n",
    "        with torch.no_grad(): self._with_events(self.all_batches, 'validate', CancelValidException)\n",
    "\n",
    "    def _do_epoch(self):\n",
    "        self._do_epoch_train()\n",
    "        self._do_epoch_validate()\n",
    "\n",
    "    def _do_fit(self):\n",
    "        for epoch in range(self.n_epoch):\n",
    "            self.epoch=epoch\n",
    "            self._with_events(self._do_epoch, 'epoch', CancelEpochException)\n",
    "\n",
    "    def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):\n",
    "        \"Fit `self.model` for `n_epoch` using `cbs`. Optionally `reset_opt`.\"\n",
    "        with self.added_cbs(cbs):\n",
    "            if reset_opt or not self.opt: self.create_opt()\n",
    "            if wd is None: wd = self.wd\n",
    "            if wd is not None: self.opt.set_hypers(wd=wd)\n",
    "            self.opt.set_hypers(lr=self.lr if lr is None else lr)\n",
    "            self.n_epoch = n_epoch\n",
    "            self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)\n",
    "\n",
    "    def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None\n",
    "    def __enter__(self): self(_before_epoch); return self\n",
    "    def __exit__(self, exc_type, exc_value, tb): self(_after_epoch)\n",
    "\n",
    "    def validation_context(self, cbs=None, inner=False):\n",
    "        \"A `ContextManagers` suitable for validation, with optional `cbs`\"\n",
    "        cms = [self.no_logging(),self.no_mbar()]\n",
    "        if cbs: cms.append(self.added_cbs(cbs))\n",
    "        if not inner: cms.append(self)\n",
    "        return ContextManagers(cms)\n",
    "\n",
    "    def validate(self, ds_idx=1, dl=None, cbs=None):\n",
    "        \"Validate on `dl` with potential new `cbs`.\"\n",
    "        if dl is None: dl = self.dls[ds_idx]\n",
    "        with self.validation_context(cbs=cbs): self._do_epoch_validate(ds_idx, dl)\n",
    "        return getattr(self, 'final_record', None)\n",
    "\n",
    "    @delegates(GatherPredsCallback.__init__)\n",
    "    def get_preds(self, ds_idx=1, dl=None, with_input=False, with_decoded=False, with_loss=False, act=None,\n",
    "                  inner=False, reorder=False, cbs=None, **kwargs):\n",
    "        \"Get the predictions and targets on the `ds_idx`-th dbunchset or `dl`, optionally `with_input` and `with_loss`\"\n",
    "        if dl is None: dl = self.dls[ds_idx]\n",
    "        else:\n",
    "            try: len(dl)\n",
    "            except TypeError as e:\n",
    "                raise TypeError(\"`dl` is something other than a single `DataLoader` object\")\n",
    "        if reorder and hasattr(dl, 'get_idxs'):\n",
    "            raise NotImplementedError(\"You're trying to use non-basic fastai functionality. You should use the entire fastai API to get this feature\")\n",
    "        cb = GatherPredsCallback(with_input=with_input, with_loss=with_loss, **kwargs)\n",
    "        ctx_mgrs = self.validation_context(cbs=L(cbs)+[cb], inner=inner)\n",
    "        if with_loss: ctx_mgrs.append(self.loss_not_reduced())\n",
    "        with ContextManagers(ctx_mgrs):\n",
    "            self._do_epoch_validate(dl=dl)\n",
    "            if act is None: act = getattr(self.loss_func, 'activation', noop)\n",
    "            res = cb.all_tensors()\n",
    "            pred_i = 1 if with_input else 0\n",
    "            if res[pred_i] is not None:\n",
    "                res[pred_i] = act(res[pred_i])\n",
    "                if with_decoded: res.insert(pred_i+2, getattr(self.loss_func, 'decodes', noop)(res[pred_i]))\n",
    "            if reorder and hasattr(dl, 'get_idxs'): res = nested_reorder(res, tensor(idxs).argsort())\n",
    "            return tuple(res)\n",
    "        self._end_cleanup()\n",
    "\n",
    "    def predict(self, item, rm_type_tfms=None, with_input=False):\n",
    "        \"Prediction on `item`, fully decoded, loss function decoded and probabilities\"\n",
    "        raise NotImplementedError(\"You're trying to use non-basic fastai functionality. You should use the entire fastai API to get this feature\")\n",
    "\n",
    "    def show_results(self, ds_idx=1, dl=None, max_n=9, shuffle=True, **kwargs):\n",
    "        \"Show some predictions on `ds_idx`-th dataset or `dl`\"\n",
    "        raise NotImplementedError(\"You're trying to use non-basic fastai functionality. You should use the entire fastai API to get this feature\")\n",
    "\n",
    "    def show_training_loop(self):\n",
    "        \"Show each step in the training loop\"\n",
    "        indent = 0\n",
    "        for s in _loop:\n",
    "            if s.startswith('Start'): print(f'{\" \"*indent}{s}'); indent += 2\n",
    "            elif s.startswith('End'): indent -= 2; print(f'{\" \"*indent}{s}')\n",
    "            else: print(f'{\" \"*indent} - {s:15}:', self.ordered_cbs(s))\n",
    "\n",
    "    @contextmanager\n",
    "    def no_logging(self): \n",
    "        \"Context manager to temporarily remove `logger`\"\n",
    "        return replacing_yield(self, 'logger', noop)\n",
    "    @contextmanager\n",
    "    def no_mbar(self):    \n",
    "        \"Context manager to temporarily prevent the master progress bar from being created\"\n",
    "        return replacing_yield(self, 'create_mbar', False)\n",
    "\n",
    "    @contextmanager\n",
    "    def loss_not_reduced(self):\n",
    "        \"A context manager to evaluate `loss_func` with reduction set to none.\"\n",
    "        if hasattr(self.loss_func, 'reduction'): return replacing_yield(self.loss_func, 'reduction', 'none')\n",
    "        else: return replacing_yield(self, 'loss_func', partial(self.loss_func, reduction='none'))\n",
    "\n",
    "    def to_detach(self,b,cpu=True,gather=True):\n",
    "        \"Calls `to_detach` if `self.dl` provides a `.to_detach` function otherwise calls global `to_detach`\"\n",
    "        return self.dl.to_detach(b,cpu,gather) if hasattr(getattr(self,'dl',None),'to_detach') else to_detach(b,cpu,gather)\n",
    "\n",
    "Learner.x,Learner.y = add_props(lambda i,x: detuplify((x.xb,x.yb)[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h2 id=\"Learner\" class=\"doc_header\"><code>class</code> <code>Learner</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h2>\n",
       "\n",
       "> <code>Learner</code>(**`dls`**, **`model`**, **`loss_func`**=*`None`*, **`opt_func`**=*`Adam`*, **`lr`**=*`0.001`*, **`splitter`**=*`trainable_params`*, **`cbs`**=*`None`*, **`metrics`**=*`None`*, **`path`**=*`None`*, **`model_dir`**=*`'models'`*, **`wd`**=*`None`*, **`wd_bn_bias`**=*`False`*, **`train_bn`**=*`True`*, **`moms`**=*`(0.95, 0.85, 0.95)`*) :: `GetAttr`\n",
       "\n",
       "Inherit from this to have all attr accesses in `self._xtra` passed down to `self.default`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`opt_func` will be used to create an optimizer when `Learner.fit` is called, with `lr` as a default learning rate. `splitter` is a function that takes `self.model` and returns a list of parameter groups (or just one parameter group if there are no different parameter groups). The default is `trainable_params`, which returns all trainable parameters of the model.\n",
    "\n",
    "`cbs` is one or a list of `Callback`s  to pass to the `Learner`. `Callback`s are used for every tweak of the training loop. Each `Callback` is registered as an attribute of `Learner` (with camel case). At creation, all the callbacks in `defaults.callbacks` (`TrainEvalCallback`, `Recorder` and `ProgressCallback`) are associated to the `Learner`.\n",
    "\n",
    "`metrics` is an optional list of metrics, that can be either functions or `Metric`s (see below). \n",
    "\n",
    "`path` and `model_dir` are used to save and/or load models. Often `path` will be inferred from `dls`, but you can override it or pass a `Path`  object to `model_dir`. Make sure you can write in `path/model_dir`!\n",
    "\n",
    "`wd` is the default weight decay used when training the model; `moms`, the default momentums used in `Learner.fit_one_cycle`. `wd_bn_bias` controls if weight decay is applied to `BatchNorm` layers and bias. \n",
    "\n",
    "Lastly, `train_bn` controls if `BatchNorm` layers are trained even when they are supposed to be frozen according to the `splitter`. Our empirical experiments have shown that it's the best behavior for those layers in transfer learning."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch interop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can use regular PyTorch functionality for most of the arguments of the `Learner`, although the experience will be smoother with pure fastai objects and you will be able to use the full functionality of the library. The expectation is that the training loop will work smoothly even if you did not use fastai end to end. What you might lose are interpretation objects or showing functionality. The list below explains how to use plain PyTorch objects for all the arguments and what you might lose.\n",
    "\n",
    "The most important is `opt_func`. If you are not using a fastai optimizer, you will need to write a function that wraps your PyTorch optimizer in an `OptimWrapper`. See the [optimizer module](http://docs.fast.ai/optimizer) for more details. This is to ensure the library's schedulers/freeze API work with your code.\n",
    "\n",
    "- `dls` is a `DataLoaders` object, that you can create from standard PyTorch dataloaders. By doing so, you will lose all showing functionality like `show_batch`/`show_results`. You can check the [data block API](http://docs.fast.ai/tutorial.datablock) or the [mid-level data API tutorial](http://docs.fast.ai/tutorial.pets) to learn how to use fastai to gather your data!\n",
    "- `model` is a standard PyTorch model. You can use anyone you like, just make sure it accepts the number of inputs you have in your `DataLoaders` and returns as many outputs as you have targets.\n",
    "- `loss_func` can be any loss function you like. It needs to be one of fastai's if you want to use `Learn.predict` or `Learn.get_preds`, or you will have to implement special methods (see more details after the `BaseLoss` documentation)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training loop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's look at the main thing the `Learner` class implements: the training loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.fit\" class=\"doc_header\"><code>Learner.fit</code><a href=\"__main__.py#L137\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.fit</code>(**`n_epoch`**, **`lr`**=*`None`*, **`wd`**=*`None`*, **`cbs`**=*`None`*, **`reset_opt`**=*`False`*)\n",
       "\n",
       "Fit `self.model` for `n_epoch` using `cbs`. Optionally `reset_opt`."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.fit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uses `lr` and `wd` if they are provided, otherwise use the defaults values given by the `lr` and `wd` attributes of `Learner`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the examples use `synth_learner` which is a simple `Learner` training a linear regression model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "def synth_learner(n_train=10, n_valid=2, lr=defaults.lr, **kwargs):\n",
    "    data = synth_dbunch(n_train=n_train,n_valid=n_valid)\n",
    "    return Learner(data, RegModel(), loss_func=nn.MSELoss(), lr=lr, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/d/lib/python3.7/site-packages/torch/autograd/__init__.py:132: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
      "  allow_unreachable=True)  # allow_unreachable flag\n"
     ]
    }
   ],
   "source": [
    "#Training a few epochs should make the model better\n",
    "learn = synth_learner(lr=0.1)\n",
    "learn(_before_epoch)\n",
    "learn.model = learn.model.cpu()\n",
    "xb,yb = learn.dls.one_batch()\n",
    "init_loss = learn.loss_func(learn.model(xb), yb)\n",
    "learn.fit(10)\n",
    "xb,yb = learn.dls.one_batch()\n",
    "final_loss = learn.loss_func(learn.model(xb), yb)\n",
    "assert final_loss < init_loss, (final_loss,init_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "class TestTrainEvalCallback(Callback):\n",
    "    run_after,run_valid = TrainEvalCallback,False\n",
    "    def before_fit(self): \n",
    "        test_eq([self.pct_train,self.train_iter], [0., 0])\n",
    "        self.old_pct_train,self.old_train_iter = self.pct_train,self.train_iter\n",
    "    \n",
    "    def before_batch(self): test_eq(next(self.parameters()).device, find_device(self.xb))\n",
    "    \n",
    "    def after_batch(self):\n",
    "        assert self.training\n",
    "        test_eq(self.pct_train , self.old_pct_train+1/(self.n_iter*self.n_epoch))\n",
    "        test_eq(self.train_iter, self.old_train_iter+1)\n",
    "        self.old_pct_train,self.old_train_iter = self.pct_train,self.train_iter\n",
    "    \n",
    "    def before_train(self):\n",
    "        assert self.training and self.model.training\n",
    "        test_eq(self.pct_train, self.epoch/self.n_epoch)\n",
    "        self.old_pct_train = self.pct_train\n",
    "    \n",
    "    def before_validate(self):\n",
    "        assert not self.training and not self.model.training\n",
    "        \n",
    "learn = synth_learner(cbs=TestTrainEvalCallback)\n",
    "learn.fit(1)\n",
    "#Check order is properly taken into account\n",
    "learn.cbs = L(reversed(learn.cbs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.one_batch\" class=\"doc_header\"><code>Learner.one_batch</code><a href=\"__main__.py#L113\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.one_batch</code>(**`i`**, **`b`**)\n",
       "\n",
       "Train or evaluate `self.model` on batch `(xb,yb)`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.one_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is an internal method called by `Learner.fit`. If passed, `i` is the index of this iteration in the epoch. In training mode, this does a full training step on the batch (compute predictions, loss, gradients, update the model parameters and zero the gradients). In validation mode, it stops at the loss computation. Training or validation is controlled internally by the `TrainEvalCallback` through the `training` attribute.\n",
    "\n",
    "Nothing is returned, but the attributes `x`, `y`, `pred`, `loss` of the `Learner` are set with the proper values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = learn.dls.one_batch()\n",
    "learn.one_batch(0, b)\n",
    "test_eq(learn.x, b[0])\n",
    "test_eq(learn.y, b[1])\n",
    "out = learn.model(learn.x)\n",
    "test_eq(learn.pred, out)\n",
    "test_eq(learn.loss, learn.loss_func(out, b[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "class VerboseCallback(Callback):\n",
    "    \"Callback that prints the name of each event called\"\n",
    "    def __call__(self, event_name):\n",
    "        print(event_name)\n",
    "        super().__call__(event_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "class TestOneBatch(VerboseCallback):\n",
    "    def __init__(self, xb, yb, i):\n",
    "        self.save_xb,self.save_yb,self.i = xb,yb,i\n",
    "        self.old_pred,self.old_loss = None,tensor(0.)\n",
    "        \n",
    "    def before_batch(self):\n",
    "        self.old_a,self.old_b = self.a.data.clone(),self.b.data.clone()\n",
    "        test_eq(self.iter,    self.i)\n",
    "        test_eq(self.save_xb, *self.xb)\n",
    "        test_eq(self.save_yb, *self.yb)\n",
    "        if hasattr(self.learn, 'pred'): test_eq(self.pred, self.old_pred)\n",
    "    \n",
    "    def after_pred(self):\n",
    "        self.old_pred = self.pred\n",
    "        test_eq(self.pred, self.a.data * self.x + self.b.data)\n",
    "        test_eq(self.loss, self.old_loss)\n",
    "    \n",
    "    def after_loss(self):\n",
    "        self.old_loss = self.loss\n",
    "        test_eq(self.loss, self.loss_func(self.old_pred, self.save_yb))\n",
    "        for p in self.parameters(): \n",
    "            if not hasattr(p, 'grad') or p.grad is not None: test_eq(p.grad, tensor([0.]))\n",
    "    \n",
    "    def before_step(self):\n",
    "        self.grad_a = (2 * self.x * (self.pred.data - self.y)).mean()\n",
    "        self.grad_b = 2 * (self.pred.data - self.y).mean()\n",
    "        test_close(self.a.grad.data, self.grad_a)\n",
    "        test_close(self.b.grad.data, self.grad_b)\n",
    "        test_eq(self.a.data, self.old_a)\n",
    "        test_eq(self.b.data, self.old_b)\n",
    "        \n",
    "    def after_step(self):\n",
    "        test_close(self.a.data, self.old_a - self.lr * self.grad_a)\n",
    "        test_close(self.b.data, self.old_b - self.lr * self.grad_b)\n",
    "        self.old_a,self.old_b = self.a.data.clone(),self.b.data.clone()\n",
    "        test_close(self.a.grad.data, self.grad_a)\n",
    "        test_close(self.b.grad.data, self.grad_b)\n",
    "    \n",
    "    def after_batch(self):\n",
    "        for p in self.parameters(): test_eq(p.grad, tensor([0.]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after_create\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from fastai_minima.optimizer import SGD\n",
    "learn = synth_learner()\n",
    "b = learn.dls.one_batch()\n",
    "learn = synth_learner(cbs=TestOneBatch(*b, 42), lr=1e-2)\n",
    "#Remove train/eval\n",
    "learn.cbs = learn.cbs[1:]\n",
    "#Setup\n",
    "learn.loss,learn.training = tensor(0.),True\n",
    "learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
    "learn.model.train()\n",
    "batch_events = ['before_batch', 'after_pred', 'after_loss', 'before_backward', 'before_step', 'after_step', 'after_batch']\n",
    "test_stdout(lambda: learn.one_batch(42, b), '\\n'.join(batch_events))\n",
    "test_stdout(lambda: learn.one_batch(42, b), '\\n'.join(batch_events)) #Check it works for a second batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.all_batches\" class=\"doc_header\"><code>Learner.all_batches</code><a href=\"__main__.py#L95\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.all_batches</code>()\n",
       "\n",
       "Train or evaluate `self.model` on all the batches of `self.dl`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.all_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after_create\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from contextlib import redirect_stdout\n",
    "import io\n",
    "learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
    "learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
    "with redirect_stdout(io.StringIO()): \n",
    "    learn(_before_epoch)\n",
    "    learn.epoch,learn.dl = 0,learn.dls.train\n",
    "    learn('before_train')\n",
    "test_stdout(learn.all_batches, '\\n'.join(batch_events * 5))\n",
    "test_eq(learn.train_iter, 5)\n",
    "\n",
    "valid_events = ['before_batch', 'after_pred', 'after_loss', 'after_batch']\n",
    "with redirect_stdout(io.StringIO()): \n",
    "    learn.dl = learn.dls.valid\n",
    "    learn('before_validate')\n",
    "test_stdout(learn.all_batches, '\\n'.join(valid_events * 2))\n",
    "test_eq(learn.train_iter, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after_create\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
    "test_stdout(lambda: learn(_before_epoch), 'before_fit\\nbefore_epoch')\n",
    "test_eq(learn.loss, tensor(0.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
    "learn.epoch = 0\n",
    "test_stdout(lambda: learn._do_epoch_train(), '\\n'.join(['before_train'] + batch_events * 5 + ['after_train']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "test_stdout(learn._do_epoch_validate, '\\n'.join(['before_validate'] + valid_events * 2+ ['after_validate']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.create_opt\" class=\"doc_header\"><code>Learner.create_opt</code><a href=\"__main__.py#L78\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.create_opt</code>()\n",
       "\n",
       "Create an optimizer with default hyper-parameters"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.create_opt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This method is called internally to create the optimizer, the hyper-parameters are then adjusted by what you pass to `Learner.fit` or your particular schedulers (see `callback.schedule`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after_create\n"
     ]
    }
   ],
   "source": [
    "learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
    "assert learn.opt is None\n",
    "learn.create_opt()\n",
    "assert learn.opt is not None\n",
    "test_eq(learn.opt.hypers[0]['lr'], learn.lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Callback handling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only describe the basic functionality linked to `Callback`s here. To learn more about `Callback`s an how to write them, check the [callback.core](http://docs.fast.ai/callback.core) module documentation.\n",
    "\n",
    "Let's first see how the `Callback`s become attributes of `Learner`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test init with callbacks\n",
    "class TstCallback(Callback):\n",
    "    def batch_begin(self): self.learn.a = self.a + 1\n",
    "\n",
    "tst_learn = synth_learner()\n",
    "test_eq(len(tst_learn.cbs), 1)\n",
    "assert isinstance(tst_learn.cbs[0], TrainEvalCallback)\n",
    "assert hasattr(tst_learn, ('train_eval'))\n",
    "\n",
    "tst_learn = synth_learner(cbs=TstCallback())\n",
    "test_eq(len(tst_learn.cbs), 2)\n",
    "assert isinstance(tst_learn.cbs[1], TstCallback)\n",
    "assert hasattr(tst_learn, ('tst'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.__call__\" class=\"doc_header\"><code>Learner.__call__</code><a href=\"__main__.py#L69\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.__call__</code>(**`event_name`**)\n",
       "\n",
       "Call `event_name` for all [`Callback`](/fastai_minima/callback.core.html#Callback)s in `self.cbs`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.__call__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This how the `Callback`s are called internally. For instance a `VerboseCallback` just prints the event names (can be useful for debugging):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after_create\n",
      "after_fit\n"
     ]
    }
   ],
   "source": [
    "learn = synth_learner(cbs=VerboseCallback())\n",
    "learn('after_fit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.add_cb\" class=\"doc_header\"><code>Learner.add_cb</code><a href=\"__main__.py#L35\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.add_cb</code>(**`cb`**)\n",
       "\n",
       "Add `cb` to the list of [`Callback`](/fastai_minima/callback.core.html#Callback) and register `self` as their learner"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.add_cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cb(TestTrainEvalCallback())\n",
    "test_eq(len(learn.cbs), 2)\n",
    "assert isinstance(learn.cbs[1], TestTrainEvalCallback)\n",
    "test_eq(learn.train_eval.learn, learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.add_cbs\" class=\"doc_header\"><code>Learner.add_cbs</code><a href=\"__main__.py#L25\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.add_cbs</code>(**`cbs`**)\n",
       "\n",
       "Add `cbs` to the list of [`Callback`](/fastai_minima/callback.core.html#Callback) and register `self` as their learner"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.add_cbs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.add_cbs([TestTrainEvalCallback(), TestTrainEvalCallback()])\n",
    "test_eq(len(learn.cbs), 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.added_cbs\" class=\"doc_header\"><code>Learner.added_cbs</code><a href=\"__main__.py#L52\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.added_cbs</code>(**`cbs`**)\n",
       "\n",
       "Context manage that temporarily adds `cbs`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.added_cbs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "test_eq(len(learn.cbs), 1)\n",
    "with learn.added_cbs(TestTrainEvalCallback()):\n",
    "    test_eq(len(learn.cbs), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.ordered_cbs\" class=\"doc_header\"><code>Learner.ordered_cbs</code><a href=\"__main__.py#L66\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.ordered_cbs</code>(**`event`**)\n",
       "\n",
       "List of [`Callback`](/fastai_minima/callback.core.html#Callback)s, in order, for an `event` in the training loop"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.ordered_cbs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By order, we mean using the internal ordering of the `Callback`s (see `callback.core` for more information on how it works)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TrainEvalCallback, TestTrainEvalCallback]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cb(TestTrainEvalCallback())\n",
    "learn.ordered_cbs('before_fit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.remove_cb\" class=\"doc_header\"><code>Learner.remove_cb</code><a href=\"__main__.py#L43\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.remove_cb</code>(**`cb`**)\n",
       "\n",
       "Add `cb` from the list of [`Callback`](/fastai_minima/callback.core.html#Callback) and deregister `self` as their learner"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.remove_cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cb(TestTrainEvalCallback())\n",
    "cb = learn.cbs[1]\n",
    "learn.remove_cb(learn.cbs[1])\n",
    "test_eq(len(learn.cbs), 1)\n",
    "assert cb.learn is None\n",
    "assert not getattr(learn,'test_train_eval',None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`cb` can simply be the class of the `Callback` we want to remove (in which case all instances of that callback are removed)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cbs([TestTrainEvalCallback(), TestTrainEvalCallback()])\n",
    "learn.remove_cb(TestTrainEvalCallback)\n",
    "test_eq(len(learn.cbs), 1)\n",
    "assert not getattr(learn,'test_train_eval',None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.remove_cbs\" class=\"doc_header\"><code>Learner.remove_cbs</code><a href=\"__main__.py#L30\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.remove_cbs</code>(**`cbs`**)\n",
       "\n",
       "Remove `cbs` from the list of [`Callback`](/fastai_minima/callback.core.html#Callback) and deregister `self` as their learner"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.remove_cbs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Elements of `cbs` can either be types of callbacks or actual callbacks of the `Learner`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cbs([TestTrainEvalCallback() for _ in range(3)])\n",
    "cb = learn.cbs[1]\n",
    "learn.remove_cbs(learn.cbs[1:])\n",
    "test_eq(len(learn.cbs), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.removed_cbs\" class=\"doc_header\"><code>Learner.removed_cbs</code><a href=\"__main__.py#L59\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.removed_cbs</code>(**`cbs`**)\n",
       "\n",
       "Context manage that temporarily removes `cbs`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.removed_cbs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Elements of `cbs` can either be types of callbacks or actual callbacks of the `Learner`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "learn.add_cb(TestTrainEvalCallback())\n",
    "with learn.removed_cbs(learn.cbs[1]):\n",
    "    test_eq(len(learn.cbs), 1)\n",
    "test_eq(len(learn.cbs), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.show_training_loop\" class=\"doc_header\"><code>Learner.show_training_loop</code><a href=\"__main__.py#L198\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.show_training_loop</code>()\n",
       "\n",
       "Show each step in the training loop"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.show_training_loop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At each step, callbacks are shown in order, which can help debugging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Fit\n",
      "   - before_fit     : [TrainEvalCallback]\n",
      "  Start Epoch Loop\n",
      "     - before_epoch   : []\n",
      "    Start Train\n",
      "       - before_train   : [TrainEvalCallback]\n",
      "      Start Batch Loop\n",
      "         - before_batch   : []\n",
      "         - after_pred     : []\n",
      "         - after_loss     : []\n",
      "         - before_backward: []\n",
      "         - before_step    : []\n",
      "         - after_step     : []\n",
      "         - after_cancel_batch: []\n",
      "         - after_batch    : [TrainEvalCallback]\n",
      "      End Batch Loop\n",
      "    End Train\n",
      "     - after_cancel_train: []\n",
      "     - after_train    : []\n",
      "    Start Valid\n",
      "       - before_validate: [TrainEvalCallback]\n",
      "      Start Batch Loop\n",
      "         - **CBs same as train batch**: []\n",
      "      End Batch Loop\n",
      "    End Valid\n",
      "     - after_cancel_validate: []\n",
      "     - after_validate : []\n",
      "  End Epoch Loop\n",
      "   - after_cancel_epoch: []\n",
      "   - after_epoch    : []\n",
      "End Fit\n",
      " - after_cancel_fit: []\n",
      " - after_fit      : []\n"
     ]
    }
   ],
   "source": [
    "learn = synth_learner()\n",
    "learn.show_training_loop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _before_batch_cb(f, self):\n",
    "    xb,yb = f(self, self.xb, self.yb)\n",
    "    self.learn.xb,self.learn.yb = xb,yb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def before_batch_cb(f):\n",
    "    \"Shortcut for creating a Callback on the `before_batch` event, which takes and returns `xb,yb`\"\n",
    "    return Callback(before_batch=partial(_before_batch_cb, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to change the data passed to your model, you will generally want to hook into the `before_batch` event, like so:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TstCallback(Callback):\n",
    "    def before_batch(self):\n",
    "        self.learn.xb = self.xb + 1000\n",
    "        self.learn.yb = self.yb - 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since that is so common, we provide the `before_batch_cb` decorator to make it easier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@before_batch_cb\n",
    "def cb(self, xb, yb): return xb+1000,yb-1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Serializing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@patch\n",
    "@delegates(save_model)\n",
    "def save(self:Learner, file, **kwargs):\n",
    "    \"Save model and optimizer state (if `with_opt`) to `self.path/self.model_dir/file`\"\n",
    "    file = join_path_file(file, self.path/self.model_dir, ext='.pth')\n",
    "    save_model(file, self.model, getattr(self,'opt',None), **kwargs)\n",
    "    return file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` can be a `Path`, a `string` or a buffer. `pickle_protocol` is passed along to `torch.save`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@patch\n",
    "@delegates(load_model)\n",
    "def load(self:Learner, file, device=None, **kwargs):\n",
    "    \"Load model and optimizer state (if `with_opt`) from `self.path/self.model_dir/file` using `device`\"\n",
    "    if device is None and hasattr(self.dls, 'device'): device = self.dls.device\n",
    "    if self.opt is None: self.create_opt()\n",
    "    file = join_path_file(file, self.path/self.model_dir, ext='.pth')\n",
    "    load_model(file, self.model, self.opt, device=device, **kwargs)\n",
    "    return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` can be a `Path`, a `string` or a buffer. Use `device` to load the model/optimizer state on a device different from the one it was saved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    learn = synth_learner(path=d)\n",
    "    learn.fit(1)\n",
    "    \n",
    "    #Test save created a file\n",
    "    learn.save('tmp')\n",
    "    assert (Path(d)/'models/tmp.pth').exists()\n",
    "    \n",
    "    #Test load did load the model\n",
    "    learn1 = synth_learner(path=d)\n",
    "    learn1 = learn1.load('tmp')\n",
    "    test_eq(learn.a, learn1.a)\n",
    "    test_eq(learn.b, learn1.b)\n",
    "    test_eq(learn.opt.state_dict(), learn1.opt.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test load works when the model is saved without opt\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    learn = synth_learner(path=d)\n",
    "    learn.fit(1)\n",
    "    learn.save('tmp', with_opt=False)\n",
    "    learn1 = synth_learner(path=d)\n",
    "    learn1 = learn1.load('tmp', with_opt=False)\n",
    "    test_eq(learn.a, learn1.a)\n",
    "    test_eq(learn.b, learn1.b)\n",
    "    test_ne(learn.opt.state_dict(), learn1.opt.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@patch\n",
    "def export(self:Learner, fname='export.pkl', pickle_module=pickle, pickle_protocol=2):\n",
    "    \"Export the content of `self` without the items and the optimizer state for inference\"\n",
    "    if rank_distrib(): return # don't export if child proc\n",
    "    self._end_cleanup()\n",
    "    old_dbunch = self.dls\n",
    "    self.dls = self.dls.new_empty()\n",
    "    state = self.opt.state_dict() if self.opt is not None else None\n",
    "    self.opt = None\n",
    "    with warnings.catch_warnings():\n",
    "        #To avoid the warning that come from PyTorch about model not being checked\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        torch.save(self, self.path/fname, pickle_module=pickle_module, pickle_protocol=pickle_protocol)\n",
    "    self.create_opt()\n",
    "    if state is not None: self.opt.load_state_dict(state)\n",
    "    self.dls = old_dbunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `Learner` is saved in `self.path/fname`, using `pickle_protocol`. Note that serialization in Python saves the names of functions, not the code itself. Therefore, any custom code you have for models, data transformation, loss function etc... should be put in a module that you will import in your training environment before exporting, and in your deployment environment before loading it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def load_learner(fname, cpu=True, pickle_module=pickle):\n",
    "    \"Load a `Learner` object in `fname`, optionally putting it on the `cpu`\"\n",
    "    distrib_barrier()\n",
    "    res = torch.load(fname, map_location='cpu' if cpu else None, pickle_module=pickle_module)\n",
    "    if hasattr(res, 'to_fp32'): res = res.to_fp32()\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Warning: `load_learner` requires all your custom code be in the exact same place as when exporting your `Learner` (the main script, or the module you imported it from)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DataLoader aware `to_detach` -"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fastai provides `to_detach` which by default detachs tensor gradients, and gathers (calling `maybe_gather`) tensors from all ranks if running in distributed data parallel (DDP) mode.\n",
    "\n",
    "When running in DDP mode all ranks need to have the same batch size, and `DistributedDL` takes care of padding batches as needed; however when gathering all tensors (e.g. for calculating metrics, inference, etc.) we need to discard the padded items. `DistributedDL` provides a method `to_detach` that removes padding appropriately.\n",
    "\n",
    "Calling `to_detach_from_dl` with `learn` as a learner will attempt to find a `to_detach` method in the learner's last used `DataLoader` `dl` and use that one if found, otherwise it will resort to the vanilla `to_detach`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def to_detach_from_dl(learn:(Learner,NoneType),b:object,cpu:bool=True,gather:bool=True):\n",
    "    return learn.dl.to_detach(b,cpu,gather) if hasattr(getattr(learn,'dl',None),'to_detach') else to_detach(b,cpu,gather)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from torch import Tensor\n",
    "learn = synth_learner()\n",
    "test_eq(to_detach_from_dl(learn,Tensor([123])),Tensor([123]))\n",
    "learn.dl = learn.dls[0]\n",
    "test_eq(to_detach_from_dl(learn,Tensor([123])),Tensor([123]))\n",
    "learn.dl.to_detach = lambda b,cpu,gather: b-100\n",
    "test_eq(to_detach_from_dl(learn,Tensor([123.])),Tensor([23.]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Metric():\n",
    "    \"Blueprint for defining a metric\"\n",
    "    def reset(self): \n",
    "        \"Reset inner state to prepare for new computation\"\n",
    "        pass\n",
    "    \n",
    "    def accumulate(self, learn): \n",
    "        \"Use `learn` to update the state with new results\"\n",
    "        pass\n",
    "    @property\n",
    "    def value(self): \n",
    "        \"The value of the metric\"\n",
    "        raise NotImplementedError\n",
    "\n",
    "    @property\n",
    "    def name(self): \n",
    "        \"Name of the `Metric`, camel-cased and with Metric removed\"\n",
    "        return class2attr(self, 'Metric')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h3 id=\"Metric\" class=\"doc_header\"><code>class</code> <code>Metric</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h3>\n",
       "\n",
       "> <code>Metric</code>()\n",
       "\n",
       "Blueprint for defining a metric"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Metric, title_level=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Metrics can be simple averages (like accuracy) but sometimes their computation is a little bit more complex and can't be averaged over batches (like precision or recall), which is why we need a special class for them. For simple functions that can be computed as averages over batches, we can use the class `AvgMetric`, otherwise you'll need to implement the following methods.\n",
    "\n",
    "> Note: If your <code>Metric</code> has state depending on tensors, don't forget to store it on the CPU to avoid any potential memory leaks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Metric.reset\" class=\"doc_header\"><code>Metric.reset</code><a href=\"__main__.py#L4\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Metric.reset</code>()\n",
       "\n",
       "Reset inner state to prepare for new computation"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Metric.reset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Metric.accumulate\" class=\"doc_header\"><code>Metric.accumulate</code><a href=\"__main__.py#L8\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Metric.accumulate</code>(**`learn`**)\n",
       "\n",
       "Use `learn` to update the state with new results"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Metric.accumulate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Metric.value\" class=\"doc_header\"><code>Metric.value</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "The value of the metric"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Metric.value, name='Metric.value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Metric.name\" class=\"doc_header\"><code>Metric.name</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "Name of the [`Metric`](/fastai_minima/learner.html#Metric), camel-cased and with Metric removed"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Metric.name, name='Metric.name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _maybe_reduce(val):\n",
    "    if num_distrib()>1:\n",
    "        val = val.clone()\n",
    "        torch.distributed.all_reduce(val, op=torch.distributed.ReduceOp.SUM)\n",
    "        val /= num_distrib()\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AvgMetric(Metric):\n",
    "    \"Average the values of `func` taking into account potential different batch sizes\"\n",
    "    def __init__(self, func):  self.func = func\n",
    "    def reset(self):           self.total,self.count = 0.,0\n",
    "    def accumulate(self, learn):\n",
    "        bs = find_bs(learn.yb)\n",
    "        self.total += learn.to_detach(self.func(learn.pred, *learn.yb))*bs\n",
    "        self.count += bs\n",
    "    @property\n",
    "    def value(self): return self.total/self.count if self.count != 0 else None\n",
    "    @property\n",
    "    def name(self):  return self.func.func.__name__ if hasattr(self.func, 'func') else  self.func.__name__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h3 id=\"AvgMetric\" class=\"doc_header\"><code>class</code> <code>AvgMetric</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h3>\n",
       "\n",
       "> <code>AvgMetric</code>(**`func`**) :: [`Metric`](/fastai_minima/learner.html#Metric)\n",
       "\n",
       "Average the values of `func` taking into account potential different batch sizes"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(AvgMetric, title_level=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = synth_learner()\n",
    "tst = AvgMetric(lambda x,y: (x-y).abs().mean())\n",
    "t,u = torch.randn(100),torch.randn(100)\n",
    "tst.reset()\n",
    "for i in range(0,100,25): \n",
    "    learn.pred,learn.yb = t[i:i+25],(u[i:i+25],)\n",
    "    tst.accumulate(learn)\n",
    "test_close(tst.value, (t-u).abs().mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#With varying batch size\n",
    "tst.reset()\n",
    "splits = [0, 30, 50, 60, 100]\n",
    "for i in range(len(splits )-1): \n",
    "    learn.pred,learn.yb = t[splits[i]:splits[i+1]],(u[splits[i]:splits[i+1]],)\n",
    "    tst.accumulate(learn)\n",
    "test_close(tst.value, (t-u).abs().mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AvgLoss(Metric):\n",
    "    \"Average the losses taking into account potential different batch sizes\"\n",
    "    def reset(self):           self.total,self.count = 0.,0\n",
    "    def accumulate(self, learn):\n",
    "        bs = find_bs(learn.yb)\n",
    "        self.total += learn.to_detach(learn.loss.mean())*bs\n",
    "        self.count += bs\n",
    "    @property\n",
    "    def value(self): return self.total/self.count if self.count != 0 else None\n",
    "    @property\n",
    "    def name(self):  return \"loss\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h3 id=\"AvgLoss\" class=\"doc_header\"><code>class</code> <code>AvgLoss</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h3>\n",
       "\n",
       "> <code>AvgLoss</code>() :: [`Metric`](/fastai_minima/learner.html#Metric)\n",
       "\n",
       "Average the losses taking into account potential different batch sizes"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(AvgLoss, title_level=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst = AvgLoss()\n",
    "t = torch.randn(100)\n",
    "tst.reset()\n",
    "for i in range(0,100,25): \n",
    "    learn.yb,learn.loss = t[i:i+25],t[i:i+25].mean()\n",
    "    tst.accumulate(learn)\n",
    "test_close(tst.value, t.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#With varying batch size\n",
    "tst.reset()\n",
    "splits = [0, 30, 50, 60, 100]\n",
    "for i in range(len(splits )-1): \n",
    "    learn.yb,learn.loss = t[splits[i]:splits[i+1]],t[splits[i]:splits[i+1]].mean()\n",
    "    tst.accumulate(learn)\n",
    "test_close(tst.value, t.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AvgSmoothLoss(Metric):\n",
    "    \"Smooth average of the losses (exponentially weighted with `beta`)\"\n",
    "    def __init__(self, beta=0.98): self.beta = beta\n",
    "    def reset(self):               self.count,self.val = 0,tensor(0.)\n",
    "    def accumulate(self, learn):\n",
    "        self.count += 1\n",
    "        self.val = torch.lerp(to_detach(learn.loss.mean(), gather=False), self.val, self.beta)\n",
    "    @property\n",
    "    def value(self): return self.val/(1-self.beta**self.count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h3 id=\"AvgSmoothLoss\" class=\"doc_header\"><code>class</code> <code>AvgSmoothLoss</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h3>\n",
       "\n",
       "> <code>AvgSmoothLoss</code>(**`beta`**=*`0.98`*) :: [`Metric`](/fastai_minima/learner.html#Metric)\n",
       "\n",
       "Smooth average of the losses (exponentially weighted with `beta`)"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(AvgSmoothLoss, title_level=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst = AvgSmoothLoss()\n",
    "t = torch.randn(100)\n",
    "tst.reset()\n",
    "val = tensor(0.)\n",
    "for i in range(4): \n",
    "    learn.loss = t[i*25:(i+1)*25].mean()\n",
    "    tst.accumulate(learn)\n",
    "    val = val*0.98 + t[i*25:(i+1)*25].mean()*(1-0.98)\n",
    "    test_close(val/(1-0.98**(i+1)), tst.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ValueMetric(Metric):\n",
    "    \"Use to include a pre-calculated metric value (for instance calculated in a `Callback`) and returned by `func`\"\n",
    "    def __init__(self, func, metric_name=None): store_attr('func, metric_name')\n",
    "\n",
    "    @property\n",
    "    def value(self): return self.func()\n",
    "\n",
    "    @property\n",
    "    def name(self): return self.metric_name if self.metric_name else self.func.__name__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h3 id=\"ValueMetric\" class=\"doc_header\"><code>class</code> <code>ValueMetric</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h3>\n",
       "\n",
       "> <code>ValueMetric</code>(**`func`**, **`metric_name`**=*`None`*) :: [`Metric`](/fastai_minima/learner.html#Metric)\n",
       "\n",
       "Use to include a pre-calculated metric value (for instance calculated in a [`Callback`](/fastai_minima/callback.core.html#Callback)) and returned by `func`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ValueMetric, title_level=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric_value_fn(): return 5e-3\n",
    "\n",
    "vm = ValueMetric(metric_value_fn, 'custom_value_metric')\n",
    "test_eq(vm.value, 5e-3)\n",
    "test_eq(vm.name, 'custom_value_metric')\n",
    "\n",
    "vm = ValueMetric(metric_value_fn)\n",
    "test_eq(vm.name, 'metric_value_fn')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recorder --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from fastprogress.fastprogress import format_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _maybe_item(t):\n",
    "    t = t.value\n",
    "    try: return t.item()\n",
    "    except: return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Recorder(Callback):\n",
    "    \"Callback that registers statistics (lr, loss and metrics) during training\"\n",
    "    _stateattrs=('lrs','iters','losses','values')\n",
    "    remove_on_fetch,order = True,50\n",
    "\n",
    "    def __init__(self, add_time=True, train_metrics=False, valid_metrics=True, beta=0.98):\n",
    "        store_attr('add_time,train_metrics,valid_metrics')\n",
    "        self.loss,self.smooth_loss = AvgLoss(),AvgSmoothLoss(beta=beta)\n",
    "\n",
    "    def before_fit(self):\n",
    "        \"Prepare state for training\"\n",
    "        self.lrs,self.iters,self.losses,self.values = [],[],[],[]\n",
    "        names = self.metrics.attrgot('name')\n",
    "        if self.train_metrics and self.valid_metrics:\n",
    "            names = L('loss') + names\n",
    "            names = names.map('train_{}') + names.map('valid_{}')\n",
    "        elif self.valid_metrics: names = L('train_loss', 'valid_loss') + names\n",
    "        else: names = L('train_loss') + names\n",
    "        if self.add_time: names.append('time')\n",
    "        self.metric_names = 'epoch'+names\n",
    "        self.smooth_loss.reset()\n",
    "\n",
    "    def after_batch(self):\n",
    "        \"Update all metrics and records lr and smooth loss in training\"\n",
    "        if len(self.yb) == 0: return\n",
    "        mets = self._train_mets if self.training else self._valid_mets\n",
    "        for met in mets: met.accumulate(self.learn)\n",
    "        if not self.training: return\n",
    "        self.lrs.append(self.opt.hypers[-1]['lr'])\n",
    "        self.losses.append(self.smooth_loss.value)\n",
    "        self.learn.smooth_loss = self.smooth_loss.value\n",
    "\n",
    "    def before_epoch(self):\n",
    "        \"Set timer if `self.add_time=True`\"\n",
    "        self.cancel_train,self.cancel_valid = False,False\n",
    "        if self.add_time: self.start_epoch = time.time()\n",
    "        self.log = L(getattr(self, 'epoch', 0))\n",
    "\n",
    "    def before_train   (self): \n",
    "        \"Reset loss and metrics state\"\n",
    "        self._train_mets[1:].map(Self.reset())\n",
    "        \n",
    "    def before_validate(self): \n",
    "        \"Reset loss and metrics state\"\n",
    "        self._valid_mets.map(Self.reset())\n",
    "        \n",
    "    def after_train   (self): \n",
    "        \"Log loss and metric values on the training set (if `self.training_metrics=True`)\"\n",
    "        self.log += self._train_mets.map(_maybe_item)\n",
    "        \n",
    "    def after_validate(self): \n",
    "        \"Log loss and metric values on the validation set\"\n",
    "        self.log += self._valid_mets.map(_maybe_item)\n",
    "        \n",
    "    def after_cancel_train(self):    \n",
    "        \"Ignore training metrics for this epoch\"\n",
    "        self.cancel_train = True\n",
    "        \n",
    "    def after_cancel_validate(self): \n",
    "        \"Ignore validation metrics for this epoch\"\n",
    "        self.cancel_valid = True\n",
    "\n",
    "    def after_epoch(self):\n",
    "        \"Store and log the loss/metric values\"\n",
    "        self.learn.final_record = self.log[1:].copy()\n",
    "        self.values.append(self.learn.final_record)\n",
    "        if self.add_time: self.log.append(format_time(time.time() - self.start_epoch))\n",
    "        self.logger(self.log)\n",
    "        self.iters.append(self.smooth_loss.count)\n",
    "\n",
    "    @property\n",
    "    def _train_mets(self):\n",
    "        if getattr(self, 'cancel_train', False): return L()\n",
    "        return L(self.smooth_loss) + (self.metrics if self.train_metrics else L())\n",
    "\n",
    "    @property\n",
    "    def _valid_mets(self):\n",
    "        if getattr(self, 'cancel_valid', False): return L()\n",
    "        return (L(self.loss) + self.metrics if self.valid_metrics else L())\n",
    "\n",
    "    def plot_loss(self, skip_start=5, with_valid=True):\n",
    "        \"Plot the losses from `skip_start` and onward\"\n",
    "        plt.plot(list(range(skip_start, len(self.losses))), self.losses[skip_start:], label='train')\n",
    "        if with_valid:\n",
    "            idx = (np.array(self.iters)<skip_start).sum()\n",
    "            plt.plot(self.iters[idx:], L(self.values[idx:]).itemgot(1), label='valid')\n",
    "            plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "if Recorder not in defaults.callbacks: defaults.callbacks.append(Recorder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, metrics are computed on the validation set only, although that can be changed by adjusting `train_metrics` and `valid_metrics`. `beta` is the weight used to compute the exponentially weighted average of the losses (which gives the `smooth_loss` attribute to `Learner`).\n",
    "\n",
    "The `logger` attribute of a `Learner` determines what happens to those metrics. By default, it just print them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test printed output\n",
    "import torch.nn.functional as F\n",
    "def tst_metric(out, targ): return F.mse_loss(out, targ)\n",
    "learn = synth_learner(n_train=5, metrics=tst_metric)\n",
    "# pat = r\"[tensor\\(\\d.\\d*\\), tensor\\(\\d.\\d*\\), tensor\\(\\d.\\d*\\), 'dd:dd']\"\n",
    "pat = r\"\\[\\d, \\d+.\\d+, \\d+.\\d+, \\d+.\\d+, '\\d\\d:\\d\\d'\\]\"\n",
    "test_stdout(lambda: learn.fit(1), pat, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "class TestRecorderCallback(Callback):\n",
    "    order=51\n",
    "    def before_fit(self): \n",
    "        self.train_metrics,self.add_time = self.recorder.train_metrics,self.recorder.add_time\n",
    "        self.beta = self.recorder.smooth_loss.beta\n",
    "        for m in self.metrics: assert isinstance(m, Metric)\n",
    "        test_eq(self.recorder.smooth_loss.val, 0.)\n",
    "        #To test what the recorder logs, we use a custom logger function.\n",
    "        self.learn.logger = self.test_log\n",
    "        self.old_smooth,self.count = tensor(0.),0\n",
    "    \n",
    "    def after_batch(self):\n",
    "        if self.training:\n",
    "            self.count += 1\n",
    "            test_eq(len(self.recorder.lrs), self.count)\n",
    "            test_eq(self.recorder.lrs[-1], self.opt.hypers[-1]['lr'])\n",
    "            test_eq(len(self.recorder.losses), self.count)\n",
    "            smooth = (1 - self.beta**(self.count-1)) * self.old_smooth * self.beta + self.loss * (1-self.beta)\n",
    "            smooth /= 1 - self.beta**self.count\n",
    "            test_close(self.recorder.losses[-1], smooth, eps=1e-4)\n",
    "            test_close(self.smooth_loss, smooth, eps=1e-4)\n",
    "            self.old_smooth = self.smooth_loss\n",
    "        self.bs += find_bs(self.yb)\n",
    "        if not self.training: test_eq(self.recorder.loss.count, self.bs)\n",
    "        if self.train_metrics or not self.training: \n",
    "            for m in self.metrics: test_eq(m.count, self.bs)\n",
    "        self.losses.append(self.loss.detach().cpu())\n",
    "    \n",
    "    def before_epoch(self): \n",
    "        if self.add_time: self.start_epoch = time.time()\n",
    "        self.log = [self.epoch]\n",
    "\n",
    "    def before_train(self):\n",
    "        self.bs = 0\n",
    "        self.losses = []\n",
    "        for m in self.recorder._train_mets: test_eq(m.count, self.bs)\n",
    "            \n",
    "    def after_train(self):\n",
    "        mean = tensor(self.losses).mean()\n",
    "        self.log += [self.smooth_loss, mean] if self.train_metrics else [self.smooth_loss]\n",
    "        test_close(self.log, self.recorder.log)\n",
    "        self.losses = []\n",
    "    \n",
    "    def before_validate(self):\n",
    "        self.bs = 0\n",
    "        self.losses = []\n",
    "        for m in [self.recorder.loss] + self.metrics: test_eq(m.count, self.bs)\n",
    "    \n",
    "    def test_log(self, log):\n",
    "        res = tensor(self.losses).mean()\n",
    "        self.log += [res, res]\n",
    "        if self.add_time: self.log.append(format_time(time.time() - self.start_epoch))\n",
    "        test_close(log[:-1], self.log[:-1])\n",
    "        test_eq(log[-1], self.log[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "learn = synth_learner(n_train=5, metrics = tst_metric, cbs = TestRecorderCallback)\n",
    "learn.fit(1)\n",
    "test_eq(learn.recorder.metric_names, ['epoch', 'train_loss', 'valid_loss', 'tst_metric', 'time'])\n",
    "\n",
    "learn = synth_learner(n_train=5, metrics = tst_metric, cbs = TestRecorderCallback)\n",
    "learn.recorder.train_metrics=True\n",
    "learn.fit(1)\n",
    "test_eq(learn.recorder.metric_names, \n",
    "        ['epoch', 'train_loss', 'train_tst_metric', 'valid_loss', 'valid_tst_metric', 'time'])\n",
    "\n",
    "learn = synth_learner(n_train=5, metrics = tst_metric, cbs = TestRecorderCallback)\n",
    "learn.recorder.add_time=False\n",
    "learn.fit(1)\n",
    "test_eq(learn.recorder.metric_names, ['epoch', 'train_loss', 'valid_loss', 'tst_metric'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 17.738677978515625, 17.051448822021484, 17.051448822021484, '00:00']\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "#Test numpy metric\n",
    "def tst_metric_np(out, targ): return F.mse_loss(out, targ).numpy()\n",
    "learn = synth_learner(n_train=5, metrics=tst_metric_np)\n",
    "learn.fit(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Internals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.before_fit\" class=\"doc_header\"><code>Recorder.before_fit</code><a href=\"__main__.py#L11\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.before_fit</code>()\n",
       "\n",
       "Prepare state for training"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.before_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.before_epoch\" class=\"doc_header\"><code>Recorder.before_epoch</code><a href=\"__main__.py#L34\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.before_epoch</code>()\n",
       "\n",
       "Set timer if `self.add_time=True`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.before_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.before_validate\" class=\"doc_header\"><code>Recorder.before_validate</code><a href=\"__main__.py#L44\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.before_validate</code>()\n",
       "\n",
       "Reset loss and metrics state"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.before_validate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.after_batch\" class=\"doc_header\"><code>Recorder.after_batch</code><a href=\"__main__.py#L24\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.after_batch</code>()\n",
       "\n",
       "Update all metrics and records lr and smooth loss in training"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.after_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.after_epoch\" class=\"doc_header\"><code>Recorder.after_epoch</code><a href=\"__main__.py#L64\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.after_epoch</code>()\n",
       "\n",
       "Store and log the loss/metric values"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.after_epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Recorder.plot_loss\" class=\"doc_header\"><code>Recorder.plot_loss</code><a href=\"__main__.py#L82\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Recorder.plot_loss</code>(**`skip_start`**=*`5`*, **`with_valid`**=*`True`*)\n",
       "\n",
       "Plot the losses from `skip_start` and onward"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Recorder.plot_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl4lfWZ//H3nYUlEEhCWAIJHGRHDAHCoghuuKGVVilC3aAorbbutXXmN+10ps5cnWoda60yLiAqIgxotZUKojBYQSCsImEJEkhYEyDsgSzf3x85RIoJhCQnz1k+r+vK5TnPcs6dr5zPefJstznnEBGRyBHldQEiItKwFPwiIhFGwS8iEmEU/CIiEUbBLyISYRT8IiIRRsEvIhJhFPwiIhFGwS8iEmFivC6gKsnJyc7n83ldhohIyFi5cmWhc651TZYNyuD3+XxkZWV5XYaISMgws+01XVa7ekREIoyCX0Qkwij4RUQiTFDu4xcRuRAlJSXk5+dTXFzsdSkB16RJE1JTU4mNja31ayj4RSTk5efnEx8fj8/nw8y8LidgnHPs37+f/Px8OnfuXOvX0a4eEQl5xcXFtGrVKqxDH8DMaNWqVZ3/slHwi0hYCPfQP60+fk8Fv4gExKJN+3hn+Q6vy5AqKPhFpN59sHYX907LYvqyHZSWlXtdTsAVFRXx4osvXvB6I0eOpKioKAAVnZuCX0Tq1ZtfbOfhd1bTv1Mi0+8bTEx0+MdMdcFfVlZ2zvXmzp1LQkJCoMqqls7qEZF64ZzjhU9z+P3Hm7mmZxv+dEd/msRGe11Wg3jyySfZunUrGRkZxMbG0rx5c1JSUlizZg0bNmzgu9/9Lnl5eRQXF/Pwww8zadIk4Jvb0xw9epQbb7yRyy+/nCVLltChQwfef/99mjZtGpB6FfwiUmfl5Y6nPsxmyufb+F6/DvxudDqxHm3p/9tfvmLDrsP1+pq927fgX79zcbXzf/vb37J+/XrWrFnDokWLuOmmm1i/fn3lKZdTpkwhKSmJEydOMHDgQG677TZatWr1D6+xZcsWZsyYwSuvvMKYMWOYM2cOd955Z73+Hqcp+EWkTkrLyvnFnC+Zsyqf8Zf5+NXNvYmKiowzbKozaNCgfzjP/vnnn+e9994DIC8vjy1btnwr+Dt37kxGRgYAAwYMIDc3N2D1KfhFpNaKS8r46durWZC9l8eu7c6DV3f1/LTKc22ZN5RmzZpVPl60aBELFixg6dKlxMXFceWVV1Z5Hn7jxo0rH0dHR3PixImA1afgF5FaOVJcwr3Tsli27QD/Pupi7r7U53VJnomPj+fIkSNVzjt06BCJiYnExcWxceNGvvjiiwau7tsU/CJywfYfPck9U5ezcfcR/jA2g1EZHbwuyVOtWrVi6NCh9OnTh6ZNm9K2bdvKeTfccAOTJ08mPT2dHj16MGTIEA8rrWDOOa9r+JbMzEynRiwiwWln0QnuenUZO4tOMPnOAVzVs43XJZGdnU2vXr28LqPBVPX7mtlK51xmTdbXFr+I1FjOvqPc9doyjp4s5a17BzPQl+R1SVILCn4RqZF1+UXcM2U50VFRzJx0Kb3bt/C6JKklBb+InNeSrYXcNy2LxGaNeGviYHzJzc6/kgQtBb+InNO8r/bw4Nur8SXH8cYPB9OuZROvS5I6UvCLSLVmZeXx5Jx1pKcm8PqEgSTENfK6JKkHCn4RqdKrn33NUx9mM6xbMpPvHECzxoqLcBH+t80TkQvinOPpeRt56sNsbrokhVfvyVToB0Dz5s0B2LVrF6NHj65ymSuvvJJAnNqu/5siUqms3PHL99fz9rIdjBuUxlPfvYToCL/vTqC1b9+e2bNnN+h7KvhFBIBTpeU8OmsNH67bzf1XduHn1/fw/L47oeQXv/gFnTp14oEHHgDg17/+NWbG4sWLOXjwICUlJTz11FOMGjXqH9bLzc3l5ptvZv369Zw4cYIJEyawYcMGevXqFbD79Sj4RYTjp0r50Zsr+WxLIf88sieThnfxuqTa+9uTsOfL+n3NdpfAjb895yJjx47lkUceqQz+WbNm8dFHH/Hoo4/SokULCgsLGTJkCLfccku1X6gvvfQScXFxrFu3jnXr1tG/f//6/T38FPwiEa7o+CkmvL6CtXlF/O62dMYMTPO6pJDUr18/9u3bx65duygoKCAxMZGUlBQeffRRFi9eTFRUFDt37mTv3r20a9euytdYvHgxDz30EADp6emkp6cHpFYFv0gE23u4mLtfW862wmO8eEd/buiT4nVJdXeeLfNAGj16NLNnz2bPnj2MHTuW6dOnU1BQwMqVK4mNjcXn81V5S+YzNcTuNZ3VIxKhcguPMXryEvIPHmfqhIHhEfoeGzt2LO+88w6zZ89m9OjRHDp0iDZt2hAbG8vChQvZvn37OdcfPnw406dPB2D9+vWsW7cuIHVqi18kAmXvPsxdry2nrLyct+8bQt+0hm/4HY4uvvhijhw5QocOHUhJSeGOO+7gO9/5DpmZmWRkZNCzZ89zrn///fczYcIE0tPTycjIYNCgQQGpU7dlFokwWbkHmPD6Cpo1iuGtewfRtU281yXVmW7LrNsyi0g1Fm7ax/1vrSSlZVPenDiI1MQ4r0sSDyj4RSLE+2t28vistfRoF8+0Hw4iuXnj868kYUnBLxIB3lyay68++IqBviRevSeTFk1ivS6p3jnnIuKCs/rYPa+zekTCmHOO5z/Zwi/f/4prerbhjR8OCsvQb9KkCfv376+XUAxmzjn2799PkyZ1uzX2ebf4zWwKcDOwzznXxz9tJtDDv0gCUOScy6hi3QTgVaAP4IAfOueW1qliEamR8nLHbz7cwNTPc7m1Xwf+a3Q6sdHhua2XmppKfn4+BQUFXpcScE2aNCE1NbVOr1GTXT2vAy8Ab5ye4Jy7/fRjM/s9cKiadf8AfOScG21mjQAdSRJpACVl5fxizjreXbWTCUN9/PKm3kSF8c3WYmNj6dy5s9dlhIzzBr9zbrGZ+aqaZxU71MYAV1cxrwUwHBjvf51TwKnalyoiNVFcUsZP317Fgux9PHZtdx68umtE7PuWmqvrwd1hwF7n3JYq5l0EFABTzawvsBJ42Dl3rI7vKSLVOFxcwn3Tsliee4B/H3Uxd1/q87okCUJ13eE3DphRzbwYoD/wknOuH3AMeLK6FzKzSWaWZWZZkbCfTqS+FR49ybiXv2Dl9oM8d3uGQl+qVevgN7MY4FZgZjWL5AP5zrll/uezqfgiqJJz7mXnXKZzLrN169a1LUskIuUfPM6YyUvZWnCUV+7OZFRGB69LkiBWly3+EcBG51x+VTOdc3uAPDM7ffbPNcCGOryfiFQhZ98Rvj95KQVHT/LmxMFc1bON1yVJkDtv8JvZDGAp0MPM8s1son/WWM7azWNm7c1s7hmTHgSmm9k6IAP4z/opW0QA1uYV8f3JSykpc8ycdCkDfUlelyQhoCZn9YyrZvr4KqbtAkae8XwNUKObBonIhVmSU8h9b2SR2KwRb00cjC+5mdclSYjQLRtEQtBH6/fw0IzV+JLjeHPiYNq2qNuVnBJZFPwiIWbWijyefHcdfdMSmDp+IAlxjbwuSUKMgl8khLyy+Gv+Y242w7olM/nOATRrrI+wXDj9qxEJAc45np63iRcXbeWmS1J49va+NI6J9rosCVEKfpEgV1bu+Jc/r2fG8h2MG9SRp77bh+gwvu+OBJ6CXySInSot59GZa/jwy908cGUXnri+h+67I3Wm4BcJUsdOlvLjt1by2ZZC/nlkTyYN7+J1SRImFPwiQajo+CkmvL6CtXlF/O62dMYMTPO6JAkjCn6RILP3cDF3v7acbYXHePGOAdzQp53XJUmYUfCLBJHcwmPc+doyDh47xdQJAxnaNdnrkiQMKfhFgsSGXYe5e8pyysrLefu+IfRNS/C6JAlTCn6RIJCVe4AJr6+gWaMY3pl0KV3bxHtdkoQxBb+IxxZu3Mf901fSvmVT3pg4iNREtaaWwFLwi3jo/TU7eXzWWnq0i2faDweR3Lyx1yVJBFDwRzDnHIAuCPLIm0tz+dUHXzHQl8Sr92TSokms1yVJhFDwR6jSsnK+88Ln7D50gsxOSQz0JTKwcxJ92rekUUxdWzHLuTjn+OOnOTz78WZG9GrDCz/oT5NY3XdHGo6CP0LNWZVP9u7DjOjVhq0FR1mQvReAJrFRZKQlMNCXxEBfEv07JdJcd4CsN+Xljt98uIGpn+dya78O/NfodGKj9UUrDUuf6AhUXFLGHxZsoW9aAq/cnYmZUXDkJFm5B1iRe5AVuQf408Icyh1EGfRu34LMTkkM6pxEpi+RNvFq+lEbJWXl/GL2Ot5dvZMJQ3388qbeROlma+IBBX8EenvZDnYdKubp7/et3L/fOr4xN16Swo2XpABw9GQpq3ccZMW2ii+Dd1bs4PUluQD4WsVV/kUwsHMSvlZxOk5wHsUlZfz07VUsyN7HY9d258Gru2rMxDMK/ghz7GQpf1qYw2VdWp3zqtDmjWMY1q01w7q1BiruEvnVrkOs8P9VsCB7L/+7Mh+A5OaNGehLJNOXxCBfEr1S4onR7otKh4tLuHdaFityD/CbURdz16U+r0uSCKfgjzBTP9/G/mOneOL6Hhe0XqOYKPp1TKRfx0QmDa/YV/114VGWbzvo/zI4wN/W7wGgWaNo+ndKrDho3DmRfmmJNG0UmQcvC4+e5J4py9m05wjP3Z7BqIwOXpckouCPJEXHT/E/i7/m2t5t6dcxsU6vFRVldG0TT9c28fxgcEcAdh86UXGMYFvFF8Fzn2zGOYiJMvp0aFlx5pAviUxfEknNwr9PbP7B49z12nJ2HzrBK3dnclXPNl6XJAIo+CPK/yz+mqMnS3n8uu4Bef2Ulk25pW9TbunbHoBDx0tYteMgy3MPkJV7gGlLtvPKZ9sA6Nqmuf84QcWXQWpi07Da571l7xHuem05x06V8ubEwQz0JXldkkglBX+E2He4mKmfb2NU3/b0bNeiQd6zZVwsV/VsU7mlW1xSxpc7D7F8W8UXwV/X7WLG8h0AtGvRhIGdv/ki6N42PmTbC67JK2LC1OVER0Uxc9Kl9G7fMOMtUlMK/gjxwsIcSsscj4wIzNZ+TTSJja48Gwgqeslu2nOErO0HWL7tAMu37ecva3cBEN8khsxO/gPGnZO4pEPLkLjI6fOcQu57I4ukZo14a+JgfMnNvC5J5FsU/BEg78BxZizfwZiBaUEVRNFRRu/2LejdvgV3X+rDOUf+wROVB4tX5B5k4aZNADSKjqJvWsvKM4f6d0qkZdPgusXBR+t389CMNfiS43hz4mDattD1DhKcFPwR4LkFW4gy46Gru3ldyjmZGWlJcaQlxXFr/1QA9h89ycrtByu/CF5Z/DUvLdqKGfRoG++/qKziy6BdS++CdtaKPJ58dx190xKYOn4gCXHhf/BaQpeCP8xt2XuE91bnc++wizwNxtpq1bwx113cjusurmg/ePxUKWt2FFVeYTx7ZT5vLN0OQGpiUwb5zxoa1DmRLq2bN8gB45cXb+U/525kWLdk/ueuAcQ10sdKgpv+hYa5Zz/eTFyjGH58RRevS6kXcY1iuKxrMpf5Lz4rLStnw+7DlaeRLt5SwLurdwKQGBdL5hlnDvXp0LJe74vjnON38zbx0qKt3JSewn+PydAN7iQkKPjD2Lr8Iv62fg+PjOgWtufNx0RHkZ6aQHpqAhMv74xzjm2Fx8jK/eY00o83fHMDun5piZV3Iu3XsfY3oCsrd/zLn79kxvI8fjC4I78Z1Sdkz0KSyKPgD2NPz9tEYlwsEy/v7HUpDcbMuKh1cy5q3ZwxA9OAilNZs7YfZLn/wrIXFuZQ/qn/4HJKCzJ9iZW7iFrHn78RysnSMh6buZYPv9zNT67qws+u6xFW1yBI+FPwh6mlW/fz2ZZC/t/IXsRHeIOPNi2aMPKSFEb6b0B3pLiEVTuKyMqtOI307WU7mPp5LgCdk5v9w32HOp11A7pjJ0v58VsrK8f2vuEXefEridSJgj8MOed4Zv4m2rZozF2XdvK6nKAT3ySWK7q35oru39yA7sudh/y3pT7AvK/2Miur4gZ0reP9N6DrlER6akue+jCbdflF/G50OmMy07z8NURqTcEfhhZu2sfK7Qf5j+/1CYmLnrzWKCaKAZ0SGdApkR9d0YXyckdOwdHKK4xX5B5k7pcVN6BrFB3Fi3cM4IY+7TyuWqT2FPxhprzc8fS8zXRqFact0lqKijK6t42ne9t47hxS8RfTzqITrNp+kK5tmtMrRbdgkNCm4A8zH365m+zdh/nD2Ay19KtHHRKa0iGhqddliNQLJUMYKS0r59mPN9OzXTzfSW/vdTkiEqTOG/xmNsXM9pnZ+jOmzTSzNf6fXDNbc471o81stZn9tb6KlqrNWZXPtsJjPH5dD/VyFZFq1WSL/3XghjMnOOdud85lOOcygDnAu+dY/2Egu9YVSo0Ul5Tx3IItZKQlMKKXGn6ISPXOG/zOucXAgarmWcUJzmOAGdXMTwVuAl6tQ41SA9OX7WD3oWJ+fr0uJhKRc6vrPv5hwF7n3JZq5j8H/BwoP98LmdkkM8sys6yCgoI6lhVZjp4s5cWFOQzt2qryHjYiItWpa/CPo/qt/ZuBfc65lTV5Iefcy865TOdcZuvWretYVmSZ+veKBuo/u+7CGqiLSGSq9emcZhYD3AoMqGaRocAtZjYSaAK0MLO3nHN31vY95duKjp/i5cVfc109NFAXkchQly3+EcBG51x+VTOdc//knEt1zvmAscCnCv36N/n/vuboqVIe19a+iNRQTU7nnAEsBXqYWb6ZTfTPGstZu3nMrL2Zza3/MqUq+w4X8/qSbXw3owM92sV7XY6IhIjz7upxzo2rZvr4KqbtAkZWMX0RsOiCq5Nz+uOnpxuoB3dLRREJLrpyN0Tt2F/RQP32gWl0ahU8DdRFJPgp+EPUc59sJjrKeDDIG6iLSPBR8IegzXuP8N7qndxzmS8kG6iLiLcU/CHo2fmbadYohvvDpIG6iDQsBX+IWZtXxEdf7eG+YReRGKYN1EUksBT8IeaZ+ZtIataIicMip4G6iNQvBX8IOd1A/YEru9C8sXroiEjtKPhDhHOOp+dtpF2LJpXtAEVEakPBHyI+3biPVTuKeOiabmqgLiJ1ouAPARUN1DfRqVUc389M9bocEQlxCv4Q8Ncvd7NxzxEeu7a7GqiLSJ0pRYJcSVk5z87fpAbqIlJvFPxBbs7KfHL3H+dnaqAuIvVEwR/EikvK+MMnW+jXMYFr1EBdROqJgj+IvfXFdnYfKuYJNVAXkXqk4A9SR0+W8uKirVzeNZnLuqiBuojUHwV/kJry920cOHaKn12vlooiUr8U/EHo4LFTvOJvoJ6RluB1OSISZhT8QWjy4q0cPVWqrX0RCQgFf5DZe7iYaUty+V5GB7q3VQN1Eal/Cv4g80JlA/XuXpciImFKwR9ETjdQHzsojY6t4rwuR0TClII/iDy3QA3URSTwFPxBYvPeI7y3ZifjL/PRtoUaqItI4Cj4g8Tv52+ieaMYfqwG6iISYAr+ILA2r4h5X+3lvuFqoC4igafgDwKnG6j/8HI1UBeRwFPwe2zJ1kI1UBeRBqXg95BzjmfmbSKlpRqoi0jDUfB76JNsNVAXkYan4PdIebnjmfmb8LWKY/QANVAXkYaj4PfIX9btYuOeIzyqBuoi0sCUOB4oKSvnvz/erAbqIuIJBb8HZvsbqD9xvRqoi0jDU/A3sOKSMv6wYAv9OyZwdU81UBeRhqfgb2BvfbGdPYeLeeL6nmqgLiKeOG/wm9kUM9tnZuvPmDbTzNb4f3LNbE0V66WZ2UIzyzazr8zs4fouPtScbqA+rFsyl3Zp5XU5IhKhanKp6OvAC8Abpyc4524//djMfg8cqmK9UuBx59wqM4sHVprZx865DXUrOXS99pm/gfp1aqkoIt457xa/c24xcKCqeVaxr2IMMKOK9XY751b5Hx8BsoEOdao2hB08dopXPvua6y9uS181UBcRD9V1H/8wYK9zbsu5FjIzH9APWFbH9wtZk/9vK8dOlfK4tvZFxGN1Df5xVLG1fyYzaw7MAR5xzh0+x3KTzCzLzLIKCgrqWFZw2Xu4mNeX5PK9fmqgLiLeq3Xwm1kMcCsw8xzLxFIR+tOdc++e6/Wccy875zKdc5mtW7eubVlB6Y+fbqHcOR5VA3URCQJ12eIfAWx0zuVXNdO///81INs592wd3iek7dh/nHeW5zF2YEfSktRAXUS8V5PTOWcAS4EeZpZvZhP9s8Zy1m4eM2tvZnP9T4cCdwFXn3Hq58h6rD0kPLdgMzHRxoNXd/W6FBERoAanczrnxlUzfXwV03YBI/2P/w5E9BVKm/ZUNFCfNPwi2qiBuogECV25G0CVDdSHq4G6iAQPBX+ArMkrYv4GNVAXkeCj4A+QZ+ZtopUaqItIEFLwB8CSnEL+nlPIA1d1VQN1EQk6Cv565pzj6fkVDdTvGNzR63JERL5FwV/PPsnex+odRTysBuoiEqQU/PXodAP1zsnNuE0N1EUkSCn465EaqItIKFA61ZOSsnKe9TdQv/mSFK/LERGploK/nvxvVj7b1UBdREKAgr8eFJeU8fwnWxjQKVEN1EUk6Cn468E3DdR7qIG6iAQ9BX8dHSku4U8LcxjWLZkhF6mBuogEPwV/HU35ey4Hj5fwxPVqqSgioUHBXwcH/A3Ub7i4HempaqAuIqFBwV8H3zRQV0tFEQkdCv5a2nOomGn+Burd1EBdREKIgr+W1EBdREKVgr8Wtu8/xswVeYwbpAbqIhJ6FPy18NyCLcREGz+9Sg3URST0KPgv0KY9R/jzmp2Mv6yzGqiLSEhS8F+gZ043UL/iIq9LERGpFQX/BVi94yAfb9jLpOEXkRCnBuoiEpoU/BfgmfkVDdQnqIG6iIQwBX8NfZ5TyOc5+/mJGqiLSIhT8NeAc46n522ifcsm/EAN1EUkxCn4a2BB9j7W5BXx8Ag1UBeR0KfgP4/ycscz8/wN1PurgbqIhD4F/3l8sHYXm/Ye4bFruxOjBuoiEgaUZOdwuoF6r5QW3KQG6iISJhT85zArK48dB47zxPXd1UBdRMKGgr8aZzZQv6qHGqiLSPhQ8FfjzaXb2Xv4JD9XA3URCTMK/iocKS7hxUU5DO/emsFqoC4iYUbBX4XX/r6tooH6dWqgLiLhR8F/lgPHTvHqZ9u4sU87Lklt6XU5IiL1TsF/lpcW5XD8VCmPXauWiiISns4b/GY2xcz2mdn6M6bNNLM1/p9cM1tTzbo3mNkmM8sxsyfrs/BA2H3oBNOWbud7/VLVQF1EwlZNtvhfB244c4Jz7nbnXIZzLgOYA7x79kpmFg38CbgR6A2MM7Peda44gP74aQ7OOR4Z0c3rUkREAua8we+cWwwcqGqeVZznOAaYUcXsQUCOc+5r59wp4B1gVB1qDajcwmPMWpHHD9RAXUTCXF338Q8D9jrntlQxrwOQd8bzfP+0KpnZJDPLMrOsgoKCOpZ14Z5bsJmYaOMnV6uBuoiEt7oG/ziq3toHqOqqJ1fdCznnXnbOZTrnMlu3bl3Hsi7Mxj2HeX/tLiYM7UybeDVQF5HwVutWUmYWA9wKDKhmkXwg7YznqcCu2r5fIP1+/maaN47hR8PVQF1Ewl9dtvhHABudc/nVzF8BdDOzzmbWCBgLfFCH9wuIVf4G6j9SA3URiRA1OZ1zBrAU6GFm+WY20T9rLGft5jGz9mY2F8A5Vwr8FJgHZAOznHNf1Wfx9eGZef4G6kPVQF1EIsN5d/U458ZVM318FdN2ASPPeD4XmFuH+gLq85xClmzdz69u7k0zNVAXkQgRsVfuOuf4nb+B+h1D1EBdRCJHxAb/xxv2sjaviEdGdKdxjBqoi0jkiMjgLyt3/H7+Zi5Kbsat/au9tEBEJCxFZPD/5XQD9evUQF1EIk/Epd7pBuq9U1owso8aqItI5Im44J+54nQD9R5qoC4iESmigv90A/XMTolc2aNhbwshIhIsIir431iay74jJ3lCDdRFJIJFTPBXNFDfyhVqoC4iES5igv/Vz7ZRdLyEn6mBuohEuIgI/ooG6l8z8hI1UBcRiYjgf2lRDidKytRAXUSECAj+0w3Ub+2fStc2aqAuIhL2wf/8JxUN1B++Rg3URUQgzIM/t/AYs7LUQF1E5ExhHfz/vWAzjaKj1EBdROQMYRv82bsP88HaXUwY6lMDdRGRM4Rt8H/TQL2L16WIiASVsAz+VTsOsiB7Lz++ogst42K9LkdEJKiEXfA753j6o00kN2/E+Mt8XpcjIhJ0wi74P8/Zz9Kv9/OTq7qqgbqISBXCKvidczw9byMdEpryg8FqoC4iUpWwCv75G/ayNv8QD4/opgbqIiLVCJvgr2igvomLWjfj1n5qoC4iUp2w2Ql+oqSMfmkVnbXUQF1EpHphE/zNG8fwX6PTvS5DRCToadNYRCTCKPhFRCKMgl9EJMIo+EVEIoyCX0Qkwij4RUQijIJfRCTCKPhFRCKMOee8ruFbzKwA2F7L1ZOBwnosp76orgujui6M6row4VhXJ+dc65osGJTBXxdmluWcy/S6jrOprgujui6M6rowkV6XdvWIiEQYBb+ISIQJx+B/2esCqqG6LozqujCq68JEdF1ht49fRETOLRy3+EVE5BxCMvjNbIqZ7TOz9dXMNzN73sxyzGydmfUPkrquNLNDZrbG//OrBqorzcwWmlm2mX1lZg9XsUyDj1kN62rwMTOzJma23MzW+uv6tyqWaWxmM/3jtczMfEFS13gzKzhjvO4NdF1nvHe0ma02s79WMa/Bx6uGdXkyXmaWa2Zf+t8zq4r5gf08OudC7gcYDvQH1lczfyTwN8CAIcCyIKnrSuCvHoxXCtDf/zge2Az09nrMalhXg4+Zfwya+x/HAsuAIWct8wAw2f94LDAzSOoaD7zQ0P/G/O/9GPB2Vf+/vBivGtblyXgBuUDyOeYH9PMYklv8zrnFwIFzLDIKeMNV+AJIMLOUIKjLE8653c65Vf7HR4Bs4OzGxA0+ZjWsq8H5x+Co/2ms/+fsg2GjgGn+x7OBa8zMgqDjaJqqAAACzklEQVQuT5hZKnAT8Go1izT4eNWwrmAV0M9jSAZ/DXQA8s54nk8QBIrfpf4/1f9mZhc39Jv7/8TuR8XW4pk8HbNz1AUejJl/98AaYB/wsXOu2vFyzpUCh4BWQVAXwG3+3QOzzSwt0DX5PQf8HCivZr4n41WDusCb8XLAfDNbaWaTqpgf0M9juAZ/VVsSwbBltIqKy6r7An8E/tyQb25mzYE5wCPOucNnz65ilQYZs/PU5cmYOefKnHMZQCowyMz6nLWIJ+NVg7r+Avicc+nAAr7Zyg4YM7sZ2OecW3muxaqYFtDxqmFdDT5efkOdc/2BG4GfmNnws+YHdLzCNfjzgTO/uVOBXR7VUsk5d/j0n+rOublArJklN8R7m1ksFeE63Tn3bhWLeDJm56vLyzHzv2cRsAi44axZleNlZjFASxpwN191dTnn9jvnTvqfvgIMaIByhgK3mFku8A5wtZm9ddYyXozXeevyaLxwzu3y/3cf8B4w6KxFAvp5DNfg/wC4239kfAhwyDm32+uizKzd6f2aZjaIivHf3wDva8BrQLZz7tlqFmvwMatJXV6MmZm1NrME/+OmwAhg41mLfQDc4388GvjU+Y/KeVnXWfuBb6HiuElAOef+yTmX6pzzUXHg9lPn3J1nLdbg41WTurwYLzNrZmbxpx8D1wFnnwkY0M9jTH29UEMysxlUnO2RbGb5wL9ScaAL59xkYC4VR8VzgOPAhCCpazRwv5mVAieAsYH+x+83FLgL+NK/fxjgn4GOZ9TmxZjVpC4vxiwFmGZm0VR80cxyzv3VzP4dyHLOfUDFF9abZpZDxZbr2ADXVNO6HjKzW4BSf13jG6CuKgXBeNWkLi/Gqy3wnn97JgZ42zn3kZn9GBrm86grd0VEIky47uoREZFqKPhFRCKMgl9EJMIo+EVEIoyCX0Qkwij4RUQijIJfRCTCKPhFRCLM/wevk+WHBGLRigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#hide\n",
    "learn.recorder.plot_loss(skip_start=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.validate\" class=\"doc_header\"><code>Learner.validate</code><a href=\"__main__.py#L158\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.validate</code>(**`ds_idx`**=*`1`*, **`dl`**=*`None`*, **`cbs`**=*`None`*)\n",
       "\n",
       "Validate on `dl` with potential new `cbs`."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.validate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test result\n",
    "learn = synth_learner(n_train=5, metrics=tst_metric)\n",
    "res = learn.validate()\n",
    "test_eq(res[0], res[1])\n",
    "x,y = learn.dls.valid_ds.tensors\n",
    "test_close(res[0], F.mse_loss(learn.model(x), y), 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test other dl\n",
    "res = learn.validate(dl=learn.dls.train)\n",
    "test_eq(res[0], res[1])\n",
    "x,y = learn.dls.train_ds.tensors\n",
    "test_close(res[0], F.mse_loss(learn.model(x), y), 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.get_preds\" class=\"doc_header\"><code>Learner.get_preds</code><a href=\"__main__.py#L164\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.get_preds</code>(**`ds_idx`**=*`1`*, **`dl`**=*`None`*, **`with_input`**=*`False`*, **`with_decoded`**=*`False`*, **`with_loss`**=*`False`*, **`act`**=*`None`*, **`inner`**=*`False`*, **`reorder`**=*`False`*, **`cbs`**=*`None`*, **`save_preds`**=*`None`*, **`save_targs`**=*`None`*, **`concat_dim`**=*`0`*)\n",
       "\n",
       "Get the predictions and targets on the `ds_idx`-th dbunchset or `dl`, optionally `with_input` and `with_loss`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.get_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`with_decoded` will also return the decoded predictions using the <code>decodes</code> function of the loss function (if it exists). For instance, fastai's `CrossEntropyFlat` takes the argmax or predictions in its decodes. \n",
    "\n",
    "Depending on the `loss_func` attribute of `Learner`, an activation function will be picked automatically so that the predictions make sense. For instance if the loss is a case of cross-entropy, a softmax will be applied, or if the loss is binary cross entropy with logits, a sigmoid will be applied. If you want to make sure a certain activation function is applied, you can pass it with `act`.\n",
    "\n",
    "`save_preds` and `save_targs` should be used when your predictions are too big to fit all in memory. Give a `Path` object that points to a folder where the predictions and targets will be saved.\n",
    "\n",
    "`concat_dim` is the batch dimension, where all the tensors will be concatenated.\n",
    "\n",
    "`inner` is an internal attribute that tells `get_preds` it's called internally, inside another training loop, to avoid recursion errors."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Note: If you want to use the option `with_loss=True` on a custom loss function, make sure you have implemented a `reduction` attribute that supports 'none' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test result\n",
    "learn = synth_learner(n_train=5, metrics=tst_metric)\n",
    "preds,targs = learn.get_preds()\n",
    "x,y = learn.dls.valid_ds.tensors\n",
    "test_eq(targs, y)\n",
    "test_close(preds, learn.model(x))\n",
    "\n",
    "preds,targs = learn.get_preds(act = torch.sigmoid)\n",
    "test_eq(targs, y)\n",
    "test_close(preds, torch.sigmoid(learn.model(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test get_preds work with ds not evenly divisible by bs\n",
    "learn = synth_learner(n_train=2.5, metrics=tst_metric)\n",
    "preds,targs = learn.get_preds(ds_idx=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test other dataset\n",
    "x = torch.randn(16*5)\n",
    "y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
    "dl = DataLoader(TensorDataset(x, y), batch_size=16)\n",
    "preds,targs = learn.get_preds(dl=dl)\n",
    "test_eq(targs, y)\n",
    "test_close(preds, learn.model(x))\n",
    "\n",
    "#Test with loss\n",
    "preds,targs,losses = learn.get_preds(dl=dl, with_loss=True)\n",
    "test_eq(targs, y)\n",
    "test_close(preds, learn.model(x))\n",
    "test_close(losses, F.mse_loss(preds, targs, reduction='none'))\n",
    "\n",
    "#Test with inputs\n",
    "inps,preds,targs = learn.get_preds(dl=dl, with_input=True)\n",
    "test_eq(inps,x)\n",
    "test_eq(targs, y)\n",
    "test_close(preds, learn.model(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test with no target\n",
    "learn = synth_learner(n_train=5)\n",
    "x = torch.randn(16*5)\n",
    "dl = DataLoader(TensorDataset(x), batch_size=16)\n",
    "preds,targs = learn.get_preds(dl=dl)\n",
    "assert targs is None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test with targets that are tuples\n",
    "def _fake_loss(x,y,z,reduction=None): return F.mse_loss(x,y)\n",
    "\n",
    "learn = synth_learner(n_train=5)\n",
    "x = torch.randn(16*5)\n",
    "y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
    "learn.dls.n_inp=1\n",
    "learn.loss_func = _fake_loss\n",
    "dl = DataLoader(TensorDataset(x, y, y), batch_size=16)\n",
    "preds,targs = learn.get_preds(dl=dl)\n",
    "test_eq(targs, [y,y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "#Test with inputs that are tuples\n",
    "class _TupleModel(nn.Module):\n",
    "    def __init__(self, model): \n",
    "        super().__init__()\n",
    "        self.model=model\n",
    "    def forward(self, x1, x2): return self.model(x1)\n",
    "\n",
    "learn = synth_learner(n_train=5)\n",
    "#learn.dls.n_inp=2\n",
    "x = torch.randn(16*5)\n",
    "y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
    "learn.model = _TupleModel(learn.model)\n",
    "learn.dls = DataLoaders(DataLoader(TensorDataset(x, x, y), batch_size=16),DataLoader(TensorDataset(x, x, y), batch_size=16))\n",
    "inps,preds,targs = learn.get_preds(ds_idx=0, with_input=True)\n",
    "test_eq(inps, [x,x])\n",
    "t = learn.get_preds(ds_idx=0, with_input=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "inps,preds,targs = learn.get_preds(ds_idx=0, with_input=True)\n",
    "tst = learn.get_preds(ds_idx=0, with_input=True, with_decoded=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"Learner.predict\" class=\"doc_header\"><code>Learner.predict</code><a href=\"__main__.py#L190\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>Learner.predict</code>(**`item`**, **`rm_type_tfms`**=*`None`*, **`with_input`**=*`False`*)\n",
       "\n",
       "Prediction on `item`, fully decoded, loss function decoded and probabilities"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(Learner.predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It returns a tuple of three elements with, in reverse order,\n",
    "- the prediction from the model, potentially passed through the activation of the loss function (if it has one)\n",
    "- the decoded prediction, using the potential <code>decodes</code> method from it\n",
    "- the fully decoded prediction, using the transforms used to build the `Datasets`/`DataLoaders`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`rm_type_tfms` is a deprecated argument that should not be used and will be removed in a future version. `with_input` will add the decoded inputs to the result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Note: To utilize `predict` you should use the entire `fastai` `DataBlock` API, as `predict` will not work with raw pytorch `DataLoaders` (and in turn, this sublibrary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@patch\n",
    "def freeze_to(self:Learner, n):\n",
    "    \"Freeze parameter groups up to `n`\"\n",
    "    if self.opt is None: self.create_opt()\n",
    "    self.opt.freeze_to(n)\n",
    "    self.opt.clear_state()\n",
    "\n",
    "@patch\n",
    "def freeze(self:Learner): \n",
    "    \"Freeze up to last parameter group\"\n",
    "    self.freeze_to(-1)\n",
    "\n",
    "@patch\n",
    "def unfreeze(self:Learner): \n",
    "    \"Unfreeze the entire model\"\n",
    "    self.freeze_to(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2.98773193359375, 2.9508399963378906, '00:00']\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "class _TstModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.a,self.b = nn.Parameter(torch.randn(1)),nn.Parameter(torch.randn(1))\n",
    "        self.tst = nn.Sequential(nn.Linear(4,5), nn.BatchNorm1d(3))\n",
    "        self.tst[0].bias.data,self.tst[1].bias.data = torch.randn(5),torch.randn(3) \n",
    "    def forward(self, x): return x * self.a + self.b\n",
    "    \n",
    "class _PutGrad(Callback):\n",
    "    def before_step(self):\n",
    "        for p in self.learn.tst.parameters():\n",
    "            if p.requires_grad: p.grad = torch.ones_like(p.data)\n",
    "                \n",
    "from fastai_minima.optimizer import convert_params\n",
    "\n",
    "def _splitter(m): return convert_params([list(m.tst[0].parameters()), list(m.tst[1].parameters()), [m.a,m.b]])\n",
    "            \n",
    "learn = synth_learner(n_train=5, opt_func = partial(SGD), cbs=_PutGrad, splitter=_splitter, lr=1e-2)\n",
    "learn.model = _TstModel()\n",
    "learn.freeze()\n",
    "init = [p.clone() for p in learn.tst.parameters()]\n",
    "learn.fit(1, wd=0.)\n",
    "end = list(learn.tst.parameters())\n",
    "#linear was not trained\n",
    "for i in [0,1]: test_close(end[i],init[i])\n",
    "#bn was trained even frozen since `train_bn=True` by default\n",
    "for i in [2,3]: test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 14.640519142150879, 12.946287155151367, '00:00']\n",
      "[0, 12.028812408447266, 10.639350891113281, '00:00']\n",
      "[0, 9.865816116333008, 8.751302719116211, '00:00']\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "learn = synth_learner(n_train=5, opt_func = partial(SGD), cbs=_PutGrad, splitter=_splitter, train_bn=False, lr=1e-2)\n",
    "learn.model = _TstModel()\n",
    "learn.freeze()\n",
    "init = [p.clone() for p in learn.tst.parameters()]\n",
    "learn.fit(1, wd=0.)\n",
    "end = list(learn.tst.parameters())\n",
    "#linear and bn were not trained\n",
    "for i in range(4): test_close(end[i],init[i])\n",
    "\n",
    "learn.freeze_to(-2)\n",
    "init = [p.clone() for p in learn.tst.parameters()]\n",
    "learn.fit(1, wd=0.)\n",
    "end = list(learn.tst.parameters())\n",
    "#linear was not trained\n",
    "for i in [0,1]: test_close(end[i],init[i])\n",
    "#bn was trained \n",
    "for i in [2,3]: test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]))\n",
    "    \n",
    "learn.unfreeze()\n",
    "init = [p.clone() for p in learn.tst.parameters()]\n",
    "learn.fit(1, wd=0.)\n",
    "end = list(learn.tst.parameters())\n",
    "#linear and bn were trained\n",
    "for i in range(4): test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]), 1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@patch\n",
    "def tta(self:Learner, ds_idx=1, dl=None, n=4, item_tfms=None, batch_tfms=None, beta=0.25, use_max=False):\n",
    "    \"Return predictions on the `ds_idx` dataset or `dl` using Test Time Augmentation\"\n",
    "    raise NotImplementedError(\"You're trying to use non-basic fastai functionality. You should use the entire fastai API to get this feature\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In practice, we get the predictions `n` times with the transforms of the training set and average those. The final predictions are `(1-beta)` multiplied by this average + `beta` multiplied by the predictions obtained with the transforms of the dataset. Set `beta` to `None` to get a tuple of the predictions and tta results. You can also use the maximum of all predictions instead of an average by setting `use_max=True`.\n",
    "\n",
    "If you want to use new transforms, you can pass them with `item_tfms` and `batch_tfms`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Note: To use `tta` you need to utilize the entire `fastai` `DataBlock` API, as a result it is unsupported in this sublibrary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "split_at_heading": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}