CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import warnings\n",
    "\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import models, Model\n",
    "from tensorflow.keras.applications import MobileNetV2\n",
    "from tensorflow.keras.layers import Dense, Conv2D, Flatten, MaxPool2D, BatchNormalization, Dropout, Input, Concatenate, GlobalAvgPool2D\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.utils import Sequence\n",
    "#from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "import json\n",
    "from evaluator import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "VARIABLES = ['ActivityCounts', 'Barometer', 'BloodPerfusion',\n",
    "             'BloodPulseWave', 'EnergyExpenditure', 'GalvanicSkinResponse', 'HR',\n",
    "             'HRV', 'RESP', 'Steps', 'SkinTemperature', 'ActivityClass']"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [],
   "source": [
    "GRAYSCALE = False # grayscale or rgb # TODO: currently not working (MobileNetV2 does not support grayscale)\n",
    "TRANSFORMER_IMPUTATION = True # whether to use transformer imputation or linear interpolation\n",
    "FULL_TRANSFORMER_IMPUTATION = False # whether to use FULL transformer imputation or linear interpolation\n",
    "BACKGROUND_FEATURES = False"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Import data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "'./Output/transformer_imputation'"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# file path to data folder\n",
    "path = './Output' if not TRANSFORMER_IMPUTATION else './Output/transformer_imputation'\n",
    "path = './Output/full_transformer_imputation' if FULL_TRANSFORMER_IMPUTATION else path\n",
    "path"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "613 370 497 30\n"
     ]
    }
   ],
   "source": [
    "# dimensions\n",
    "N, HEIGHT, WIDTH, CHANNELS = sum([1 for p in os.listdir(path) if (p[:14] == 'feature_vector' and p[:19] != 'feature_vector_stat')]), \\\n",
    "                             *np.load(path + '/feature_vector0.npy').shape\n",
    "CHANNELS = len(VARIABLES) if GRAYSCALE else CHANNELS # reduce channels for grayscale\n",
    "\n",
    "print(N, HEIGHT, WIDTH, CHANNELS)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Metadata (subjectID etc.)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [],
   "source": [
    "with open(path + '/metadata.txt') as f:\n",
    "    metadata = f.read()\n",
    "\n",
    "metadata = json.loads(metadata.replace('\\'', '\\\"').replace('False', 'false').replace('True', 'true')) # doesn't accept other chars"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [],
   "source": [
    "subjects = [meta['subjectID'] for meta in metadata]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [],
   "source": [
    "# age: 26-35: [0, 0], 36-45: [0, 1], 46-55: [1, 0] (subject 22: unknown -> randomly assigned)\n",
    "age_ranges = [np.array([0, 0]), np.array([0, 1]), np.array([1, 0])]\n",
    "age = {1: age_ranges[1],\n",
    "       2: age_ranges[1],\n",
    "       3: age_ranges[1],\n",
    "       4: age_ranges[1],\n",
    "       5: age_ranges[1],\n",
    "       6: age_ranges[2],\n",
    "       7: age_ranges[1],\n",
    "       8: age_ranges[2],\n",
    "       9: age_ranges[1],\n",
    "       10: age_ranges[1],\n",
    "       11: age_ranges[1],\n",
    "       12: age_ranges[2],\n",
    "       13: age_ranges[2],\n",
    "       14: age_ranges[1],\n",
    "       15: age_ranges[2],\n",
    "       16: age_ranges[0],\n",
    "       17: age_ranges[2],\n",
    "       18: age_ranges[2],\n",
    "       19: age_ranges[1],\n",
    "       20: age_ranges[2],\n",
    "       21: age_ranges[2],\n",
    "       22: age_ranges[np.random.choice((0, 1, 2))],\n",
    "       23: age_ranges[0],\n",
    "       24: age_ranges[0],\n",
    "       25: age_ranges[2],\n",
    "       26: age_ranges[0],\n",
    "       27: age_ranges[1],\n",
    "       28: age_ranges[0]}\n",
    "age = [age[subject] for subject in subjects]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [],
   "source": [
    "# gender: 0: male, 1: female (subject 22: unknown -> randomly assigned)\n",
    "gender = {1: 0,\n",
    "          2: 0,\n",
    "          3: 0,\n",
    "          4: 1,\n",
    "          5: 1,\n",
    "          6: 1,\n",
    "          7: 0,\n",
    "          8: 0,\n",
    "          9: 0,\n",
    "          10: 1,\n",
    "          11: 1,\n",
    "          12: 1,\n",
    "          13: 0,\n",
    "          14: 1,\n",
    "          15: 1,\n",
    "          16: 1,\n",
    "          17: 0,\n",
    "          18: 0,\n",
    "          19: 0,\n",
    "          20: 0,\n",
    "          21: 0,\n",
    "          22: np.random.choice((0, 1)),\n",
    "          23: 1,\n",
    "          24: 0,\n",
    "          25: 0,\n",
    "          26: 0,\n",
    "          27: 1,\n",
    "          28: 0}\n",
    "gender = [gender[subject] for subject in subjects]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# CNN"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Addditional functions"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [],
   "source": [
    "# image-wise transformer\n",
    "def rgb2gray(rgb):\n",
    "    \"\"\"greyscale = 0.2989 * red + 0.5870 * green + 0.1140 * blue\"\"\"\n",
    "    return np.dot(rgb[:, :, :3], [0.2989, 0.5870, 0.1140])\n",
    "\n",
    "# loss function\n",
    "def weighted_cross_entropy(weight):\n",
    "    def weighted_cross_entropy_with_logits(labels, logits):\n",
    "        loss = tf.nn.weighted_cross_entropy_with_logits(\n",
    "            labels, logits, weight\n",
    "        )\n",
    "        return loss\n",
    "    return weighted_cross_entropy_with_logits\n",
    "\n",
    "# weight (imbalanced classes)\n",
    "def check_imbalance(path_to_labels, indices, variable):\n",
    "    \"\"\"Returns indices of positives/negatives\"\"\"\n",
    "    y = np.empty((len(indices), 2), dtype=int)\n",
    "    for i, index in enumerate(indices):\n",
    "        y[i, ] = np.load(path_to_labels + f'/labels{index}.npy', allow_pickle=True)\n",
    "\n",
    "    positives = np.where(y[:, variable] == 1)[0]\n",
    "    negatives = np.where(y[:, variable] == 0)[0]\n",
    "\n",
    "    return np.array(indices)[positives], np.array(indices)[negatives]\n",
    "\n",
    "def get_weighting_factor(path, train_set_indices, variable):\n",
    "    positives, negatives = check_imbalance(path, train_set_indices, variable)\n",
    "    sample_weight = len(negatives) / len(positives) # for weighted cross-entropy\n",
    "    return sample_weight"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Dataloader (dataset with images too large)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [],
   "source": [
    "class DataGenerator(Sequence):\n",
    "\n",
    "    def __init__(self, data_path: str, indices_dataset: list, variable, batch_size=32, dim=(HEIGHT, WIDTH), n_channels=CHANNELS, shuffle=True):\n",
    "        self.data_path = data_path # path to full dataset\n",
    "        self.dim = dim # image dimension\n",
    "        self.batch_size = batch_size\n",
    "        self.indices_dataset = indices_dataset # indices of full dataset (different for train/validation/test set)\n",
    "        self.n_channels = n_channels\n",
    "        self.shuffle = shuffle\n",
    "        assert variable in (0, 1)\n",
    "        self.variable = variable\n",
    "\n",
    "        self.on_epoch_end() # shuffle data for each epoch\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        \"\"\"\n",
    "        Shuffle data for each epoch\n",
    "        \"\"\"\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(self.indices_dataset)\n",
    "\n",
    "    def __data_generation(self, indices):\n",
    "        \"\"\"\n",
    "        Loads and returns datapoints[indices]\n",
    "        \"\"\"\n",
    "        # init\n",
    "        X = np.empty((self.batch_size, *self.dim, self.n_channels))\n",
    "        y = np.empty(self.batch_size, dtype=float) # float: logits, int: non-logits\n",
    "\n",
    "        # load individual datapoints\n",
    "        for i, index in enumerate(indices):\n",
    "            images = np.load(self.data_path + f'/feature_vector{index}.npy', allow_pickle=True)\n",
    "            if GRAYSCALE:\n",
    "                images_gray = np.empty((HEIGHT, WIDTH, self.n_channels))\n",
    "                for j in range(len(VARIABLES)):\n",
    "                    image_rgb = images[:, :, (3 * j): (3 * (j + 1))]\n",
    "                    image_gray = rgb2gray(image_rgb)\n",
    "                    images_gray[:, :, j] = image_gray\n",
    "                images = images_gray\n",
    "\n",
    "            X[i, ] = images\n",
    "            y[i] = np.load(self.data_path + f'/labels{index}.npy', allow_pickle=True)[self.variable]\n",
    "\n",
    "        if BACKGROUND_FEATURES:\n",
    "            global age, gender\n",
    "            background_features = np.empty((self.batch_size, *self.dim, 3)) # create pseudo-image to comply with network\n",
    "            for i, index in enumerate(indices):\n",
    "                age_i, gender_i = age[i], gender[i]\n",
    "                background_features[i, ] = np.dstack([np.ones(self.dim) * age_i[0],\n",
    "                                                     np.ones(self.dim) * age_i[1],\n",
    "                                                     np.ones(self.dim) * gender_i])\n",
    "\n",
    "            X = np.concatenate([X, background_features], axis=-1)\n",
    "\n",
    "        return X, y\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"\n",
    "        Number of batches per epoch\n",
    "        \"\"\"\n",
    "        return int(np.floor(len(self.indices_dataset) / self.batch_size))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"\n",
    "        Generates batch[index]\n",
    "        \"\"\"\n",
    "        # calculate indices of batch\n",
    "        indices = self.indices_dataset[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "\n",
    "        # generate batch\n",
    "        X, y = self.__data_generation(indices)\n",
    "\n",
    "        return X, y"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Architecture"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [],
   "source": [
    "class ConvNet_larger(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, name='cnn', **kwargs):\n",
    "        super(ConvNet_larger, self).__init__(name, **kwargs)\n",
    "\n",
    "        self.in_shape = (HEIGHT, WIDTH, CHANNELS) if not BACKGROUND_FEATURES else (HEIGHT, WIDTH, CHANNELS + 3)\n",
    "        self.in_shape_mobilenet = (HEIGHT, WIDTH, 3) if not GRAYSCALE else (HEIGHT, WIDTH, 1)\n",
    "\n",
    "        # MobileNetV2 embedding\n",
    "        self.mobilenet = MobileNetV2(input_shape=self.in_shape_mobilenet, weights='imagenet', include_top=False)\n",
    "        self.mobilenet._name = 'mobilenet'\n",
    "        self.mobilenet.trainable = False\n",
    "        self.finetuning = False\n",
    "        self.out_shape_mobilenet = self.mobilenet.layers[-1].output_shape # for one spectrogram\n",
    "\n",
    "        # Concatenation\n",
    "        self.concat = Concatenate(name='concat')\n",
    "\n",
    "        # Global pooling\n",
    "        '''self.pool = GlobalAvgPool2D(name='global_avg_pool')'''\n",
    "\n",
    "        # TODO: more sophisticated dense (dropout, regularizer, init., ...)\n",
    "        self.conv0 = Conv2D(filters=4, kernel_size=(1, 1), activation='relu', padding='same', name='conv0')\n",
    "        self.conv1 = Conv2D(filters=8, kernel_size=(3, 3), activation='relu', padding='same', name='conv1')\n",
    "        self.batch_norm1 = BatchNormalization(name='batch_norm1') # try for incr. stability\n",
    "        self.pool1 = MaxPool2D(strides=(2, 2), name='pool1')\n",
    "\n",
    "        self.conv2 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same', name='conv2')\n",
    "        self.batch_norm2 = BatchNormalization(name='batch_norm2') # try for incr. stability\n",
    "        self.pool2 = MaxPool2D(strides=(2, 2), name='pool2')\n",
    "\n",
    "        self.flatten = Flatten(name='flatten')\n",
    "\n",
    "        self.dense = Dense(10, name='dense')\n",
    "\n",
    "        # Fully-connected network\n",
    "        if BACKGROUND_FEATURES:\n",
    "            self.concat2 = Concatenate(name='concat2', axis=-1)\n",
    "        self.dense2 = Dense(1, name='dense2') # keep logits\n",
    "        self.out_shape = 1\n",
    "\n",
    "        # build graph\n",
    "        self.build_graph()\n",
    "\n",
    "    def build_graph(self):\n",
    "        self.build(input_shape=(None, *self.in_shape))\n",
    "        x = Input(shape=self.in_shape)\n",
    "        Model(inputs=[x], outputs=self.call(x))\n",
    "\n",
    "    def set_finetuning(self, mode=True):\n",
    "        self.finetuning = mode\n",
    "        self.mobilenet.trainable = mode\n",
    "\n",
    "        for layers in self.mobilenet.layers:\n",
    "            layers.trainable = False\n",
    "\n",
    "        # \"activate\" last conv layer of MobileNet\n",
    "        self.mobilenet.layers[-3].trainable = mode\n",
    "        self.mobilenet.layers[-2].trainable = mode\n",
    "\n",
    "    def call(self, inputs, background_features=None):\n",
    "        \"\"\"\n",
    "        Model predictions (logits)\n",
    "        :param background_features: additional features of shape (3)\n",
    "        :param inputs: all spectrograms of shape (HEIGHT, WIDTH, CHANNELS)\n",
    "        :return: class prediction (logits)\n",
    "        \"\"\"\n",
    "        if BACKGROUND_FEATURES:\n",
    "            inputs = inputs[:, :, :, :-3]\n",
    "            background_features = inputs[:, 0, 0, -3:]\n",
    "\n",
    "        # MobileNetV2 embeddings\n",
    "        x = [self.mobilenet(inputs[..., i:i+3], training=self.finetuning) for i in range(0, CHANNELS, 3)]\n",
    "\n",
    "        # Concatenation\n",
    "        x = self.concat(x)\n",
    "\n",
    "        # Global pooling\n",
    "        '''x = self.pool(x)'''\n",
    "        x= self.conv0(x)\n",
    "        x = self.conv1(x)\n",
    "        x = self.batch_norm1(x)\n",
    "        x = self.pool1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = self.batch_norm2(x)\n",
    "        x = self.pool2(x)\n",
    "        x = self.flatten(x)\n",
    "\n",
    "        if background_features is not None:\n",
    "            x = self.concat2([x, background_features])\n",
    "\n",
    "        # Fully-connected network\n",
    "        x = self.dense(x)\n",
    "        x = self.dense2(x)\n",
    "\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [],
   "source": [
    "class ConvNet(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, name='cnn', **kwargs):\n",
    "        super(ConvNet, self).__init__(name, **kwargs)\n",
    "\n",
    "        self.in_shape = (HEIGHT, WIDTH, CHANNELS) if not BACKGROUND_FEATURES else (HEIGHT, WIDTH, CHANNELS + 3)\n",
    "        self.in_shape_mobilenet = (HEIGHT, WIDTH, 3) if not GRAYSCALE else (HEIGHT, WIDTH, 1)\n",
    "\n",
    "        # MobileNetV2 embedding\n",
    "        self.mobilenet = MobileNetV2(input_shape=self.in_shape_mobilenet, weights='imagenet', include_top=False)\n",
    "        self.mobilenet._name = 'mobilenet'\n",
    "        self.mobilenet.trainable = False\n",
    "        self.finetuning = False\n",
    "        self.out_shape_mobilenet = self.mobilenet.layers[-1].output_shape # for one spectrogram\n",
    "\n",
    "        # Concatenation\n",
    "        self.concat = Concatenate(name='concat')\n",
    "\n",
    "        # Global pooling\n",
    "        self.pool = GlobalAvgPool2D(name='global_avg_pool')\n",
    "\n",
    "        # TODO: more sophisticated dense (dropout, regularizer, init., ...)\n",
    "        # Fully-connected network\n",
    "        if BACKGROUND_FEATURES:\n",
    "            self.concat2 = Concatenate(name='concat2', axis=-1)\n",
    "        self.dense = Dense(1, name='dense') # keep logits\n",
    "        self.out_shape = 1\n",
    "\n",
    "        # build graph\n",
    "        self.build_graph()\n",
    "\n",
    "    def build_graph(self):\n",
    "        self.build(input_shape=(None, *self.in_shape))\n",
    "        x = Input(shape=self.in_shape)\n",
    "        Model(inputs=[x], outputs=self.call(x))\n",
    "\n",
    "    def set_finetuning(self, mode=True):\n",
    "        self.finetuning = mode\n",
    "        self.mobilenet.trainable = mode\n",
    "\n",
    "        for layers in self.mobilenet.layers:\n",
    "            layers.trainable = False\n",
    "\n",
    "        # \"activate\" last conv layer of MobileNet\n",
    "        self.mobilenet.layers[-3].trainable = mode\n",
    "        self.mobilenet.layers[-2].trainable = mode\n",
    "\n",
    "    def call(self, inputs, background_features=None):\n",
    "        \"\"\"\n",
    "        Model predictions (logits)\n",
    "        :param background_features: additional features of shape (3)\n",
    "        :param inputs: all spectrograms of shape (HEIGHT, WIDTH, CHANNELS)\n",
    "        :return: class prediction (logits)\n",
    "        \"\"\"\n",
    "        if BACKGROUND_FEATURES:\n",
    "            inputs = inputs[:, :, :, :-3]\n",
    "            background_features = inputs[:, 0, 0, -3:]\n",
    "\n",
    "        # MobileNetV2 embeddings\n",
    "        x = [self.mobilenet(inputs[..., i:i+3], training=self.finetuning) for i in range(0, CHANNELS, 3)]\n",
    "\n",
    "        # Concatenation\n",
    "        x = self.concat(x)\n",
    "\n",
    "        # Global pooling\n",
    "        x = self.pool(x)\n",
    "\n",
    "        if background_features is not None:\n",
    "            x = self.concat2([x, background_features])\n",
    "\n",
    "        # Fully-connected network\n",
    "        x = self.dense(x)\n",
    "\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [],
   "source": [
    "class CNN:\n",
    "\n",
    "    def __init__(self, path, variable, epochs, learning_rate, batch_size):\n",
    "        self.model = ConvNet()\n",
    "        self.path = path\n",
    "        assert variable in (0, 1)\n",
    "        self.variable = variable\n",
    "        self.epochs = epochs\n",
    "        self.learning_rate = learning_rate\n",
    "        self.batch_size = batch_size\n",
    "        self.history = None\n",
    "        self.train_indices = None\n",
    "        self.test_indices = None\n",
    "\n",
    "    def fit(self, train_indices, validation_indices=None):\n",
    "        self.train_indices = train_indices\n",
    "\n",
    "        # training set\n",
    "        train_dataloader = DataGenerator(self.path, train_indices, batch_size=self.batch_size, variable=self.variable)\n",
    "\n",
    "        # validation set\n",
    "        if validation_indices is not None:\n",
    "            val_dataloader = DataGenerator(self.path, validation_indices, batch_size=self.batch_size, variable=self.variable)\n",
    "\n",
    "        # weights for loss function\n",
    "        sample_weights = get_weighting_factor(self.path, train_indices, self.variable)\n",
    "\n",
    "        # build model\n",
    "        self.model.compile(optimizer=Adam(learning_rate=self.learning_rate),\n",
    "                           loss=weighted_cross_entropy(sample_weights))\n",
    "\n",
    "        # training\n",
    "        if validation_indices is not None:\n",
    "            self.history = self.model.fit_generator(generator=train_dataloader,\n",
    "                                                    validation_data=val_dataloader,\n",
    "                                                    epochs=self.epochs).history\n",
    "        else:\n",
    "            self.history = self.model.fit_generator(generator=train_dataloader,\n",
    "                                                    epochs=self.epochs).history\n",
    "\n",
    "    def reset(self):\n",
    "        \"\"\"Resets model weights\"\"\"\n",
    "        self.model = ConvNet()\n",
    "\n",
    "    def predict(self, test_indices, logits=False):\n",
    "        \"\"\"Predicts actual class labels by default (not logits/probability values)\"\"\"\n",
    "        self.test_indices = test_indices\n",
    "\n",
    "        # TODO: make more efficient\n",
    "        # test set + predict\n",
    "        y_pred = np.empty(len(test_indices), dtype=float)\n",
    "\n",
    "        for i, index in enumerate(test_indices):\n",
    "            X_i = np.load(path + f'/feature_vector{index}.npy', allow_pickle=True)\n",
    "            X_i = tf.expand_dims(X_i, axis=0) # add \"batch dimension\"\n",
    "\n",
    "            if BACKGROUND_FEATURES:\n",
    "                global age, gender\n",
    "                age_i, gender_i = age[index], gender[index]\n",
    "                background_features = np.dstack([np.ones((X_i.shape[1], X_i.shape[2])) * age_i[0],\n",
    "                                                 np.ones((X_i.shape[1], X_i.shape[2])) * age_i[1],\n",
    "                                                 np.ones((X_i.shape[1], X_i.shape[2])) * gender_i])\n",
    "                background_features = tf.expand_dims(background_features, axis=0) # add \"batch dimension\"\n",
    "\n",
    "                X_i = np.concatenate([X_i, background_features], axis=-1)\n",
    "\n",
    "            logits_pred_i = self.model.predict(X_i)\n",
    "\n",
    "            y_pred[i] = logits_pred_i\n",
    "\n",
    "        if logits:\n",
    "            return y_pred\n",
    "\n",
    "        y_probs = tf.math.sigmoid(y_pred) # logits to probs\n",
    "        y_pred = tf.round(y_probs) # probs to labels\n",
    "\n",
    "        return y_pred\n",
    "\n",
    "    def summary(self):\n",
    "        return self.model.summary()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Model: \"conv_net_81\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenet (Functional)       (None, 12, 16, 1280)      2257984   \n",
      "_________________________________________________________________\n",
      "concat (Concatenate)         (None, 12, 16, 12800)     0         \n",
      "_________________________________________________________________\n",
      "global_avg_pool (GlobalAvera (None, 12800)             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 12801     \n",
      "=================================================================\n",
      "Total params: 2,270,785\n",
      "Trainable params: 12,801\n",
      "Non-trainable params: 2,257,984\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "CNN(path, variable=0, epochs=10, learning_rate=1e-3, batch_size=16).summary()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# CV"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Starting stratified group 5-fold for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "36/36 [==============================] - 73s 1s/step - loss: 0.5009\n",
      "Epoch 2/10\n",
      "36/36 [==============================] - 24s 644ms/step - loss: 0.3349\n",
      "Epoch 3/10\n",
      "36/36 [==============================] - 24s 643ms/step - loss: 0.3002\n",
      "Epoch 4/10\n",
      "36/36 [==============================] - 24s 644ms/step - loss: 0.2833\n",
      "Epoch 5/10\n",
      "36/36 [==============================] - 24s 643ms/step - loss: 0.2754\n",
      "Epoch 6/10\n",
      "36/36 [==============================] - 24s 643ms/step - loss: 0.2541\n",
      "Epoch 7/10\n",
      "36/36 [==============================] - 24s 670ms/step - loss: 0.2329\n",
      "Epoch 8/10\n",
      "36/36 [==============================] - 23s 640ms/step - loss: 0.2273\n",
      "Epoch 9/10\n",
      "36/36 [==============================] - 23s 641ms/step - loss: 0.2093\n",
      "Epoch 10/10\n",
      "36/36 [==============================] - 23s 640ms/step - loss: 0.2111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.8148148148148148:  20%|██        | 1/5 [05:09<20:39, 309.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "24/24 [==============================] - 40s 640ms/step - loss: 0.5588\n",
      "Epoch 2/10\n",
      "24/24 [==============================] - 16s 638ms/step - loss: 0.4697\n",
      "Epoch 3/10\n",
      "24/24 [==============================] - 16s 638ms/step - loss: 0.3709\n",
      "Epoch 4/10\n",
      "24/24 [==============================] - 16s 639ms/step - loss: 0.2781\n",
      "Epoch 5/10\n",
      "24/24 [==============================] - 16s 639ms/step - loss: 0.2627\n",
      "Epoch 6/10\n",
      "24/24 [==============================] - 16s 638ms/step - loss: 0.2699\n",
      "Epoch 7/10\n",
      "24/24 [==============================] - 16s 639ms/step - loss: 0.2347\n",
      "Epoch 8/10\n",
      "24/24 [==============================] - 16s 641ms/step - loss: 0.2252\n",
      "Epoch 9/10\n",
      "24/24 [==============================] - 16s 642ms/step - loss: 0.2164\n",
      "Epoch 10/10\n",
      "24/24 [==============================] - 16s 640ms/step - loss: 0.2311\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.6212211466865226:  40%|████      | 2/5 [09:01<13:11, 263.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 44s 635ms/step - loss: 0.3576\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.2695\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.2517\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.2666\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.2242\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.2122\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.1915\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.1892\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 637ms/step - loss: 0.1645\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.1551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.5638822860686236:  60%|██████    | 3/5 [13:14<08:38, 259.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "25/25 [==============================] - 39s 647ms/step - loss: 0.6345\n",
      "Epoch 2/10\n",
      "25/25 [==============================] - 17s 648ms/step - loss: 0.4443\n",
      "Epoch 3/10\n",
      "25/25 [==============================] - 17s 648ms/step - loss: 0.3817\n",
      "Epoch 4/10\n",
      "25/25 [==============================] - 17s 647ms/step - loss: 0.3440\n",
      "Epoch 5/10\n",
      "25/25 [==============================] - 17s 647ms/step - loss: 0.3199\n",
      "Epoch 6/10\n",
      "25/25 [==============================] - 17s 646ms/step - loss: 0.3383\n",
      "Epoch 7/10\n",
      "25/25 [==============================] - 17s 646ms/step - loss: 0.3071\n",
      "Epoch 8/10\n",
      "25/25 [==============================] - 17s 646ms/step - loss: 0.2843\n",
      "Epoch 9/10\n",
      "25/25 [==============================] - 17s 647ms/step - loss: 0.2926\n",
      "Epoch 10/10\n",
      "25/25 [==============================] - 17s 647ms/step - loss: 0.2495\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.525308660078008:  80%|████████  | 4/5 [17:10<04:09, 249.95s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "36/36 [==============================] - 47s 636ms/step - loss: 0.4424\n",
      "Epoch 2/10\n",
      "36/36 [==============================] - 23s 638ms/step - loss: 0.3028\n",
      "Epoch 3/10\n",
      "36/36 [==============================] - 23s 632ms/step - loss: 0.3172\n",
      "Epoch 4/10\n",
      "36/36 [==============================] - 23s 619ms/step - loss: 0.2588\n",
      "Epoch 5/10\n",
      "36/36 [==============================] - 23s 621ms/step - loss: 0.2488\n",
      "Epoch 6/10\n",
      "36/36 [==============================] - 23s 620ms/step - loss: 0.2381\n",
      "Epoch 7/10\n",
      "36/36 [==============================] - 23s 622ms/step - loss: 0.2182\n",
      "Epoch 8/10\n",
      "36/36 [==============================] - 23s 620ms/step - loss: 0.2178\n",
      "Epoch 9/10\n",
      "36/36 [==============================] - 23s 620ms/step - loss: 0.2436\n",
      "Epoch 10/10\n",
      "36/36 [==============================] - 23s 622ms/step - loss: 0.1996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.5232076044089392: 100%|██████████| 5/5 [21:40<00:00, 260.20s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.593 +- 0.114 \n",
      "\n",
      " balanced_accuracy: 0.585 +- 0.034 \n",
      "\n",
      " f1: 0.61 +- 0.109 \n",
      "\n",
      " recall: 0.593 +- 0.114 \n",
      "\n",
      " precision: 0.702 +- 0.105 \n",
      "\n",
      "Starting leave-one-subject-out for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/27 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 46s 625ms/step - loss: 0.5018\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3580\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3528\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2921\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.2792\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2509\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2523\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2158\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2228\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2151\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.8888888888888888:   4%|▎         | 1/27 [04:34<1:58:44, 274.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "38/38 [==============================] - 47s 626ms/step - loss: 0.4788\n",
      "Epoch 2/10\n",
      "38/38 [==============================] - 24s 626ms/step - loss: 0.3417\n",
      "Epoch 3/10\n",
      "38/38 [==============================] - 24s 625ms/step - loss: 0.3166\n",
      "Epoch 4/10\n",
      "38/38 [==============================] - 24s 624ms/step - loss: 0.2969\n",
      "Epoch 5/10\n",
      "38/38 [==============================] - 24s 626ms/step - loss: 0.2668\n",
      "Epoch 6/10\n",
      "38/38 [==============================] - 24s 624ms/step - loss: 0.2619\n",
      "Epoch 7/10\n",
      "38/38 [==============================] - 24s 625ms/step - loss: 0.2861\n",
      "Epoch 8/10\n",
      "38/38 [==============================] - 24s 625ms/step - loss: 0.2347\n",
      "Epoch 9/10\n",
      "38/38 [==============================] - 24s 624ms/step - loss: 0.2188\n",
      "Epoch 10/10\n",
      "38/38 [==============================] - 24s 626ms/step - loss: 0.1928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.0:   7%|▋         | 2/27 [09:14<1:55:44, 277.77s/it]               "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 653ms/step - loss: 0.5195\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 30s 795ms/step - loss: 0.3519\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 25s 659ms/step - loss: 0.3043\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 25s 653ms/step - loss: 0.2850\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 646ms/step - loss: 0.2436\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.2653\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.2275\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.2463\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2286\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2050\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.36199095022624433:  11%|█         | 3/27 [14:00<1:52:42, 281.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 642ms/step - loss: 0.4778\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3208\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2966\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2736\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2784\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.2463\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.2179\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.2195\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.2120\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.1971\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.6666666666666666:  15%|█▍        | 4/27 [18:41<1:47:46, 281.13s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 49s 640ms/step - loss: 0.5330\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3344\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3083\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3111\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2707\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2787\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 644ms/step - loss: 0.2610\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 624ms/step - loss: 0.2536\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.2201\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.1996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.2857142857142857:  19%|█▊        | 5/27 [23:19<1:42:40, 280.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 625ms/step - loss: 0.5106\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.3608\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3691\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3000\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3054\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.2595\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3000\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.2297\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.2230\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.2315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 6 F1: 0.7575757575757575:  22%|██▏       | 6/27 [27:51<1:37:07, 277.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "38/38 [==============================] - 49s 634ms/step - loss: 0.4012\n",
      "Epoch 2/10\n",
      "38/38 [==============================] - 24s 634ms/step - loss: 0.3272\n",
      "Epoch 3/10\n",
      "38/38 [==============================] - 25s 636ms/step - loss: 0.3552\n",
      "Epoch 4/10\n",
      "38/38 [==============================] - 24s 635ms/step - loss: 0.2662\n",
      "Epoch 5/10\n",
      "38/38 [==============================] - 25s 636ms/step - loss: 0.3108\n",
      "Epoch 6/10\n",
      "38/38 [==============================] - 25s 635ms/step - loss: 0.2830\n",
      "Epoch 7/10\n",
      "38/38 [==============================] - 25s 635ms/step - loss: 0.2828\n",
      "Epoch 8/10\n",
      "38/38 [==============================] - 25s 635ms/step - loss: 0.2288\n",
      "Epoch 9/10\n",
      "38/38 [==============================] - 24s 635ms/step - loss: 0.2450\n",
      "Epoch 10/10\n",
      "38/38 [==============================] - 24s 635ms/step - loss: 0.1940\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 7 F1: 0.5333333333333333:  26%|██▌       | 7/27 [32:34<1:33:04, 279.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 627ms/step - loss: 0.4858\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3625\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3308\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.2847\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.2608\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.2635\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.2218\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.2332\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 632ms/step - loss: 0.2199\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.2579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 8 F1: 0.03571428571428571:  30%|██▉       | 8/27 [37:09<1:28:01, 277.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 622ms/step - loss: 0.4266\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3892\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2775\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.2656\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2493\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2626\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.2419\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 619ms/step - loss: 0.2624\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 620ms/step - loss: 0.2381\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.1997\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 9 F1: 0.7499999999999999:  33%|███▎      | 9/27 [41:42<1:22:52, 276.24s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 623ms/step - loss: 0.4508\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3354\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 620ms/step - loss: 0.3241\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.2990\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2655\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.2766\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2290\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 30s 800ms/step - loss: 0.2168\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.2078\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.2058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 10 F1: 0.06666666666666668:  37%|███▋      | 10/27 [46:22<1:18:35, 277.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 49s 646ms/step - loss: 0.5341\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.4487\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 647ms/step - loss: 0.3094\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 646ms/step - loss: 0.2709\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.2724\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 644ms/step - loss: 0.2416\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 643ms/step - loss: 0.2321\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.2250\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.2167\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.2040\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 11 F1: 0.5142857142857142:  41%|████      | 11/27 [51:04<1:14:23, 278.97s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 638ms/step - loss: 0.4039\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.3163\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.2935\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.3141\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.2620\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.2928\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 647ms/step - loss: 0.2190\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.2050\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.2314\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.1850\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 12 F1: 0.6588235294117648:  44%|████▍     | 12/27 [55:43<1:09:42, 278.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 46s 625ms/step - loss: 0.4924\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.3329\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2883\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2673\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2538\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2450\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2353\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2085\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.2237\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 25s 660ms/step - loss: 0.3135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 13 F1: 0.2571428571428571:  48%|████▊     | 13/27 [1:00:20<1:04:55, 278.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 54s 716ms/step - loss: 0.4609\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 25s 669ms/step - loss: 0.3338\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 25s 673ms/step - loss: 0.3044\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 25s 659ms/step - loss: 0.2916\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 25s 658ms/step - loss: 0.3223\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 25s 659ms/step - loss: 0.2492\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 25s 666ms/step - loss: 0.2611\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 25s 663ms/step - loss: 0.2346\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 25s 654ms/step - loss: 0.2148\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 25s 664ms/step - loss: 0.2130\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 14 F1: 0.8000000000000002:  52%|█████▏    | 14/27 [1:05:19<1:01:38, 284.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 50s 662ms/step - loss: 0.4549\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 25s 671ms/step - loss: 0.3159\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 26s 698ms/step - loss: 0.3020\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 25s 660ms/step - loss: 0.2866\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 25s 659ms/step - loss: 0.2702\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 25s 656ms/step - loss: 0.2521\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 25s 656ms/step - loss: 0.2421\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 25s 654ms/step - loss: 0.2645\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.2894\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 25s 657ms/step - loss: 0.2291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 15 F1: 0.3212121212121213:  56%|█████▌    | 15/27 [1:10:11<57:21, 286.81s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 50s 646ms/step - loss: 0.4028\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.3309\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3055\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 25s 661ms/step - loss: 0.2838\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 27s 720ms/step - loss: 0.3255\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 25s 656ms/step - loss: 0.2388\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 646ms/step - loss: 0.2789\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2927\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.2791\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 16 F1: 0.38333333333333336:  59%|█████▉    | 16/27 [1:14:59<52:39, 287.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 641ms/step - loss: 0.5003\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.3195\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 27s 729ms/step - loss: 0.2917\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 28s 750ms/step - loss: 0.2768\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 27s 718ms/step - loss: 0.3057\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2288\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.2278\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.2226\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.2123\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.1940\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 17 F1: 0.6692307692307692:  63%|██████▎   | 17/27 [1:19:52<48:09, 288.99s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 49s 661ms/step - loss: 0.4031\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 25s 652ms/step - loss: 0.3128\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 29s 768ms/step - loss: 0.3360\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 25s 661ms/step - loss: 0.2577\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 25s 662ms/step - loss: 0.2419\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 25s 662ms/step - loss: 0.2796\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 652ms/step - loss: 0.2065\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 648ms/step - loss: 0.2203\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 648ms/step - loss: 0.2220\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 644ms/step - loss: 0.1842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 18 F1: 0.04102564102564103:  67%|██████▋   | 18/27 [1:24:44<43:29, 289.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 52s 658ms/step - loss: 0.3886\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 647ms/step - loss: 0.3147\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 644ms/step - loss: 0.2914\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.3403\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 27s 722ms/step - loss: 0.2660\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 27s 723ms/step - loss: 0.2272\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 30s 806ms/step - loss: 0.2314\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 30s 795ms/step - loss: 0.2069\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 30s 791ms/step - loss: 0.2249\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 30s 799ms/step - loss: 0.1860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 19 F1: 0.9:  70%|███████   | 19/27 [1:30:04<39:50, 298.85s/it]                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 51s 654ms/step - loss: 0.4330\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.3280\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 25s 660ms/step - loss: 0.3169\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.3038\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.2660\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.2771\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.2342\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.2109\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.2127\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.2282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 20 F1: 1.0:  74%|███████▍  | 20/27 [1:34:56<34:38, 296.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 648ms/step - loss: 0.4265\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 645ms/step - loss: 0.3502\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.3361\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.2837\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 26s 700ms/step - loss: 0.2496\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 26s 678ms/step - loss: 0.2480\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.2490\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 25s 678ms/step - loss: 0.2102\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.2076\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.2013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 21 F1: 0.5804195804195803:  78%|███████▊  | 21/27 [1:39:44<29:25, 294.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 49s 653ms/step - loss: 0.5011\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.3304\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 649ms/step - loss: 0.3259\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 651ms/step - loss: 0.2911\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 648ms/step - loss: 0.3223\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 648ms/step - loss: 0.3233\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 25s 655ms/step - loss: 0.2273\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 650ms/step - loss: 0.2658\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 25s 677ms/step - loss: 0.2161\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 26s 704ms/step - loss: 0.2450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 22 F1: 0.6656611362493715:  81%|████████▏ | 22/27 [1:44:36<24:27, 293.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "31/31 [==============================] - 49s 788ms/step - loss: 0.4973\n",
      "Epoch 2/10\n",
      "31/31 [==============================] - 24s 754ms/step - loss: 0.2741\n",
      "Epoch 3/10\n",
      "31/31 [==============================] - 23s 742ms/step - loss: 0.2630\n",
      "Epoch 4/10\n",
      "31/31 [==============================] - 24s 772ms/step - loss: 0.2350\n",
      "Epoch 5/10\n",
      "31/31 [==============================] - 23s 735ms/step - loss: 0.2388\n",
      "Epoch 6/10\n",
      "31/31 [==============================] - 22s 705ms/step - loss: 0.2059\n",
      "Epoch 7/10\n",
      "31/31 [==============================] - 21s 676ms/step - loss: 0.1806\n",
      "Epoch 8/10\n",
      "31/31 [==============================] - 21s 675ms/step - loss: 0.1788\n",
      "Epoch 9/10\n",
      "31/31 [==============================] - 21s 674ms/step - loss: 0.1788\n",
      "Epoch 10/10\n",
      "31/31 [==============================] - 21s 674ms/step - loss: 0.2098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 23 F1: 0.5345433199837154:  85%|████████▌ | 23/27 [1:49:26<19:30, 292.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "29/29 [==============================] - 49s 700ms/step - loss: 0.7928\n",
      "Epoch 2/10\n",
      "29/29 [==============================] - 21s 704ms/step - loss: 0.4914\n",
      "Epoch 3/10\n",
      "29/29 [==============================] - 20s 684ms/step - loss: 0.3906\n",
      "Epoch 4/10\n",
      "29/29 [==============================] - 22s 732ms/step - loss: 0.4032\n",
      "Epoch 5/10\n",
      "29/29 [==============================] - 20s 683ms/step - loss: 0.3460\n",
      "Epoch 6/10\n",
      "29/29 [==============================] - 20s 680ms/step - loss: 0.3232\n",
      "Epoch 7/10\n",
      "29/29 [==============================] - 20s 679ms/step - loss: 0.3323\n",
      "Epoch 8/10\n",
      "29/29 [==============================] - 20s 680ms/step - loss: 0.4170\n",
      "Epoch 9/10\n",
      "29/29 [==============================] - 20s 689ms/step - loss: 0.3074\n",
      "Epoch 10/10\n",
      "29/29 [==============================] - 20s 682ms/step - loss: 0.2958\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 24 F1: 0.9008015368309488:  89%|████████▉ | 24/27 [1:54:03<14:23, 287.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "36/36 [==============================] - 51s 681ms/step - loss: 0.4570\n",
      "Epoch 2/10\n",
      "36/36 [==============================] - 25s 679ms/step - loss: 0.3134\n",
      "Epoch 3/10\n",
      "36/36 [==============================] - 25s 680ms/step - loss: 0.2833\n",
      "Epoch 4/10\n",
      "36/36 [==============================] - 25s 678ms/step - loss: 0.2821\n",
      "Epoch 5/10\n",
      "36/36 [==============================] - 25s 679ms/step - loss: 0.2756\n",
      "Epoch 6/10\n",
      "36/36 [==============================] - 25s 680ms/step - loss: 0.3190\n",
      "Epoch 7/10\n",
      "36/36 [==============================] - 25s 679ms/step - loss: 0.2612\n",
      "Epoch 8/10\n",
      "36/36 [==============================] - 25s 680ms/step - loss: 0.2201\n",
      "Epoch 9/10\n",
      "36/36 [==============================] - 25s 680ms/step - loss: 0.2036\n",
      "Epoch 10/10\n",
      "36/36 [==============================] - 25s 681ms/step - loss: 0.2136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 25 F1: 0.5729166666666666:  93%|█████████▎| 25/27 [1:59:01<09:41, 290.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "34/34 [==============================] - 50s 679ms/step - loss: 0.5597\n",
      "Epoch 2/10\n",
      "34/34 [==============================] - 24s 681ms/step - loss: 0.3784\n",
      "Epoch 3/10\n",
      "34/34 [==============================] - 24s 680ms/step - loss: 0.3991\n",
      "Epoch 4/10\n",
      "34/34 [==============================] - 23s 680ms/step - loss: 0.3449\n",
      "Epoch 5/10\n",
      "34/34 [==============================] - 23s 678ms/step - loss: 0.3045\n",
      "Epoch 6/10\n",
      "34/34 [==============================] - 23s 675ms/step - loss: 0.3155\n",
      "Epoch 7/10\n",
      "34/34 [==============================] - 24s 686ms/step - loss: 0.3784\n",
      "Epoch 8/10\n",
      "34/34 [==============================] - 23s 678ms/step - loss: 0.2916\n",
      "Epoch 9/10\n",
      "34/34 [==============================] - 23s 678ms/step - loss: 0.3056\n",
      "Epoch 10/10\n",
      "34/34 [==============================] - 24s 679ms/step - loss: 0.2400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 26 F1: 0.8376811594202898:  96%|█████████▋| 26/27 [2:03:51<04:50, 290.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "35/35 [==============================] - 50s 680ms/step - loss: 0.4380\n",
      "Epoch 2/10\n",
      "35/35 [==============================] - 24s 669ms/step - loss: 0.3090\n",
      "Epoch 3/10\n",
      "35/35 [==============================] - 24s 670ms/step - loss: 0.3237\n",
      "Epoch 4/10\n",
      "35/35 [==============================] - 24s 670ms/step - loss: 0.2707\n",
      "Epoch 5/10\n",
      "35/35 [==============================] - 25s 691ms/step - loss: 0.2462\n",
      "Epoch 6/10\n",
      "35/35 [==============================] - 24s 671ms/step - loss: 0.2544\n",
      "Epoch 7/10\n",
      "35/35 [==============================] - 24s 671ms/step - loss: 0.2986\n",
      "Epoch 8/10\n",
      "35/35 [==============================] - 24s 671ms/step - loss: 0.2044\n",
      "Epoch 9/10\n",
      "35/35 [==============================] - 24s 670ms/step - loss: 0.1850\n",
      "Epoch 10/10\n",
      "35/35 [==============================] - 24s 671ms/step - loss: 0.1837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 27 F1: 0.3285356695869837: 100%|██████████| 27/27 [2:08:43<00:00, 286.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.531 +- 0.264 \n",
      "\n",
      " balanced_accuracy: 0.548 +- 0.222 \n",
      "\n",
      " f1: 0.53 +- 0.287 \n",
      "\n",
      " recall: 0.531 +- 0.264 \n",
      "\n",
      " precision: 0.656 +- 0.327 \n",
      "\n",
      "Starting stratified 5-fold for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 48s 688ms/step - loss: 0.4419\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 21s 685ms/step - loss: 0.3682\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 21s 702ms/step - loss: 0.2965\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 23s 763ms/step - loss: 0.2821\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 24s 776ms/step - loss: 0.2693\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 24s 778ms/step - loss: 0.2747\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 24s 793ms/step - loss: 0.2547\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 23s 740ms/step - loss: 0.2727\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 21s 679ms/step - loss: 0.2261\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 20s 663ms/step - loss: 0.2550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.6252147028512538:  20%|██        | 1/5 [04:45<19:03, 285.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 47s 646ms/step - loss: 0.5568\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 20s 645ms/step - loss: 0.3583\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 20s 646ms/step - loss: 0.2893\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 20s 648ms/step - loss: 0.2987\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 20s 646ms/step - loss: 0.2972\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 20s 648ms/step - loss: 0.2568\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 20s 647ms/step - loss: 0.2509\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 20s 663ms/step - loss: 0.2361\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.2191\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 20s 639ms/step - loss: 0.2181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.7314320877705044:  40%|████      | 2/5 [09:06<13:32, 270.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 45s 638ms/step - loss: 0.4080\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.4567\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.3668\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.2754\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 20s 639ms/step - loss: 0.2988\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.3177\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.2343\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 20s 639ms/step - loss: 0.2408\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.2104\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.2307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.7098685348555615:  60%|██████    | 3/5 [13:20<08:46, 263.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 44s 644ms/step - loss: 0.4335\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 20s 647ms/step - loss: 0.4089\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 20s 645ms/step - loss: 0.3030\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 20s 644ms/step - loss: 0.2840\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 20s 644ms/step - loss: 0.2605\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 20s 643ms/step - loss: 0.2676\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 20s 643ms/step - loss: 0.2411\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 20s 644ms/step - loss: 0.2315\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 20s 644ms/step - loss: 0.2289\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 20s 644ms/step - loss: 0.2195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.8323829888334521:  80%|████████  | 4/5 [17:37<04:20, 260.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 45s 656ms/step - loss: 0.3944\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 20s 656ms/step - loss: 0.3374\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 20s 655ms/step - loss: 0.3018\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 20s 656ms/step - loss: 0.2730\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 20s 655ms/step - loss: 0.2576\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 21s 675ms/step - loss: 0.2373\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 20s 641ms/step - loss: 0.2824\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 638ms/step - loss: 0.2239\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 20s 639ms/step - loss: 0.2111\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 20s 638ms/step - loss: 0.1795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.762496971654688: 100%|██████████| 5/5 [21:53<00:00, 262.77s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.746 +- 0.05 \n",
      "\n",
      " balanced_accuracy: 0.692 +- 0.1 \n",
      "\n",
      " f1: 0.732 +- 0.068 \n",
      "\n",
      " recall: 0.746 +- 0.05 \n",
      "\n",
      " precision: 0.749 +- 0.103 \n",
      "\n",
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting stratified group 5-fold for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "35/35 [==============================] - 47s 630ms/step - loss: 0.7908\n",
      "Epoch 2/10\n",
      "35/35 [==============================] - 22s 627ms/step - loss: 0.5501\n",
      "Epoch 3/10\n",
      "35/35 [==============================] - 22s 631ms/step - loss: 0.4484\n",
      "Epoch 4/10\n",
      "35/35 [==============================] - 22s 625ms/step - loss: 0.3994\n",
      "Epoch 5/10\n",
      "35/35 [==============================] - 22s 628ms/step - loss: 0.3737\n",
      "Epoch 6/10\n",
      "35/35 [==============================] - 22s 630ms/step - loss: 0.3676\n",
      "Epoch 7/10\n",
      "35/35 [==============================] - 22s 626ms/step - loss: 0.3608\n",
      "Epoch 8/10\n",
      "35/35 [==============================] - 22s 630ms/step - loss: 0.3306\n",
      "Epoch 9/10\n",
      "35/35 [==============================] - 22s 631ms/step - loss: 0.3219\n",
      "Epoch 10/10\n",
      "35/35 [==============================] - 22s 631ms/step - loss: 0.3000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.47470238095238093:  20%|██        | 1/5 [04:28<17:55, 268.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "28/28 [==============================] - 42s 640ms/step - loss: 0.6141\n",
      "Epoch 2/10\n",
      "28/28 [==============================] - 18s 639ms/step - loss: 0.4159\n",
      "Epoch 3/10\n",
      "28/28 [==============================] - 18s 641ms/step - loss: 0.3908\n",
      "Epoch 4/10\n",
      "28/28 [==============================] - 18s 641ms/step - loss: 0.3596\n",
      "Epoch 5/10\n",
      "28/28 [==============================] - 18s 639ms/step - loss: 0.3364\n",
      "Epoch 6/10\n",
      "28/28 [==============================] - 18s 640ms/step - loss: 0.3822\n",
      "Epoch 7/10\n",
      "28/28 [==============================] - 18s 639ms/step - loss: 0.4662\n",
      "Epoch 8/10\n",
      "28/28 [==============================] - 18s 640ms/step - loss: 0.3092\n",
      "Epoch 9/10\n",
      "28/28 [==============================] - 18s 640ms/step - loss: 0.2840\n",
      "Epoch 10/10\n",
      "28/28 [==============================] - 18s 641ms/step - loss: 0.3138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.5006735958268388:  40%|████      | 2/5 [08:32<12:42, 254.01s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "31/31 [==============================] - 44s 623ms/step - loss: 0.5027\n",
      "Epoch 2/10\n",
      "31/31 [==============================] - 20s 626ms/step - loss: 0.4038\n",
      "Epoch 3/10\n",
      "31/31 [==============================] - 20s 625ms/step - loss: 0.3571\n",
      "Epoch 4/10\n",
      "31/31 [==============================] - 20s 624ms/step - loss: 0.3371\n",
      "Epoch 5/10\n",
      "31/31 [==============================] - 20s 626ms/step - loss: 0.3262\n",
      "Epoch 6/10\n",
      "31/31 [==============================] - 20s 625ms/step - loss: 0.2875\n",
      "Epoch 7/10\n",
      "31/31 [==============================] - 20s 624ms/step - loss: 0.2787\n",
      "Epoch 8/10\n",
      "31/31 [==============================] - 20s 623ms/step - loss: 0.3249\n",
      "Epoch 9/10\n",
      "31/31 [==============================] - 20s 625ms/step - loss: 0.2916\n",
      "Epoch 10/10\n",
      "31/31 [==============================] - 20s 624ms/step - loss: 0.2573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.5156469429248149:  60%|██████    | 3/5 [12:45<08:27, 253.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 628ms/step - loss: 0.6362\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.4425\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.4342\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3713\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3528\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3505\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.3403\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.2955\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.2889\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.2723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.5882352941176471:  80%|████████  | 4/5 [17:21<04:22, 262.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "20/20 [==============================] - 37s 625ms/step - loss: 0.8536\n",
      "Epoch 2/10\n",
      "20/20 [==============================] - 13s 629ms/step - loss: 0.6221\n",
      "Epoch 3/10\n",
      "20/20 [==============================] - 13s 624ms/step - loss: 0.5877\n",
      "Epoch 4/10\n",
      "20/20 [==============================] - 13s 628ms/step - loss: 0.5642\n",
      "Epoch 5/10\n",
      "20/20 [==============================] - 13s 624ms/step - loss: 0.5765\n",
      "Epoch 6/10\n",
      "20/20 [==============================] - 13s 628ms/step - loss: 0.4709\n",
      "Epoch 7/10\n",
      "20/20 [==============================] - 13s 627ms/step - loss: 0.4591\n",
      "Epoch 8/10\n",
      "20/20 [==============================] - 13s 626ms/step - loss: 0.4351\n",
      "Epoch 9/10\n",
      "20/20 [==============================] - 13s 625ms/step - loss: 0.4066\n",
      "Epoch 10/10\n",
      "20/20 [==============================] - 13s 626ms/step - loss: 0.3831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.7224161033856414: 100%|██████████| 5/5 [20:50<00:00, 250.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.562 +- 0.107 \n",
      "\n",
      " balanced_accuracy: 0.476 +- 0.066 \n",
      "\n",
      " f1: 0.56 +- 0.089 \n",
      "\n",
      " recall: 0.562 +- 0.107 \n",
      "\n",
      " precision: 0.646 +- 0.198 \n",
      "\n",
      "Starting leave-one-subject-out for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/27 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 629ms/step - loss: 0.5579\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.4200\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3954\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3980\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3829\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3242\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3354\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3555\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3449\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.7777777777777779:   4%|▎         | 1/27 [04:36<1:59:43, 276.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "38/38 [==============================] - 49s 636ms/step - loss: 0.6229\n",
      "Epoch 2/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.4438\n",
      "Epoch 3/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.4417\n",
      "Epoch 4/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.4396\n",
      "Epoch 5/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.3546\n",
      "Epoch 6/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.3452\n",
      "Epoch 7/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.3364\n",
      "Epoch 8/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.3119\n",
      "Epoch 9/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.2919\n",
      "Epoch 10/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.2820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.0:   7%|▋         | 2/27 [09:19<1:56:52, 280.51s/it]               "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 621ms/step - loss: 0.5415\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.4422\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.4198\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4052\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3768\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3716\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 620ms/step - loss: 0.3283\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2964\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2903\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.2723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.3328671328671329:  11%|█         | 3/27 [13:52<1:50:47, 277.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 622ms/step - loss: 0.6965\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4958\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4608\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4193\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3816\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3455\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3552\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3756\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3115\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.2552447552447552:  15%|█▍        | 4/27 [18:27<1:45:53, 276.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 46s 627ms/step - loss: 0.6187\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.4676\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.4566\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.4275\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.4169\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.3621\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3660\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3683\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3059\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2876\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.9090909090909091:  19%|█▊        | 5/27 [23:01<1:40:57, 275.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 625ms/step - loss: 0.5513\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4934\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.4142\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.4096\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3985\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3576\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3674\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3272\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3366\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 6 F1: 0.047619047619047616:  22%|██▏       | 6/27 [27:34<1:36:07, 274.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "38/38 [==============================] - 48s 639ms/step - loss: 0.6763\n",
      "Epoch 2/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.6002\n",
      "Epoch 3/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.4360\n",
      "Epoch 4/10\n",
      "38/38 [==============================] - 25s 637ms/step - loss: 0.4158\n",
      "Epoch 5/10\n",
      "38/38 [==============================] - 25s 639ms/step - loss: 0.3888\n",
      "Epoch 6/10\n",
      "38/38 [==============================] - 25s 639ms/step - loss: 0.3898\n",
      "Epoch 7/10\n",
      "38/38 [==============================] - 25s 639ms/step - loss: 0.3380\n",
      "Epoch 8/10\n",
      "38/38 [==============================] - 25s 639ms/step - loss: 0.3372\n",
      "Epoch 9/10\n",
      "38/38 [==============================] - 25s 638ms/step - loss: 0.2997\n",
      "Epoch 10/10\n",
      "38/38 [==============================] - 25s 640ms/step - loss: 0.3069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 7 F1: 0.0:  26%|██▌       | 7/27 [32:18<1:32:33, 277.65s/it]                 "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 627ms/step - loss: 0.5639\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.4739\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.4134\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3698\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3665\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3769\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3493\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3029\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3099\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.2994\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 8 F1: 0.2857142857142857:  30%|██▉       | 8/27 [36:55<1:27:53, 277.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 46s 619ms/step - loss: 0.6074\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 620ms/step - loss: 0.4815\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.4243\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3933\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3510\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3679\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3250\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3051\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3333\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.2918\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 9 F1: 0.2222222222222222:  33%|███▎      | 9/27 [41:28<1:22:49, 276.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 630ms/step - loss: 0.6258\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.4729\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.4405\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 632ms/step - loss: 0.3788\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.4536\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3893\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.3441\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.3367\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.3104\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.2797\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 10 F1: 0.3333333333333333:  37%|███▋      | 10/27 [46:04<1:18:14, 276.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 637ms/step - loss: 0.5665\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.4499\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.4148\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.3848\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.3655\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.3577\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.3375\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.3558\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.3311\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 639ms/step - loss: 0.2867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 11 F1: 0.34949494949494947:  41%|████      | 11/27 [50:44<1:13:56, 277.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 629ms/step - loss: 0.6167\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.4741\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.4129\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.4383\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.4591\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.3884\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.3663\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 632ms/step - loss: 0.3248\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.3503\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 630ms/step - loss: 0.3366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 12 F1: 1.0:  44%|████▍     | 12/27 [55:21<1:09:15, 277.01s/it]                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 622ms/step - loss: 0.5507\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.5041\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4168\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.4094\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.5006\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3420\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3329\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3080\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3021\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 13 F1: 0.05128205128205127:  48%|████▊     | 13/27 [59:54<1:04:23, 275.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 635ms/step - loss: 0.6566\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.4513\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.4550\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 635ms/step - loss: 0.5178\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.4143\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.4173\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 635ms/step - loss: 0.3375\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.3513\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 635ms/step - loss: 0.3299\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 637ms/step - loss: 0.3556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 14 F1: 0.7575757575757575:  52%|█████▏    | 14/27 [1:04:32<59:55, 276.57s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 631ms/step - loss: 0.6054\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.4323\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.4160\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.4219\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.4137\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 636ms/step - loss: 0.3643\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 635ms/step - loss: 0.3263\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.3186\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.3104\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.3062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 15 F1: 0.8084415584415584:  56%|█████▌    | 15/27 [1:09:12<55:29, 277.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 627ms/step - loss: 0.5783\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.4355\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4569\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4146\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3925\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3627\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3324\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3224\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3087\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 16 F1: 0.8366013071895425:  59%|█████▉    | 16/27 [1:13:46<50:40, 276.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 627ms/step - loss: 0.5476\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.4608\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.5510\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3917\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3901\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3553\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3872\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 628ms/step - loss: 0.3277\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 629ms/step - loss: 0.3144\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 17 F1: 0.37692307692307697:  63%|██████▎   | 17/27 [1:18:22<46:02, 276.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 624ms/step - loss: 0.6174\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4555\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.4137\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.4094\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3632\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3541\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3411\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 626ms/step - loss: 0.3231\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.2942\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3335\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 18 F1: 0.4423076923076923:  67%|██████▋   | 18/27 [1:22:58<41:26, 276.28s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 624ms/step - loss: 0.6099\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 625ms/step - loss: 0.4961\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4522\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3965\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3819\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 622ms/step - loss: 0.3501\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3436\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3198\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 23s 621ms/step - loss: 0.3349\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.3074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 19 F1: 0.6558441558441559:  70%|███████   | 19/27 [1:27:33<36:46, 275.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 48s 633ms/step - loss: 0.5893\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.5400\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.4894\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.4315\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.4227\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.4310\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.3581\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 632ms/step - loss: 0.3423\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 635ms/step - loss: 0.3278\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.3100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 20 F1: 0.8333333333333333:  74%|███████▍  | 20/27 [1:32:09<32:12, 276.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 625ms/step - loss: 0.6313\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.4726\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 23s 623ms/step - loss: 0.4705\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.4016\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3658\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 23s 624ms/step - loss: 0.3630\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 23s 625ms/step - loss: 0.3462\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 631ms/step - loss: 0.3097\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.2947\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 24s 627ms/step - loss: 0.3041\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 21 F1: 0.5882352941176471:  78%|███████▊  | 21/27 [1:36:45<27:35, 276.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "37/37 [==============================] - 47s 639ms/step - loss: 0.6276\n",
      "Epoch 2/10\n",
      "37/37 [==============================] - 24s 634ms/step - loss: 0.4712\n",
      "Epoch 3/10\n",
      "37/37 [==============================] - 24s 633ms/step - loss: 0.4599\n",
      "Epoch 4/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3895\n",
      "Epoch 5/10\n",
      "37/37 [==============================] - 24s 640ms/step - loss: 0.3828\n",
      "Epoch 6/10\n",
      "37/37 [==============================] - 24s 643ms/step - loss: 0.3799\n",
      "Epoch 7/10\n",
      "37/37 [==============================] - 24s 642ms/step - loss: 0.3589\n",
      "Epoch 8/10\n",
      "37/37 [==============================] - 24s 641ms/step - loss: 0.3215\n",
      "Epoch 9/10\n",
      "37/37 [==============================] - 24s 638ms/step - loss: 0.3255\n",
      "Epoch 10/10\n",
      "37/37 [==============================] - 25s 653ms/step - loss: 0.3241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 22 F1: 1.0:  81%|████████▏ | 22/27 [1:41:27<23:08, 277.61s/it]               "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "31/31 [==============================] - 43s 630ms/step - loss: 0.5192\n",
      "Epoch 2/10\n",
      "31/31 [==============================] - 20s 628ms/step - loss: 0.4397\n",
      "Epoch 3/10\n",
      "31/31 [==============================] - 20s 627ms/step - loss: 0.4290\n",
      "Epoch 4/10\n",
      "31/31 [==============================] - 20s 630ms/step - loss: 0.3899\n",
      "Epoch 5/10\n",
      "31/31 [==============================] - 20s 627ms/step - loss: 0.3677\n",
      "Epoch 6/10\n",
      "31/31 [==============================] - 20s 629ms/step - loss: 0.3672\n",
      "Epoch 7/10\n",
      "31/31 [==============================] - 20s 628ms/step - loss: 0.3163\n",
      "Epoch 8/10\n",
      "31/31 [==============================] - 20s 631ms/step - loss: 0.3084\n",
      "Epoch 9/10\n",
      "31/31 [==============================] - 20s 628ms/step - loss: 0.2767\n",
      "Epoch 10/10\n",
      "31/31 [==============================] - 20s 629ms/step - loss: 0.3484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 23 F1: 0.44419306184012064:  85%|████████▌ | 23/27 [1:45:41<18:03, 270.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "29/29 [==============================] - 42s 634ms/step - loss: 0.7786\n",
      "Epoch 2/10\n",
      "29/29 [==============================] - 19s 633ms/step - loss: 0.6225\n",
      "Epoch 3/10\n",
      "29/29 [==============================] - 19s 634ms/step - loss: 0.5142\n",
      "Epoch 4/10\n",
      "29/29 [==============================] - 19s 633ms/step - loss: 0.6185\n",
      "Epoch 5/10\n",
      "29/29 [==============================] - 19s 634ms/step - loss: 0.4662\n",
      "Epoch 6/10\n",
      "29/29 [==============================] - 19s 634ms/step - loss: 0.4325\n",
      "Epoch 7/10\n",
      "29/29 [==============================] - 19s 634ms/step - loss: 0.3836\n",
      "Epoch 8/10\n",
      "29/29 [==============================] - 19s 635ms/step - loss: 0.4052\n",
      "Epoch 9/10\n",
      "29/29 [==============================] - 19s 633ms/step - loss: 0.3526\n",
      "Epoch 10/10\n",
      "29/29 [==============================] - 19s 633ms/step - loss: 0.3375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 24 F1: 0.3873276261335963:  89%|████████▉ | 24/27 [1:49:49<13:11, 263.96s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "36/36 [==============================] - 48s 635ms/step - loss: 0.6826\n",
      "Epoch 2/10\n",
      "36/36 [==============================] - 23s 624ms/step - loss: 0.4744\n",
      "Epoch 3/10\n",
      "36/36 [==============================] - 23s 625ms/step - loss: 0.4483\n",
      "Epoch 4/10\n",
      "36/36 [==============================] - 23s 626ms/step - loss: 0.4004\n",
      "Epoch 5/10\n",
      "36/36 [==============================] - 23s 623ms/step - loss: 0.3673\n",
      "Epoch 6/10\n",
      "36/36 [==============================] - 23s 625ms/step - loss: 0.3736\n",
      "Epoch 7/10\n",
      "36/36 [==============================] - 23s 625ms/step - loss: 0.3277\n",
      "Epoch 8/10\n",
      "36/36 [==============================] - 23s 624ms/step - loss: 0.3362\n",
      "Epoch 9/10\n",
      "36/36 [==============================] - 23s 625ms/step - loss: 0.3187\n",
      "Epoch 10/10\n",
      "36/36 [==============================] - 23s 624ms/step - loss: 0.2877\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 25 F1: 0.5694444444444444:  93%|█████████▎| 25/27 [1:54:23<08:53, 266.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "34/34 [==============================] - 46s 630ms/step - loss: 0.6109\n",
      "Epoch 2/10\n",
      "34/34 [==============================] - 22s 628ms/step - loss: 0.5019\n",
      "Epoch 3/10\n",
      "34/34 [==============================] - 22s 629ms/step - loss: 0.4824\n",
      "Epoch 4/10\n",
      "34/34 [==============================] - 22s 627ms/step - loss: 0.4561\n",
      "Epoch 5/10\n",
      "34/34 [==============================] - 22s 629ms/step - loss: 0.4290\n",
      "Epoch 6/10\n",
      "34/34 [==============================] - 22s 627ms/step - loss: 0.3811\n",
      "Epoch 7/10\n",
      "34/34 [==============================] - 22s 630ms/step - loss: 0.4851\n",
      "Epoch 8/10\n",
      "34/34 [==============================] - 22s 630ms/step - loss: 0.3780\n",
      "Epoch 9/10\n",
      "34/34 [==============================] - 22s 630ms/step - loss: 0.4153\n",
      "Epoch 10/10\n",
      "34/34 [==============================] - 22s 631ms/step - loss: 0.3411\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 26 F1: 0.7470735785953176:  96%|█████████▋| 26/27 [1:58:51<04:27, 267.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "35/35 [==============================] - 47s 635ms/step - loss: 0.5603\n",
      "Epoch 2/10\n",
      "35/35 [==============================] - 22s 623ms/step - loss: 0.4380\n",
      "Epoch 3/10\n",
      "35/35 [==============================] - 22s 624ms/step - loss: 0.3759\n",
      "Epoch 4/10\n",
      "35/35 [==============================] - 22s 624ms/step - loss: 0.4042\n",
      "Epoch 5/10\n",
      "35/35 [==============================] - 22s 624ms/step - loss: 0.3407\n",
      "Epoch 6/10\n",
      "35/35 [==============================] - 22s 623ms/step - loss: 0.3255\n",
      "Epoch 7/10\n",
      "35/35 [==============================] - 22s 624ms/step - loss: 0.3152\n",
      "Epoch 8/10\n",
      "35/35 [==============================] - 22s 623ms/step - loss: 0.3256\n",
      "Epoch 9/10\n",
      "35/35 [==============================] - 22s 623ms/step - loss: 0.2758\n",
      "Epoch 10/10\n",
      "35/35 [==============================] - 22s 624ms/step - loss: 0.2796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 27 F1: 0.38904935817724295: 100%|██████████| 27/27 [2:03:20<00:00, 274.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.504 +- 0.292 \n",
      "\n",
      " balanced_accuracy: 0.505 +- 0.258 \n",
      "\n",
      " f1: 0.496 +- 0.301 \n",
      "\n",
      " recall: 0.504 +- 0.292 \n",
      "\n",
      " precision: 0.611 +- 0.341 \n",
      "\n",
      "Starting stratified 5-fold for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 44s 637ms/step - loss: 0.6746\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.5492\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.4092\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3965\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.3898\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.4128\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3864\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 636ms/step - loss: 0.3690\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3105\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 1 F1: 0.6516371443907676:  20%|██        | 1/5 [04:13<16:53, 253.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 44s 634ms/step - loss: 0.6916\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.4965\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.4486\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 633ms/step - loss: 0.3901\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 633ms/step - loss: 0.4038\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3665\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 633ms/step - loss: 0.3349\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 633ms/step - loss: 0.4467\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 632ms/step - loss: 0.3527\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 2 F1: 0.7059925093632959:  40%|████      | 2/5 [08:25<12:37, 252.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 43s 624ms/step - loss: 0.7009\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 624ms/step - loss: 0.4510\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 624ms/step - loss: 0.4587\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 623ms/step - loss: 0.4032\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 624ms/step - loss: 0.3768\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 623ms/step - loss: 0.3595\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 623ms/step - loss: 0.3978\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 623ms/step - loss: 0.3348\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 623ms/step - loss: 0.3441\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 624ms/step - loss: 0.3421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 3 F1: 0.3735159817351598:  60%|██████    | 3/5 [12:35<08:23, 251.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 43s 632ms/step - loss: 0.7618\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.4495\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.4696\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.4488\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.3908\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.4061\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3811\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 635ms/step - loss: 0.3237\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 633ms/step - loss: 0.3650\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 634ms/step - loss: 0.3047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 4 F1: 0.6727726750663083:  80%|████████  | 4/5 [16:48<04:11, 251.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Epoch 1/10\n",
      "30/30 [==============================] - 44s 625ms/step - loss: 0.6075\n",
      "Epoch 2/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.4564\n",
      "Epoch 3/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.5315\n",
      "Epoch 4/10\n",
      "30/30 [==============================] - 19s 624ms/step - loss: 0.4404\n",
      "Epoch 5/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.4280\n",
      "Epoch 6/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.3831\n",
      "Epoch 7/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.4317\n",
      "Epoch 8/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.3522\n",
      "Epoch 9/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.3508\n",
      "Epoch 10/10\n",
      "30/30 [==============================] - 19s 625ms/step - loss: 0.3303\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Fold 5 F1: 0.8152532135174373: 100%|██████████| 5/5 [20:56<00:00, 251.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.654 +- 0.126 \n",
      "\n",
      " balanced_accuracy: 0.673 +- 0.084 \n",
      "\n",
      " f1: 0.644 +- 0.146 \n",
      "\n",
      " recall: 0.654 +- 0.126 \n",
      "\n",
      " precision: 0.727 +- 0.053 \n",
      "\n",
      "CPU times: total: 6h 14min 57s\n",
      "Wall time: 5h 37min 42s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scores_strat_group_k_fold = [None]*2\n",
    "scores_loso = [None]*2\n",
    "scores_strat_k_fold = [None]*2\n",
    "\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings('ignore')\n",
    "\n",
    "    for variable in (0, 1): # phF, MF\n",
    "        model = CNN(path, variable=variable, epochs=10, learning_rate=1e-3, batch_size=16)\n",
    "\n",
    "        scores_strat_group_k_fold[variable] = stratified_group_k_fold(path=path,\n",
    "                                                            groups=subjects,\n",
    "                                                            model=model,\n",
    "                                                            folds=5,\n",
    "                                                            images=True,\n",
    "                                                            verbose=True,\n",
    "                                                            variable=variable)\n",
    "\n",
    "        scores_loso[variable] = leave_one_subject_out(path=path,\n",
    "                                            groups=subjects,\n",
    "                                            model=model,\n",
    "                                            images=True,\n",
    "                                            verbose=True,\n",
    "                                            variable=variable)\n",
    "\n",
    "        scores_strat_k_fold[variable] = stratified_k_fold(path=path,\n",
    "                                                          model=model,\n",
    "                                                          folds=5,\n",
    "                                                          images=True,\n",
    "                                                          verbose=True,\n",
    "                                                          variable=variable)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Save scores"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "outputs": [],
   "source": [
    "path_scores = './Scores'\n",
    "model_name = 'cnn'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "outputs": [],
   "source": [
    "# stratified group 5-fold\n",
    "with open(f'{path_scores}/strat_group_5_fold//{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_strat_group_k_fold))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "outputs": [],
   "source": [
    "# stratified 5-fold\n",
    "with open(f'{path_scores}/strat_5_fold/{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_strat_group_k_fold))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "outputs": [],
   "source": [
    "# LOSO\n",
    "with open(f'{path_scores}/loso//{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_loso))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}