00_core.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Core\n",
    "> Core of this library: DataChunk, RecordMaster, Data_Pipe, and import/export of reM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "from nbdev.test import test_eq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import h5py\n",
    "import json, re\n",
    "import numpy as np\n",
    "from collections import namedtuple \n",
    "from typing import Dict, Tuple, Sequence, Union\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.patches import Patch\n",
    "\n",
    "class DataChunk(np.ndarray):\n",
    "    \"\"\"Base brick of data. Derived from np.ndarray\n",
    "    params:\n",
    "        - data: The ndarray with shape (time, ...)\n",
    "        - idx: Index of the start of the DataChunk in the record.\n",
    "        - group: group of the DataChunk in {stim, sync, cell, data}\n",
    "        - fill: Default filling value.\"\"\"\n",
    "    def __new__(cls, data, idx, group, fill=0):\n",
    "        # See https://docs.scipy.org/doc/numpy-1.11.0/user/basics.subclassing.html#basics-subclassing\n",
    "        # for explanation on subclassing numpy arrays\n",
    "        obj = np.asarray(data).view(cls)\n",
    "        obj.idx = idx\n",
    "        obj.group = group\n",
    "        obj.fill = fill\n",
    "        \n",
    "        obj.attrs = {}\n",
    "        \n",
    "        return obj\n",
    "\n",
    "    def __array_finalize__(self, obj):\n",
    "        if obj is None: return\n",
    "        self.idx   = getattr(obj, 'idx', None)\n",
    "        self.group = getattr(obj, 'group', None)\n",
    "        self.fill  = getattr(obj, 'fill', 0)\n",
    "        self.attrs = getattr(obj, 'attrs', {})\n",
    "        \n",
    "\n",
    "    @property\n",
    "    def range(self):\n",
    "        return range(self.idx, self.idx + len(self))\n",
    "    \n",
    "    @property\n",
    "    def slice(self):\n",
    "        return slice(self.idx, self.idx + len(self))\n",
    "\n",
    "    def __str__(self):\n",
    "        return (\"Group: \"+str(self.group)\n",
    "                +\"\\nStarting index: \"+str(self.idx)\n",
    "                +\"\\nFilling value: \"+str(self.fill)\n",
    "                +\"\\n\"+super().__str__())\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"DataChunk(%s,%s,%s,%s)\"%(self.shape, self.idx, self.group, self.fill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "dc = DataChunk(np.random.rand(100), 0, \"data\", fill=0)\n",
    "test_eq(len(dc), 100)\n",
    "test_eq(dc.idx, 0)\n",
    "test_eq(dc.group, \"data\")\n",
    "test_eq(dc.fill, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ContiguousRecord():\n",
    "    \"\"\"Representation of a contiguous recording session to store DataChunk\n",
    "    of various sources under a single time reference. DataChunk are stored\n",
    "    under a name in one of the groups \"sync\",\"stim\",\"data\" and \"cell\". \n",
    "    \n",
    "    A name can contain multiple DataChunk if those are not overlapping in time.\n",
    "    \n",
    "    Each ContiguousRecord contains in the group \"sync\" two master DataChunk,\n",
    "    one for signals to be recorded across acquisition device to syncronize them,\n",
    "    one for timepoints of these signals for the main device and are called \n",
    "    respectively \"signals\" and \"main_tp\".\n",
    "    \n",
    "    \"\"\"\n",
    "    MAIN_TP = \"main_tp\"\n",
    "    SIGNALS = \"signals\"\n",
    "    def __init__(self, length:int, signals:DataChunk, main_tp:DataChunk, frame_rate:int=60 ):\n",
    "        \"\"\"Instanciate a ContiguousRecord.\n",
    "        \n",
    "        Parameters:\n",
    "            - length (int): Number of bins of this record\n",
    "            - signals (DataChunk): Signals for this record\n",
    "            - main_tp (DataChunk): Timepoints of the signals for the main device\n",
    "            - frame_rate(int): Frame rate in Hz\n",
    "        \"\"\"\n",
    "        self.length = length\n",
    "        self._frame_time = 1/frame_rate\n",
    "        self._data_dict = {}  \n",
    "        \n",
    "        self[self.SIGNALS] = signals\n",
    "        self[self.MAIN_TP] = main_tp\n",
    "        \n",
    "        self._slice = slice(0,self.length,1)\n",
    "      \n",
    "    def dataset_intersect(self, existing_datachunk:list, new_datachunk:DataChunk):\n",
    "        \"\"\"Check for timepoint intersections of two DataChunks\"\"\"\n",
    "        range_new = set(new_datachunk.range)\n",
    "        intersect = False\n",
    "        for range_existing in existing_datachunk:\n",
    "            intersect |= len(range_new.intersection(range_existing.range)) > 0\n",
    "        return intersect\n",
    "    \n",
    "    def keys(self):\n",
    "        \"\"\"Retrieves the existing keyys inside this ContiguousRecord\"\"\"\n",
    "        return self._data_dict.keys()\n",
    "    \n",
    "    def get_slice(self, datachunk_name:str) -> list:\n",
    "        \"\"\"Returns the slices of the DataChunk corresponding to the given key\"\"\"\n",
    "        if datachunk_name in self._data_dict.keys():\n",
    "            return [chunk.slice for chunk in self._data_dict[datachunk_name]]\n",
    "        else:\n",
    "            return []\n",
    "        \n",
    "    def set_slice(self, slice_):\n",
    "        \"\"\"Set the slice to restrict the size of the DataChunk returned\"\"\"\n",
    "        if slice_ is None:\n",
    "            self._slice = slice(0,self.length,1)\n",
    "        else:\n",
    "            start, stop, step = slice_.start, slice_.stop, slice_.step\n",
    "            if start is None:\n",
    "                start = 0\n",
    "            if stop is None:\n",
    "                stop = self.length\n",
    "            if step is None:\n",
    "                step = 1\n",
    "            if step!=1:\n",
    "                print(\"Step in slice is currently not supported.\")\n",
    "                self._slice = slice(0,self.length,1)\n",
    "            else:\n",
    "                self._slice = slice(start,stop,step)\n",
    "                    \n",
    "    \n",
    "    def get_names_group(self, group_name:str) -> list:\n",
    "        names = []\n",
    "        for key, dChunk_l in self._data_dict.items():\n",
    "            if dChunk_l[0].group == group_name:\n",
    "                names.append(key)\n",
    "        return names\n",
    "    \n",
    "    def to_s(self, n_frame):\n",
    "        return round(self._frame_time*n_frame,2)\n",
    "    \n",
    "    def to_time_str(self, n_frame):\n",
    "        s = int(self.to_s(n_frame))\n",
    "        m, s = s//60, str(s%60)\n",
    "        h, m = str(m//60), str(m%60)\n",
    "        return \"{0}:{1}:{2}\".format('0'*(2-len(h))+h, '0'*(2-len(m))+m, '0'*(2-len(s))+s)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.length\n",
    "    \n",
    "    def __setitem__(self, key, value:DataChunk):\n",
    "        if isinstance(key, str):\n",
    "            if key not in self._data_dict.keys():\n",
    "                self._data_dict[key] = []\n",
    "                \n",
    "            if not self.dataset_intersect(self._data_dict[key], value):\n",
    "                self._data_dict[key].append(value)\n",
    "            else:\n",
    "                raise ValueError(\"Data with the same name already exists and intersect with the one provided\")\n",
    "        else:\n",
    "            raise KeyError(\"Cannot set data with an integer index, it needs a name\")\n",
    "\n",
    "    def __getitem__(self, key):\n",
    "        if isinstance(key, str):\n",
    "            l_datachunk = self._data_dict[key]\n",
    "            fill_value  = l_datachunk[0].fill\n",
    "            shape       = l_datachunk[0].shape\n",
    "            \n",
    "            full_sequence = DataChunk(np.zeros((len(range(*self._slice.indices(self.length))), *shape[1:]), \n",
    "                                               dtype=l_datachunk[0].dtype)+fill_value, \n",
    "                                      self._slice.start if self._slice.start is not None else 0, \n",
    "                                      fill_value)\n",
    "            for datachunk in l_datachunk:\n",
    "                dc_slice = datachunk.slice\n",
    "                if dc_slice.start>=self._slice.stop or dc_slice.stop<=self._slice.start:\n",
    "                    continue\n",
    "                \n",
    "                start = max(dc_slice.start, self._slice.start) #flooring to the maximum of both start\n",
    "                stop  = min(dc_slice.stop, self._slice.stop) # and capping to the min of both end\n",
    "                    \n",
    "                new_dc_slice  = slice(start-datachunk.idx, stop-datachunk.idx)\n",
    "                res_start = self._slice.start if self._slice.start is not None else 0\n",
    "                res_slice = slice(start-res_start, stop-res_start)\n",
    "                full_sequence[res_slice] = datachunk.data[new_dc_slice]\n",
    "                full_sequence.attrs.update(datachunk.attrs)\n",
    "            \n",
    "            return full_sequence\n",
    "                \n",
    "    def __iter__(self):             \n",
    "        groups = {\"sync\":[],\"stim\":[],\"data\":[],\"cell\":[]}\n",
    "        for key, dChunk_l in self._data_dict.items():\n",
    "            groups[dChunk_l[0].group].append((dChunk_l[0].idx, key))\n",
    "\n",
    "        self._iter_order = []\n",
    "        for group_name in [\"sync\",\"stim\",\"data\",\"cell\"]:\n",
    "            sorted_ = sorted(groups[group_name], key=lambda e:(e[0],))\n",
    "            self._iter_order.extend([key for _, key in sorted_])\n",
    "        self._n = 0\n",
    "        \n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        if self._n < len(self._iter_order):\n",
    "            key = self._iter_order[self._n]\n",
    "            dChunk_l = self._data_dict[key]\n",
    "            self._n += 1\n",
    "            return (key, dChunk_l)\n",
    "        else:\n",
    "            raise StopIteration\n",
    "            \n",
    "    def __delitem__(self, key):\n",
    "        del self._data_dict[key]\n",
    "        \n",
    "    def __str__(self):\n",
    "        res = \"ContiguousRecord:\\n\"\n",
    "        for k,v in self._data_dict.items():\n",
    "            res += k+\" : \"+\" \".join([str(dc.shape) for dc in v]) +\"\\n\"\n",
    "        return res\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return self._data_dict.__repr__()\n",
    "    \n",
    "    def __delete__(self, instance):\n",
    "        for k, v in self._data_dict.items():\n",
    "            del v\n",
    "            self._data_dict[k] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "dc_tp      = DataChunk(np.arange(0,10000,50), 0, \"sync\", fill=0)\n",
    "dc_signals = DataChunk(np.random.rand(100), 10, \"sync\", fill=0)\n",
    "dc         = DataChunk(np.random.rand(100), 111, \"cell\", fill=0)\n",
    "\n",
    "cr = ContiguousRecord(len(dc_tp), dc_signals, dc_tp)\n",
    "test_eq(cr.dataset_intersect([dc_signals], dc_tp), True)\n",
    "test_eq(cr.dataset_intersect([dc_signals], dc),    False)\n",
    "\n",
    "test_eq(cr.get_slice(\"signals\"),    [slice(10, 110, None)])\n",
    "cr[\"test\"] = dc\n",
    "test_eq(cr.get_names_group(\"cell\"),    ['test'])\n",
    "test_eq(cr.get_names_group(\"sync\"),    ['signals', 'main_tp'])\n",
    "\n",
    "test_eq(len(cr[\"main_tp\"]),    200)\n",
    "cr.set_slice(slice(100,200,1))\n",
    "test_eq(len(cr[\"main_tp\"]),    100)\n",
    "cr.set_slice(None)\n",
    "test_eq(len(cr[\"main_tp\"]),    200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class RecordMaster(list):\n",
    "    \"\"\"\n",
    "    The RecordMaster class is the top level object managing all\n",
    "    timeseries. It uses a list of ContiguousRecord to represent\n",
    "    possible discontinuted data records.\n",
    "    \n",
    "    The main aim of the RecordMaster is to store the various data\n",
    "    stream of an experiment under a unique time reference, to ease\n",
    "    the processing of the data.\n",
    "    \n",
    "    A RecordMaster is created by providing to it a list of (timepoints, signals) arrays of\n",
    "    identical lenght. This serves as reference to align other DataChunks with the signals\n",
    "    at the given timepoints. Multiple tuples represent uncontiguous sequences of data in\n",
    "    a same record. The timepoints must be evenly spaced (regular).\n",
    "    \n",
    "    params:\n",
    "        - reference_data_list: list of (timepoints, signals) arrays.\n",
    "        - frame_rate: Frame rate in Hz, or list of frame rates matching len(reference_data_list)\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, reference_data_list: Sequence[Tuple[DataChunk, DataChunk]], frame_rate=60):\n",
    "        \n",
    "        if not hasattr(frame_rate, '__iter__'):\n",
    "            frame_rate = [frame_rate]*len(reference_data_list)\n",
    "            \n",
    "        self._sep_size   = 1000 #Used for the plotting of multiple sequences\n",
    "        self._sequences = []\n",
    "        for (ref_timepoints, ref_signals), fr in zip(reference_data_list, frame_rate):\n",
    "            cs = ContiguousRecord(len(ref_timepoints), ref_signals, ref_timepoints, fr)\n",
    "            self._sequences.append(cs)\n",
    "            \n",
    "    def set_datachunk(self, dc:DataChunk, name:str, sequence_idx=0):\n",
    "        \"\"\"Set the given DataChunk dc for the sequence at sequence_idx under name.\"\"\"\n",
    "        self._sequences[sequence_idx][name] = dc\n",
    "        \n",
    "    def append(self, ref_timepoints:DataChunk, ref_signals:DataChunk, frame_rate=60):\n",
    "        cs = ContiguousRecord(len(ref_timepoints), ref_signals, ref_timepoints, frame_rate)\n",
    "        self._sequences.append(cs)\n",
    "        \n",
    "    def insert(self, idx:int, ref_timepoints:DataChunk, ref_signals:DataChunk, frame_rate=60):\n",
    "        cs = ContiguousRecord(len(ref_timepoints), ref_signals, ref_timepoints, frame_rate)\n",
    "        self._sequences.insert(idx, cs)\n",
    "        \n",
    "    def keys(self):\n",
    "        keys = []\n",
    "        for seq in self._sequences:\n",
    "            keys.extend(list(seq.keys()))\n",
    "        return set(keys)\n",
    "        \n",
    "    def __setitem__(self, key, value:DataChunk):\n",
    "        \"\"\"Setting an item directly to the record_master place it in the first sequence\"\"\"\n",
    "        if isinstance(key, str):\n",
    "            self._sequences[0][key] = value\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        if isinstance(key, (int, np.integer)):\n",
    "            return self._sequences[key]\n",
    "        elif isinstance(key, str):\n",
    "            #We want all the data of that name\n",
    "            res = []\n",
    "            for seq in self._sequences:\n",
    "                res.append(seq[key])\n",
    "            return res\n",
    "\n",
    "        raise TypeError(\"Indexing not understood\")\n",
    "        \n",
    "    def __iter__(self):\n",
    "        self._n = 0\n",
    "        return self\n",
    "    \n",
    "    def __next__(self):\n",
    "        if self._n < len(self):\n",
    "            res = self._sequences[self._n]\n",
    "            self._n += 1\n",
    "            return res\n",
    "        else:\n",
    "            raise StopIteration \n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self._sequences)\n",
    "        \n",
    "    def plot(self, ax=None, show_time=True, sort_by_name=False):\n",
    "        colors = {\"sync\":\"cornflowerblue\", \"stim\":\"orange\", \"data\":\"yellowgreen\", \"cell\":\"plum\"}\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(figsize=(10, 5))\n",
    "        ax.invert_yaxis()\n",
    "        ax.xaxis.set_visible(False)\n",
    "        \n",
    "        #A first iteration through the ContiguousRecords to have all the DataChunk for the sorting\n",
    "        groups = {\"sync\":[],\"stim\":[],\"data\":[],\"cell\":[]}\n",
    "        frame_times = []\n",
    "        for seq in self._sequences:\n",
    "            frame_times.append(seq._frame_time)\n",
    "            for name, dChunk_l in seq:\n",
    "                dc = dChunk_l[0]\n",
    "                if name not in groups[dc.group]:\n",
    "                    groups[dc.group].append(name)\n",
    "                    \n",
    "        all_names = []\n",
    "        y_pos_dict = {}\n",
    "        y_count    = 0\n",
    "        for group_name in [\"sync\",\"stim\",\"data\",\"cell\"]:\n",
    "            if sort_by_name:\n",
    "                groups[group_name].sort()\n",
    "            all_names += groups[group_name]\n",
    "            for dc_name in groups[group_name]:\n",
    "                y_pos_dict[dc_name] = y_count\n",
    "                y_count+=1           \n",
    "        \n",
    "        frametime_ratios = [fr_t/max(frame_times) for fr_t in frame_times]\n",
    "        cursor = 0\n",
    "        for i, seq in enumerate(self._sequences):\n",
    "            fr_ratio = frametime_ratios[i]\n",
    "            for name, dChunk_l in seq:\n",
    "                y_pos = y_pos_dict[name]\n",
    "                for dChunk in dChunk_l:\n",
    "                    pos = dChunk.idx + cursor\n",
    "                    ax.barh(y_pos, len(dChunk)*fr_ratio, left=pos*fr_ratio, height=0.8, color=colors[dChunk.group], label=dChunk.group)              \n",
    "                    if show_time:\n",
    "                        x = pos + len(dChunk)/2\n",
    "                        text = \"{0} -> {1} \".format(seq.to_time_str(dChunk.idx), seq.to_time_str(dChunk.idx+len(dChunk)))\n",
    "                        ax.text(x*fr_ratio, y_pos, text, ha='center', va='center')\n",
    "            cursor += (len(seq))*fr_ratio + self._sep_size\n",
    "            \n",
    "        legend_elements = [Patch(facecolor=colors[\"sync\"],label='Synchro'),\n",
    "                           Patch(facecolor=colors[\"data\"],label='Data'),\n",
    "                           Patch(facecolor=colors[\"stim\"],label='Stimulus'),\n",
    "                           Patch(facecolor=colors[\"cell\"],label='Cell'),]\n",
    "        ax.legend(handles=legend_elements, ncol=5, bbox_to_anchor=(0, 1), loc='lower left', fontsize='small')\n",
    "        ax.set_yticks(range(len(all_names)))\n",
    "        ax.set_yticklabels(all_names)\n",
    "        ax.set_xlim(-100,cursor)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return \"[\"+\",\\n\".join([repr(seq) for seq in self._sequences])+\"]\"\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"[\"+\", \".join([repr(seq) for seq in self._sequences])+\"]\"\n",
    "    \n",
    "    def __delete__(self, instance):\n",
    "        for seq in self._sequences:\n",
    "            del seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAE1CAYAAAC4IKf+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAh10lEQVR4nO3deViWVeL/8c8BAgWVRUkFFDUQc18AzclSy9R+DlZmOk1ZaXWlNdU0NjWt2q+vpX1bxnHSyazMGh1zrJxyLWdsMcHd0kRKUgEtFHABBXw43z9ABmURSeKo79d1dQXPc5/znEOkb+/7fsRYawUAAAA3edX1AgAAAFA5Yg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAc5lPXC/ilbNiw4TofH59HrbXN6notAH5Zxph9x48ff7579+4f1PVaAOBMXTCx5u3t/eIll1xy1N/ff78xpq6XA+AXYq1VXl5eg+++++5FSR/U9XoA4ExdSJdBvQMCAo4RasCFxRijgICAY5K863otAFATF1KsAQAAnHMumMugp7rr1aweP2f8zHEh66t6Pjs722vYsGFtjhw54l1QUGAee+yxjJtuuulQTV+vY8eOl37zzTff1nR8Tc3c0ONnfZ0k6a7u6yv9WiUnJ/v26tXr0piYmKPHjx833bp1y33ppZcyGjZsWHTqsXPmzAnq37//kfDw8OM/d02V+rv52fvVzbbS/Vb0fbF7927f8ePH75ek0aNHt5g6dWpagwYNavRz4H7p75Ods3f+7K9Xm9vaVPn/0ooVKwIef/zxCGutvL297ZQpU9KuuOKKvLLHJCcn+z7wwAMRS5cu3RkfHx+zYsWKlMDAwHLfQwBwLuLMWi2ZMWNG4wEDBhxMSkpK3rBhw/b+/fvn1tZrFRUVqajo3P19qWfPnofXrFmzIykpKdnf379o/PjxYRUd98EHHwRlZGSc03/AqOj74q233go98fwbb7yxp6ahdj768ccfve+7777I9957b+fatWuTP/744++9vPhlC8CFhV/1akn9+vWLEhMTG+zZs8fHy8tLixcvbvjoo482k6SsrCyv3r17t5WkNm3adLjhhhtatWvXrv306dNDJOmbb77xu+yyy9rGx8fH3HXXXRGSlJeX53XqccOGDWt16623trz88suj09PTfYYOHdo6Li4upm/fvlGZmZnn3P05Xl5emjx58t5ly5YFPfnkk03j4+Nj2rdvf+n777/faPv27b6rVq0KvOOOO1rfc889EUlJSfXj4uJiunbt2m7UqFEt63rt1XXq98WsWbNCUlNT68XHx8csWrSoYXx8fMzBgwe9pk6d2viaa665pH///lFdu3ZtN23atMa9e/du26tXr7b5+fnmo48+anj33XdHSNLatWvrDRs2rFXZ13nooYfC5s6dGyhJkyZNCp06dWrjrVu3+nXr1q1dz5492958882RdbD9M7ZgwYLAa6+9NicyMrJQkho3buzp3r370aFDh7bu1atX2/79+0dlZWXx6xiA8xq/yNWSsWPHZrVt2/bYgAED2nbt2rVdly5djn7yySeBkjRnzpzg6667LluS9u/ff9Hrr7++e/Xq1dunT59+sSQ99NBDEVOmTElLSkpKnjFjRlplx0lS9+7d81avXp3yySefNAwPDy9Yu3Zt8vDhw7MmT558cUXrcl29evVsYWGh+eMf/5iZlJSUvGLFipRJkyY1b9euXcGVV1558M0330ydMWNGWvv27Y8lJiYmb9q0aXt6evpFX3/9tV9dr706Tv2+GDRo0KHWrVsfS0pKSk5ISDhc9tiQkJDjK1eu/O7yyy8/vHHjRv/Vq1fv6NChw9Fly5Y1qMlrL1++vOGIESMOJCYm7pgzZ86us7Oj2pWRkeEbFhZWUPaxV155pUm/fv0OrVmzZsdvfvObA6+88kpoZeMB4HxArNUSPz8/O2XKlL3bt2/f9vTTT2c8/vjj4R06dMj74osv/BcuXBh82223ZUlSixYt8kNCQopCQkKKioqKjFT8G1SfPn3yJMnbu/gEWUXHSVLv3r1zJSklJcUvPj4+t+SxvO+++67eL7zls+Lo0aPG19e3aMaMGSGxsbEx119//SWZmZkXnXpccnKyX9++faPj4uJitm7dGrB79+5yx7ioou+Lyo7t1KnTUUkKCwsr6Ny5c54khYeHF2RlZXkbY0ovlVpry73FuaLnb7/99qzU1FS/hISE1tOnT298NvdVW8LCwgrS09N9yz62bdu2+q+//vrF8fHxMdOnT2+6f//+c/rSOACcDrFWS3bs2OF77NgxI0nNmjUrtNZqzJgxB5577rlm9erVs02bNvVIxX+twKnCwsIKvvjiC39J8ng8quw4SfLy8rKSFB0dnZ+YmBggSatXr/aPioo6VgvbqnWPP/5488GDB+e89tprTdesWZP8z3/+83tri7vjoosussePHzeSNHXq1NCHHnpo39q1a5Pbt2+fV1GwuKii74vKlA2usv/9rbWmSZMmnoyMDF9JWrduXf1TxwYHB3v27NnjK0lbtmypLxV//f72t7+lLVq0KPWVV15pduJ7y2U33njjwSVLlgTt2rXrIqn4FoJmzZoVjh079scT9/29/PLLGXW9TgCoTRfsn0hP927On2v9+vX1hw8f3sbPz89aa/Xqq6/uiouLOzZ69Oj6TzzxRHpVY1966aW0MWPGRFprTZcuXXJnzpyZdrrXu+WWW7IXLlwYFBsbGxMQEOCZP39+6tnYR1Xv5DxbEhMTG/bs2bOtx+MxPXr0yH3xxRfT7777bu/Y2Nh2PXr0yPX39y+SpMGDBx988MEHW/Tt2/fQ0KFDD44fP77l9OnTj1UVPGesindyng0VfV88++yzzQcOHHjJH/7wh33VnScuLu5oXl6eV+/evaMvvfTScmF+yy23ZA8ZMiRq6dKlgQ0aNPBI0ty5c4NmzJhxsST17dv34Imztj/H6d7J+XM1bdrUM23atF3Dhw9vc+LdoM8880z69OnTQ+fMmdNEkh588MEfu3XrdrQ21wEAdcmc1d/oHLZ58+YfunTpsr+u1xEfHx+zatWqHfXr178wvvCAIzZv3tykS5curep6HQBwprgM+gvZv3+/d+/evaOHDBmSTagBAIDqumAvg/7SmjRp4lm9enVKXa8DAACcWy6kM2ue3NzcehfKZV8Axay1ys3NrSfJ/XdUAEAFLpgzax6P5w/ff//9o9bappLOiXcOAjgrrDHmR4/H80RdLwQAauKCeYMBAADAuehCugwKAABwziHWAAAAHHbexpoxZmldrwEAAKA6quqW8/YNBo0aNRoYGxvLDXkAAOBccKiyJ87bWIuOjta6devqehkAAACnZYyp9O9iPW8vgwIAAJwPiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAh523P27qXHbXq1l1vYQLzsxxIXW9BAAAKsSZNQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIf51PUCLlRLly7VAw88II/HozvvvFOPPvqoJCk1NVUfTrlR+bnZatyyi/reNl3ePr7lxm9a9rJ2rH5XxstLlw1/XhHt+0uS9mz9VGsW/Em2qEgxv7pFXa55sNxYT2G+/vP2OB3YvVl+AcHqP2aWGjZuWeW8Z8v+3Zu0as598hQcU0SHq3XZ8OdkjNGx3GytfGOMjhzYowaNW+iqMW/Izz+o3Pgda+Zq09KXJEldBz2ktr1+U+W8ZVlr9dV7f1La1k/k7VtfV946TU1adtHhA3vUvfvVKioqUmFhoX73u9/pnnvuOav7BgCgps7amTVjzOvGmPZna74y8x4523PWNY/Ho3vvvVdLlizRtm3bNHfuXG3btk2S9Mgjj6hj/7G6aeI6+fkHKXn1O+XGZ+/drp3r39ewJ77UoHvf05f/eFhFRR4VFXm0ev4fNfDe+Rr25Gp9v26hsvduLzc++at35OcfpJsmrlPH/mOV9MHEKuetSmF+roo8hdXe+5fzxqvPzS9r+IS1OpS5U2nbPpUkbV7+Z4XHXKGbJqxVeMwV2rz8lXJjj+Vma+PiF5Tw8HIN/eMKbVz8gvLzcqqct6y0rZ/oUOZODZ+wVn1ufklfzhsvSfIPbKqvvvpKmzZtUmJiop5//nllZGRUe08AANSmsxZr1to7rbXbztZ857OkpCRFRUWpTZs28vX11ciRI/Xhhx/KWquVK1eqdbcESVJ0z5HatWVxufG7tixRmx7Xy/siPzVsEqlGoa2V+cMGZf6wQY1CW6tRk1by9vFVmx7Xa9eWJRWOj+45UpLUuluCMpI/k7W20nmrcvCn7zV/YrwSFz6l7H3JVR6bd3CfCo4d1sWt42SMUXTPEdq1uXh/u7csLl1TdM+RpY+Xlf7tSoW366t6AcHy8w9SeLu+Stv2aZXzlt/3CBljdHHrOBUcPai8g/vk7eMrPz8/SVJ+fr6Kioqq3AcAAL+kGsWaMSbAGPOxMWazMeYbY8wIY8x/jDGxJc+PMcbsMMYkGWNmGmOmlTz+ljFmqjFmtTFmpzHmxpLHGxhjPjXGbDDGfG2MGVrBazY3xnxmjNlU8pp9fs7G61J6erpatGhR+nlERITS09N14MABBQUFycu7+Op0QHCY8nL2SioOjfUfPSdJysvZq4Dg8NLxAUHFx1X2uCSt/+i50nDLy9mrBsFhkiQvbx/51m+k/NysKsdXpkmLzrrhsc8U2DRan7/7oP714rXa8dW7KszPLXdsbs5eBQSFnTR/7sHi+Y8ezpR/YDNJUv1GTXX0cKYkKXPXRn327gP/HR9cZnxwmHJz9lY577efv6lvP3+zePzBvQoIOnl/uSX727Nnjzp37qwWLVrokUceUVjYf+cDAKAu1fSetUGSMqy1/0+SjDGBksaWfBwm6UlJ3SUdlrRS0uYyY5tLulxSO0mLJC2QdEzS9dbaQ8aYJpLWGGMWWWttmXE3S1pmrf0fY4y3JP8arv2cFNl5sCI7D67x+B5D/nQWV3My33oN1e5Xt6rdr25V9r5kff7OA/pqwWO67cVdNZqv+F6z4vvNQiO7KTSyW43XdmmfO6p1XIsWLbRlyxZlZGTouuuu04033qimTZvW+HUBADhbanoZ9GtJA4wxk40xfay1B8s8Fy9plbU2y1pbKOm9U8Z+YK0tKrlkeuJ3QyNpkjFmi6RPJIWXee6EtZLuMMZMkNTJWnv41EUZY+42xqwzxqzLzMys4dZqX3h4uPbs2VP6eVpamsLDw9W4cWPl5OSoyHNckpSbnSH/oOblxvsHNVdudnrp57k5xcdV9nhF449kF9+TVeQ5roKjh+QXEFKt8T9s+kgLJ12phZOuVOaujaWPHz6wWxs+nqJPXhulgOBwXXXnW+VeNyCouXJz/nsvWG5OhgICi+ev3zBUeQf3SSq+XFq/YZOKx2eXGZ+doYCg5lXOe9L4wObKzTl5fwGn7C8sLEwdO3bU559/Xm48AAB1oUaxZq3doeIzZ19LetYY89QZDM8v8/GJt+v9VlKopB7W2q6SfpRU75TX/EzSFZLSJb1ljBlVwbpes9bGWmtjQ0NDz2BJv6y4uDilpKQoNTVVBQUFmjdvnhISEmSMUb9+/ZS6cZEkKSVxXoVn0yI7DdbO9e/LU5ivw/t36dBPOxXaqrtCI7vp0E87dXj/LnmOF2jn+vcV2ami8YOUkjhPkpS6cZHC2vaRMabSectq1XWIbnhslW54bJVCI7vp8IHdWjz1eq34263y9Q/Urx9aoqvGzFLEpf3Kva5/YDP51muon1LXylqrlMR/lO6vZafBpWtKSZynlp2vLTc+/NL+Stv+b+Xn5Sg/L0dp2/+t8Ev7VzlvWS07D1JK4j9krdVPqWvlW7+R/AObKTc7XUePHpUkZWdn64svvlBMTEyV/w0BAPil1OgyaMmlzixr7TvGmBxJd5Z5eq2kV4wxwSq+DDpMxVFXlUBJP1lrC40x/SRFVvCakZLSrLUzjTF+Ko7Ft2uy/rrm4+OjadOmaeDAgfJ4PBo9erQ6dOggSZo8ebJ69b9R6/81SY1bdFLMZbdIKr5nbf/uTeox5E8KDmun1t2HasGzveXl5a3eI6bIy8tbktT7psla8tfhskUetb3sZgWHtZNUfM9ak5ZdFdl5sNr2vkWrZo/V/Kdj5RcQpH6jX5ekKuetjPHyVmzCE7q4VY9q7b33iBf02Zz7dLzwmFq0v0oRHa6WJHW55gGtnDVayavfVYOQCPUf84ak4nvWvv3iLV3x2z+rXkCwug0arw8nF4/pPni86gUEVznvifvVLu1zh1p0GKA9W1do/oRY+fjW1xW3/EWSlL1vh3r2/K2MMbLWavz48erUqVO19gMAQG0zJ98WVs1BxgyU9IKkIkmFKr5f7X8ljbfWrjPG3C3pYUlZkrarOLIeN8a8Jekja+2CknmOWGsblNyn9i9JDSStk9RL0mBr7Q9ljrmtZM5CSUckjbLWpla2xtjYWLtu3boz3psL7no1q66XcMGZOS6krpcAALiAGWPWW2tjK3quRmfWrLXLJC075eG+ZT7+u7X2NWOMj6T3JX1QMu72U+ZpUPLv/ZIuq+S1ThwzW9LsmqwXAADgXFVbP25qgjFmk6RvJKWqJNYAAABwZmrlx01Za8fXxrwAAAAXGn6QOwAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGE+db0AlDdzXEhdLwEAADiCM2sAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAc5lPXC8Dp3fVqVl0vocZmjgup6yUAAHBO48waAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrZ2Dp0qWKiYlRVFSUnn/++dLHU1NT1bNnT0VFRWnEiBEqKCiocPxzzz2nqKgoxcTEaNmyZaedtyxPYb4+nTVG85+O1YdTBujwgd2lz21a9rLmPx2r9ybGK23byrO02//av3uT/vk/l2v+07FaPf9RWWslScdys7X4Lzdo/oQ4Lf7LDcrPy6lw/OzZsxUdHa3o6GjNnj279PH169erU6dOioqK0v333186b1nWWt1///2KiopS586dtWHDhtLnBg0apKCgIA0ZMuTsbhgAAIfUaqwZY2KNMVNrMC7IGDOuNtZUUx6PR/fee6+WLFmibdu2ae7cudq2bZsk6ZFHHtHvf/97fffddwoODtasWbPKjd+2bZvmzZunrVu3aunSpRo3bpw8Hk+V85aV/NU78vMP0k0T16lj/7FK+mCiJCl773btXP++hj3xpQbd+56+/MfDKiryVLmXwvxcFXkKq733L+eNV5+bX9bwCWt1KHOn0rZ9KknavPzPCo+5QjdNWKvwmCu0efkr5cZmZWVp4sSJSkxMVFJSkiZOnKjs7GxJ0tixYzVz5kylpKQoJSVFS5cuLTd+yZIlpc+/9tprGjt2bOlzDz/8sObMmVPtfQAAcC6q1Viz1q6z1t5fg6FBkpyKtaSkJEVFRalNmzby9fXVyJEj9eGHH8paq5UrV+rGG2+UJN1222364IMPyo3/8MMPNXLkSPn5+al169aKiopSUlJSpfOeateWJYruOVKS1LpbgjKSP5O1Vru2LFGbHtfL+yI/NWwSqUahrZX5w4Zy48s6+NP3mj8xXokLn1L2vuQqj807uE8Fxw7r4tZxMsYouucI7dq8WJK0e8vi0jVF9xxZ+nhZy5Yt04ABAxQSEqLg4GANGDBAS5cu1d69e3Xo0CH16tVLxhiNGjWq0q/bqFGjZIxRr169lJOTo71790qSrrrqKjVs2LDK9QMAcK47bawZY1oZY7YbY94yxuwwxrxrjLnaGPOlMSbFGBNf8s9XxpiNxpjVxpiYkrF9jTEflXw8wRjzhjHmP8aYncaYqiLueUmXGGM2GWNeKJnnM2PMx8aYZGPMDGPML3oJNz09XS1atCj9PCIiQunp6Tpw4ICCgoLk4+Nz0uOStGjRIj311FNVjq/scUl66qmntGjRIklSXs5eNQgOkyR5efvIt34j5edmKS9nrwKCw0vHBwSFKS9nb5V7adKis2547DMFNo3W5+8+qH+9eK12fPWuCvNzyx2bm7NXAUFhJ82fe7B4/qOHM+Uf2EySVL9RUx09nClJyty1UZ+9+8Bp9x0REVHhvmfMmKEZM2ZUOR4AgAuFTzWPi5I0XNJoSWsl3SzpckkJkh6TNEpSH2vtcWPM1ZImSRpWwTztJPWT1FBSsjFmurW2outxj0rqaK3tKhVHn6R4Se0l7ZK0VNINkhaUHWSMuVvS3ZLUsmXLam6t9iQkJCghIaHG45955hlJ0r9ezTpbSyrlW6+h2v3qVrX71a3K3pesz995QF8teEy3vbirRvMZYyQZSVJoZDeFRnar8druueeeGo8FAOB8U92zU6nW2q+ttUWStkr61BbfDf61pFaSAiW9Z4z5RtLLkjpUMs/H1tp8a+1+ST9JanoGa02y1u601nokzVVxLJ7EWvuatTbWWhsbGhp6BlOfXnh4uPbs2VP6eVpamsLDw9W4cWPl5OTo+PHjJz1e3fGVPX4q/6DmOpKdIUkq8hxXwdFD8gsIkX9Qc+Vm//dMU25OhvyDmp809odNH2nhpCu1cNKVyty1sfTxwwd2a8PHU/TJa6MUEByuq+58q9zrBgQ1V25OxknzBwQWz1+/YajyDu6TVHy5tH7DJme077S0tNPuu7pfHwAAzlfVjbX8Mh8Xlfm8SMVn5/6/pH9baztK+rWketWYx6Pqn9mTpFPfKlj+rYO1KC4uTikpKUpNTVVBQYHmzZunhIQEGWPUr18/LVhQfJJv9uzZGjp0aLnxCQkJmjdvnvLz85WamqqUlBTFx8dXOu+pIjsNUkriPElS6sZFCmvbR8YYRXYarJ3r35enMF+H9+/SoZ92KrRV95PGtuo6RDc8tko3PLZKoZHddPjAbi2eer1W/O1W+foH6tcPLdFVY2Yp4tJ+5V7XP7CZfOs11E+pa2WtVUriPxTZebAkqWWnwaVrSkmcp5adry03fuDAgVq+fLmys7OVnZ2t5cuXa+DAgWrevLkaNWqkNWvWyFqrt99+u9Kv29tvvy1rrdasWaPAwEA1b9683HEAAJyvziSWqhIo6cTpndvPwnyHVXyptKx4Y0xrFV8GHSHptbPwOtXm4+OjadOmaeDAgfJ4PBo9erQ6dCg+gTh58mSNHDlSTzzxhLp166YxY8ZIKr5nbd26dXrmmWfUoUMH3XTTTWrfvr18fHz017/+Vd7e3pJU6bxPPfWUYmNjJV2utr1v0arZYzX/6Vj5BQSp3+jXJUnBYe3UuvtQLXi2t7y8vNV7xBR5eXlXuRfj5a3YhCd0case1dp77xEv6LM59+l44TG1aH+VIjpcLUnqcs0DWjlrtJJXv6sGIRHqP+YNScX3rH37xVu64rd/VkhIiJ588knFxcWV7ikkJESS9Oqrr+r222/X0aNHNXjwYA0eXByBJ+5Xu+eee3Tttddq8eLFioqKkr+/v958883SdfXp00fbt2/XkSNHFBERoVmzZmngwIHV2hMAAOcKU9HfbXXSAca0kvRRyVkzGWPeKvl8wYnnJN0labakXEkfS7rFWtuq5F6z8dbaIcaYCZKOWGv/t2SebyQNsdb+UMnr/l1SZ0lLSuZ8RsURFyXp35LGlVyWrVBsbKxdt27dab8A54K7auGetV/KzHEhdb0EAACcZ4xZb62Nrei5055ZK4mpjmU+v72S59qWGfZEyfP/kfSfko8nnDJvR1XBWnvziY9Lou+QtZa//RQAAFxQ+AkGAAAADjtb96zViDGmsaRPK3jqKmvtgROflD1DBwAAcCGp01grCbKudbkGAAAAl3EZFAAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcBixBgAA4DBiDQAAwGHEGgAAgMOINQAAAIcRawAAAA4j1gAAABxGrAEAADiMWAMAAHAYsQYAAOAwYg0AAMBhxBoAAIDDiDUAAACHEWsAAAAOI9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOMynrheA05s5LqSulwAAAOoIZ9YAAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHEasAQAAOIxYAwAAcJix1tb1GmqFMSZT0q66XgcAAEA1RFprQyt64ryNNQAAgPMBl0EBAAAcRqwBAAA4jFgDAABwGLEGAADgMGINAADAYcQaAACAw4g1AAAAhxFrAAAADiPWAAAAHPZ/O+AC98vNZccAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "dc_tp      = DataChunk(np.arange(0,10000,50), 0, \"sync\", fill=0)\n",
    "dc_signals = DataChunk(np.random.rand(100), 10, \"sync\", fill=0)\n",
    "dc         = DataChunk(np.random.rand(100), 111, \"cell\", fill=0)\n",
    "\n",
    "reM = RecordMaster([(dc_signals, dc_tp)])\n",
    "reM.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Data_Pipe():\n",
    "    \"\"\"\n",
    "    A Data_Pipe is used to query data from a RecordMaster. By adding/substracting portions\n",
    "    of the record using DataChunk names, it creates a mask over which specified DataChunks \n",
    "    are retrivied as a dictionary from the Data_Pipe.\n",
    "    \n",
    "    params:\n",
    "        - record_master: the RecordMaster from which to retrieve data\n",
    "        - data_names: Name, or list of names of the DataChunk to retrieve once the masking process is done.\n",
    "        - target_names: Alias, or list of alias to give to the DataChunk in the retrieved dictionnary.\n",
    "        - cast_to_np: boolean to cast the piped data into a numpy array. Can be set directly too in self.cast_to_np\n",
    "    \"\"\"\n",
    "    def __init__(self, record_master:RecordMaster, data_names:Union[str,list], target_names:Union[str,list]=None, cast_to_np=False):\n",
    "        self.record_master = record_master\n",
    "        if isinstance(data_names, str):\n",
    "            data_names = [data_names]\n",
    "        \n",
    "        if isinstance(target_names, str):\n",
    "            target_names = [target_names]\n",
    "            \n",
    "        if target_names is None:\n",
    "            target_names = data_names\n",
    "        elif len(data_names) != len(target_names):\n",
    "            raise Exception(\"data_names and target_names length must match\")\n",
    "        \n",
    "        self.target_names = target_names\n",
    "        self.data_names   = data_names\n",
    "        self._masks       = [np.zeros(len(seq), dtype=bool) for seq in record_master]\n",
    "        self._slices      = []\n",
    "        \n",
    "        self.cast_to_np   = cast_to_np\n",
    "        \n",
    "    def plot(self, newfig=False):\n",
    "        \"\"\"\n",
    "        Add the mask of the Pipe to a RecordMaster plot.\n",
    "        Usage:\n",
    "            reM.plot()\n",
    "            pipe.plot()\n",
    "        \"\"\"\n",
    "        factor = -1\n",
    "        if newfig:\n",
    "            plt.figure()\n",
    "            factor = 1\n",
    "        cursor = 0\n",
    "        frame_times      = [seq._frame_time for seq in self.record_master._sequences]\n",
    "        frametime_ratios = [fr_t/max(frame_times) for fr_t in frame_times]\n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            fr = frametime_ratios[i] \n",
    "            x  = np.linspace(cursor, cursor+len(seq)*fr, len(seq), endpoint=False)\n",
    "            plt.plot(x, self._masks[i]*factor-1, c='tab:blue')\n",
    "            cursor += len(seq)*fr + self.record_master._sep_size\n",
    "            \n",
    "    def copy(self):\n",
    "        \"\"\"\n",
    "        Duplicate the Pipe and its mask, and return the new pipe.\n",
    "        \"\"\"\n",
    "        new_pipe =  Data_Pipe(record_master=self.record_master, \n",
    "                         data_names=self.data_names,\n",
    "                         target_names=self.target_names)\n",
    "        new_pipe._masks = np.copy(self._masks)\n",
    "        new_pipe._slices = self._slices.copy()\n",
    "        return new_pipe\n",
    "        \n",
    "    def _get_dchunk_names(self, names):\n",
    "        if isinstance(names, str):\n",
    "            names = [names]\n",
    "        dchunk_name = []\n",
    "        for seq in self.record_master:\n",
    "            for name in names:\n",
    "                if name in [\"sync\", \"cell\", \"data\", \"stim\"]:\n",
    "                    dchunk_name.extend(seq.get_names_group(name))\n",
    "                else:\n",
    "                    dchunk_name.append(name)\n",
    "        return list(set(dchunk_name))\n",
    "    \n",
    "    def _intersect_names(self):        \n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            for name in self.data_names:\n",
    "                if name not in seq.keys():\n",
    "                    self._masks[i] = np.zeros(len(seq), dtype=bool)\n",
    "                    break\n",
    "            \n",
    "    def _update_slices(self):\n",
    "#         self._intersect_names() #Always intersect the names we wanna retrieve ? Might be not needed (even cause bug)\n",
    "        self._slices = []\n",
    "        #Iterating the list of mask (one per seq of the record_master)\n",
    "        for j, mask in enumerate(self._masks):\n",
    "            seq = self.record_master[j]\n",
    "            mask = np.concatenate(([0],mask,[0])) #Putting zeros on the side in case the limits would be ones\n",
    "            for start, stop in np.where(mask[1:]-mask[:-1])[0].reshape(-1,2):#Split the mask where 0 separates the 1\n",
    "                self._slices.append((j, slice(start,stop)))\n",
    "        \n",
    "    def __ior__(self, names:Union[str, list]):\n",
    "        return self.__iadd__(names)\n",
    "    def __or__(self, names:Union[str, list]):\n",
    "        return self.copy().__ior__(names)\n",
    "    \n",
    "    def __iand__(self, names:Union[str, list]):\n",
    "        dchunk_name = self._get_dchunk_names(names)\n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            new_mask = np.zeros(len(seq), dtype=bool)\n",
    "            for name in dchunk_name:\n",
    "                for slice_ in seq.get_slice(name):\n",
    "                    new_mask[slice_] = 1\n",
    "            self._masks[i] &= new_mask                \n",
    "        self._update_slices()\n",
    "        return self\n",
    "    def __and__(self, names:Union[str, list]):\n",
    "        return self.copy().__iand__(names)\n",
    "    \n",
    "    def __ixor__(self, names:Union[str, list]):\n",
    "        dchunk_name = self._get_dchunk_names(names)\n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            new_mask = np.zeros(len(seq), dtype=bool)\n",
    "            for name in dchunk_name:\n",
    "                for slice_ in seq.get_slice(name):\n",
    "                    new_mask[slice_] = 1\n",
    "            self._masks[i] ^= new_mask\n",
    "        self._update_slices()\n",
    "        return self\n",
    "    def __xor__(self, names:Union[str, list]):\n",
    "        return self.copy().__ixor__(names)\n",
    "        \n",
    "    def __iadd__(self, names:Union[str, list]):\n",
    "        dchunk_name = self._get_dchunk_names(names)\n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            for name in dchunk_name:\n",
    "                for slice_ in seq.get_slice(name):\n",
    "                    self._masks[i][slice_] = 1\n",
    "        self._update_slices()\n",
    "        return self\n",
    "    def __add__(self, names:Union[str, list]):\n",
    "        return self.copy().__iadd__(names)\n",
    "    \n",
    "    def __isub__(self, names:Union[str, list]):\n",
    "        dchunk_name = self._get_dchunk_names(names)\n",
    "        for i, seq in enumerate(self.record_master):\n",
    "            for name in dchunk_name:\n",
    "                for slice_ in seq.get_slice(name):\n",
    "                    self._masks[i][slice_] = 0\n",
    "        self._update_slices()\n",
    "        return self\n",
    "    def __sub__(self, names:Union[str, list]):\n",
    "        return self.copy().__isub__(names)\n",
    "                \n",
    "    def __iter__(self):\n",
    "        self._n = 0\n",
    "        return self\n",
    "    \n",
    "    def __next__(self):\n",
    "        if self._n < len(self):\n",
    "            res = {}\n",
    "            seq_idx, _slice = self._slices[self._n]\n",
    "            self.record_master[seq_idx].set_slice(_slice)\n",
    "            for i, name in enumerate(self.data_names):\n",
    "                if self.cast_to_np:\n",
    "                    res[self.target_names[i]] = np.array(self.record_master[seq_idx][name])\n",
    "                else:\n",
    "                    res[self.target_names[i]] = self.record_master[seq_idx][name]\n",
    "            self.record_master[seq_idx].set_slice(None)\n",
    "            self._n += 1\n",
    "            return res\n",
    "        else:\n",
    "            raise StopIteration \n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self._slices)\n",
    "            \n",
    "    def __getitem__(self, key):\n",
    "        if isinstance(key, (int, np.integer)):\n",
    "            seq_idx, _slice = self._slices[key]\n",
    "            self.record_master[seq_idx].set_slice(_slice)\n",
    "            res = {}\n",
    "            for i, name in enumerate(self.data_names):\n",
    "                if self.cast_to_np:\n",
    "                    res[self.target_names[i]] = np.array(self.record_master[seq_idx][name])\n",
    "                else:\n",
    "                    res[self.target_names[i]] = self.record_master[seq_idx][name]\n",
    "            self.record_master[seq_idx].set_slice(None)\n",
    "            return res\n",
    "        elif isinstance(key, slice):\n",
    "            l_res = []\n",
    "            for seq_idx, _slice in self._slices[key]:\n",
    "                res = {}\n",
    "                for i, name in enumerate(self.data_names):\n",
    "                    if self.cast_to_np:\n",
    "                        res[self.target_names[i]] = np.array(self.record_master[seq_idx][name][_slice])\n",
    "                    else:\n",
    "                        res[self.target_names[i]] = self.record_master[seq_idx][name][_slice]\n",
    "                    \n",
    "                l_res.append(res)\n",
    "            return l_res\n",
    "        else:\n",
    "            raise IndexError (\"only integers and slices (`:`) are valid indices\")\n",
    "      \n",
    "    def __str__(self):\n",
    "        return \"(datachunks, targets, slices), \"+self.__repr__()\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"Pipe(%s)\"%(repr(self.data_names)+\", \"+repr(self.target_names)+\", \"+repr(self._slices))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def export_record(path, record_master):\n",
    "    \"\"\"Export a Record_Master object to an h5 file, readable outside of this library.\n",
    "    \n",
    "    params:\n",
    "        - path: path of the file to be saved\n",
    "        - record_master: RecordMaster to save\n",
    "    \"\"\"\n",
    "    print(\"Exporting the record master\")\n",
    "    with h5py.File(path, mode=\"w\") as h5_f:\n",
    "        fr = None\n",
    "        if hasattr(record_master, '_frame_time'):\n",
    "            fr = record_master._frame_time #_frame_time was moved to Contigous_Record\n",
    "        h5_f.attrs[\"_sep_size\"]   = record_master._sep_size\n",
    "        for i, contig in enumerate(record_master):\n",
    "            #create contig\n",
    "            print(\"Contiguous sequence\",i)\n",
    "            cntig_ref = h5_f.create_group(str(i))\n",
    "            cntig_ref.attrs[\"length\"] = contig.length\n",
    "            if fr is not None:\n",
    "                cntig_ref.attrs[\"_frame_time\"] = fr\n",
    "            else:\n",
    "                cntig_ref.attrs[\"_frame_time\"] = contig._frame_time\n",
    "            for key, dc_list in contig._data_dict.items():\n",
    "                #create datastream\n",
    "                print(\"...Entering stream\",key)\n",
    "                stream_ref = cntig_ref.create_group(key)\n",
    "                for datachunk in dc_list:\n",
    "                    print(\"......\",str(datachunk.idx)+\"->\"+str(datachunk.idx+len(datachunk)))\n",
    "                    dset = stream_ref.create_dataset(str(datachunk.idx), data=np.array(datachunk), compression=\"gzip\", compression_opts=4)\n",
    "                    ndarray_ref = stream_ref.create_group(\"__ndarray_\"+str(datachunk.idx))\n",
    "                    for attr_k, attr_v in datachunk.attrs.items():\n",
    "                        if isinstance(attr_v, (np.ndarray,)):\n",
    "                            ndarray_ref.create_dataset(attr_k, data=attr_v, compression=\"gzip\", compression_opts=4)\n",
    "                        else:\n",
    "                            dset.attrs[attr_k] = json.dumps(attr_v)\n",
    "                    dset.attrs[\"__fill\"] = datachunk.fill\n",
    "                    dset.attrs[\"__group\"] = datachunk.group\n",
    "    print()\n",
    "                    \n",
    "def import_record(path):\n",
    "    \"\"\"Import a Record_Master from an h5 file saved by the export_record function of this library.\n",
    "    \n",
    "    params:\n",
    "        - path: path of the RecordMaster to import\n",
    "    \"\"\"\n",
    "    print(\"Importing the record master\")\n",
    "    with h5py.File(path, mode=\"r\") as h5_f:\n",
    "        record_master    = None\n",
    "        frame_rate = None\n",
    "        reg         = re.compile(\"frame_time\")\n",
    "        fr_time_key = list(filter(reg.search, h5_f.attrs.keys()))\n",
    "        if len(fr_time_key)==1:\n",
    "            frame_rate = round(1/h5_f.attrs[fr_time_key[0]])\n",
    "        keys = sorted(h5_f.keys(), key=int)\n",
    "        for j, key_contig in enumerate(keys):\n",
    "            ref_contig = h5_f[key_contig]\n",
    "            stream_d   = {}\n",
    "            keys = sorted(h5_f.keys(), key=int)\n",
    "            for i, key_dstream in enumerate(ref_contig.keys()):\n",
    "                ref_dstream = ref_contig[key_dstream]\n",
    "                dchunk_l = []\n",
    "                for key_dc in ref_dstream.keys():\n",
    "                    if key_dc.startswith(\"__ndarray_\"): #skipping array attributes for later\n",
    "                        continue\n",
    "                    data = ref_dstream[key_dc]\n",
    "                    idx  = int(key_dc)\n",
    "                    attrs = {}\n",
    "                    for k,v in data.attrs.items():\n",
    "                        if k not in  [\"__fill\", \"__group\"]:\n",
    "                            attrs[k] = json.loads(v)\n",
    "                        elif k == \"__fill\":\n",
    "                            fill = v\n",
    "                        elif k == \"__group\":\n",
    "                            group = v\n",
    "                    if \"__ndarray_\"+str(idx) in ref_dstream.keys():\n",
    "                        # if condition for backward support\n",
    "                        for k,v in ref_dstream[\"__ndarray_\"+str(idx)].items():\n",
    "                            attrs[k] = v[:]\n",
    "                    dchunk = DataChunk(data=data[:], idx=idx, group=group)\n",
    "                    dchunk.attrs = attrs\n",
    "                    dchunk_l.append(dchunk)\n",
    "                    \n",
    "                stream_d[key_dstream] = dchunk_l\n",
    "            if len(fr_time_key)==0:\n",
    "                frame_rate = round(1/ref_contig.attrs[\"_frame_time\"])\n",
    "            if record_master is None:\n",
    "                record_master = RecordMaster([(stream_d[\"main_tp\"][0],stream_d[\"signals\"][0])], frame_rate=frame_rate)\n",
    "            else:\n",
    "                record_master.append(stream_d[\"main_tp\"][0],stream_d[\"signals\"][0], frame_rate=frame_rate)\n",
    "            for kstream, vstream in stream_d.items():\n",
    "                for k, dc in enumerate(vstream):\n",
    "                    if kstream in [\"main_tp\", \"signals\"] and k==0:\n",
    "                        continue\n",
    "                    record_master.set_datachunk(dc, name=kstream, sequence_idx=j)\n",
    "    print()\n",
    "    return record_master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 00_core.ipynb.\n",
      "Converted 01_utils.ipynb.\n",
      "Converted 02_processing.ipynb.\n",
      "Converted 03_modelling.ipynb.\n",
      "Converted 04_plotting.ipynb.\n",
      "Converted 05_database.ipynb.\n",
      "Converted 06_eyetrack.ipynb.\n",
      "Converted 10_synchro.io.ipynb.\n",
      "Converted 11_synchro.extracting.ipynb.\n",
      "Converted 12_synchro.processing.ipynb.\n",
      "Converted 13_leddome.ipynb.\n",
      "Converted 14_synchro.nested_stims.ipynb.\n",
      "Converted 99_testdata.ipynb.\n",
      "Converted index.ipynb.\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from nbdev.export import *\n",
    "notebook2script()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}