filegroup_and_database.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FitsFileGroup and Database\n",
    "\n",
    "Currently astropop uses simple [Astropy Table](https://docs.astropy.org/en/stable/table/) to store headers data in `FitsFileGroup`. This can be fast, but have the problem to be in-memory only.\n",
    "\n",
    "So, we will investigate the use of SQL database to store the header table and filtering.\n",
    "\n",
    "Fortunately, Python has a built-in sqlite module called [sqlite3](https://docs.python.org/3/library/sqlite3.html), that we can use here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQLite3 Wrapper\n",
    "\n",
    "First of all, lest implement a wrapper to do a more object-oriented sql work. This is limited to a single table database and just some functions and interactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.io import fits\n",
    "from astropop.file_collection import list_fits_files\n",
    "from astropop.logger import logger\n",
    "from astropop._db import SQLDatabase\n",
    "\n",
    "logger.setLevel('INFO')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is everything working?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = SQLDatabase(':memory:')\n",
    "db.add_table('files')\n",
    "tab = db['files']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = list_fits_files('/home/julio/19jan30', fits_extensions=['fits.gz'])\n",
    "for i, f in enumerate(files):\n",
    "    logger.debug(\"file %i from %i\", i+1, len(files))\n",
    "    header = dict(fits.getheader(f))\n",
    "    header.pop('COMMENT', None)\n",
    "    header.pop('HISTORY', None)\n",
    "    db.add_row('files', dict(header), add_columns=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "columns:  ['simple', 'bitpix', 'naxis', 'naxis1', 'naxis2', 'extend', 'bzero', 'bscale', 'head', 'acqmode', 'readmode', 'imgrect', 'hbin', 'vbin', 'subrect', 'datatype', 'xtype', 'xunit', 'raywave', 'calbwvnm', 'trigger', 'calib', 'dllver', 'exposure', 'temp', 'readtime', 'operatn', 'gain', 'emrealgn', 'vclkamp', 'vshift', 'outptamp', 'preamp', 'serno', 'unsttemp', 'blclamp', 'precan', 'flipx', 'flipy', 'cntcvtmd', 'cntcvt', 'dtnwlgth', 'sntvty', 'spsnfltr', 'thrshld', 'pcntenld', 'nsethsld', 'ptnthld1', 'ptnthld2', 'ptnthld3', 'ptnthld4', 'avgftrmd', 'avgfctr', 'frmcnt', 'portmode', 'lsheight', 'lsspeed', 'lsaltdir', 'lsctrl', 'lsdir', 'fksmode', 'fktmode', 'usertxt1', 'usertxt2', 'usertxt3', 'usertxt4', 'date', 'frame', 'eshtmode', 'preampgaintext', 'spectrographserial', 'shamrockisactive', 'spectrographname', 'spectrographisactive', 'irigdataavailable', 'irigdata', 'detector', 'exptime', 'camgain', 'date_obs', 'ra', 'dec', 'epoch', 'airmass', 'jd', 'st', 'ha', 'image', 'telescop', 'platescl', 'instrume', 'object', 'observer', 'rdnoise', 'lamina', 'lam_pos', 'filter', 'fil_pos', 'focusval', 'w_bar', 'w_temp', 'w_hum']\n",
      "files: 427\n"
     ]
    }
   ],
   "source": [
    "print('columns: ', db.column_names('files'))\n",
    "print('files:', len(db['files']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2458514.73231', '2458514.73240', '2458514.73250', '2458514.73259', '2458514.73268', '2458514.73277', '2458514.73286', '2458514.73295', '2458514.73304', '2458514.73313']\n",
      "SQLRow 34 in table 'files' {'simple': 1, 'bitpix': 16, 'naxis': 2, 'naxis1': 1024, 'naxis2': 1024, 'extend': 1, 'bzero': 32768, 'bscale': 1, 'head': '', 'acqmode': 'Single', 'readmode': 'Image', 'imgrect': '1, 1024,1024, 1', 'hbin': '1', 'vbin': '1', 'subrect': '1, 1024,1024,1', 'datatype': 'Counts', 'xtype': 'Pixel number', 'xunit': 0, 'raywave': 0.0, 'calbwvnm': 0, 'trigger': 'Internal', 'calib': '0.,1.,0.,0.', 'dllver': '2.100.30039.0', 'exposure': '0,00001', 'temp': '-74,1', 'readtime': 1e-06, 'operatn': 4, 'gain': '1.5', 'emrealgn': 0, 'vclkamp': 0, 'vshift': '12.9E-06', 'outptamp': 'Conventional', 'preamp': '2.5x', 'serno': '4335', 'unsttemp': '-999', 'blclamp': 0, 'precan': 0, 'flipx': 0, 'flipy': 0, 'cntcvtmd': 0, 'cntcvt': 0, 'dtnwlgth': 500.0, 'sntvty': 0.0, 'spsnfltr': 0, 'thrshld': 0.0, 'pcntenld': 0, 'nsethsld': 0, 'ptnthld1': 0.0, 'ptnthld2': 0.0, 'ptnthld3': 0.0, 'ptnthld4': 2.802597e-45, 'avgftrmd': 0, 'avgfctr': 1, 'frmcnt': 1, 'portmode': -1, 'lsheight': 0, 'lsspeed': 0.0, 'lsaltdir': 0, 'lsctrl': 0, 'lsdir': 0, 'fksmode': 0, 'fktmode': 0, 'usertxt1': '', 'usertxt2': '', 'usertxt3': '', 'usertxt4': '', 'date': '2019-01-30T21:54:19', 'frame': '2019-01-30T21:54:19.228', 'eshtmode': 0, 'preampgaintext': '', 'spectrographserial': '', 'shamrockisactive': 0, 'spectrographname': '', 'spectrographisactive': 0, 'irigdataavailable': 0, 'irigdata': '', 'detector': '', 'exptime': '0,00001', 'camgain': '2.5x', 'date_obs': '2019-01-30T21:54:19.228', 'ra': '03:06:33', 'dec': '-06:05:19', 'epoch': '2000.0', 'airmass': '1.000', 'jd': '2458514.41270', 'st': '03:31:18', 'ha': '00:00:12', 'image': 'BIAS_0018', 'telescop': '0.60m(BC)', 'platescl': '0.34', 'instrume': 'Ikon 19002', 'object': 'BIAS', 'observer': 'Julio Cesar Neves Campagnolo', 'rdnoise': '6.3', 'lamina': 'L1', 'lam_pos': '1', 'filter': 'F1', 'fil_pos': '1', 'focusval': '2256', 'w_bar': '757,9', 'w_temp': '20,1', 'w_hum': '67'}\n",
      "['L0', 'L1', 'L2', 'L3', 'L4', 'L5', 'L6', 'L7', 'L8', 'L9']\n",
      "L0\n",
      "['L0', 'L1', 'L2', 'L3', 'L4', 'L5', 'L6', 'L7', 'L8', 'L9']\n"
     ]
    }
   ],
   "source": [
    "print(db['files', 'jd'][0:10])\n",
    "print(db['files'][34])\n",
    "print(db['files']['lamina'][0:10])\n",
    "print(db['files', 0]['lamina'])\n",
    "print(db['files', 'lamina'][0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like everything is working properly. All 427 files were added and all columns are present. What is int seems to be returned as int and what is float seems to be float. The filtering is also working too.\n",
    "\n",
    "Notes:\n",
    "\n",
    "- SQL tables don't like `-` character in name. So it is replaced by `_`. Same for spaces. This create possible conflicts, but may be a smaller problem in practice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FitsFileGroup implementations\n",
    "\n",
    "Lets try our two `FitsFileGroup` implementations. One using `astropy.table.Table`, another using our SQL wrapper."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table\n",
    "from astropop.file_collection import list_fits_files\n",
    "\n",
    "from astropop.fits_utils import _fits_extensions, \\\n",
    "                                _fits_extensions_with_compress\n",
    "from astropop.framedata import check_framedata\n",
    "from astropop.py_utils import check_iterable\n",
    "from astropop.logger import logger\n",
    "\n",
    "\n",
    "def create_table_summary(headers, n):\n",
    "    \"\"\"Create a table summary of headers.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    headers: iterator\n",
    "        Iterator for a list of header files.\n",
    "    n: int\n",
    "        Number of headers to iterate.\n",
    "    \"\"\"\n",
    "    summary_dict = {}\n",
    "    for i, head in enumerate(headers):\n",
    "        logger.debug('Reading file %i from %i', i, n)\n",
    "        keys = head.keys()\n",
    "        for k in keys:\n",
    "            k_lower = k.lower()\n",
    "            if k_lower in ('history', 'comment'):\n",
    "                logger.debug('%s key ignored', k)\n",
    "                continue\n",
    "            if k_lower not in summary_dict.keys():\n",
    "                summary_dict[k_lower] = [None]*n\n",
    "            summary_dict[k_lower][i] = head.get(k)\n",
    "\n",
    "    return Table(summary_dict)\n",
    "\n",
    "\n",
    "def gen_mask(table, keywords):\n",
    "    \"\"\"Generate a mask to be applyed in the filtering.\"\"\"\n",
    "    if len(table) == 0:\n",
    "        return []\n",
    "\n",
    "    t = Table(table)\n",
    "\n",
    "    mask = np.ones(len(t), dtype=bool)\n",
    "    for k, v in keywords.items():\n",
    "        if not check_iterable(v):\n",
    "            v = [v]\n",
    "        k = k.lower()\n",
    "        if k not in t.colnames:\n",
    "            t[k] = [None]*len(t)\n",
    "        nmask = [t[k][i] in v for i in range(len(t))]\n",
    "        mask &= np.array(nmask)\n",
    "\n",
    "    return mask\n",
    "\n",
    "\n",
    "class FitsFileGroup_Table():\n",
    "    \"\"\"Easy handle groups of fits files.\"\"\"\n",
    "\n",
    "    def __init__(self, location=None, files=None, ext=0,\n",
    "                 compression=False, **kwargs):\n",
    "        self._ext = ext\n",
    "        self._extensions = kwargs.get('fits_ext',\n",
    "                                      _fits_extensions_with_compress\n",
    "                                      if compression else _fits_extensions)\n",
    "\n",
    "        self._include = kwargs.get('glob_include')\n",
    "        self._exclude = kwargs.get('glob_exclude')\n",
    "        self._keywords = kwargs.get('keywords')\n",
    "\n",
    "        if location is None and files is None:\n",
    "            raise ValueError(\"You must specify a 'location'\"\n",
    "                             \"or a list of 'files'\")\n",
    "        if files is None and location is not None:\n",
    "            files = list_fits_files(location, self._extensions,\n",
    "                                    self._include, self._exclude)\n",
    "\n",
    "        self._files = files\n",
    "        self._location = location\n",
    "\n",
    "        self._summary = create_table_summary(self.headers(), len(self))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.files)\n",
    "\n",
    "    @property\n",
    "    def files(self):\n",
    "        return self._files.copy()\n",
    "\n",
    "    @property\n",
    "    def location(self):\n",
    "        return self._location\n",
    "\n",
    "    @property\n",
    "    def keywords(self):\n",
    "        return self._keywords\n",
    "\n",
    "    @property\n",
    "    def summary(self):\n",
    "        return Table(self._summary)\n",
    "\n",
    "    def __copy__(self, files=None, summary=None):\n",
    "        nfg = FitsFileGroup_Table.__new__(FitsFileGroup_Table)\n",
    "        for k, v in self.__dict__.items():\n",
    "            if k == '_summary':\n",
    "                nfg._summary = summary or self._summary\n",
    "            elif k == '_files':\n",
    "                nfg._files = files if files is not None else self._files\n",
    "            else:\n",
    "                nfg.__dict__[k] = v\n",
    "        return nfg\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        if isinstance(item, str):\n",
    "            # string will be interpreted as collumn name\n",
    "            if item.lower() not in self._summary.colnames:\n",
    "                raise KeyError(f'Column {item} not found.')\n",
    "            return self._summary.columns[item.lower()]\n",
    "\n",
    "        # returning FitsFileGroups\n",
    "        if isinstance(item, (int, np.integer)):\n",
    "            # single index will be interpreted as a single file group\n",
    "            return self.__copy__(files=[self._files[item]],\n",
    "                                 summary=self._summary[item])\n",
    "        if (isinstance(item, slice)):\n",
    "            files = self._files[item]\n",
    "            summ = self._summary[item]\n",
    "            return self.__copy__(files=files, summary=summ)\n",
    "        if isinstance(item, (np.ndarray, list, tuple)):\n",
    "            item = np.array(item)\n",
    "            if len(item) == 0:\n",
    "                return self.__copy__(files=[], summary=self._summary[item])\n",
    "            files = list(np.take(self._files, item))\n",
    "            summ = self._summary[item]\n",
    "            return self.__copy__(files=files, summary=summ)\n",
    "\n",
    "        raise KeyError(f'{item}')\n",
    "\n",
    "    def filtered(self, keywords=None):\n",
    "        \"\"\"Create a new FileGroup with only filtered files.\"\"\"\n",
    "        where = np.where(gen_mask(self._summary, keywords))[0]\n",
    "        return self[where]\n",
    "\n",
    "    def values(self, keyword, unique=False):\n",
    "        \"\"\"Return the values of a keyword in the summary.\n",
    "\n",
    "        If unique, only unique values returned.\n",
    "        \"\"\"\n",
    "        if keyword not in self.summary.colnames:\n",
    "            if unique:\n",
    "                n = 1\n",
    "            else:\n",
    "                n = len(self.summary)\n",
    "            return [None]*n\n",
    "        if unique:\n",
    "            return list(set(self.summary[keyword].tolist()))\n",
    "        return self.summary[keyword].tolist()\n",
    "\n",
    "    def add_column(self, name, values, mask=None):\n",
    "        \"\"\"Add a new column to the summary.\"\"\"\n",
    "        if not check_iterable(values):\n",
    "            values = [values]*len(self.summary)\n",
    "        elif len(values) != len(self.summary):\n",
    "            values = [values]*len(self.summary)\n",
    "\n",
    "        self.summary[name] = values\n",
    "        self.summary[name].mask = mask\n",
    "\n",
    "    def _intern_yelder(self, files=None, ext=None, ret_type=None,\n",
    "                       **kwargs):\n",
    "        \"\"\"Iter over files.\"\"\"\n",
    "        ext = ext if ext is not None else self._ext\n",
    "        files = files if files is not None else self._files\n",
    "        for i in files:\n",
    "            if ret_type == 'header':\n",
    "                yield fits.open(i, **kwargs)[ext].header\n",
    "            if ret_type == 'data':\n",
    "                yield fits.open(i, **kwargs)[ext].data\n",
    "            if ret_type == 'hdu':\n",
    "                yield fits.open(i, **kwargs)[ext]\n",
    "            if ret_type == 'framedata':\n",
    "                yield check_framedata(i, hdu=ext, **kwargs)\n",
    "\n",
    "    def hdus(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their HDUs.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='hdu', **kwargs)\n",
    "\n",
    "    def headers(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their headers.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='header', **kwargs)\n",
    "\n",
    "    def data(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='data', **kwargs)\n",
    "\n",
    "    def framedata(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='framedata', **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = FitsFileGroup_Table(files=files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    }
   ],
   "source": [
    "print(len(f.filtered({'object': 'HD126593'}).summary))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQL implementation\n",
    "\n",
    "*Temporary implementation, for full implementation, look to astropop*\n",
    "\n",
    "The SQL backend is working as expected and has almost the same performance as table implementation. However, has the advantage of persistent data.\n",
    "\n",
    "Filtering seems to work faster too, specially for very large datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "_headers = 'headers'\n",
    "_metadata = 'astropop_metadata'\n",
    "_files_col = '__file'\n",
    "\n",
    "class FitsFileGroup_SQL():\n",
    "    \"\"\"Easy handle groups of fits files.\"\"\"\n",
    "\n",
    "    def __init__(self, location=None, files=None, ext=0,\n",
    "                 compression=False, database=':memory:', **kwargs):\n",
    "        self._ext = ext\n",
    "        self._extensions = kwargs.get('fits_ext')\n",
    "        self._include = kwargs.get('glob_include')\n",
    "        self._exclude = kwargs.get('glob_exclude')\n",
    "\n",
    "        self._db = SQLDatabase(database)        \n",
    "        if database == ':memory:':\n",
    "            self._db_dir = None\n",
    "        else:\n",
    "            self._db_dir = Path(database).resolve().parent\n",
    "\n",
    "        self._read_db(files, location, compression, kwargs.get('update', 0))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.files)\n",
    "\n",
    "    def _list_files(self, files, location, compression):\n",
    "        extensions = self._extensions\n",
    "        if extensions is None:\n",
    "            if compression:\n",
    "                extensions = _fits_extensions_with_compress\n",
    "            else:\n",
    "                extensions = _fits_extensions\n",
    "\n",
    "        if files is not None and location is not None:\n",
    "            raise ValueError('You can only specify either files or location.')\n",
    "        if files is None and location is not None:\n",
    "            files = list_fits_files(location, extensions,\n",
    "                                    self._include, self._exclude)\n",
    "        return files\n",
    "    \n",
    "    def _read_db(self, files, location, compression, update=False):\n",
    "        \"\"\"Read the database and generate the summary if needed.\"\"\"\n",
    "        initialized = _metadata in self._db.table_names\n",
    "\n",
    "        if not initialized:\n",
    "            self._db.add_table(_metadata)\n",
    "            self._db.add_row(_metadata, {'DB_API_MAJ': 1,\n",
    "                                         'DB_API_MIN': 0,\n",
    "                                         'GLOB_INCLUDE': self._include,\n",
    "                                         'GLOB_EXCLUDE': self._exclude,\n",
    "                                         'LOCATION': location,\n",
    "                                         'COMPRESSION': compression,\n",
    "                                         'FITS_EXT': self._extensions,\n",
    "                                         'EXT': self._ext},\n",
    "                             add_columns=True)\n",
    "\n",
    "        self._include = self._db[_metadata, 'glob_include'][0]\n",
    "        self._exclude = self._db[_metadata, 'glob_exclude'][0]\n",
    "        self._extensions = self._db[_metadata, 'fits_ext'][0]\n",
    "        self._ext = self._db[_metadata, 'ext'][0]\n",
    "        self._location = self._db[_metadata, 'location'][0]\n",
    "        self._compression = self._db[_metadata, 'compression'][0]\n",
    "\n",
    "        if update or not initialized:\n",
    "            self.update(files, location, compression)\n",
    "\n",
    "    @property\n",
    "    def files(self):\n",
    "        files = self._db[_headers, _files_col].values\n",
    "        if self._db_dir is not None:\n",
    "            return [os.path.join(self._db_dir, f) for f in files]\n",
    "        return files\n",
    "\n",
    "    @property\n",
    "    def summary(self):\n",
    "        return self._db[_headers].as_table()\n",
    "\n",
    "    def __copy__(self):\n",
    "        db = self._db.copy()\n",
    "        nfg = object.__new__(FitsFileGroup_SQL)\n",
    "        nfg._db = db\n",
    "        nfg._db_dir = None\n",
    "        nfg._read_db(None, None, None, False)\n",
    "        return nfg\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._db[_headers])\n",
    "\n",
    "    def filtered(self, keywords):\n",
    "        \"\"\"Create a new FileGroup with only filtered files.\"\"\"\n",
    "        db = SQLDatabase(':memory:')\n",
    "        db.add_table(_metadata, data=self._db[_metadata].as_table())\n",
    "        db.add_table(_headers, columns=self._db.column_names(_headers))\n",
    "        data = self._db.select(_headers, where=keywords)\n",
    "        for i in data:\n",
    "            db.add_row(_headers, dict(zip(self._db.column_names(_headers), i)))\n",
    "        nfg = self.__copy__()\n",
    "        nfg._db = db\n",
    "        return nfg\n",
    "    \n",
    "    def update(self, files=None, location=None, compression=False):\n",
    "        \"\"\"Update the database with the current files.\"\"\"\n",
    "        if _headers in self._db.table_names:\n",
    "            self._db.drop_table(_headers)\n",
    "\n",
    "        self._db.add_table(_headers)\n",
    "        location = location or self._location\n",
    "        compression = compression or self._compression\n",
    "        files = self._list_files(files, location, compression)\n",
    "        for i, f in enumerate(files):\n",
    "            logger.debug('reading file %i from %i', i, len(files))\n",
    "            self.add_file(f)\n",
    "\n",
    "    def values(self, keyword, unique=False):\n",
    "        \"\"\"Return the values of a keyword in the summary.\n",
    "\n",
    "        If unique, only unique values returned.\n",
    "        \"\"\"\n",
    "        vals = self._db[_headers, keyword].values()\n",
    "        if unique:\n",
    "            vals = list(set(vals))\n",
    "        return vals\n",
    "\n",
    "    def add_column(self, name, values=None):\n",
    "        \"\"\"Add a new column to the summary.\"\"\"\n",
    "        self._db.add_column(_headers, name, data=values)\n",
    "    \n",
    "    def add_file(self, file):\n",
    "        \"\"\"Add a new file to the group.\"\"\"\n",
    "        header = fits.getheader(file, ext=self._ext)\n",
    "        logger.debug('reading file %s', file)\n",
    "        if self._db_dir is not None:\n",
    "            file = os.path.relpath(file, self._db_dir)\n",
    "        hdr = {_files_col: file}\n",
    "        hdr.update(dict(header))\n",
    "        hdr.pop('COMMENT', None)\n",
    "        hdr.pop('HISTORY', None)\n",
    "        self._db.add_row(_headers,  hdr, add_columns=True)\n",
    "\n",
    "    def _intern_yelder(self, ext=None, ret_type=None, **kwargs):\n",
    "        \"\"\"Iter over files.\"\"\"\n",
    "        ext = ext if ext is not None else self._ext\n",
    "        for i in self.files:\n",
    "            if ret_type == 'header':\n",
    "                yield fits.open(i, **kwargs)[ext].header\n",
    "            if ret_type == 'data':\n",
    "                yield fits.open(i, **kwargs)[ext].data\n",
    "            if ret_type == 'hdu':\n",
    "                yield fits.open(i, **kwargs)[ext]\n",
    "            if ret_type == 'framedata':\n",
    "                yield check_framedata(i, hdu=ext, **kwargs)\n",
    "\n",
    "    def hdus(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their HDUs.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='hdu', **kwargs)\n",
    "\n",
    "    def headers(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their headers.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='header', **kwargs)\n",
    "\n",
    "    def data(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='data', **kwargs)\n",
    "\n",
    "    def framedata(self, ext=None, **kwargs):\n",
    "        \"\"\"Read the files and iterate over their data.\"\"\"\n",
    "        return self._intern_yelder(ext=ext, ret_type='framedata', **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.setLevel('INFO')\n",
    "fg = FitsFileGroup_SQL(location='/home/julio/19jan30', database='/home/julio/19jan30/test.db', compression=True, update=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    }
   ],
   "source": [
    "print(len(fg.filtered({'object': 'HD126593'}).summary))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}