haystack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install shaystack module from test.pypi.org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-22T09:37:21.958877Z",
     "start_time": "2022-07-22T09:37:21.953070Z"
    }
   },
   "outputs": [],
   "source": [
    "# import sys\n",
    "# !{sys.executable} -m pip install shaystack --upgrade "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-22T08:01:49.874426Z",
     "start_time": "2022-07-22T08:01:49.869156Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "import shaystack\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-22T09:37:35.927634Z",
     "start_time": "2022-07-22T09:37:35.905854Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ver:\"3.0\"\n",
      "name,site,vax,summary\n",
      "\"Site 2\",M,0.0,\"Entities are separated by one more dashes\\n\\n\"\n",
      "\n",
      "Grid\n",
      "\tVersion: 3.0\n",
      "\tColumns:\n",
      "\t\tname\n",
      "\t\tsite\n",
      "\t\tvax\n",
      "\t\tsummary\n",
      "\t---- Row    0:\n",
      "\tname='Site 2'\n",
      "\tsite=MARKER\n",
      "\tvax=0.0\n",
      "\tsummary='Entities are separated by one more dashes\\n\\n'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def test_filter():\n",
    "    g = shaystack.parse(\n",
    "        \"\"\"\n",
    "        name: \"Site 2\"\n",
    "        site\n",
    "        vax: 0\n",
    "        summary:\n",
    "          Entities are separated by one more dashes\n",
    "        \"\"\"\n",
    "        , mode=shaystack.MODE_TRIO)\n",
    "    print(shaystack.dump(g))\n",
    "    print(g.filter('vax==0'))\n",
    "test_filter()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a grid with metadata and columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = shaystack.Grid()\n",
    "g.metadata['aMarker'] = shaystack.MARKER\n",
    "g.metadata['today'] = datetime.date.today()\n",
    "g.column['firstColumn'] = {'metaData':'in no particular order', 'abc': 123}\n",
    "g.column['secondColumn'] = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The same with the constructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = shaystack.Grid(metadata=\n",
    "                     {\n",
    "                         \"aMarker\":shaystack.MARKER,\n",
    "                         \"today\":datetime.date.today()\n",
    "                     },\n",
    "                    columns=\n",
    "                     {\n",
    "                         \"firstColumn\":{'metaData':'in no particular order', 'abc': 123},\n",
    "                         \"secondColumn\": {}\n",
    "                     }\n",
    "                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add some entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g.extend([\n",
    "    {'firstColumn': shaystack.Quantity(154, 'kg'), 'secondColumn': 'and counting'},\n",
    "    {'firstColumn': shaystack.MARKER, 'secondColumn': 'supported on Python 3.7+'},\n",
    "    {'firstColumn': shaystack.Coordinate(-27.4725,153.003),\n",
    "     'secondColumn': 'Made in Australia from local and imported ingredients'},\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump to zinc format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(shaystack.dump(g))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump to JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "json.loads(shaystack.dump(g,mode=shaystack.MODE_JSON))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load haystack file from URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "base_url= \"https://shaystack.s3.eu-west-3.amazonaws.com/\"\n",
    "url = base_url + \"carytown.zinc\"\n",
    "with urllib.request.urlopen(url) as f:\n",
    "    g = shaystack.parse(f.read(),shaystack.MODE_ZINC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filter some entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "site = g.filter(\"site\")[0]\n",
    "site"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read time-series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with_his = g.filter(\"his\")[0]\n",
    "ts_uri = base_url + with_his[\"hisURI\"]\n",
    "with urllib.request.urlopen(ts_uri) as f:\n",
    "    ts = shaystack.parse(f.read(),shaystack.MODE_ZINC)\n",
    "    print(ts)  # Print associated time-series"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open(\"ontology.csv\",\"w\") as f:\n",
    "    f.write(shaystack.dump(g,shaystack.MODE_CSV))\n",
    "with open(\"ontology.json\",\"w\") as f:\n",
    "    f.write(shaystack.dump(g,shaystack.MODE_JSON))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python [conda env:etl_helper_env]",
   "language": "python",
   "name": "conda-env-etl_helper_env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}