polars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting Acquainted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/getting-started/\n",
    "\n",
    "https://docs.pola.rs/api/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import polars.selectors as cs\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I/O"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Polars has all the usual suspects for I/O include parquet, csv, json, etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/io/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A couple datasets to play with. You are encouraged to change an example to an alternated dataset to further explore the capabilities of Polars."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_census = pl.read_csv('data/census_income.csv')\n",
    "# df_census.to_pandas() # if you need your pandas fix\n",
    "df_pokemon = pl.read_csv(\n",
    "    \"https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BUT! We can use the Lazy API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_0 = (\n",
    "    pl.scan_csv('data/census_income.csv')\n",
    "    .with_columns(pl.col('age').cast(pl.Int32))\n",
    "    .filter(pl.col(\"age\") > 25)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's not executed at this stage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>NAIVE QUERY PLAN</h4><p>run <b>LazyFrame.show_graph()</b> to see the optimized version</p><?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 9.0.0 (20230911.1827)\n",
       " -->\n",
       "<!-- Title: polars_query Pages: 1 -->\n",
       "<svg width=\"224pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 223.50 209.50\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 205.5)\">\n",
       "<title>polars_query</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-205.5 219.5,-205.5 219.5,4 -4,4\"/>\n",
       "<!-- FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)] -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"203.12,-201.5 12.38,-201.5 12.38,-165.5 203.12,-165.5 203.12,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"107.75\" y=\"-178.45\" font-family=\"Times,serif\" font-size=\"14.00\">FILTER BY (col(&quot;age&quot;)) &gt; (25)</text>\n",
       "</g>\n",
       "<!-- WITH COLUMNS [&quot;age&quot;] [(0, 1)] -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>WITH COLUMNS [&quot;age&quot;] [(0, 1)]</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"190.75,-129.5 24.75,-129.5 24.75,-93.5 190.75,-93.5 190.75,-129.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"107.75\" y=\"-106.45\" font-family=\"Times,serif\" font-size=\"14.00\">WITH COLUMNS [&quot;age&quot;]</text>\n",
       "</g>\n",
       "<!-- FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]&#45;&#45;WITH COLUMNS [&quot;age&quot;] [(0, 1)] -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]&#45;&#45;WITH COLUMNS [&quot;age&quot;] [(0, 1)]</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M107.75,-165.2C107.75,-154.35 107.75,-140.42 107.75,-129.6\"/>\n",
       "</g>\n",
       "<!-- Csv SCAN data/census_income.csv;\n",
       "π */15;\n",
       "σ &#45; [(0, 2)] -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>Csv SCAN data/census_income.csv;\n",
       "π */15;\n",
       "σ &#45; [(0, 2)]</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"215.5,-57.5 0,-57.5 0,0 215.5,0 215.5,-57.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"107.75\" y=\"-40.2\" font-family=\"Times,serif\" font-size=\"14.00\">Csv SCAN data/census_income.csv;</text>\n",
       "<text text-anchor=\"middle\" x=\"107.75\" y=\"-23.7\" font-family=\"Times,serif\" font-size=\"14.00\">π */15;</text>\n",
       "<text text-anchor=\"middle\" x=\"107.75\" y=\"-7.2\" font-family=\"Times,serif\" font-size=\"14.00\">σ &#45;</text>\n",
       "</g>\n",
       "<!-- WITH COLUMNS [&quot;age&quot;] [(0, 1)]&#45;&#45;Csv SCAN data/census_income.csv;\n",
       "π */15;\n",
       "σ &#45; [(0, 2)] -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>WITH COLUMNS [&quot;age&quot;] [(0, 1)]&#45;&#45;Csv SCAN data/census_income.csv;\n",
       "π */15;\n",
       "σ &#45; [(0, 2)]</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M107.75,-93.38C107.75,-83.1 107.75,-69.77 107.75,-57.97\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<LazyFrame [15 cols, {\"age\": Int32 … \"income\": String}] at 0x1092E9760>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also convert a current dataframe to lazy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>NAIVE QUERY PLAN</h4><p>run <b>LazyFrame.show_graph()</b> to see the optimized version</p><?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 9.0.0 (20230911.1827)\n",
       " -->\n",
       "<!-- Title: polars_query Pages: 1 -->\n",
       "<svg width=\"199pt\" height=\"138pt\"\n",
       " viewBox=\"0.00 0.00 198.75 137.50\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 133.5)\">\n",
       "<title>polars_query</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-133.5 194.75,-133.5 194.75,4 -4,4\"/>\n",
       "<!-- FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)] -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"190.75,-129.5 0,-129.5 0,-93.5 190.75,-93.5 190.75,-129.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"95.38\" y=\"-106.45\" font-family=\"Times,serif\" font-size=\"14.00\">FILTER BY (col(&quot;age&quot;)) &gt; (25)</text>\n",
       "</g>\n",
       "<!-- TABLE\n",
       "π */15;\n",
       "σ &#45; [(0, 1)] -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>TABLE\n",
       "π */15;\n",
       "σ &#45; [(0, 1)]</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"125.12,-57.5 65.62,-57.5 65.62,0 125.12,0 125.12,-57.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"95.38\" y=\"-40.2\" font-family=\"Times,serif\" font-size=\"14.00\">TABLE</text>\n",
       "<text text-anchor=\"middle\" x=\"95.38\" y=\"-23.7\" font-family=\"Times,serif\" font-size=\"14.00\">π */15;</text>\n",
       "<text text-anchor=\"middle\" x=\"95.38\" y=\"-7.2\" font-family=\"Times,serif\" font-size=\"14.00\">σ &#45;</text>\n",
       "</g>\n",
       "<!-- FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]&#45;&#45;TABLE\n",
       "π */15;\n",
       "σ &#45; [(0, 1)] -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>FILTER BY (col(&quot;age&quot;)) &gt; (25) [(0, 0)]&#45;&#45;TABLE\n",
       "π */15;\n",
       "σ &#45; [(0, 1)]</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M95.38,-93.38C95.38,-83.1 95.38,-69.77 95.38,-57.97\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<LazyFrame [15 cols, {\"age\": Int64 … \"income\": String}] at 0x156426CC0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.lazy().filter(pl.col(\"age\") > 25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When ready, just collect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>i32</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>90</td><td>&quot;?&quot;</td><td>77053</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>82</td><td>&quot;Private&quot;</td><td>132870</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;Exec-manageria…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>18</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>66</td><td>&quot;?&quot;</td><td>186061</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 15)\n",
       "┌─────┬───────────┬────────┬──────────────┬───┬──────────────┬──────────────┬─────────────┬────────┐\n",
       "│ age ┆ workclass ┆ fnlwgt ┆ education    ┆ … ┆ capital.loss ┆ hours.per.we ┆ native.coun ┆ income │\n",
       "│ --- ┆ ---       ┆ ---    ┆ ---          ┆   ┆ ---          ┆ ek           ┆ try         ┆ ---    │\n",
       "│ i32 ┆ str       ┆ i64    ┆ str          ┆   ┆ i64          ┆ ---          ┆ ---         ┆ str    │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆ i64          ┆ str         ┆        │\n",
       "╞═════╪═══════════╪════════╪══════════════╪═══╪══════════════╪══════════════╪═════════════╪════════╡\n",
       "│ 90  ┆ ?         ┆ 77053  ┆ HS-grad      ┆ … ┆ 4356         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 82  ┆ Private   ┆ 132870 ┆ HS-grad      ┆ … ┆ 4356         ┆ 18           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 66  ┆ ?         ┆ 186061 ┆ Some-college ┆ … ┆ 4356         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "└─────┴───────────┴────────┴──────────────┴───┴──────────────┴──────────────┴─────────────┴────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_0.limit(3).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Creation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Typical data creation can be done similar to pandas with a dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string_a</th><th>string_b</th><th>string_c</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>u8</td><td>u8</td><td>u8</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>1</td><td>0</td><td>0</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>0</td><td>1</td><td>0</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>0</td><td>0</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 6)\n",
       "┌─────────┬─────────────────────┬───────┬──────────┬──────────┬──────────┐\n",
       "│ integer ┆ date                ┆ float ┆ string_a ┆ string_b ┆ string_c │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---      ┆ ---      ┆ ---      │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ u8       ┆ u8       ┆ u8       │\n",
       "╞═════════╪═════════════════════╪═══════╪══════════╪══════════╪══════════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ 1        ┆ 0        ┆ 0        │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ 0        ┆ 1        ┆ 0        │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ 0        ┆ 0        ┆ 1        │\n",
       "└─────────┴─────────────────────┴───────┴──────────┴──────────┴──────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"integer\": [1, 2, 3],\n",
    "        \"date\": [\n",
    "            datetime(2025, 1, 1),\n",
    "            datetime(2025, 1, 2),\n",
    "            datetime(2025, 1, 3),\n",
    "        ],\n",
    "        \"float\": [4.0, 5.0, 6.0],\n",
    "        \"string\": [\"a\", \"b\", \"c\"],\n",
    "    }\n",
    ")\n",
    "\n",
    "df.to_dummies('string')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice anything missing? An index! Yay!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32561, 15)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Int64,\n",
       " String,\n",
       " Int64,\n",
       " String,\n",
       " Int64,\n",
       " String,\n",
       " String,\n",
       " String,\n",
       " String,\n",
       " String,\n",
       " Int64,\n",
       " Int64,\n",
       " Int64,\n",
       " String,\n",
       " String]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use pandas `describe` to get a quick overview of the data all the time. Polars has a describe also which is very similar to pandas, but honestly doesn't look as great as it by default mixes dtypes, with no way within the function to choose which columns. However, we'll see how to use selectors to get around that issue, but it still displays non-applicable stats for strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>str</td><td>f64</td><td>str</td><td>f64</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>32561.0</td><td>&quot;32561&quot;</td><td>32561.0</td><td>&quot;32561&quot;</td><td>32561.0</td><td>&quot;32561&quot;</td><td>&quot;32561&quot;</td><td>&quot;32561&quot;</td><td>&quot;32561&quot;</td><td>&quot;32561&quot;</td><td>32561.0</td><td>32561.0</td><td>32561.0</td><td>&quot;32561&quot;</td><td>&quot;32561&quot;</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>&quot;0&quot;</td><td>0.0</td><td>&quot;0&quot;</td><td>0.0</td><td>&quot;0&quot;</td><td>&quot;0&quot;</td><td>&quot;0&quot;</td><td>&quot;0&quot;</td><td>&quot;0&quot;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&quot;0&quot;</td><td>&quot;0&quot;</td></tr><tr><td>&quot;mean&quot;</td><td>38.581647</td><td>null</td><td>189778.366512</td><td>null</td><td>10.080679</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1077.648844</td><td>87.30383</td><td>40.437456</td><td>null</td><td>null</td></tr><tr><td>&quot;std&quot;</td><td>13.640433</td><td>null</td><td>105549.977697</td><td>null</td><td>2.57272</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>7385.292085</td><td>402.960219</td><td>12.347429</td><td>null</td><td>null</td></tr><tr><td>&quot;min&quot;</td><td>17.0</td><td>&quot;?&quot;</td><td>12285.0</td><td>&quot;10th&quot;</td><td>1.0</td><td>&quot;Divorced&quot;</td><td>&quot;?&quot;</td><td>&quot;Husband&quot;</td><td>&quot;Amer-Indian-Es…</td><td>&quot;Female&quot;</td><td>0.0</td><td>0.0</td><td>1.0</td><td>&quot;?&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>&quot;25%&quot;</td><td>28.0</td><td>null</td><td>117827.0</td><td>null</td><td>9.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>0.0</td><td>40.0</td><td>null</td><td>null</td></tr><tr><td>&quot;50%&quot;</td><td>37.0</td><td>null</td><td>178356.0</td><td>null</td><td>10.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>0.0</td><td>40.0</td><td>null</td><td>null</td></tr><tr><td>&quot;75%&quot;</td><td>48.0</td><td>null</td><td>237051.0</td><td>null</td><td>12.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>0.0</td><td>45.0</td><td>null</td><td>null</td></tr><tr><td>&quot;max&quot;</td><td>90.0</td><td>&quot;Without-pay&quot;</td><td>1.484705e6</td><td>&quot;Some-college&quot;</td><td>16.0</td><td>&quot;Widowed&quot;</td><td>&quot;Transport-movi…</td><td>&quot;Wife&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>99999.0</td><td>4356.0</td><td>99.0</td><td>&quot;Yugoslavia&quot;</td><td>&quot;&gt;50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 16)\n",
       "┌────────────┬───────────┬────────────┬───────────┬───┬───────────┬───────────┬───────────┬────────┐\n",
       "│ statistic  ┆ age       ┆ workclass  ┆ fnlwgt    ┆ … ┆ capital.l ┆ hours.per ┆ native.co ┆ income │\n",
       "│ ---        ┆ ---       ┆ ---        ┆ ---       ┆   ┆ oss       ┆ .week     ┆ untry     ┆ ---    │\n",
       "│ str        ┆ f64       ┆ str        ┆ f64       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ str    │\n",
       "│            ┆           ┆            ┆           ┆   ┆ f64       ┆ f64       ┆ str       ┆        │\n",
       "╞════════════╪═══════════╪════════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪════════╡\n",
       "│ count      ┆ 32561.0   ┆ 32561      ┆ 32561.0   ┆ … ┆ 32561.0   ┆ 32561.0   ┆ 32561     ┆ 32561  │\n",
       "│ null_count ┆ 0.0       ┆ 0          ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0         ┆ 0      │\n",
       "│ mean       ┆ 38.581647 ┆ null       ┆ 189778.36 ┆ … ┆ 87.30383  ┆ 40.437456 ┆ null      ┆ null   │\n",
       "│            ┆           ┆            ┆ 6512      ┆   ┆           ┆           ┆           ┆        │\n",
       "│ std        ┆ 13.640433 ┆ null       ┆ 105549.97 ┆ … ┆ 402.96021 ┆ 12.347429 ┆ null      ┆ null   │\n",
       "│            ┆           ┆            ┆ 7697      ┆   ┆ 9         ┆           ┆           ┆        │\n",
       "│ min        ┆ 17.0      ┆ ?          ┆ 12285.0   ┆ … ┆ 0.0       ┆ 1.0       ┆ ?         ┆ <=50K  │\n",
       "│ 25%        ┆ 28.0      ┆ null       ┆ 117827.0  ┆ … ┆ 0.0       ┆ 40.0      ┆ null      ┆ null   │\n",
       "│ 50%        ┆ 37.0      ┆ null       ┆ 178356.0  ┆ … ┆ 0.0       ┆ 40.0      ┆ null      ┆ null   │\n",
       "│ 75%        ┆ 48.0      ┆ null       ┆ 237051.0  ┆ … ┆ 0.0       ┆ 45.0      ┆ null      ┆ null   │\n",
       "│ max        ┆ 90.0      ┆ Without-pa ┆ 1.484705e ┆ … ┆ 4356.0    ┆ 99.0      ┆ Yugoslavi ┆ >50K   │\n",
       "│            ┆           ┆ y          ┆ 6         ┆   ┆           ┆           ┆ a         ┆        │\n",
       "└────────────┴───────────┴────────────┴───────────┴───┴───────────┴───────────┴───────────┴────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.describe()  # too many features require scrolling, bah!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even when using pandas `describe`, it is almost always better displayed as transposed with even just a few columns, since more info can be displayed without being hidden. The structure of the polars describe doesn't lend itself to using `transpose`, and I found that there is a bug when trying to do it via `pivot`, but it's [known](https://github.com/pola-rs/polars/issues/11592), so will hopefully get fixed in the near future."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is what I'm looking for, but it's obviously clunky, and they need to [get rid of quotes in the printout](https://github.com/pola-rs/polars/issues/6510)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>column</th><th>count</th><th>null_count</th><th>mean</th><th>std</th><th>min</th><th>25%</th><th>50%</th><th>75%</th><th>max</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;age&quot;</td><td>32561.0</td><td>0.0</td><td>38.581647</td><td>13.640433</td><td>17.0</td><td>28.0</td><td>37.0</td><td>48.0</td><td>90.0</td></tr><tr><td>&quot;fnlwgt&quot;</td><td>32561.0</td><td>0.0</td><td>189778.366512</td><td>105549.977697</td><td>12285.0</td><td>117827.0</td><td>178356.0</td><td>237051.0</td><td>1.484705e6</td></tr><tr><td>&quot;education.num&quot;</td><td>32561.0</td><td>0.0</td><td>10.080679</td><td>2.57272</td><td>1.0</td><td>9.0</td><td>10.0</td><td>12.0</td><td>16.0</td></tr><tr><td>&quot;capital.gain&quot;</td><td>32561.0</td><td>0.0</td><td>1077.648844</td><td>7385.292085</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>99999.0</td></tr><tr><td>&quot;capital.loss&quot;</td><td>32561.0</td><td>0.0</td><td>87.30383</td><td>402.960219</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>4356.0</td></tr><tr><td>&quot;hours.per.week…</td><td>32561.0</td><td>0.0</td><td>40.437456</td><td>12.347429</td><td>1.0</td><td>40.0</td><td>40.0</td><td>45.0</td><td>99.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 10)\n",
       "┌────────────┬─────────┬────────────┬────────────┬───┬──────────┬──────────┬──────────┬────────────┐\n",
       "│ column     ┆ count   ┆ null_count ┆ mean       ┆ … ┆ 25%      ┆ 50%      ┆ 75%      ┆ max        │\n",
       "│ ---        ┆ ---     ┆ ---        ┆ ---        ┆   ┆ ---      ┆ ---      ┆ ---      ┆ ---        │\n",
       "│ str        ┆ f64     ┆ f64        ┆ f64        ┆   ┆ f64      ┆ f64      ┆ f64      ┆ f64        │\n",
       "╞════════════╪═════════╪════════════╪════════════╪═══╪══════════╪══════════╪══════════╪════════════╡\n",
       "│ age        ┆ 32561.0 ┆ 0.0        ┆ 38.581647  ┆ … ┆ 28.0     ┆ 37.0     ┆ 48.0     ┆ 90.0       │\n",
       "│ fnlwgt     ┆ 32561.0 ┆ 0.0        ┆ 189778.366 ┆ … ┆ 117827.0 ┆ 178356.0 ┆ 237051.0 ┆ 1.484705e6 │\n",
       "│            ┆         ┆            ┆ 512        ┆   ┆          ┆          ┆          ┆            │\n",
       "│ education. ┆ 32561.0 ┆ 0.0        ┆ 10.080679  ┆ … ┆ 9.0      ┆ 10.0     ┆ 12.0     ┆ 16.0       │\n",
       "│ num        ┆         ┆            ┆            ┆   ┆          ┆          ┆          ┆            │\n",
       "│ capital.ga ┆ 32561.0 ┆ 0.0        ┆ 1077.64884 ┆ … ┆ 0.0      ┆ 0.0      ┆ 0.0      ┆ 99999.0    │\n",
       "│ in         ┆         ┆            ┆ 4          ┆   ┆          ┆          ┆          ┆            │\n",
       "│ capital.lo ┆ 32561.0 ┆ 0.0        ┆ 87.30383   ┆ … ┆ 0.0      ┆ 0.0      ┆ 0.0      ┆ 4356.0     │\n",
       "│ ss         ┆         ┆            ┆            ┆   ┆          ┆          ┆          ┆            │\n",
       "│ hours.per. ┆ 32561.0 ┆ 0.0        ┆ 40.437456  ┆ … ┆ 40.0     ┆ 40.0     ┆ 45.0     ┆ 99.0       │\n",
       "│ week       ┆         ┆            ┆            ┆   ┆          ┆          ┆          ┆            │\n",
       "└────────────┴─────────┴────────────┴────────────┴───┴──────────┴──────────┴──────────┴────────────┘"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.numeric()).describe().transpose(include_header=True, column_names='statistic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# currently a bug in polars\n",
    "# df_census.describe().pivot(columns='statistic', values = 'value', index=None)  # index should take None but doesn't"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, dtype should be an arg to describe in my opinion."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's some more stuff..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 15)\n",
       "┌─────┬───────────┬────────┬───────────┬───┬──────────────┬───────────────┬───────────────┬────────┐\n",
       "│ age ┆ workclass ┆ fnlwgt ┆ education ┆ … ┆ capital.loss ┆ hours.per.wee ┆ native.countr ┆ income │\n",
       "│ --- ┆ ---       ┆ ---    ┆ ---       ┆   ┆ ---          ┆ k             ┆ y             ┆ ---    │\n",
       "│ u32 ┆ u32       ┆ u32    ┆ u32       ┆   ┆ u32          ┆ ---           ┆ ---           ┆ u32    │\n",
       "│     ┆           ┆        ┆           ┆   ┆              ┆ u32           ┆ u32           ┆        │\n",
       "╞═════╪═══════════╪════════╪═══════════╪═══╪══════════════╪═══════════════╪═══════════════╪════════╡\n",
       "│ 0   ┆ 0         ┆ 0      ┆ 0         ┆ … ┆ 0            ┆ 0             ┆ 0             ┆ 0      │\n",
       "└─────┴───────────┴────────┴───────────┴───┴──────────────┴───────────────┴───────────────┴────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.null_count()  # come back after replacing ? with null later for a more interesting result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>73</td><td>9</td><td>21648</td><td>16</td><td>16</td><td>7</td><td>15</td><td>6</td><td>5</td><td>2</td><td>119</td><td>92</td><td>94</td><td>42</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 15)\n",
       "┌─────┬───────────┬────────┬───────────┬───┬──────────────┬───────────────┬───────────────┬────────┐\n",
       "│ age ┆ workclass ┆ fnlwgt ┆ education ┆ … ┆ capital.loss ┆ hours.per.wee ┆ native.countr ┆ income │\n",
       "│ --- ┆ ---       ┆ ---    ┆ ---       ┆   ┆ ---          ┆ k             ┆ y             ┆ ---    │\n",
       "│ u32 ┆ u32       ┆ u32    ┆ u32       ┆   ┆ u32          ┆ ---           ┆ ---           ┆ u32    │\n",
       "│     ┆           ┆        ┆           ┆   ┆              ┆ u32           ┆ u32           ┆        │\n",
       "╞═════╪═══════════╪════════╪═══════════╪═══╪══════════════╪═══════════════╪═══════════════╪════════╡\n",
       "│ 73  ┆ 9         ┆ 21648  ┆ 16        ┆ … ┆ 92           ┆ 94            ┆ 42            ┆ 2      │\n",
       "└─────┴───────────┴────────┴───────────┴───┴──────────────┴───────────────┴───────────────┴────────┘"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(pl.all().n_unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td><td>32561</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 15)\n",
       "┌───────┬───────────┬────────┬───────────┬───┬──────────────┬──────────────┬──────────────┬────────┐\n",
       "│ age   ┆ workclass ┆ fnlwgt ┆ education ┆ … ┆ capital.loss ┆ hours.per.we ┆ native.count ┆ income │\n",
       "│ ---   ┆ ---       ┆ ---    ┆ ---       ┆   ┆ ---          ┆ ek           ┆ ry           ┆ ---    │\n",
       "│ u32   ┆ u32       ┆ u32    ┆ u32       ┆   ┆ u32          ┆ ---          ┆ ---          ┆ u32    │\n",
       "│       ┆           ┆        ┆           ┆   ┆              ┆ u32          ┆ u32          ┆        │\n",
       "╞═══════╪═══════════╪════════╪═══════════╪═══╪══════════════╪══════════════╪══════════════╪════════╡\n",
       "│ 32561 ┆ 32561     ┆ 32561  ┆ 32561     ┆ … ┆ 32561        ┆ 32561        ┆ 32561        ┆ 32561  │\n",
       "└───────┴───────────┴────────┴───────────┴───┴──────────────┴──────────────┴──────────────┴────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.count() # come back after replacing ? with null"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selection and Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/expressions/column-selections/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can think of selection in two ways. We can think of it as filtering columns to those selected, or choosing which columns to work on to manipulate the data. The basic `.select` method is used primarily for the former, and the `.with_columns` function is for the latter, which we'll see shortly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>education</th></tr><tr><td>i64</td><td>str</td></tr></thead><tbody><tr><td>90</td><td>&quot;HS-grad&quot;</td></tr><tr><td>82</td><td>&quot;HS-grad&quot;</td></tr><tr><td>66</td><td>&quot;Some-college&quot;</td></tr><tr><td>54</td><td>&quot;7th-8th&quot;</td></tr><tr><td>41</td><td>&quot;Some-college&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────┬──────────────┐\n",
       "│ age ┆ education    │\n",
       "│ --- ┆ ---          │\n",
       "│ i64 ┆ str          │\n",
       "╞═════╪══════════════╡\n",
       "│ 90  ┆ HS-grad      │\n",
       "│ 82  ┆ HS-grad      │\n",
       "│ 66  ┆ Some-college │\n",
       "│ 54  ┆ 7th-8th      │\n",
       "│ 41  ┆ Some-college │\n",
       "└─────┴──────────────┘"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(\"age\", \"education\").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>education</th></tr><tr><td>i64</td><td>str</td></tr></thead><tbody><tr><td>90</td><td>&quot;HS-grad&quot;</td></tr><tr><td>82</td><td>&quot;HS-grad&quot;</td></tr><tr><td>66</td><td>&quot;Some-college&quot;</td></tr><tr><td>54</td><td>&quot;7th-8th&quot;</td></tr><tr><td>41</td><td>&quot;Some-college&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────┬──────────────┐\n",
       "│ age ┆ education    │\n",
       "│ --- ┆ ---          │\n",
       "│ i64 ┆ str          │\n",
       "╞═════╪══════════════╡\n",
       "│ 90  ┆ HS-grad      │\n",
       "│ 82  ┆ HS-grad      │\n",
       "│ 66  ┆ Some-college │\n",
       "│ 54  ┆ 7th-8th      │\n",
       "│ 41  ┆ Some-college │\n",
       "└─────┴──────────────┘"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census[['age', 'education']].head() # still fine, but please don't do this."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use selectors to make things easier!\n",
    "\n",
    "https://docs.pola.rs/py-polars/html/reference/selectors.html\n",
    "\n",
    "https://docs.pola.rs/user-guide/expressions/column-selections/#using-selectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[String, String, String, String, String, String, String, String, String]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.string()).dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>capital.gain</th><th>capital.loss</th><th>education</th><th>education.num</th><th>income</th></tr><tr><td>i64</td><td>i64</td><td>str</td><td>i64</td><td>str</td></tr></thead><tbody><tr><td>0</td><td>4356</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>4356</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>4356</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3900</td><td>&quot;7th-8th&quot;</td><td>4</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3900</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3770</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3770</td><td>&quot;10th&quot;</td><td>6</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3683</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>0</td><td>3683</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>3004</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>0</td><td>3004</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>0</td><td>2824</td><td>&quot;Prof-school&quot;</td><td>15</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>0</td><td>0</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;10th&quot;</td><td>6</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Assoc-voc&quot;</td><td>11</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;Assoc-acdm&quot;</td><td>12</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>0</td><td>0</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;&lt;=50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 5)\n",
       "┌──────────────┬──────────────┬──────────────┬───────────────┬────────┐\n",
       "│ capital.gain ┆ capital.loss ┆ education    ┆ education.num ┆ income │\n",
       "│ ---          ┆ ---          ┆ ---          ┆ ---           ┆ ---    │\n",
       "│ i64          ┆ i64          ┆ str          ┆ i64           ┆ str    │\n",
       "╞══════════════╪══════════════╪══════════════╪═══════════════╪════════╡\n",
       "│ 0            ┆ 4356         ┆ HS-grad      ┆ 9             ┆ <=50K  │\n",
       "│ 0            ┆ 4356         ┆ HS-grad      ┆ 9             ┆ <=50K  │\n",
       "│ 0            ┆ 4356         ┆ Some-college ┆ 10            ┆ <=50K  │\n",
       "│ 0            ┆ 3900         ┆ 7th-8th      ┆ 4             ┆ <=50K  │\n",
       "│ 0            ┆ 3900         ┆ Some-college ┆ 10            ┆ <=50K  │\n",
       "│ …            ┆ …            ┆ …            ┆ …             ┆ …      │\n",
       "│ 0            ┆ 0            ┆ Some-college ┆ 10            ┆ <=50K  │\n",
       "│ 0            ┆ 0            ┆ Assoc-acdm   ┆ 12            ┆ <=50K  │\n",
       "│ 0            ┆ 0            ┆ HS-grad      ┆ 9             ┆ >50K   │\n",
       "│ 0            ┆ 0            ┆ HS-grad      ┆ 9             ┆ <=50K  │\n",
       "│ 0            ┆ 0            ┆ HS-grad      ┆ 9             ┆ <=50K  │\n",
       "└──────────────┴──────────────┴──────────────┴───────────────┴────────┘"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.starts_with(\"capital\"), cs.matches('educ|inc'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>fnlwgt</th><th>education.num</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>90</td><td>77053</td><td>9</td><td>0</td><td>4356</td><td>40</td></tr><tr><td>82</td><td>132870</td><td>9</td><td>0</td><td>4356</td><td>18</td></tr><tr><td>66</td><td>186061</td><td>10</td><td>0</td><td>4356</td><td>40</td></tr><tr><td>54</td><td>140359</td><td>4</td><td>0</td><td>3900</td><td>40</td></tr><tr><td>41</td><td>264663</td><td>10</td><td>0</td><td>3900</td><td>40</td></tr><tr><td>34</td><td>216864</td><td>9</td><td>0</td><td>3770</td><td>45</td></tr><tr><td>38</td><td>150601</td><td>6</td><td>0</td><td>3770</td><td>40</td></tr><tr><td>74</td><td>88638</td><td>16</td><td>0</td><td>3683</td><td>20</td></tr><tr><td>68</td><td>422013</td><td>9</td><td>0</td><td>3683</td><td>40</td></tr><tr><td>41</td><td>70037</td><td>10</td><td>0</td><td>3004</td><td>60</td></tr><tr><td>45</td><td>172274</td><td>16</td><td>0</td><td>3004</td><td>35</td></tr><tr><td>38</td><td>164526</td><td>15</td><td>0</td><td>2824</td><td>45</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>43</td><td>260761</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>255835</td><td>10</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>27242</td><td>10</td><td>0</td><td>0</td><td>50</td></tr><tr><td>32</td><td>34066</td><td>6</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>84661</td><td>11</td><td>0</td><td>0</td><td>45</td></tr><tr><td>32</td><td>116138</td><td>14</td><td>0</td><td>0</td><td>11</td></tr><tr><td>53</td><td>321865</td><td>14</td><td>0</td><td>0</td><td>40</td></tr><tr><td>22</td><td>310152</td><td>10</td><td>0</td><td>0</td><td>40</td></tr><tr><td>27</td><td>257302</td><td>12</td><td>0</td><td>0</td><td>38</td></tr><tr><td>40</td><td>154374</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>58</td><td>151910</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>22</td><td>201490</td><td>9</td><td>0</td><td>0</td><td>20</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 6)\n",
       "┌─────┬────────┬───────────────┬──────────────┬──────────────┬────────────────┐\n",
       "│ age ┆ fnlwgt ┆ education.num ┆ capital.gain ┆ capital.loss ┆ hours.per.week │\n",
       "│ --- ┆ ---    ┆ ---           ┆ ---          ┆ ---          ┆ ---            │\n",
       "│ i64 ┆ i64    ┆ i64           ┆ i64          ┆ i64          ┆ i64            │\n",
       "╞═════╪════════╪═══════════════╪══════════════╪══════════════╪════════════════╡\n",
       "│ 90  ┆ 77053  ┆ 9             ┆ 0            ┆ 4356         ┆ 40             │\n",
       "│ 82  ┆ 132870 ┆ 9             ┆ 0            ┆ 4356         ┆ 18             │\n",
       "│ 66  ┆ 186061 ┆ 10            ┆ 0            ┆ 4356         ┆ 40             │\n",
       "│ 54  ┆ 140359 ┆ 4             ┆ 0            ┆ 3900         ┆ 40             │\n",
       "│ 41  ┆ 264663 ┆ 10            ┆ 0            ┆ 3900         ┆ 40             │\n",
       "│ …   ┆ …      ┆ …             ┆ …            ┆ …            ┆ …              │\n",
       "│ 22  ┆ 310152 ┆ 10            ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 27  ┆ 257302 ┆ 12            ┆ 0            ┆ 0            ┆ 38             │\n",
       "│ 40  ┆ 154374 ┆ 9             ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 58  ┆ 151910 ┆ 9             ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 22  ┆ 201490 ┆ 9             ┆ 0            ┆ 0            ┆ 20             │\n",
       "└─────┴────────┴───────────────┴──────────────┴──────────────┴────────────────┘"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.integer())  # compare with cs.float or cs.numeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>fnlwgt</th><th>education.num</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>90</td><td>77053</td><td>9</td><td>0</td><td>4356</td><td>40</td></tr><tr><td>82</td><td>132870</td><td>9</td><td>0</td><td>4356</td><td>18</td></tr><tr><td>66</td><td>186061</td><td>10</td><td>0</td><td>4356</td><td>40</td></tr><tr><td>54</td><td>140359</td><td>4</td><td>0</td><td>3900</td><td>40</td></tr><tr><td>41</td><td>264663</td><td>10</td><td>0</td><td>3900</td><td>40</td></tr><tr><td>34</td><td>216864</td><td>9</td><td>0</td><td>3770</td><td>45</td></tr><tr><td>38</td><td>150601</td><td>6</td><td>0</td><td>3770</td><td>40</td></tr><tr><td>74</td><td>88638</td><td>16</td><td>0</td><td>3683</td><td>20</td></tr><tr><td>68</td><td>422013</td><td>9</td><td>0</td><td>3683</td><td>40</td></tr><tr><td>41</td><td>70037</td><td>10</td><td>0</td><td>3004</td><td>60</td></tr><tr><td>45</td><td>172274</td><td>16</td><td>0</td><td>3004</td><td>35</td></tr><tr><td>38</td><td>164526</td><td>15</td><td>0</td><td>2824</td><td>45</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>43</td><td>260761</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>255835</td><td>10</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>27242</td><td>10</td><td>0</td><td>0</td><td>50</td></tr><tr><td>32</td><td>34066</td><td>6</td><td>0</td><td>0</td><td>40</td></tr><tr><td>43</td><td>84661</td><td>11</td><td>0</td><td>0</td><td>45</td></tr><tr><td>32</td><td>116138</td><td>14</td><td>0</td><td>0</td><td>11</td></tr><tr><td>53</td><td>321865</td><td>14</td><td>0</td><td>0</td><td>40</td></tr><tr><td>22</td><td>310152</td><td>10</td><td>0</td><td>0</td><td>40</td></tr><tr><td>27</td><td>257302</td><td>12</td><td>0</td><td>0</td><td>38</td></tr><tr><td>40</td><td>154374</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>58</td><td>151910</td><td>9</td><td>0</td><td>0</td><td>40</td></tr><tr><td>22</td><td>201490</td><td>9</td><td>0</td><td>0</td><td>20</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 6)\n",
       "┌─────┬────────┬───────────────┬──────────────┬──────────────┬────────────────┐\n",
       "│ age ┆ fnlwgt ┆ education.num ┆ capital.gain ┆ capital.loss ┆ hours.per.week │\n",
       "│ --- ┆ ---    ┆ ---           ┆ ---          ┆ ---          ┆ ---            │\n",
       "│ i64 ┆ i64    ┆ i64           ┆ i64          ┆ i64          ┆ i64            │\n",
       "╞═════╪════════╪═══════════════╪══════════════╪══════════════╪════════════════╡\n",
       "│ 90  ┆ 77053  ┆ 9             ┆ 0            ┆ 4356         ┆ 40             │\n",
       "│ 82  ┆ 132870 ┆ 9             ┆ 0            ┆ 4356         ┆ 18             │\n",
       "│ 66  ┆ 186061 ┆ 10            ┆ 0            ┆ 4356         ┆ 40             │\n",
       "│ 54  ┆ 140359 ┆ 4             ┆ 0            ┆ 3900         ┆ 40             │\n",
       "│ 41  ┆ 264663 ┆ 10            ┆ 0            ┆ 3900         ┆ 40             │\n",
       "│ …   ┆ …      ┆ …             ┆ …            ┆ …            ┆ …              │\n",
       "│ 22  ┆ 310152 ┆ 10            ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 27  ┆ 257302 ┆ 12            ┆ 0            ┆ 0            ┆ 38             │\n",
       "│ 40  ┆ 154374 ┆ 9             ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 58  ┆ 151910 ┆ 9             ┆ 0            ┆ 0            ┆ 40             │\n",
       "│ 22  ┆ 201490 ┆ 9             ┆ 0            ┆ 0            ┆ 20             │\n",
       "└─────┴────────┴───────────────┴──────────────┴──────────────┴────────────────┘"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.exclude(cs.string()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>education</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>native.country</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;?&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Widowed&quot;</td><td>&quot;Exec-manageria…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;?&quot;</td><td>&quot;Some-college&quot;</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;7th-8th&quot;</td><td>&quot;Divorced&quot;</td><td>&quot;Machine-op-ins…</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Some-college&quot;</td><td>&quot;Separated&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Divorced&quot;</td><td>&quot;Other-service&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;10th&quot;</td><td>&quot;Separated&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;State-gov&quot;</td><td>&quot;Doctorate&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Other-relative…</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Some-college&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Craft-repair&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;?&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Doctorate&quot;</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>&quot;Prof-school&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;Mexico&quot;</td></tr><tr><td>&quot;State-gov&quot;</td><td>&quot;Some-college&quot;</td><td>&quot;Divorced&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Other-relative…</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>&quot;Some-college&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Craft-repair&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;10th&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Handlers-clean…</td><td>&quot;Husband&quot;</td><td>&quot;Amer-Indian-Es…</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Assoc-voc&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Sales&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Masters&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Tech-support&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;Asian-Pac-Isla…</td><td>&quot;Male&quot;</td><td>&quot;Taiwan&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Masters&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Some-college&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Protective-ser…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;Assoc-acdm&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Tech-support&quot;</td><td>&quot;Wife&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Widowed&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>&quot;United-States&quot;</td></tr><tr><td>&quot;Private&quot;</td><td>&quot;HS-grad&quot;</td><td>&quot;Never-married&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>&quot;United-States&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 8)\n",
       "┌───────────┬─────────────┬─────────────┬─────────────┬─────────────┬───────┬────────┬─────────────┐\n",
       "│ workclass ┆ education   ┆ marital.sta ┆ occupation  ┆ relationshi ┆ race  ┆ sex    ┆ native.coun │\n",
       "│ ---       ┆ ---         ┆ tus         ┆ ---         ┆ p           ┆ ---   ┆ ---    ┆ try         │\n",
       "│ str       ┆ str         ┆ ---         ┆ str         ┆ ---         ┆ str   ┆ str    ┆ ---         │\n",
       "│           ┆             ┆ str         ┆             ┆ str         ┆       ┆        ┆ str         │\n",
       "╞═══════════╪═════════════╪═════════════╪═════════════╪═════════════╪═══════╪════════╪═════════════╡\n",
       "│ ?         ┆ HS-grad     ┆ Widowed     ┆ ?           ┆ Not-in-fami ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆             ┆             ┆             ┆ ly          ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ HS-grad     ┆ Widowed     ┆ Exec-manage ┆ Not-in-fami ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆             ┆             ┆ rial        ┆ ly          ┆       ┆        ┆ es          │\n",
       "│ ?         ┆ Some-colleg ┆ Widowed     ┆ ?           ┆ Unmarried   ┆ Black ┆ Female ┆ United-Stat │\n",
       "│           ┆ e           ┆             ┆             ┆             ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ 7th-8th     ┆ Divorced    ┆ Machine-op- ┆ Unmarried   ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆             ┆             ┆ inspct      ┆             ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ Some-colleg ┆ Separated   ┆ Prof-specia ┆ Own-child   ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆ e           ┆             ┆ lty         ┆             ┆       ┆        ┆ es          │\n",
       "│ …         ┆ …           ┆ …           ┆ …           ┆ …           ┆ …     ┆ …      ┆ …           │\n",
       "│ Private   ┆ Some-colleg ┆ Never-marri ┆ Protective- ┆ Not-in-fami ┆ White ┆ Male   ┆ United-Stat │\n",
       "│           ┆ e           ┆ ed          ┆ serv        ┆ ly          ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ Assoc-acdm  ┆ Married-civ ┆ Tech-suppor ┆ Wife        ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆             ┆ -spouse     ┆ t           ┆             ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ HS-grad     ┆ Married-civ ┆ Machine-op- ┆ Husband     ┆ White ┆ Male   ┆ United-Stat │\n",
       "│           ┆             ┆ -spouse     ┆ inspct      ┆             ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ HS-grad     ┆ Widowed     ┆ Adm-clerica ┆ Unmarried   ┆ White ┆ Female ┆ United-Stat │\n",
       "│           ┆             ┆             ┆ l           ┆             ┆       ┆        ┆ es          │\n",
       "│ Private   ┆ HS-grad     ┆ Never-marri ┆ Adm-clerica ┆ Own-child   ┆ White ┆ Male   ┆ United-Stat │\n",
       "│           ┆             ┆ ed          ┆ l           ┆             ┆       ┆        ┆ es          │\n",
       "└───────────┴─────────────┴─────────────┴─────────────┴─────────────┴───────┴────────┴─────────────┘"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(cs.string() - cs.last()) # all strings but drop the last one"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lots of handy selectors are available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>date</th></tr><tr><td>datetime[μs]</td></tr></thead><tbody><tr><td>2025-01-01 00:00:00</td></tr><tr><td>2025-01-02 00:00:00</td></tr><tr><td>2025-01-03 00:00:00</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌─────────────────────┐\n",
       "│ date                │\n",
       "│ ---                 │\n",
       "│ datetime[μs]        │\n",
       "╞═════════════════════╡\n",
       "│ 2025-01-01 00:00:00 │\n",
       "│ 2025-01-02 00:00:00 │\n",
       "│ 2025-01-03 00:00:00 │\n",
       "└─────────────────────┘"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(cs.temporal()) # also date, datetime, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (163, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Name</th><th>Type 1</th><th>Type 2</th></tr><tr><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;Bulbasaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td></tr><tr><td>&quot;Ivysaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td></tr><tr><td>&quot;Venusaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td></tr><tr><td>&quot;VenusaurMega V…</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td></tr><tr><td>&quot;Charmander&quot;</td><td>&quot;Fire&quot;</td><td>null</td></tr><tr><td>&quot;Charmeleon&quot;</td><td>&quot;Fire&quot;</td><td>null</td></tr><tr><td>&quot;Charizard&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Dragon&quot;</td></tr><tr><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Squirtle&quot;</td><td>&quot;Water&quot;</td><td>null</td></tr><tr><td>&quot;Wartortle&quot;</td><td>&quot;Water&quot;</td><td>null</td></tr><tr><td>&quot;Blastoise&quot;</td><td>&quot;Water&quot;</td><td>null</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Kabuto&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td></tr><tr><td>&quot;Kabutops&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td></tr><tr><td>&quot;Aerodactyl&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;AerodactylMega…</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Snorlax&quot;</td><td>&quot;Normal&quot;</td><td>null</td></tr><tr><td>&quot;Articuno&quot;</td><td>&quot;Ice&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Zapdos&quot;</td><td>&quot;Electric&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Moltres&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Dratini&quot;</td><td>&quot;Dragon&quot;</td><td>null</td></tr><tr><td>&quot;Dragonair&quot;</td><td>&quot;Dragon&quot;</td><td>null</td></tr><tr><td>&quot;Dragonite&quot;</td><td>&quot;Dragon&quot;</td><td>&quot;Flying&quot;</td></tr><tr><td>&quot;Mewtwo&quot;</td><td>&quot;Psychic&quot;</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (163, 3)\n",
       "┌───────────────────────┬─────────┬────────┐\n",
       "│ Name                  ┆ Type 1  ┆ Type 2 │\n",
       "│ ---                   ┆ ---     ┆ ---    │\n",
       "│ str                   ┆ str     ┆ str    │\n",
       "╞═══════════════════════╪═════════╪════════╡\n",
       "│ Bulbasaur             ┆ Grass   ┆ Poison │\n",
       "│ Ivysaur               ┆ Grass   ┆ Poison │\n",
       "│ Venusaur              ┆ Grass   ┆ Poison │\n",
       "│ VenusaurMega Venusaur ┆ Grass   ┆ Poison │\n",
       "│ Charmander            ┆ Fire    ┆ null   │\n",
       "│ …                     ┆ …       ┆ …      │\n",
       "│ Moltres               ┆ Fire    ┆ Flying │\n",
       "│ Dratini               ┆ Dragon  ┆ null   │\n",
       "│ Dragonair             ┆ Dragon  ┆ null   │\n",
       "│ Dragonite             ┆ Dragon  ┆ Flying │\n",
       "│ Mewtwo                ┆ Psychic ┆ null   │\n",
       "└───────────────────────┴─────────┴────────┘"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pokemon.select(cs.temporal() | cs.string()) # can use set operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Legendary</th><th>HP_float</th></tr><tr><td>bool</td><td>f64</td></tr></thead><tbody><tr><td>false</td><td>45.0</td></tr><tr><td>false</td><td>60.0</td></tr><tr><td>false</td><td>80.0</td></tr><tr><td>false</td><td>80.0</td></tr><tr><td>false</td><td>39.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────┬──────────┐\n",
       "│ Legendary ┆ HP_float │\n",
       "│ ---       ┆ ---      │\n",
       "│ bool      ┆ f64      │\n",
       "╞═══════════╪══════════╡\n",
       "│ false     ┆ 45.0     │\n",
       "│ false     ┆ 60.0     │\n",
       "│ false     ┆ 80.0     │\n",
       "│ false     ┆ 80.0     │\n",
       "│ false     ┆ 39.0     │\n",
       "└───────────┴──────────┘"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_pokemon\n",
    "    .with_columns(HP_float = pl.col('HP').cast(pl.Float64()))\n",
    "    .select(~cs.integer() & ~cs.string())\n",
    "    .head()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The nice thing about Polars is that it doesn't conflate filtering with selection, and also the lack of index means we don't need additional/special methods to get around it. No `.loc`, `.iloc`, for example. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (28, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>i64</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>90</td><td>&quot;?&quot;</td><td>77053</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>51744</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Never-married&quot;</td><td>&quot;Other-service&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;Black&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>2206</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Local-gov&quot;</td><td>227796</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>20051</td><td>0</td><td>60</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>87372</td><td>&quot;Prof-school&quot;</td><td>15</td><td>&quot;Married-civ-sp…</td><td>&quot;Prof-specialty…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>20051</td><td>0</td><td>72</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>90</td><td>&quot;Self-emp-not-i…</td><td>155981</td><td>&quot;Bachelors&quot;</td><td>13</td><td>&quot;Married-civ-sp…</td><td>&quot;Prof-specialty…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>10566</td><td>0</td><td>50</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>175491</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Craft-repair&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>9386</td><td>0</td><td>50</td><td>&quot;Ecuador&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>90</td><td>&quot;Local-gov&quot;</td><td>153602</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Other-service&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>6767</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Local-gov&quot;</td><td>214594</td><td>&quot;7th-8th&quot;</td><td>4</td><td>&quot;Married-civ-sp…</td><td>&quot;Protective-ser…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>2653</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>137018</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Never-married&quot;</td><td>&quot;Other-service&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>221832</td><td>&quot;Bachelors&quot;</td><td>13</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>45</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>171956</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Separated&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;Puerto-Rico&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>313986</td><td>&quot;11th&quot;</td><td>7</td><td>&quot;Never-married&quot;</td><td>&quot;Handlers-clean…</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>40388</td><td>&quot;Bachelors&quot;</td><td>13</td><td>&quot;Never-married&quot;</td><td>&quot;Exec-manageria…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>55</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>250832</td><td>&quot;10th&quot;</td><td>6</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>90523</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;Transport-movi…</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>99</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>88991</td><td>&quot;Bachelors&quot;</td><td>13</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Wife&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;England&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>115306</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;Never-married&quot;</td><td>&quot;Exec-manageria…</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>84553</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>226968</td><td>&quot;7th-8th&quot;</td><td>4</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>206667</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;Married-civ-sp…</td><td>&quot;Prof-specialty…</td><td>&quot;Wife&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>52386</td><td>&quot;Bachelors&quot;</td><td>13</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;Asian-Pac-Isla…</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;?&quot;</td><td>166343</td><td>&quot;1st-4th&quot;</td><td>2</td><td>&quot;Widowed&quot;</td><td>&quot;?&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;Private&quot;</td><td>47929</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>90</td><td>&quot;?&quot;</td><td>313986</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;?&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (28, 15)\n",
       "┌─────┬──────────────┬────────┬─────────────┬───┬─────────────┬─────────────┬─────────────┬────────┐\n",
       "│ age ┆ workclass    ┆ fnlwgt ┆ education   ┆ … ┆ capital.los ┆ hours.per.w ┆ native.coun ┆ income │\n",
       "│ --- ┆ ---          ┆ ---    ┆ ---         ┆   ┆ s           ┆ eek         ┆ try         ┆ ---    │\n",
       "│ i64 ┆ str          ┆ i64    ┆ str         ┆   ┆ ---         ┆ ---         ┆ ---         ┆ str    │\n",
       "│     ┆              ┆        ┆             ┆   ┆ i64         ┆ i64         ┆ str         ┆        │\n",
       "╞═════╪══════════════╪════════╪═════════════╪═══╪═════════════╪═════════════╪═════════════╪════════╡\n",
       "│ 90  ┆ ?            ┆ 77053  ┆ HS-grad     ┆ … ┆ 4356        ┆ 40          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Private      ┆ 51744  ┆ HS-grad     ┆ … ┆ 2206        ┆ 40          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Local-gov    ┆ 227796 ┆ Masters     ┆ … ┆ 0           ┆ 60          ┆ United-Stat ┆ >50K   │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Private      ┆ 87372  ┆ Prof-school ┆ … ┆ 0           ┆ 72          ┆ United-Stat ┆ >50K   │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Self-emp-not ┆ 155981 ┆ Bachelors   ┆ … ┆ 0           ┆ 50          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆ -inc         ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ …   ┆ …            ┆ …      ┆ …           ┆ … ┆ …           ┆ …           ┆ …           ┆ …      │\n",
       "│ 90  ┆ Private      ┆ 206667 ┆ Masters     ┆ … ┆ 0           ┆ 40          ┆ United-Stat ┆ >50K   │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Private      ┆ 52386  ┆ Bachelors   ┆ … ┆ 0           ┆ 40          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ ?            ┆ 166343 ┆ 1st-4th     ┆ … ┆ 0           ┆ 40          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ Private      ┆ 47929  ┆ HS-grad     ┆ … ┆ 0           ┆ 40          ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "│ 90  ┆ ?            ┆ 313986 ┆ HS-grad     ┆ … ┆ 0           ┆ 40          ┆ United-Stat ┆ >50K   │\n",
       "│     ┆              ┆        ┆             ┆   ┆             ┆             ┆ es          ┆        │\n",
       "└─────┴──────────────┴────────┴─────────────┴───┴─────────────┴─────────────┴─────────────┴────────┘"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .filter(pl.col(\"age\") >= 90, pl.col('hours.per.week') >= 40) # are these U.S. presidential candidates or something?\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (4, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>u32</td><td>i64</td><td>str</td><td>i64</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>7</td><td>74</td><td>&quot;State-gov&quot;</td><td>88638</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Other-relative…</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3683</td><td>20</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>8</td><td>68</td><td>&quot;Federal-gov&quot;</td><td>422013</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3683</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>9</td><td>41</td><td>&quot;Private&quot;</td><td>70037</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Never-married&quot;</td><td>&quot;Craft-repair&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>3004</td><td>60</td><td>&quot;?&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>10</td><td>45</td><td>&quot;Private&quot;</td><td>172274</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3004</td><td>35</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (4, 16)\n",
       "┌───────┬─────┬─────────────┬────────┬───┬──────────────┬────────────────┬────────────────┬────────┐\n",
       "│ index ┆ age ┆ workclass   ┆ fnlwgt ┆ … ┆ capital.loss ┆ hours.per.week ┆ native.country ┆ income │\n",
       "│ ---   ┆ --- ┆ ---         ┆ ---    ┆   ┆ ---          ┆ ---            ┆ ---            ┆ ---    │\n",
       "│ u32   ┆ i64 ┆ str         ┆ i64    ┆   ┆ i64          ┆ i64            ┆ str            ┆ str    │\n",
       "╞═══════╪═════╪═════════════╪════════╪═══╪══════════════╪════════════════╪════════════════╪════════╡\n",
       "│ 7     ┆ 74  ┆ State-gov   ┆ 88638  ┆ … ┆ 3683         ┆ 20             ┆ United-States  ┆ >50K   │\n",
       "│ 8     ┆ 68  ┆ Federal-gov ┆ 422013 ┆ … ┆ 3683         ┆ 40             ┆ United-States  ┆ <=50K  │\n",
       "│ 9     ┆ 41  ┆ Private     ┆ 70037  ┆ … ┆ 3004         ┆ 60             ┆ ?              ┆ >50K   │\n",
       "│ 10    ┆ 45  ┆ Private     ┆ 172274 ┆ … ┆ 3004         ┆ 35             ┆ United-States  ┆ >50K   │\n",
       "└───────┴─────┴─────────────┴────────┴───┴──────────────┴────────────────┴────────────────┴────────┘"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .with_row_index()\n",
    "    .slice(7, 4) # start, length\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating New/Transforming Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/expressions/functions/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wherein we waste a lot of programming time relative to tidyverse..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we create a couple new columns based on existing ones in the DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>capital.gain</th><th>capital.loss</th><th>capital_total</th><th>capital_any</th><th>age_sc</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>bool</td><td>f64</td></tr></thead><tbody><tr><td>90</td><td>0</td><td>4356</td><td>4356</td><td>true</td><td>3.769554</td></tr><tr><td>82</td><td>0</td><td>4356</td><td>4356</td><td>true</td><td>3.183063</td></tr><tr><td>66</td><td>0</td><td>4356</td><td>4356</td><td>true</td><td>2.010079</td></tr><tr><td>54</td><td>0</td><td>3900</td><td>3900</td><td>true</td><td>1.130342</td></tr><tr><td>41</td><td>0</td><td>3900</td><td>3900</td><td>true</td><td>0.177293</td></tr><tr><td>34</td><td>0</td><td>3770</td><td>3770</td><td>true</td><td>-0.335887</td></tr><tr><td>38</td><td>0</td><td>3770</td><td>3770</td><td>true</td><td>-0.042641</td></tr><tr><td>74</td><td>0</td><td>3683</td><td>3683</td><td>true</td><td>2.596571</td></tr><tr><td>68</td><td>0</td><td>3683</td><td>3683</td><td>true</td><td>2.156702</td></tr><tr><td>41</td><td>0</td><td>3004</td><td>3004</td><td>true</td><td>0.177293</td></tr><tr><td>45</td><td>0</td><td>3004</td><td>3004</td><td>true</td><td>0.470539</td></tr><tr><td>38</td><td>0</td><td>2824</td><td>2824</td><td>true</td><td>-0.042641</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>43</td><td>0</td><td>0</td><td>0</td><td>false</td><td>0.323916</td></tr><tr><td>43</td><td>0</td><td>0</td><td>0</td><td>false</td><td>0.323916</td></tr><tr><td>43</td><td>0</td><td>0</td><td>0</td><td>false</td><td>0.323916</td></tr><tr><td>32</td><td>0</td><td>0</td><td>0</td><td>false</td><td>-0.48251</td></tr><tr><td>43</td><td>0</td><td>0</td><td>0</td><td>false</td><td>0.323916</td></tr><tr><td>32</td><td>0</td><td>0</td><td>0</td><td>false</td><td>-0.48251</td></tr><tr><td>53</td><td>0</td><td>0</td><td>0</td><td>false</td><td>1.057031</td></tr><tr><td>22</td><td>0</td><td>0</td><td>0</td><td>false</td><td>-1.215625</td></tr><tr><td>27</td><td>0</td><td>0</td><td>0</td><td>false</td><td>-0.849067</td></tr><tr><td>40</td><td>0</td><td>0</td><td>0</td><td>false</td><td>0.103982</td></tr><tr><td>58</td><td>0</td><td>0</td><td>0</td><td>false</td><td>1.423588</td></tr><tr><td>22</td><td>0</td><td>0</td><td>0</td><td>false</td><td>-1.215625</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 6)\n",
       "┌─────┬──────────────┬──────────────┬───────────────┬─────────────┬───────────┐\n",
       "│ age ┆ capital.gain ┆ capital.loss ┆ capital_total ┆ capital_any ┆ age_sc    │\n",
       "│ --- ┆ ---          ┆ ---          ┆ ---           ┆ ---         ┆ ---       │\n",
       "│ i64 ┆ i64          ┆ i64          ┆ i64           ┆ bool        ┆ f64       │\n",
       "╞═════╪══════════════╪══════════════╪═══════════════╪═════════════╪═══════════╡\n",
       "│ 90  ┆ 0            ┆ 4356         ┆ 4356          ┆ true        ┆ 3.769554  │\n",
       "│ 82  ┆ 0            ┆ 4356         ┆ 4356          ┆ true        ┆ 3.183063  │\n",
       "│ 66  ┆ 0            ┆ 4356         ┆ 4356          ┆ true        ┆ 2.010079  │\n",
       "│ 54  ┆ 0            ┆ 3900         ┆ 3900          ┆ true        ┆ 1.130342  │\n",
       "│ 41  ┆ 0            ┆ 3900         ┆ 3900          ┆ true        ┆ 0.177293  │\n",
       "│ …   ┆ …            ┆ …            ┆ …             ┆ …           ┆ …         │\n",
       "│ 22  ┆ 0            ┆ 0            ┆ 0             ┆ false       ┆ -1.215625 │\n",
       "│ 27  ┆ 0            ┆ 0            ┆ 0             ┆ false       ┆ -0.849067 │\n",
       "│ 40  ┆ 0            ┆ 0            ┆ 0             ┆ false       ┆ 0.103982  │\n",
       "│ 58  ┆ 0            ┆ 0            ┆ 0             ┆ false       ┆ 1.423588  │\n",
       "│ 22  ┆ 0            ┆ 0            ┆ 0             ┆ false       ┆ -1.215625 │\n",
       "└─────┴──────────────┴──────────────┴───────────────┴─────────────┴───────────┘"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census.with_columns(\n",
    "        capital_total=pl.sum_horizontal(['capital.gain', 'capital.loss']),\n",
    "        capital_any = pl.any_horizontal(['capital.gain', 'capital.loss']),\n",
    "        # here we would ideally use polars functions for optimized performance, e.g.\n",
    "        # as opposed to created a function\n",
    "        age_sc = (pl.col('age') - pl.col('age').mean()) / pl.col('age').std()\n",
    "    )\n",
    "    .select(cs.matches('^capital|^age'))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you are hell-bent on doing things the hard way..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th></tr><tr><td>i64</td></tr></thead><tbody><tr><td>91</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 1)\n",
       "┌─────┐\n",
       "│ age │\n",
       "│ --- │\n",
       "│ i64 │\n",
       "╞═════╡\n",
       "│ 91  │\n",
       "└─────┘"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(pl.sql_expr('MAX(age + 1)'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Unlike pandas, we don't need a lambda for everything."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>log_capital_gain</th><th>fulltime</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>bool</td></tr></thead><tbody><tr><td>0</td><td>4356</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>4356</td><td>18</td><td>0.0</td><td>false</td></tr><tr><td>0</td><td>4356</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3900</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3900</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3770</td><td>45</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3770</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3683</td><td>20</td><td>0.0</td><td>false</td></tr><tr><td>0</td><td>3683</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3004</td><td>60</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>3004</td><td>35</td><td>0.0</td><td>false</td></tr><tr><td>0</td><td>2824</td><td>45</td><td>0.0</td><td>true</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>50</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>45</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>11</td><td>0.0</td><td>false</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>38</td><td>0.0</td><td>false</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>40</td><td>0.0</td><td>true</td></tr><tr><td>0</td><td>0</td><td>20</td><td>0.0</td><td>false</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 5)\n",
       "┌──────────────┬──────────────┬────────────────┬──────────────────┬──────────┐\n",
       "│ capital.gain ┆ capital.loss ┆ hours.per.week ┆ log_capital_gain ┆ fulltime │\n",
       "│ ---          ┆ ---          ┆ ---            ┆ ---              ┆ ---      │\n",
       "│ i64          ┆ i64          ┆ i64            ┆ f64              ┆ bool     │\n",
       "╞══════════════╪══════════════╪════════════════╪══════════════════╪══════════╡\n",
       "│ 0            ┆ 4356         ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 4356         ┆ 18             ┆ 0.0              ┆ false    │\n",
       "│ 0            ┆ 4356         ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 3900         ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 3900         ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ …            ┆ …            ┆ …              ┆ …                ┆ …        │\n",
       "│ 0            ┆ 0            ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 0            ┆ 38             ┆ 0.0              ┆ false    │\n",
       "│ 0            ┆ 0            ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 0            ┆ 40             ┆ 0.0              ┆ true     │\n",
       "│ 0            ┆ 0            ┆ 20             ┆ 0.0              ┆ false    │\n",
       "└──────────────┴──────────────┴────────────────┴──────────────────┴──────────┘"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .with_columns(\n",
    "        log_capital_gain = pl.col('capital.gain').log1p(),\n",
    "        fulltime = pl.when(pl.col('hours.per.week') >= 40).then(True).otherwise(False)\n",
    "    )\n",
    "    .select(cs.matches('capital|hours|fulltime'))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more complicated approaches you might want to investigate [folds](https://docs.pola.rs/user-guide/expressions/folds/)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categoricals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Categoricals can use `Enum` or `Categorical` dtype. If you know the categories, use `Enum`. If you think they might change, use `Categorical`. But you can cast from one to the other."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some apply on selection approaches are clunky at best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_census = df_census.with_columns(\n",
    "    pl.when(cs.string() != '?').then(cs.string()).otherwise(None) # this works but is not what anyone would guess\n",
    "    # pl.when(cs.string() == '?').then(None).otherwise(cs.string()) # doesn't work\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 15)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>age</th><th>workclass</th><th>fnlwgt</th><th>education</th><th>education.num</th><th>marital.status</th><th>occupation</th><th>relationship</th><th>race</th><th>sex</th><th>capital.gain</th><th>capital.loss</th><th>hours.per.week</th><th>native.country</th><th>income</th></tr><tr><td>i64</td><td>cat</td><td>i64</td><td>cat</td><td>i64</td><td>cat</td><td>cat</td><td>cat</td><td>cat</td><td>cat</td><td>i64</td><td>i64</td><td>i64</td><td>cat</td><td>cat</td></tr></thead><tbody><tr><td>90</td><td>null</td><td>77053</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>null</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>82</td><td>&quot;Private&quot;</td><td>132870</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;Exec-manageria…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>18</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>66</td><td>null</td><td>186061</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Widowed&quot;</td><td>null</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>4356</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>54</td><td>&quot;Private&quot;</td><td>140359</td><td>&quot;7th-8th&quot;</td><td>4</td><td>&quot;Divorced&quot;</td><td>&quot;Machine-op-ins…</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3900</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>41</td><td>&quot;Private&quot;</td><td>264663</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Separated&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3900</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>34</td><td>&quot;Private&quot;</td><td>216864</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Divorced&quot;</td><td>&quot;Other-service&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3770</td><td>45</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>38</td><td>&quot;Private&quot;</td><td>150601</td><td>&quot;10th&quot;</td><td>6</td><td>&quot;Separated&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>3770</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>74</td><td>&quot;State-gov&quot;</td><td>88638</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Other-relative…</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3683</td><td>20</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>68</td><td>&quot;Federal-gov&quot;</td><td>422013</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3683</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>41</td><td>&quot;Private&quot;</td><td>70037</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Never-married&quot;</td><td>&quot;Craft-repair&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>3004</td><td>60</td><td>null</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>45</td><td>&quot;Private&quot;</td><td>172274</td><td>&quot;Doctorate&quot;</td><td>16</td><td>&quot;Divorced&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Unmarried&quot;</td><td>&quot;Black&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>3004</td><td>35</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>38</td><td>&quot;Self-emp-not-i…</td><td>164526</td><td>&quot;Prof-school&quot;</td><td>15</td><td>&quot;Never-married&quot;</td><td>&quot;Prof-specialty…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>2824</td><td>45</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>43</td><td>&quot;Private&quot;</td><td>260761</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;Mexico&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>43</td><td>&quot;State-gov&quot;</td><td>255835</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Divorced&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Other-relative…</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>43</td><td>&quot;Self-emp-not-i…</td><td>27242</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Married-civ-sp…</td><td>&quot;Craft-repair&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>50</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>32</td><td>&quot;Private&quot;</td><td>34066</td><td>&quot;10th&quot;</td><td>6</td><td>&quot;Married-civ-sp…</td><td>&quot;Handlers-clean…</td><td>&quot;Husband&quot;</td><td>&quot;Amer-Indian-Es…</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>43</td><td>&quot;Private&quot;</td><td>84661</td><td>&quot;Assoc-voc&quot;</td><td>11</td><td>&quot;Married-civ-sp…</td><td>&quot;Sales&quot;</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>45</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>32</td><td>&quot;Private&quot;</td><td>116138</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;Never-married&quot;</td><td>&quot;Tech-support&quot;</td><td>&quot;Not-in-family&quot;</td><td>&quot;Asian-Pac-Isla…</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>11</td><td>&quot;Taiwan&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>53</td><td>&quot;Private&quot;</td><td>321865</td><td>&quot;Masters&quot;</td><td>14</td><td>&quot;Married-civ-sp…</td><td>&quot;Exec-manageria…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>22</td><td>&quot;Private&quot;</td><td>310152</td><td>&quot;Some-college&quot;</td><td>10</td><td>&quot;Never-married&quot;</td><td>&quot;Protective-ser…</td><td>&quot;Not-in-family&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>27</td><td>&quot;Private&quot;</td><td>257302</td><td>&quot;Assoc-acdm&quot;</td><td>12</td><td>&quot;Married-civ-sp…</td><td>&quot;Tech-support&quot;</td><td>&quot;Wife&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>38</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>40</td><td>&quot;Private&quot;</td><td>154374</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Married-civ-sp…</td><td>&quot;Machine-op-ins…</td><td>&quot;Husband&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&gt;50K&quot;</td></tr><tr><td>58</td><td>&quot;Private&quot;</td><td>151910</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Widowed&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Unmarried&quot;</td><td>&quot;White&quot;</td><td>&quot;Female&quot;</td><td>0</td><td>0</td><td>40</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr><tr><td>22</td><td>&quot;Private&quot;</td><td>201490</td><td>&quot;HS-grad&quot;</td><td>9</td><td>&quot;Never-married&quot;</td><td>&quot;Adm-clerical&quot;</td><td>&quot;Own-child&quot;</td><td>&quot;White&quot;</td><td>&quot;Male&quot;</td><td>0</td><td>0</td><td>20</td><td>&quot;United-States&quot;</td><td>&quot;&lt;=50K&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 15)\n",
       "┌─────┬───────────┬────────┬──────────────┬───┬──────────────┬──────────────┬─────────────┬────────┐\n",
       "│ age ┆ workclass ┆ fnlwgt ┆ education    ┆ … ┆ capital.loss ┆ hours.per.we ┆ native.coun ┆ income │\n",
       "│ --- ┆ ---       ┆ ---    ┆ ---          ┆   ┆ ---          ┆ ek           ┆ try         ┆ ---    │\n",
       "│ i64 ┆ cat       ┆ i64    ┆ cat          ┆   ┆ i64          ┆ ---          ┆ ---         ┆ cat    │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆ i64          ┆ cat         ┆        │\n",
       "╞═════╪═══════════╪════════╪══════════════╪═══╪══════════════╪══════════════╪═════════════╪════════╡\n",
       "│ 90  ┆ null      ┆ 77053  ┆ HS-grad      ┆ … ┆ 4356         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 82  ┆ Private   ┆ 132870 ┆ HS-grad      ┆ … ┆ 4356         ┆ 18           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 66  ┆ null      ┆ 186061 ┆ Some-college ┆ … ┆ 4356         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 54  ┆ Private   ┆ 140359 ┆ 7th-8th      ┆ … ┆ 3900         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 41  ┆ Private   ┆ 264663 ┆ Some-college ┆ … ┆ 3900         ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ …   ┆ …         ┆ …      ┆ …            ┆ … ┆ …            ┆ …            ┆ …           ┆ …      │\n",
       "│ 22  ┆ Private   ┆ 310152 ┆ Some-college ┆ … ┆ 0            ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 27  ┆ Private   ┆ 257302 ┆ Assoc-acdm   ┆ … ┆ 0            ┆ 38           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 40  ┆ Private   ┆ 154374 ┆ HS-grad      ┆ … ┆ 0            ┆ 40           ┆ United-Stat ┆ >50K   │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 58  ┆ Private   ┆ 151910 ┆ HS-grad      ┆ … ┆ 0            ┆ 40           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "│ 22  ┆ Private   ┆ 201490 ┆ HS-grad      ┆ … ┆ 0            ┆ 20           ┆ United-Stat ┆ <=50K  │\n",
       "│     ┆           ┆        ┆              ┆   ┆              ┆              ┆ es          ┆        │\n",
       "└─────┴───────────┴────────┴──────────────┴───┴──────────────┴──────────────┴─────────────┴────────┘"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census = df_census.with_columns(cs.string().cast(pl.Categorical)) # very nice\n",
    "df_census"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm still trying to find a non-terrible way to get simple levels and category counts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is way too verbose."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (8,)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;Private&quot;</td></tr><tr><td>&quot;State-gov&quot;</td></tr><tr><td>&quot;Federal-gov&quot;</td></tr><tr><td>&quot;Self-emp-not-i…</td></tr><tr><td>&quot;Self-emp-inc&quot;</td></tr><tr><td>&quot;Local-gov&quot;</td></tr><tr><td>&quot;Without-pay&quot;</td></tr><tr><td>&quot;Never-worked&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (8,)\n",
       "Series: 'workclass' [str]\n",
       "[\n",
       "\t\"Private\"\n",
       "\t\"State-gov\"\n",
       "\t\"Federal-gov\"\n",
       "\t\"Self-emp-not-i…\n",
       "\t\"Self-emp-inc\"\n",
       "\t\"Local-gov\"\n",
       "\t\"Without-pay\"\n",
       "\t\"Never-worked\"\n",
       "]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select('workclass').to_series().cat.get_categories()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I would likely never want this. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th></tr><tr><td>struct[2]</td></tr></thead><tbody><tr><td>{&quot;Private&quot;,22696}</td></tr><tr><td>{&quot;Never-worked&quot;,7}</td></tr><tr><td>{&quot;Local-gov&quot;,2093}</td></tr><tr><td>{&quot;Self-emp-inc&quot;,1116}</td></tr><tr><td>{null,1836}</td></tr><tr><td>{&quot;State-gov&quot;,1298}</td></tr><tr><td>{&quot;Federal-gov&quot;,960}</td></tr><tr><td>{&quot;Self-emp-not-inc&quot;,2541}</td></tr><tr><td>{&quot;Without-pay&quot;,14}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 1)\n",
       "┌───────────────────────────┐\n",
       "│ workclass                 │\n",
       "│ ---                       │\n",
       "│ struct[2]                 │\n",
       "╞═══════════════════════════╡\n",
       "│ {\"Private\",22696}         │\n",
       "│ {\"Never-worked\",7}        │\n",
       "│ {\"Local-gov\",2093}        │\n",
       "│ {\"Self-emp-inc\",1116}     │\n",
       "│ {null,1836}               │\n",
       "│ {\"State-gov\",1298}        │\n",
       "│ {\"Federal-gov\",960}       │\n",
       "│ {\"Self-emp-not-inc\",2541} │\n",
       "│ {\"Without-pay\",14}        │\n",
       "└───────────────────────────┘"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select(pl.col('workclass').value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Still shouldn't need `to_series`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>count</th></tr><tr><td>cat</td><td>u32</td></tr></thead><tbody><tr><td>&quot;Self-emp-inc&quot;</td><td>1116</td></tr><tr><td>&quot;Local-gov&quot;</td><td>2093</td></tr><tr><td>null</td><td>1836</td></tr><tr><td>&quot;Private&quot;</td><td>22696</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>2541</td></tr><tr><td>&quot;State-gov&quot;</td><td>1298</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>960</td></tr><tr><td>&quot;Never-worked&quot;</td><td>7</td></tr><tr><td>&quot;Without-pay&quot;</td><td>14</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬───────┐\n",
       "│ workclass        ┆ count │\n",
       "│ ---              ┆ ---   │\n",
       "│ cat              ┆ u32   │\n",
       "╞══════════════════╪═══════╡\n",
       "│ Self-emp-inc     ┆ 1116  │\n",
       "│ Local-gov        ┆ 2093  │\n",
       "│ null             ┆ 1836  │\n",
       "│ Private          ┆ 22696 │\n",
       "│ Self-emp-not-inc ┆ 2541  │\n",
       "│ State-gov        ┆ 1298  │\n",
       "│ Federal-gov      ┆ 960   │\n",
       "│ Never-worked     ┆ 7     │\n",
       "│ Without-pay      ┆ 14    │\n",
       "└──────────────────┴───────┘"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.select('workclass').to_series().value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the least verbose, but still not great due to the groupby operation. And for some reason they have deprecated `count` for `len` (not a good idea in my opinion). At least it leads us to our next section!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>len</th></tr><tr><td>cat</td><td>u32</td></tr></thead><tbody><tr><td>null</td><td>1836</td></tr><tr><td>&quot;Without-pay&quot;</td><td>14</td></tr><tr><td>&quot;Private&quot;</td><td>22696</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>2541</td></tr><tr><td>&quot;Self-emp-inc&quot;</td><td>1116</td></tr><tr><td>&quot;Never-worked&quot;</td><td>7</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>960</td></tr><tr><td>&quot;Local-gov&quot;</td><td>2093</td></tr><tr><td>&quot;State-gov&quot;</td><td>1298</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬───────┐\n",
       "│ workclass        ┆ len   │\n",
       "│ ---              ┆ ---   │\n",
       "│ cat              ┆ u32   │\n",
       "╞══════════════════╪═══════╡\n",
       "│ null             ┆ 1836  │\n",
       "│ Without-pay      ┆ 14    │\n",
       "│ Private          ┆ 22696 │\n",
       "│ Self-emp-not-inc ┆ 2541  │\n",
       "│ Self-emp-inc     ┆ 1116  │\n",
       "│ Never-worked     ┆ 7     │\n",
       "│ Federal-gov      ┆ 960   │\n",
       "│ Local-gov        ┆ 2093  │\n",
       "│ State-gov        ┆ 1298  │\n",
       "└──────────────────┴───────┘"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.group_by('workclass').len() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Group & Windowed Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/expressions/aggregation/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One of the greatest improvements in polars over pandas is using snake case instead of the inappropriate camel case for `group_by`. It's the little things that can matter most. 🐍  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is a simple group operation done two different ways. Pretty straightforward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>mean_hours_per_week</th></tr><tr><td>cat</td><td>f64</td></tr></thead><tbody><tr><td>&quot;State-gov&quot;</td><td>39.031587</td></tr><tr><td>null</td><td>31.91939</td></tr><tr><td>&quot;Never-worked&quot;</td><td>28.428571</td></tr><tr><td>&quot;Without-pay&quot;</td><td>32.714286</td></tr><tr><td>&quot;Self-emp-inc&quot;</td><td>48.8181</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>44.421881</td></tr><tr><td>&quot;Private&quot;</td><td>40.267096</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>41.379167</td></tr><tr><td>&quot;Local-gov&quot;</td><td>40.9828</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬─────────────────────┐\n",
       "│ workclass        ┆ mean_hours_per_week │\n",
       "│ ---              ┆ ---                 │\n",
       "│ cat              ┆ f64                 │\n",
       "╞══════════════════╪═════════════════════╡\n",
       "│ State-gov        ┆ 39.031587           │\n",
       "│ null             ┆ 31.91939            │\n",
       "│ Never-worked     ┆ 28.428571           │\n",
       "│ Without-pay      ┆ 32.714286           │\n",
       "│ Self-emp-inc     ┆ 48.8181             │\n",
       "│ Self-emp-not-inc ┆ 44.421881           │\n",
       "│ Private          ┆ 40.267096           │\n",
       "│ Federal-gov      ┆ 41.379167           │\n",
       "│ Local-gov        ┆ 40.9828             │\n",
       "└──────────────────┴─────────────────────┘"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.group_by('workclass').agg(mean_hours_per_week = pl.col('hours.per.week').mean())  # either is fine\n",
    "df_census.group_by('workclass').agg(pl.mean('hours.per.week').alias('mean_hours_per_week'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The two above approaches will not return the same order of results, and actually the returned order is not guaranteed at all. Rerun it and at some point you will see the order change. If you need to guarantee the order (i.e. most of the time), you can use  the `maintain_order` arg (and take a speed hit for very large data) or `sort` the result after, though the latter is not an option if you needed the actual observed order."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>mean_hours_per_week</th></tr><tr><td>cat</td><td>f64</td></tr></thead><tbody><tr><td>null</td><td>31.91939</td></tr><tr><td>&quot;Private&quot;</td><td>40.267096</td></tr><tr><td>&quot;State-gov&quot;</td><td>39.031587</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>41.379167</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>44.421881</td></tr><tr><td>&quot;Self-emp-inc&quot;</td><td>48.8181</td></tr><tr><td>&quot;Local-gov&quot;</td><td>40.9828</td></tr><tr><td>&quot;Without-pay&quot;</td><td>32.714286</td></tr><tr><td>&quot;Never-worked&quot;</td><td>28.428571</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬─────────────────────┐\n",
       "│ workclass        ┆ mean_hours_per_week │\n",
       "│ ---              ┆ ---                 │\n",
       "│ cat              ┆ f64                 │\n",
       "╞══════════════════╪═════════════════════╡\n",
       "│ null             ┆ 31.91939            │\n",
       "│ Private          ┆ 40.267096           │\n",
       "│ State-gov        ┆ 39.031587           │\n",
       "│ Federal-gov      ┆ 41.379167           │\n",
       "│ Self-emp-not-inc ┆ 44.421881           │\n",
       "│ Self-emp-inc     ┆ 48.8181             │\n",
       "│ Local-gov        ┆ 40.9828             │\n",
       "│ Without-pay      ┆ 32.714286           │\n",
       "│ Never-worked     ┆ 28.428571           │\n",
       "└──────────────────┴─────────────────────┘"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_census.group_by('workclass').agg(mean_hours_per_week = pl.col('hours.per.week').mean()).sort('workclass')\n",
    "df_census.group_by('workclass', maintain_order=True).agg(pl.mean('hours.per.week').alias('mean_hours_per_week'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I want to see the unique values of education.num for each education category to make sure only one numeric value is associated with a education category. Since we wouldn't know in a lot of situations that the result is a single value, we can obtain list of unique values as a **list column**, which can then be explored as such. In this case we can just take the first/only value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (16, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>education</th><th>education.num</th></tr><tr><td>cat</td><td>list[i64]</td></tr></thead><tbody><tr><td>&quot;HS-grad&quot;</td><td>[9]</td></tr><tr><td>&quot;Some-college&quot;</td><td>[10]</td></tr><tr><td>&quot;7th-8th&quot;</td><td>[4]</td></tr><tr><td>&quot;10th&quot;</td><td>[6]</td></tr><tr><td>&quot;Doctorate&quot;</td><td>[16]</td></tr><tr><td>&quot;Prof-school&quot;</td><td>[15]</td></tr><tr><td>&quot;Bachelors&quot;</td><td>[13]</td></tr><tr><td>&quot;Masters&quot;</td><td>[14]</td></tr><tr><td>&quot;11th&quot;</td><td>[7]</td></tr><tr><td>&quot;Assoc-acdm&quot;</td><td>[12]</td></tr><tr><td>&quot;Assoc-voc&quot;</td><td>[11]</td></tr><tr><td>&quot;1st-4th&quot;</td><td>[2]</td></tr><tr><td>&quot;5th-6th&quot;</td><td>[3]</td></tr><tr><td>&quot;12th&quot;</td><td>[8]</td></tr><tr><td>&quot;9th&quot;</td><td>[5]</td></tr><tr><td>&quot;Preschool&quot;</td><td>[1]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (16, 2)\n",
       "┌──────────────┬───────────────┐\n",
       "│ education    ┆ education.num │\n",
       "│ ---          ┆ ---           │\n",
       "│ cat          ┆ list[i64]     │\n",
       "╞══════════════╪═══════════════╡\n",
       "│ HS-grad      ┆ [9]           │\n",
       "│ Some-college ┆ [10]          │\n",
       "│ 7th-8th      ┆ [4]           │\n",
       "│ 10th         ┆ [6]           │\n",
       "│ Doctorate    ┆ [16]          │\n",
       "│ …            ┆ …             │\n",
       "│ 1st-4th      ┆ [2]           │\n",
       "│ 5th-6th      ┆ [3]           │\n",
       "│ 12th         ┆ [8]           │\n",
       "│ 9th          ┆ [5]           │\n",
       "│ Preschool    ┆ [1]           │\n",
       "└──────────────┴───────────────┘"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .group_by('education')\n",
    "    .agg(pl.col('education.num').unique())   # add .first() to get the first value, or try age or other columns\n",
    "    # .agg(pl.col('workclass'))                # creates a list column\n",
    "    # .agg(pl.col('workclass').n_unique())     # count unique values\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given the above, we can group by college vs. not and get the proportion of those with a salary > 50k. I left a couple issues I came across when doing this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>college</th><th>income</th></tr><tr><td>i32</td><td>f64</td></tr></thead><tbody><tr><td>0</td><td>0.130066</td></tr><tr><td>1</td><td>0.332566</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 2)\n",
       "┌─────────┬──────────┐\n",
       "│ college ┆ income   │\n",
       "│ ---     ┆ ---      │\n",
       "│ i32     ┆ f64      │\n",
       "╞═════════╪══════════╡\n",
       "│ 0       ┆ 0.130066 │\n",
       "│ 1       ┆ 0.332566 │\n",
       "└─────────┴──────────┘"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .with_columns(\n",
    "        # when requires same dtype; and honestly, is not as intuitive as it should be\n",
    "        # pl.when(pl.col('education.num') >= 10).then('Some college').otherwise('HS or less') # doesn't work because of different type- boo!\n",
    "        pl.when(pl.col('education.num') >= 10).then(1).otherwise(0)\n",
    "        .alias('college')\n",
    "        # can't cast numerics to categorical, so you'd have to break it into two steps to create categorical with levels\n",
    "        # .cast(pl.String) \n",
    "        # .cast(pl.Categorical)\n",
    "    )\n",
    "    .group_by('college')\n",
    "    .agg((pl.col('income') == '>50K').drop_nulls().mean()) # annoying, no arg for drop NA w/in mean\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following represents a very \"tidy\" approach. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>education</th><th>FT</th><th>PT</th><th>prop_FT</th></tr><tr><td>cat</td><td>u32</td><td>u32</td><td>f64</td></tr></thead><tbody><tr><td>&quot;Doctorate&quot;</td><td>352</td><td>61</td><td>0.8523</td></tr><tr><td>&quot;Masters&quot;</td><td>1439</td><td>284</td><td>0.835171</td></tr><tr><td>&quot;Prof-school&quot;</td><td>480</td><td>96</td><td>0.833333</td></tr><tr><td>&quot;Bachelors&quot;</td><td>4452</td><td>903</td><td>0.831373</td></tr><tr><td>&quot;Assoc-voc&quot;</td><td>1131</td><td>251</td><td>0.818379</td></tr><tr><td>&quot;HS-grad&quot;</td><td>8223</td><td>2278</td><td>0.783068</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 4)\n",
       "┌─────────────┬──────┬──────┬──────────┐\n",
       "│ education   ┆ FT   ┆ PT   ┆ prop_FT  │\n",
       "│ ---         ┆ ---  ┆ ---  ┆ ---      │\n",
       "│ cat         ┆ u32  ┆ u32  ┆ f64      │\n",
       "╞═════════════╪══════╪══════╪══════════╡\n",
       "│ Doctorate   ┆ 352  ┆ 61   ┆ 0.8523   │\n",
       "│ Masters     ┆ 1439 ┆ 284  ┆ 0.835171 │\n",
       "│ Prof-school ┆ 480  ┆ 96   ┆ 0.833333 │\n",
       "│ Bachelors   ┆ 4452 ┆ 903  ┆ 0.831373 │\n",
       "│ Assoc-voc   ┆ 1131 ┆ 251  ┆ 0.818379 │\n",
       "│ HS-grad     ┆ 8223 ┆ 2278 ┆ 0.783068 │\n",
       "└─────────────┴──────┴──────┴──────────┘"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .group_by('education')\n",
    "    .agg(\n",
    "        FT = (pl.col(\"hours.per.week\") >=40).sum(),\n",
    "        PT = (pl.col(\"hours.per.week\") < 40).sum(),\n",
    "        prop_FT = (pl.col(\"hours.per.week\") >=40).mean()\n",
    "    )\n",
    "    .sort('prop_FT', descending=True)\n",
    "    .limit(6)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What if we want a group operation that maintains the original DataFrame shape? Here is one method using the `group_by` approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>hours_centered</th></tr><tr><td>cat</td><td>f64</td></tr></thead><tbody><tr><td>&quot;Private&quot;</td><td>-22.267096</td></tr><tr><td>&quot;Private&quot;</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>4.732904</td></tr><tr><td>&quot;Private&quot;</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>19.732904</td></tr><tr><td>&quot;Private&quot;</td><td>-5.267096</td></tr><tr><td>&quot;Private&quot;</td><td>-20.267096</td></tr><tr><td>&quot;Private&quot;</td><td>14.732904</td></tr><tr><td>&quot;Private&quot;</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>35.732904</td></tr><tr><td>&quot;Private&quot;</td><td>9.732904</td></tr><tr><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>2.620833</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-33.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>8.620833</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>-1.379167</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 2)\n",
       "┌─────────────┬────────────────┐\n",
       "│ workclass   ┆ hours_centered │\n",
       "│ ---         ┆ ---            │\n",
       "│ cat         ┆ f64            │\n",
       "╞═════════════╪════════════════╡\n",
       "│ Private     ┆ -22.267096     │\n",
       "│ Private     ┆ -0.267096      │\n",
       "│ Private     ┆ -0.267096      │\n",
       "│ Private     ┆ 4.732904       │\n",
       "│ Private     ┆ -0.267096      │\n",
       "│ …           ┆ …              │\n",
       "│ Federal-gov ┆ -1.379167      │\n",
       "│ Federal-gov ┆ -33.379167     │\n",
       "│ Federal-gov ┆ 8.620833       │\n",
       "│ Federal-gov ┆ -1.379167      │\n",
       "│ Federal-gov ┆ -1.379167      │\n",
       "└─────────────┴────────────────┘"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .group_by('workclass')\n",
    "    .agg(hours_centered = pl.col('hours.per.week') -  pl.col('hours.per.week').mean())\n",
    "    .explode('hours_centered')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>age_x</th></tr><tr><td>cat</td><td>list[f64]</td></tr></thead><tbody><tr><td>&quot;Private&quot;</td><td>[45.202415, 17.202415, … -14.797585]</td></tr><tr><td>null</td><td>[49.03976, 25.03976, … 31.03976]</td></tr><tr><td>&quot;Without-pay&quot;</td><td>[-25.785714, 17.214286, … 14.214286]</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>[-6.969697, 15.030303, … -1.969697]</td></tr><tr><td>&quot;Never-worked&quot;</td><td>[-2.571429, 2.428571, … -2.571429]</td></tr><tr><td>&quot;State-gov&quot;</td><td>[34.563945, 11.563945, … 3.563945]</td></tr><tr><td>&quot;Self-emp-inc&quot;</td><td>[-9.017025, 12.982975, … 11.982975]</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>[25.409375, 20.409375, … -3.590625]</td></tr><tr><td>&quot;Local-gov&quot;</td><td>[-1.751075, -0.751075, … -2.751075]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬───────────────────────────────────┐\n",
       "│ workclass        ┆ age_x                             │\n",
       "│ ---              ┆ ---                               │\n",
       "│ cat              ┆ list[f64]                         │\n",
       "╞══════════════════╪═══════════════════════════════════╡\n",
       "│ Private          ┆ [45.202415, 17.202415, … -14.797… │\n",
       "│ null             ┆ [49.03976, 25.03976, … 31.03976]  │\n",
       "│ Without-pay      ┆ [-25.785714, 17.214286, … 14.214… │\n",
       "│ Self-emp-not-inc ┆ [-6.969697, 15.030303, … -1.9696… │\n",
       "│ Never-worked     ┆ [-2.571429, 2.428571, … -2.57142… │\n",
       "│ State-gov        ┆ [34.563945, 11.563945, … 3.56394… │\n",
       "│ Self-emp-inc     ┆ [-9.017025, 12.982975, … 11.9829… │\n",
       "│ Federal-gov      ┆ [25.409375, 20.409375, … -3.5906… │\n",
       "│ Local-gov        ┆ [-1.751075, -0.751075, … -2.7510… │\n",
       "└──────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .group_by('workclass')\n",
    "    .agg(\n",
    "        age_x = pl.map_groups(['age'], function = lambda x: x[0] - x[0].mean())\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Windowed Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/expressions/window/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above is fine, and working with list columns can be handy for many settings, but we didn't really want to aggregate, which would require us to rejoin to the original frame, we just wanted to add a column to the existing data frame, or create a new subset of the same number of rows.  Pandas is very clunky or slow in this regard, but polars is better. Its windowed operations allow us to perform aggregations on groups in the select context."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we subtract the group mean from hours per week, and select 'workclass' and 'hours_per_week' to show the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (32_561, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>hours.per.week</th><th>hours_centered</th></tr><tr><td>cat</td><td>i64</td><td>f64</td></tr></thead><tbody><tr><td>null</td><td>40</td><td>8.08061</td></tr><tr><td>&quot;Private&quot;</td><td>18</td><td>-22.267096</td></tr><tr><td>null</td><td>40</td><td>8.08061</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>45</td><td>4.732904</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;State-gov&quot;</td><td>20</td><td>-19.031587</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>40</td><td>-1.379167</td></tr><tr><td>&quot;Private&quot;</td><td>60</td><td>19.732904</td></tr><tr><td>&quot;Private&quot;</td><td>35</td><td>-5.267096</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>45</td><td>0.578119</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;State-gov&quot;</td><td>40</td><td>0.968413</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>50</td><td>5.578119</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>45</td><td>4.732904</td></tr><tr><td>&quot;Private&quot;</td><td>11</td><td>-29.267096</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>38</td><td>-2.267096</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>40</td><td>-0.267096</td></tr><tr><td>&quot;Private&quot;</td><td>20</td><td>-20.267096</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (32_561, 3)\n",
       "┌───────────┬────────────────┬────────────────┐\n",
       "│ workclass ┆ hours.per.week ┆ hours_centered │\n",
       "│ ---       ┆ ---            ┆ ---            │\n",
       "│ cat       ┆ i64            ┆ f64            │\n",
       "╞═══════════╪════════════════╪════════════════╡\n",
       "│ null      ┆ 40             ┆ 8.08061        │\n",
       "│ Private   ┆ 18             ┆ -22.267096     │\n",
       "│ null      ┆ 40             ┆ 8.08061        │\n",
       "│ Private   ┆ 40             ┆ -0.267096      │\n",
       "│ Private   ┆ 40             ┆ -0.267096      │\n",
       "│ …         ┆ …              ┆ …              │\n",
       "│ Private   ┆ 40             ┆ -0.267096      │\n",
       "│ Private   ┆ 38             ┆ -2.267096      │\n",
       "│ Private   ┆ 40             ┆ -0.267096      │\n",
       "│ Private   ┆ 40             ┆ -0.267096      │\n",
       "│ Private   ┆ 20             ┆ -20.267096     │\n",
       "└───────────┴────────────────┴────────────────┘"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .select(\n",
    "        'workclass', \n",
    "        'hours.per.week',\n",
    "        hours_centered = \n",
    "            (pl.col('hours.per.week') -  pl.col('hours.per.week').mean())\n",
    "            .over('workclass')\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using transform with a lambda on large data is unreasonably slow in pandas, and now you have to merge it back in \n",
    "# or split out this operation to create the column and hope there aren't any indexing issues\n",
    "# df_census.to_pandas().groupby('workclass')['hours.per.week'].transform(lambda x: x - x.mean()) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Of course, still nowhere near the simplicity of the tidyverse, but take what you can get. 😄"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No quotes necessary, not even a group_by!\n",
    "# df_census |>\n",
    "#     mutate(\n",
    "#         hours_centered = hours.per.week - mean(hours.per.week),\n",
    "#         .by = workclass\n",
    "#     )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some more examples of windowed operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(shape: (5, 13)\n",
       " ┌─────┬───────────────────────┬────────┬────────┬───┬─────────┬───────┬────────────┬───────────┐\n",
       " │ #   ┆ Name                  ┆ Type 1 ┆ Type 2 ┆ … ┆ Sp. Def ┆ Speed ┆ Generation ┆ Legendary │\n",
       " │ --- ┆ ---                   ┆ ---    ┆ ---    ┆   ┆ ---     ┆ ---   ┆ ---        ┆ ---       │\n",
       " │ i64 ┆ str                   ┆ str    ┆ str    ┆   ┆ i64     ┆ i64   ┆ i64        ┆ bool      │\n",
       " ╞═════╪═══════════════════════╪════════╪════════╪═══╪═════════╪═══════╪════════════╪═══════════╡\n",
       " │ 1   ┆ Bulbasaur             ┆ Grass  ┆ Poison ┆ … ┆ 65      ┆ 45    ┆ 1          ┆ false     │\n",
       " │ 2   ┆ Ivysaur               ┆ Grass  ┆ Poison ┆ … ┆ 80      ┆ 60    ┆ 1          ┆ false     │\n",
       " │ 3   ┆ Venusaur              ┆ Grass  ┆ Poison ┆ … ┆ 100     ┆ 80    ┆ 1          ┆ false     │\n",
       " │ 3   ┆ VenusaurMega Venusaur ┆ Grass  ┆ Poison ┆ … ┆ 120     ┆ 80    ┆ 1          ┆ false     │\n",
       " │ 4   ┆ Charmander            ┆ Fire   ┆ null   ┆ … ┆ 50      ┆ 65    ┆ 1          ┆ false     │\n",
       " └─────┴───────────────────────┴────────┴────────┴───┴─────────┴───────┴────────────┴───────────┘,\n",
       " (163, 13))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pokemon.head(), df_pokemon.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (163, 12)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>#</th><th>Name</th><th>Type 1</th><th>Type 2</th><th>Total</th><th>HP</th><th>Attack</th><th>Defense</th><th>Sp. Atk</th><th>Sp. Def</th><th>Speed</th><th>attack2</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td></tr></thead><tbody><tr><td>1</td><td>&quot;Bulbasaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>318</td><td>45</td><td>49</td><td>49</td><td>65</td><td>65</td><td>45</td><td>75.8</td></tr><tr><td>2</td><td>&quot;Ivysaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>405</td><td>60</td><td>62</td><td>63</td><td>80</td><td>80</td><td>60</td><td>75.8</td></tr><tr><td>3</td><td>&quot;Venusaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>525</td><td>80</td><td>82</td><td>83</td><td>100</td><td>100</td><td>80</td><td>75.8</td></tr><tr><td>3</td><td>&quot;VenusaurMega V…</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>625</td><td>80</td><td>100</td><td>123</td><td>122</td><td>120</td><td>80</td><td>75.8</td></tr><tr><td>4</td><td>&quot;Charmander&quot;</td><td>&quot;Fire&quot;</td><td>null</td><td>309</td><td>39</td><td>52</td><td>43</td><td>60</td><td>50</td><td>65</td><td>82.3</td></tr><tr><td>5</td><td>&quot;Charmeleon&quot;</td><td>&quot;Fire&quot;</td><td>null</td><td>405</td><td>58</td><td>64</td><td>58</td><td>80</td><td>65</td><td>80</td><td>82.3</td></tr><tr><td>6</td><td>&quot;Charizard&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>534</td><td>78</td><td>84</td><td>78</td><td>109</td><td>85</td><td>100</td><td>96.0</td></tr><tr><td>6</td><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Dragon&quot;</td><td>634</td><td>78</td><td>130</td><td>111</td><td>130</td><td>85</td><td>100</td><td>130.0</td></tr><tr><td>6</td><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>634</td><td>78</td><td>104</td><td>78</td><td>159</td><td>115</td><td>100</td><td>96.0</td></tr><tr><td>7</td><td>&quot;Squirtle&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>314</td><td>44</td><td>48</td><td>65</td><td>50</td><td>64</td><td>43</td><td>67.105263</td></tr><tr><td>8</td><td>&quot;Wartortle&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>405</td><td>59</td><td>63</td><td>80</td><td>65</td><td>80</td><td>58</td><td>67.105263</td></tr><tr><td>9</td><td>&quot;Blastoise&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>530</td><td>79</td><td>83</td><td>100</td><td>85</td><td>105</td><td>78</td><td>67.105263</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>140</td><td>&quot;Kabuto&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td><td>355</td><td>30</td><td>80</td><td>90</td><td>55</td><td>45</td><td>55</td><td>73.75</td></tr><tr><td>141</td><td>&quot;Kabutops&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td><td>495</td><td>60</td><td>115</td><td>105</td><td>65</td><td>70</td><td>80</td><td>73.75</td></tr><tr><td>142</td><td>&quot;Aerodactyl&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td><td>515</td><td>80</td><td>105</td><td>65</td><td>60</td><td>75</td><td>130</td><td>120.0</td></tr><tr><td>142</td><td>&quot;AerodactylMega…</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td><td>615</td><td>80</td><td>135</td><td>85</td><td>70</td><td>95</td><td>150</td><td>120.0</td></tr><tr><td>143</td><td>&quot;Snorlax&quot;</td><td>&quot;Normal&quot;</td><td>null</td><td>540</td><td>160</td><td>110</td><td>65</td><td>65</td><td>110</td><td>30</td><td>69.615385</td></tr><tr><td>144</td><td>&quot;Articuno&quot;</td><td>&quot;Ice&quot;</td><td>&quot;Flying&quot;</td><td>580</td><td>90</td><td>85</td><td>100</td><td>95</td><td>125</td><td>85</td><td>85.0</td></tr><tr><td>145</td><td>&quot;Zapdos&quot;</td><td>&quot;Electric&quot;</td><td>&quot;Flying&quot;</td><td>580</td><td>90</td><td>90</td><td>85</td><td>125</td><td>90</td><td>100</td><td>90.0</td></tr><tr><td>146</td><td>&quot;Moltres&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>580</td><td>90</td><td>100</td><td>90</td><td>125</td><td>85</td><td>90</td><td>96.0</td></tr><tr><td>147</td><td>&quot;Dratini&quot;</td><td>&quot;Dragon&quot;</td><td>null</td><td>300</td><td>41</td><td>64</td><td>45</td><td>50</td><td>50</td><td>50</td><td>74.0</td></tr><tr><td>148</td><td>&quot;Dragonair&quot;</td><td>&quot;Dragon&quot;</td><td>null</td><td>420</td><td>61</td><td>84</td><td>65</td><td>70</td><td>70</td><td>70</td><td>74.0</td></tr><tr><td>149</td><td>&quot;Dragonite&quot;</td><td>&quot;Dragon&quot;</td><td>&quot;Flying&quot;</td><td>600</td><td>91</td><td>134</td><td>95</td><td>100</td><td>100</td><td>80</td><td>134.0</td></tr><tr><td>150</td><td>&quot;Mewtwo&quot;</td><td>&quot;Psychic&quot;</td><td>null</td><td>680</td><td>106</td><td>110</td><td>90</td><td>154</td><td>90</td><td>130</td><td>55.142857</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (163, 12)\n",
       "┌─────┬───────────────────────┬─────────┬────────┬───┬─────────┬─────────┬───────┬───────────┐\n",
       "│ #   ┆ Name                  ┆ Type 1  ┆ Type 2 ┆ … ┆ Sp. Atk ┆ Sp. Def ┆ Speed ┆ attack2   │\n",
       "│ --- ┆ ---                   ┆ ---     ┆ ---    ┆   ┆ ---     ┆ ---     ┆ ---   ┆ ---       │\n",
       "│ i64 ┆ str                   ┆ str     ┆ str    ┆   ┆ i64     ┆ i64     ┆ i64   ┆ f64       │\n",
       "╞═════╪═══════════════════════╪═════════╪════════╪═══╪═════════╪═════════╪═══════╪═══════════╡\n",
       "│ 1   ┆ Bulbasaur             ┆ Grass   ┆ Poison ┆ … ┆ 65      ┆ 65      ┆ 45    ┆ 75.8      │\n",
       "│ 2   ┆ Ivysaur               ┆ Grass   ┆ Poison ┆ … ┆ 80      ┆ 80      ┆ 60    ┆ 75.8      │\n",
       "│ 3   ┆ Venusaur              ┆ Grass   ┆ Poison ┆ … ┆ 100     ┆ 100     ┆ 80    ┆ 75.8      │\n",
       "│ 3   ┆ VenusaurMega Venusaur ┆ Grass   ┆ Poison ┆ … ┆ 122     ┆ 120     ┆ 80    ┆ 75.8      │\n",
       "│ 4   ┆ Charmander            ┆ Fire    ┆ null   ┆ … ┆ 60      ┆ 50      ┆ 65    ┆ 82.3      │\n",
       "│ …   ┆ …                     ┆ …       ┆ …      ┆ … ┆ …       ┆ …       ┆ …     ┆ …         │\n",
       "│ 146 ┆ Moltres               ┆ Fire    ┆ Flying ┆ … ┆ 125     ┆ 85      ┆ 90    ┆ 96.0      │\n",
       "│ 147 ┆ Dratini               ┆ Dragon  ┆ null   ┆ … ┆ 50      ┆ 50      ┆ 50    ┆ 74.0      │\n",
       "│ 148 ┆ Dragonair             ┆ Dragon  ┆ null   ┆ … ┆ 70      ┆ 70      ┆ 70    ┆ 74.0      │\n",
       "│ 149 ┆ Dragonite             ┆ Dragon  ┆ Flying ┆ … ┆ 100     ┆ 100     ┆ 80    ┆ 134.0     │\n",
       "│ 150 ┆ Mewtwo                ┆ Psychic ┆ null   ┆ … ┆ 154     ┆ 90      ┆ 130   ┆ 55.142857 │\n",
       "└─────┴───────────────────────┴─────────┴────────┴───┴─────────┴─────────┴───────┴───────────┘"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_pokemon\n",
    "    .select(\n",
    "        cs.exclude('Generation', 'Legendary'),\n",
    "        attack2 = pl.col('Attack').mean().over(['Type 1', 'Type 2']),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (163, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Name</th><th>Type 1</th><th>Type 2</th><th>HP</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;Venusaur&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>80</td></tr><tr><td>&quot;VenusaurMega V…</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>80</td></tr><tr><td>&quot;Victreebel&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>80</td></tr><tr><td>&quot;Vileplume&quot;</td><td>&quot;Grass&quot;</td><td>&quot;Poison&quot;</td><td>75</td></tr><tr><td>&quot;Arcanine&quot;</td><td>&quot;Fire&quot;</td><td>null</td><td>90</td></tr><tr><td>&quot;Ninetales&quot;</td><td>&quot;Fire&quot;</td><td>null</td><td>73</td></tr><tr><td>&quot;Moltres&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>90</td></tr><tr><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Dragon&quot;</td><td>78</td></tr><tr><td>&quot;Charizard&quot;</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>78</td></tr><tr><td>&quot;Vaporeon&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>130</td></tr><tr><td>&quot;Golduck&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>80</td></tr><tr><td>&quot;Seaking&quot;</td><td>&quot;Water&quot;</td><td>null</td><td>80</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Omanyte&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td><td>35</td></tr><tr><td>&quot;Kabuto&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Water&quot;</td><td>30</td></tr><tr><td>&quot;Aerodactyl&quot;</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td><td>80</td></tr><tr><td>&quot;AerodactylMega…</td><td>&quot;Rock&quot;</td><td>&quot;Flying&quot;</td><td>80</td></tr><tr><td>&quot;Rattata&quot;</td><td>&quot;Normal&quot;</td><td>null</td><td>30</td></tr><tr><td>&quot;Articuno&quot;</td><td>&quot;Ice&quot;</td><td>&quot;Flying&quot;</td><td>90</td></tr><tr><td>&quot;Zapdos&quot;</td><td>&quot;Electric&quot;</td><td>&quot;Flying&quot;</td><td>90</td></tr><tr><td>&quot;CharizardMega …</td><td>&quot;Fire&quot;</td><td>&quot;Flying&quot;</td><td>78</td></tr><tr><td>&quot;Dragonair&quot;</td><td>&quot;Dragon&quot;</td><td>null</td><td>61</td></tr><tr><td>&quot;Dratini&quot;</td><td>&quot;Dragon&quot;</td><td>null</td><td>41</td></tr><tr><td>&quot;Dragonite&quot;</td><td>&quot;Dragon&quot;</td><td>&quot;Flying&quot;</td><td>91</td></tr><tr><td>&quot;Abra&quot;</td><td>&quot;Psychic&quot;</td><td>null</td><td>25</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (163, 4)\n",
       "┌───────────────────────────┬─────────┬────────┬─────┐\n",
       "│ Name                      ┆ Type 1  ┆ Type 2 ┆ HP  │\n",
       "│ ---                       ┆ ---     ┆ ---    ┆ --- │\n",
       "│ str                       ┆ str     ┆ str    ┆ i64 │\n",
       "╞═══════════════════════════╪═════════╪════════╪═════╡\n",
       "│ Venusaur                  ┆ Grass   ┆ Poison ┆ 80  │\n",
       "│ VenusaurMega Venusaur     ┆ Grass   ┆ Poison ┆ 80  │\n",
       "│ Victreebel                ┆ Grass   ┆ Poison ┆ 80  │\n",
       "│ Vileplume                 ┆ Grass   ┆ Poison ┆ 75  │\n",
       "│ Arcanine                  ┆ Fire    ┆ null   ┆ 90  │\n",
       "│ …                         ┆ …       ┆ …      ┆ …   │\n",
       "│ CharizardMega Charizard Y ┆ Fire    ┆ Flying ┆ 78  │\n",
       "│ Dragonair                 ┆ Dragon  ┆ null   ┆ 61  │\n",
       "│ Dratini                   ┆ Dragon  ┆ null   ┆ 41  │\n",
       "│ Dragonite                 ┆ Dragon  ┆ Flying ┆ 91  │\n",
       "│ Abra                      ┆ Psychic ┆ null   ┆ 25  │\n",
       "└───────────────────────────┴─────────┴────────┴─────┘"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_pokemon.select(\n",
    "        pl.col('Name', 'Type 1', 'Type 2', 'HP')\n",
    "        .sort_by('HP', descending=True)\n",
    "        .over((['Type 1', 'Type 2']))\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Joins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/transformations/joins/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Polars, minimizes data expansion to two sensible approaches:\n",
    "- concat: for stacking dataframes \n",
    "- join: for merging dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2 = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [1, 2, 3],\n",
    "        \"b\": [4, 5, 6],\n",
    "        \"c\": [7, 8, 9],\n",
    "    }\n",
    ")\n",
    "\n",
    "df_3 = pl.DataFrame({\n",
    "    \"integer\": [1, 2,3],\n",
    "    \"date\": [datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)],\n",
    "    \"float\": [7.0, 8.0, 9.0],\n",
    "    \"string\": [\"d\", \"e\", \"f\"],\n",
    "})\n",
    "\n",
    "df_4 = df_2.with_columns(string = np.array(['a', 'b', 'd']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like pandas, polars can't simply refer to 'rows' and 'columns' (even though it is explicitly 2D)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th><th>a</th><th>b</th><th>c</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td><td>1</td><td>4</td><td>7</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td><td>2</td><td>5</td><td>8</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>&quot;c&quot;</td><td>3</td><td>6</td><td>9</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 7)\n",
       "┌─────────┬─────────────────────┬───────┬────────┬─────┬─────┬─────┐\n",
       "│ integer ┆ date                ┆ float ┆ string ┆ a   ┆ b   ┆ c   │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    ┆ --- ┆ --- ┆ --- │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    ┆ i64 ┆ i64 ┆ i64 │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╪═════╪═════╪═════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      ┆ 1   ┆ 4   ┆ 7   │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      ┆ 2   ┆ 5   ┆ 8   │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ c      ┆ 3   ┆ 6   ┆ 9   │\n",
       "└─────────┴─────────────────────┴───────┴────────┴─────┴─────┴─────┘"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.concat([df, df_2], how = \"horizontal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>&quot;c&quot;</td></tr><tr><td>1</td><td>2025-01-01 00:00:00</td><td>7.0</td><td>&quot;d&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>8.0</td><td>&quot;e&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>9.0</td><td>&quot;f&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 4)\n",
       "┌─────────┬─────────────────────┬───────┬────────┐\n",
       "│ integer ┆ date                ┆ float ┆ string │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ c      │\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 7.0   ┆ d      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 8.0   ┆ e      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 9.0   ┆ f      │\n",
       "└─────────┴─────────────────────┴───────┴────────┘"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.concat([df, df_3], how = \"vertical\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th><th>a</th><th>b</th><th>c</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td><td>1</td><td>4</td><td>7</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td><td>2</td><td>5</td><td>8</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 7)\n",
       "┌─────────┬─────────────────────┬───────┬────────┬─────┬─────┬─────┐\n",
       "│ integer ┆ date                ┆ float ┆ string ┆ a   ┆ b   ┆ c   │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    ┆ --- ┆ --- ┆ --- │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    ┆ i64 ┆ i64 ┆ i64 │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╪═════╪═════╪═════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      ┆ 1   ┆ 4   ┆ 7   │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      ┆ 2   ┆ 5   ┆ 8   │\n",
       "└─────────┴─────────────────────┴───────┴────────┴─────┴─────┴─────┘"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.join(df_4, on = 'string', how = 'inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th><th>a</th><th>b</th><th>c</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td><td>1</td><td>4</td><td>7</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td><td>2</td><td>5</td><td>8</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>&quot;c&quot;</td><td>null</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 7)\n",
       "┌─────────┬─────────────────────┬───────┬────────┬──────┬──────┬──────┐\n",
       "│ integer ┆ date                ┆ float ┆ string ┆ a    ┆ b    ┆ c    │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    ┆ ---  ┆ ---  ┆ ---  │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    ┆ i64  ┆ i64  ┆ i64  │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╪══════╪══════╪══════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      ┆ 1    ┆ 4    ┆ 7    │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      ┆ 2    ┆ 5    ┆ 8    │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ c      ┆ null ┆ null ┆ null │\n",
       "└─────────┴─────────────────────┴───────┴────────┴──────┴──────┴──────┘"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.join(df_4, on = 'string', how = 'left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is also `hstack` and `vstack` and `extend` but who knows why. Timings might be different, but shouldn't be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.hstack(df_2)\n",
    "# df.vstack(df_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concat Horizontal Time: 0.016919084009714425\n",
      "HStack Time: 0.020685667055658996\n",
      "Concat Vertical Time: 0.36969774996396154\n",
      "VStack Time: 0.009627500083297491\n"
     ]
    }
   ],
   "source": [
    "import timeit\n",
    "\n",
    "# Define the setup code\n",
    "setup_code = '''\n",
    "import polars as pl\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "\n",
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"integer\": [1, 2, 3],\n",
    "        \"date\": [\n",
    "            datetime(2025, 1, 1),\n",
    "            datetime(2025, 1, 2),\n",
    "            datetime(2025, 1, 3),\n",
    "        ],\n",
    "        \"float\": [4.0, 5.0, 6.0],\n",
    "        \"string\": [\"a\", \"b\", \"c\"],\n",
    "    }\n",
    ")\n",
    "\n",
    "df_2 = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [1, 2, 3],\n",
    "        \"b\": [4, 5, 6],\n",
    "        \"c\": [7, 8, 9],\n",
    "    }\n",
    ")\n",
    "\n",
    "df_3 = pl.DataFrame({\n",
    "    \"integer\": [1, 2,3],\n",
    "    \"date\": [datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)],\n",
    "    \"float\": [7.0, 8.0, 9.0],\n",
    "    \"string\": [\"d\", \"e\", \"f\"],\n",
    "})\n",
    "'''\n",
    "\n",
    "# Define the code for concat\n",
    "concat_code = '''\n",
    "result = pl.concat([df, df_2], how='horizontal')\n",
    "'''\n",
    "\n",
    "# Define the code for hstack\n",
    "hstack_code = '''\n",
    "result = df.hstack(df_2)\n",
    "'''\n",
    "\n",
    "\n",
    "# Measure the execution time for concat\n",
    "concat_time = timeit.timeit(concat_code, setup=setup_code, number=10000)\n",
    "\n",
    "# Measure the execution time for hstack\n",
    "hstack_time = timeit.timeit(hstack_code, setup=setup_code, number=10000)\n",
    "\n",
    "\n",
    "# Print the results\n",
    "print(f\"Concat Horizontal Time: {concat_time}\")\n",
    "print(f\"HStack Time: {hstack_time}\")\n",
    "\n",
    "\n",
    "# Define the code for concat\n",
    "concat_code = '''\n",
    "result = pl.concat([df, df_3], how='vertical')\n",
    "'''\n",
    "# Define the code for vstack\n",
    "vstack_code = '''\n",
    "result = df.vstack(df_3)\n",
    "'''\n",
    "\n",
    "# Measure the execution time for concat\n",
    "concat_time = timeit.timeit(concat_code, setup=setup_code, number=10000)\n",
    "\n",
    "# Measure the execution time for vstack\n",
    "vstack_time = timeit.timeit(vstack_code, setup=setup_code, number=10000)\n",
    "\n",
    "\n",
    "print(f\"Concat Vertical Time: {concat_time}\")\n",
    "print(f\"VStack Time: {vstack_time}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the docs for `vstack` vs. `extend`, which apparently only does vertical stacking and modifies in place."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Prefer extend over vstack when you want to do a query after a single append. For instance, during online operations where you add n rows and rerun a query.\n",
    "\n",
    "> Prefer vstack over extend when you want to append many times before doing a query. For instance, when you read in multiple files and want to store them in a single DataFrame. In the latter case, finish the sequence of vstack operations with a rechunk."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reshape/Transformations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/transformations/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Most of this will work how you expect coming from pandas."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pivots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>&quot;c&quot;</td></tr><tr><td>1</td><td>2025-01-01 00:00:00</td><td>7.0</td><td>&quot;d&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>8.0</td><td>&quot;e&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>9.0</td><td>&quot;f&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 4)\n",
       "┌─────────┬─────────────────────┬───────┬────────┐\n",
       "│ integer ┆ date                ┆ float ┆ string │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ c      │\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 7.0   ┆ d      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 8.0   ┆ e      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 9.0   ┆ f      │\n",
       "└─────────┴─────────────────────┴───────┴────────┘"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pivot = pl.concat([df, df_3], how = \"vertical\")\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>date</th><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th><th>f</th></tr><tr><td>datetime[μs]</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>2025-01-01 00:00:00</td><td>1</td><td>null</td><td>null</td><td>1</td><td>null</td><td>null</td></tr><tr><td>2025-01-02 00:00:00</td><td>null</td><td>2</td><td>null</td><td>null</td><td>2</td><td>null</td></tr><tr><td>2025-01-03 00:00:00</td><td>null</td><td>null</td><td>3</td><td>null</td><td>null</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 7)\n",
       "┌─────────────────────┬──────┬──────┬──────┬──────┬──────┬──────┐\n",
       "│ date                ┆ a    ┆ b    ┆ c    ┆ d    ┆ e    ┆ f    │\n",
       "│ ---                 ┆ ---  ┆ ---  ┆ ---  ┆ ---  ┆ ---  ┆ ---  │\n",
       "│ datetime[μs]        ┆ i64  ┆ i64  ┆ i64  ┆ i64  ┆ i64  ┆ i64  │\n",
       "╞═════════════════════╪══════╪══════╪══════╪══════╪══════╪══════╡\n",
       "│ 2025-01-01 00:00:00 ┆ 1    ┆ null ┆ null ┆ 1    ┆ null ┆ null │\n",
       "│ 2025-01-02 00:00:00 ┆ null ┆ 2    ┆ null ┆ null ┆ 2    ┆ null │\n",
       "│ 2025-01-03 00:00:00 ┆ null ┆ null ┆ 3    ┆ null ┆ null ┆ 3    │\n",
       "└─────────────────────┴──────┴──────┴──────┴──────┴──────┴──────┘"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pivot.pivot(index = 'date', columns = 'string', values = 'integer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>integer</th><th>date</th><th>float</th><th>string</th></tr><tr><td>i64</td><td>datetime[μs]</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>1</td><td>2025-01-01 00:00:00</td><td>4.0</td><td>&quot;a&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>5.0</td><td>&quot;b&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>6.0</td><td>&quot;c&quot;</td></tr><tr><td>1</td><td>2025-01-01 00:00:00</td><td>7.0</td><td>&quot;d&quot;</td></tr><tr><td>2</td><td>2025-01-02 00:00:00</td><td>8.0</td><td>&quot;e&quot;</td></tr><tr><td>3</td><td>2025-01-03 00:00:00</td><td>9.0</td><td>&quot;f&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 4)\n",
       "┌─────────┬─────────────────────┬───────┬────────┐\n",
       "│ integer ┆ date                ┆ float ┆ string │\n",
       "│ ---     ┆ ---                 ┆ ---   ┆ ---    │\n",
       "│ i64     ┆ datetime[μs]        ┆ f64   ┆ str    │\n",
       "╞═════════╪═════════════════════╪═══════╪════════╡\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 4.0   ┆ a      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 5.0   ┆ b      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 6.0   ┆ c      │\n",
       "│ 1       ┆ 2025-01-01 00:00:00 ┆ 7.0   ┆ d      │\n",
       "│ 2       ┆ 2025-01-02 00:00:00 ┆ 8.0   ┆ e      │\n",
       "│ 3       ┆ 2025-01-03 00:00:00 ┆ 9.0   ┆ f      │\n",
       "└─────────┴─────────────────────┴───────┴────────┘"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pivot = pl.concat([df, df_3], how = \"vertical\")\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>date</th><th>1</th><th>2</th><th>3</th></tr><tr><td>datetime[μs]</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>2025-01-01 00:00:00</td><td>11.0</td><td>null</td><td>null</td></tr><tr><td>2025-01-02 00:00:00</td><td>null</td><td>13.0</td><td>null</td></tr><tr><td>2025-01-03 00:00:00</td><td>null</td><td>null</td><td>15.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 4)\n",
       "┌─────────────────────┬──────┬──────┬──────┐\n",
       "│ date                ┆ 1    ┆ 2    ┆ 3    │\n",
       "│ ---                 ┆ ---  ┆ ---  ┆ ---  │\n",
       "│ datetime[μs]        ┆ f64  ┆ f64  ┆ f64  │\n",
       "╞═════════════════════╪══════╪══════╪══════╡\n",
       "│ 2025-01-01 00:00:00 ┆ 11.0 ┆ null ┆ null │\n",
       "│ 2025-01-02 00:00:00 ┆ null ┆ 13.0 ┆ null │\n",
       "│ 2025-01-03 00:00:00 ┆ null ┆ null ┆ 15.0 │\n",
       "└─────────────────────┴──────┴──────┴──────┘"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wide = (\n",
    "    df_pivot\n",
    "    .pivot(\n",
    "        index   = 'date', \n",
    "        columns = 'integer',\n",
    "        values  = 'float',\n",
    "        aggregate_function = \"sum\"\n",
    "    )\n",
    ")\n",
    "\n",
    "df_wide"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aside, the columns above are strings as they should be, not actual numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['date', '1', '2', '3']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wide.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is also `unstack` but it is currently 'unstable' and subject to change."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Melts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If your 'make wide' function is `pivot`, it doesn't make sense to call the 'make long' function `melt`, but I digress. In any case, melt takes data from wide to long format, and unlike pivot, works for lazy dataframes also."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>date</th><th>variable</th><th>sum</th></tr><tr><td>datetime[μs]</td><td>str</td><td>f64</td></tr></thead><tbody><tr><td>2025-01-01 00:00:00</td><td>&quot;1&quot;</td><td>11.0</td></tr><tr><td>2025-01-02 00:00:00</td><td>&quot;2&quot;</td><td>13.0</td></tr><tr><td>2025-01-03 00:00:00</td><td>&quot;3&quot;</td><td>15.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 3)\n",
       "┌─────────────────────┬──────────┬──────┐\n",
       "│ date                ┆ variable ┆ sum  │\n",
       "│ ---                 ┆ ---      ┆ ---  │\n",
       "│ datetime[μs]        ┆ str      ┆ f64  │\n",
       "╞═════════════════════╪══════════╪══════╡\n",
       "│ 2025-01-01 00:00:00 ┆ 1        ┆ 11.0 │\n",
       "│ 2025-01-02 00:00:00 ┆ 2        ┆ 13.0 │\n",
       "│ 2025-01-03 00:00:00 ┆ 3        ┆ 15.0 │\n",
       "└─────────────────────┴──────────┴──────┘"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_wide\n",
    "    .melt(\n",
    "        id_vars = ['date'],\n",
    "        value_vars = ['1', '2', '3'],\n",
    "        value_name = 'sum'\n",
    "    )\n",
    "    .drop_nulls()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Missing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/expressions/missing-data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Missing data is *always* `null` in polars, regardless of dtype. Yay!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/x6/4jhswqxj0sqf_gkgq6lw6l880000gn/T/ipykernel_26734/18906770.py:4: CategoricalRemappingWarning: Local categoricals have different encodings, expensive re-encoding is done to perform this merge operation. Consider using a StringCache or an Enum type if the categories are known in advance\n",
      "  .len()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>workclass</th><th>len</th></tr><tr><td>cat</td><td>u32</td></tr></thead><tbody><tr><td>&quot;Private&quot;</td><td>22696</td></tr><tr><td>&quot;Self-emp-not-i…</td><td>2541</td></tr><tr><td>&quot;Local-gov&quot;</td><td>2093</td></tr><tr><td>&quot;Other&quot;</td><td>1836</td></tr><tr><td>&quot;State-gov&quot;</td><td>1298</td></tr><tr><td>&quot;Self-emp-inc&quot;</td><td>1116</td></tr><tr><td>&quot;Federal-gov&quot;</td><td>960</td></tr><tr><td>&quot;Without-pay&quot;</td><td>14</td></tr><tr><td>&quot;Never-worked&quot;</td><td>7</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌──────────────────┬───────┐\n",
       "│ workclass        ┆ len   │\n",
       "│ ---              ┆ ---   │\n",
       "│ cat              ┆ u32   │\n",
       "╞══════════════════╪═══════╡\n",
       "│ Private          ┆ 22696 │\n",
       "│ Self-emp-not-inc ┆ 2541  │\n",
       "│ Local-gov        ┆ 2093  │\n",
       "│ Other            ┆ 1836  │\n",
       "│ State-gov        ┆ 1298  │\n",
       "│ Self-emp-inc     ┆ 1116  │\n",
       "│ Federal-gov      ┆ 960   │\n",
       "│ Without-pay      ┆ 14    │\n",
       "│ Never-worked     ┆ 7     │\n",
       "└──────────────────┴───────┘"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_census\n",
    "    .group_by(pl.col('workclass').fill_null('Other'))\n",
    "    .len()\n",
    "    .sort('len', descending=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "numpy NaN values can be used, but are considered to be a type of floating point data in Polars. MC sees no obvious benefit to using NaN over null, and there would be potential drawbacks as they are not counted as null, would have to be replaced with null to do a mean or similar function, etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time Series"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.pola.rs/user-guide/transformations/time-series/parsing/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When it comes to times series, polars can do all the usual functions like parse-on-read, `str.to_date`, extracting month/year/day from a date column, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2_068, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city</th><th>country</th><th>date.utc</th><th>location</th><th>parameter</th><th>value</th><th>unit</th></tr><tr><td>str</td><td>str</td><td>datetime[μs, UTC]</td><td>str</td><td>str</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>25.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Antwerpen&quot;</td><td>&quot;BE&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;BETR801&quot;</td><td>&quot;no2&quot;</td><td>50.5</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>23.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>27.7</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Antwerpen&quot;</td><td>&quot;BE&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;BETR801&quot;</td><td>&quot;no2&quot;</td><td>45.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 03:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>50.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 03:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 04:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>61.9</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 04:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>16.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 05:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>72.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 06:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>77.7</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 13:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 14:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.1</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 15:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.3</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 16:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 17:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.2</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 18:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.9</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 19:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>25.3</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 20:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 21:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>24.9</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 22:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>26.5</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 23:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.8</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-21 00:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.0</td><td>&quot;µg/m³&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2_068, 7)\n",
       "┌───────────┬─────────┬─────────────────────────┬────────────────────┬───────────┬───────┬───────┐\n",
       "│ city      ┆ country ┆ date.utc                ┆ location           ┆ parameter ┆ value ┆ unit  │\n",
       "│ ---       ┆ ---     ┆ ---                     ┆ ---                ┆ ---       ┆ ---   ┆ ---   │\n",
       "│ str       ┆ str     ┆ datetime[μs, UTC]       ┆ str                ┆ str       ┆ f64   ┆ str   │\n",
       "╞═══════════╪═════════╪═════════════════════════╪════════════════════╪═══════════╪═══════╪═══════╡\n",
       "│ Paris     ┆ FR      ┆ 2019-05-07 01:00:00 UTC ┆ FR04014            ┆ no2       ┆ 25.0  ┆ µg/m³ │\n",
       "│ Antwerpen ┆ BE      ┆ 2019-05-07 01:00:00 UTC ┆ BETR801            ┆ no2       ┆ 50.5  ┆ µg/m³ │\n",
       "│ London    ┆ GB      ┆ 2019-05-07 01:00:00 UTC ┆ London Westminster ┆ no2       ┆ 23.0  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-05-07 02:00:00 UTC ┆ FR04014            ┆ no2       ┆ 27.7  ┆ µg/m³ │\n",
       "│ Antwerpen ┆ BE      ┆ 2019-05-07 02:00:00 UTC ┆ BETR801            ┆ no2       ┆ 45.0  ┆ µg/m³ │\n",
       "│ …         ┆ …       ┆ …                       ┆ …                  ┆ …         ┆ …     ┆ …     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 20:00:00 UTC ┆ FR04014            ┆ no2       ┆ 21.4  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 21:00:00 UTC ┆ FR04014            ┆ no2       ┆ 24.9  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 22:00:00 UTC ┆ FR04014            ┆ no2       ┆ 26.5  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 23:00:00 UTC ┆ FR04014            ┆ no2       ┆ 21.8  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-21 00:00:00 UTC ┆ FR04014            ┆ no2       ┆ 20.0  ┆ µg/m³ │\n",
       "└───────────┴─────────┴─────────────────────────┴────────────────────┴───────────┴───────┴───────┘"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_aq = pl.read_csv(\"data/air_quality_no2_long.csv\", try_parse_dates=True)\n",
    "df_aq = df_aq.sort(\"date.utc\")\n",
    "df_aq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_aq.plot('date.utc', 'value')  # requires hvplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (812, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city</th><th>country</th><th>date.utc</th><th>location</th><th>parameter</th><th>value</th><th>unit</th></tr><tr><td>str</td><td>str</td><td>datetime[μs, UTC]</td><td>str</td><td>str</td><td>f64</td><td>str</td></tr></thead><tbody><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-01 00:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>84.7</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 00:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>22.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-01 01:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>74.8</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Antwerpen&quot;</td><td>&quot;BE&quot;</td><td>2019-06-01 01:00:00 UTC</td><td>&quot;BETR801&quot;</td><td>&quot;no2&quot;</td><td>52.5</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 01:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>22.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-01 02:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>68.1</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 02:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>16.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 03:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>16.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 04:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>11.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 05:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>11.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-01 06:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>44.6</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-06-01 06:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>4.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 13:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 14:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.1</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 15:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.3</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 16:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 17:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.2</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 18:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.9</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 19:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>25.3</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 20:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.4</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 21:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>24.9</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 22:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>26.5</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 23:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.8</td><td>&quot;µg/m³&quot;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-21 00:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.0</td><td>&quot;µg/m³&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (812, 7)\n",
       "┌───────────┬─────────┬─────────────────────────┬────────────────────┬───────────┬───────┬───────┐\n",
       "│ city      ┆ country ┆ date.utc                ┆ location           ┆ parameter ┆ value ┆ unit  │\n",
       "│ ---       ┆ ---     ┆ ---                     ┆ ---                ┆ ---       ┆ ---   ┆ ---   │\n",
       "│ str       ┆ str     ┆ datetime[μs, UTC]       ┆ str                ┆ str       ┆ f64   ┆ str   │\n",
       "╞═══════════╪═════════╪═════════════════════════╪════════════════════╪═══════════╪═══════╪═══════╡\n",
       "│ Paris     ┆ FR      ┆ 2019-06-01 00:00:00 UTC ┆ FR04014            ┆ no2       ┆ 84.7  ┆ µg/m³ │\n",
       "│ London    ┆ GB      ┆ 2019-06-01 00:00:00 UTC ┆ London Westminster ┆ no2       ┆ 22.0  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-01 01:00:00 UTC ┆ FR04014            ┆ no2       ┆ 74.8  ┆ µg/m³ │\n",
       "│ Antwerpen ┆ BE      ┆ 2019-06-01 01:00:00 UTC ┆ BETR801            ┆ no2       ┆ 52.5  ┆ µg/m³ │\n",
       "│ London    ┆ GB      ┆ 2019-06-01 01:00:00 UTC ┆ London Westminster ┆ no2       ┆ 22.0  ┆ µg/m³ │\n",
       "│ …         ┆ …       ┆ …                       ┆ …                  ┆ …         ┆ …     ┆ …     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 20:00:00 UTC ┆ FR04014            ┆ no2       ┆ 21.4  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 21:00:00 UTC ┆ FR04014            ┆ no2       ┆ 24.9  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 22:00:00 UTC ┆ FR04014            ┆ no2       ┆ 26.5  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 23:00:00 UTC ┆ FR04014            ┆ no2       ┆ 21.8  ┆ µg/m³ │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-21 00:00:00 UTC ┆ FR04014            ┆ no2       ┆ 20.0  ┆ µg/m³ │\n",
       "└───────────┴─────────┴─────────────────────────┴────────────────────┴───────────┴───────┴───────┘"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_aq.filter(\n",
    "    # pl.col(\"date.utc\").dt.date() == datetime(2019, 6, 20), \n",
    "    pl.col(\"date.utc\").dt.month() == 6\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2_068, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city</th><th>country</th><th>date.utc</th><th>location</th><th>parameter</th><th>value</th><th>unit</th><th>mo</th></tr><tr><td>str</td><td>str</td><td>datetime[μs, UTC]</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>i8</td></tr></thead><tbody><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>25.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Antwerpen&quot;</td><td>&quot;BE&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;BETR801&quot;</td><td>&quot;no2&quot;</td><td>50.5</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 01:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>23.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>27.7</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Antwerpen&quot;</td><td>&quot;BE&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;BETR801&quot;</td><td>&quot;no2&quot;</td><td>45.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 02:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 03:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>50.4</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 03:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 04:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>61.9</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;London&quot;</td><td>&quot;GB&quot;</td><td>2019-05-07 04:00:00 UTC</td><td>&quot;London Westmin…</td><td>&quot;no2&quot;</td><td>16.0</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 05:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>72.4</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-05-07 06:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>77.7</td><td>&quot;µg/m³&quot;</td><td>5</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 13:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.4</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 14:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.1</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 15:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.3</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 16:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>19.0</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 17:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.2</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 18:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>23.9</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 19:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>25.3</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 20:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.4</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 21:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>24.9</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 22:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>26.5</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-20 23:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>21.8</td><td>&quot;µg/m³&quot;</td><td>6</td></tr><tr><td>&quot;Paris&quot;</td><td>&quot;FR&quot;</td><td>2019-06-21 00:00:00 UTC</td><td>&quot;FR04014&quot;</td><td>&quot;no2&quot;</td><td>20.0</td><td>&quot;µg/m³&quot;</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2_068, 8)\n",
       "┌───────────┬─────────┬─────────────────────┬────────────────────┬───────────┬───────┬───────┬─────┐\n",
       "│ city      ┆ country ┆ date.utc            ┆ location           ┆ parameter ┆ value ┆ unit  ┆ mo  │\n",
       "│ ---       ┆ ---     ┆ ---                 ┆ ---                ┆ ---       ┆ ---   ┆ ---   ┆ --- │\n",
       "│ str       ┆ str     ┆ datetime[μs, UTC]   ┆ str                ┆ str       ┆ f64   ┆ str   ┆ i8  │\n",
       "╞═══════════╪═════════╪═════════════════════╪════════════════════╪═══════════╪═══════╪═══════╪═════╡\n",
       "│ Paris     ┆ FR      ┆ 2019-05-07 01:00:00 ┆ FR04014            ┆ no2       ┆ 25.0  ┆ µg/m³ ┆ 5   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Antwerpen ┆ BE      ┆ 2019-05-07 01:00:00 ┆ BETR801            ┆ no2       ┆ 50.5  ┆ µg/m³ ┆ 5   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ London    ┆ GB      ┆ 2019-05-07 01:00:00 ┆ London Westminster ┆ no2       ┆ 23.0  ┆ µg/m³ ┆ 5   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Paris     ┆ FR      ┆ 2019-05-07 02:00:00 ┆ FR04014            ┆ no2       ┆ 27.7  ┆ µg/m³ ┆ 5   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Antwerpen ┆ BE      ┆ 2019-05-07 02:00:00 ┆ BETR801            ┆ no2       ┆ 45.0  ┆ µg/m³ ┆ 5   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ …         ┆ …       ┆ …                   ┆ …                  ┆ …         ┆ …     ┆ …     ┆ …   │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 20:00:00 ┆ FR04014            ┆ no2       ┆ 21.4  ┆ µg/m³ ┆ 6   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 21:00:00 ┆ FR04014            ┆ no2       ┆ 24.9  ┆ µg/m³ ┆ 6   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 22:00:00 ┆ FR04014            ┆ no2       ┆ 26.5  ┆ µg/m³ ┆ 6   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-20 23:00:00 ┆ FR04014            ┆ no2       ┆ 21.8  ┆ µg/m³ ┆ 6   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "│ Paris     ┆ FR      ┆ 2019-06-21 00:00:00 ┆ FR04014            ┆ no2       ┆ 20.0  ┆ µg/m³ ┆ 6   │\n",
       "│           ┆         ┆ UTC                 ┆                    ┆           ┆       ┆       ┆     │\n",
       "└───────────┴─────────┴─────────────────────┴────────────────────┴───────────┴───────┴───────┴─────┘"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_aq.with_columns(mo = pl.col(\"date.utc\").dt.month())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Special functionality for grouped operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (89, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>date.utc</th><th>max</th><th>min</th></tr><tr><td>datetime[μs, UTC]</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>2019-05-07 00:00:00 UTC</td><td>77.7</td><td>13.0</td></tr><tr><td>2019-05-07 12:00:00 UTC</td><td>36.2</td><td>10.6</td></tr><tr><td>2019-05-08 00:00:00 UTC</td><td>36.0</td><td>13.5</td></tr><tr><td>2019-05-08 12:00:00 UTC</td><td>48.9</td><td>14.3</td></tr><tr><td>2019-05-09 00:00:00 UTC</td><td>50.7</td><td>10.0</td></tr><tr><td>2019-05-09 12:00:00 UTC</td><td>97.0</td><td>21.3</td></tr><tr><td>2019-05-10 00:00:00 UTC</td><td>60.7</td><td>10.5</td></tr><tr><td>2019-05-10 12:00:00 UTC</td><td>43.6</td><td>22.0</td></tr><tr><td>2019-05-11 00:00:00 UTC</td><td>36.8</td><td>14.9</td></tr><tr><td>2019-05-11 12:00:00 UTC</td><td>37.0</td><td>17.8</td></tr><tr><td>2019-05-12 00:00:00 UTC</td><td>44.0</td><td>11.4</td></tr><tr><td>2019-05-12 12:00:00 UTC</td><td>46.5</td><td>8.7</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>2019-06-14 12:00:00 UTC</td><td>55.0</td><td>14.2</td></tr><tr><td>2019-06-15 00:00:00 UTC</td><td>35.8</td><td>8.0</td></tr><tr><td>2019-06-15 12:00:00 UTC</td><td>22.6</td><td>9.0</td></tr><tr><td>2019-06-16 00:00:00 UTC</td><td>42.5</td><td>4.0</td></tr><tr><td>2019-06-16 12:00:00 UTC</td><td>67.3</td><td>8.1</td></tr><tr><td>2019-06-17 00:00:00 UTC</td><td>69.3</td><td>11.0</td></tr><tr><td>2019-06-17 12:00:00 UTC</td><td>73.3</td><td>9.6</td></tr><tr><td>2019-06-18 00:00:00 UTC</td><td>66.2</td><td>26.5</td></tr><tr><td>2019-06-18 12:00:00 UTC</td><td>39.3</td><td>15.3</td></tr><tr><td>2019-06-19 00:00:00 UTC</td><td>27.3</td><td>26.6</td></tr><tr><td>2019-06-20 12:00:00 UTC</td><td>26.5</td><td>19.0</td></tr><tr><td>2019-06-21 00:00:00 UTC</td><td>20.0</td><td>20.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (89, 3)\n",
       "┌─────────────────────────┬──────┬──────┐\n",
       "│ date.utc                ┆ max  ┆ min  │\n",
       "│ ---                     ┆ ---  ┆ ---  │\n",
       "│ datetime[μs, UTC]       ┆ f64  ┆ f64  │\n",
       "╞═════════════════════════╪══════╪══════╡\n",
       "│ 2019-05-07 00:00:00 UTC ┆ 77.7 ┆ 13.0 │\n",
       "│ 2019-05-07 12:00:00 UTC ┆ 36.2 ┆ 10.6 │\n",
       "│ 2019-05-08 00:00:00 UTC ┆ 36.0 ┆ 13.5 │\n",
       "│ 2019-05-08 12:00:00 UTC ┆ 48.9 ┆ 14.3 │\n",
       "│ 2019-05-09 00:00:00 UTC ┆ 50.7 ┆ 10.0 │\n",
       "│ …                       ┆ …    ┆ …    │\n",
       "│ 2019-06-18 00:00:00 UTC ┆ 66.2 ┆ 26.5 │\n",
       "│ 2019-06-18 12:00:00 UTC ┆ 39.3 ┆ 15.3 │\n",
       "│ 2019-06-19 00:00:00 UTC ┆ 27.3 ┆ 26.6 │\n",
       "│ 2019-06-20 12:00:00 UTC ┆ 26.5 ┆ 19.0 │\n",
       "│ 2019-06-21 00:00:00 UTC ┆ 20.0 ┆ 20.0 │\n",
       "└─────────────────────────┴──────┴──────┘"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_aq\n",
    "    .with_columns(\n",
    "        year = pl.col(\"date.utc\").dt.year(),\n",
    "        month = pl.col(\"date.utc\").dt.month(),\n",
    "        day = pl.col(\"date.utc\").dt.day(), \n",
    "    )\n",
    "    .group_by_dynamic(\"date.utc\", every = \"12h\")\n",
    "    .agg(\n",
    "        max = pl.col(\"value\").max(),\n",
    "        min = pl.col(\"value\").min(),\n",
    "    )\n",
    ")\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "polars",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}