From 09dc0be165f97ed0fcab5676c9afd862b46945eb Mon Sep 17 00:00:00 2001
From: nanne-aben <47976799+nanne-aben@users.noreply.github.com>
Date: Thu, 19 Oct 2023 11:55:36 +0200
Subject: [PATCH] Add Python 3.12 to ci/cd (#203)

* Add Python 3.12 to ci/cd

* update

* update
---
 .github/workflows/build.yml                |  8 ++-
 docs/remove_metadata.py                    | 13 ++--
 docs/run_notebooks.sh                      | 13 +++-
 docs/source/column_ambiguity.ipynb         | 12 ++--
 docs/source/create_empty_datasets.ipynb    | 28 ++++----
 docs/source/generating_schemas.ipynb       |  2 +-
 docs/source/structtype_columns.ipynb       |  6 +-
 docs/source/structtypes_in_notebooks.ipynb |  4 +-
 docs/source/transforming_datasets.ipynb    | 10 +--
 docs/source/type_checking.ipynb            |  2 +-
 docs/videos/ide.ipynb                      |  3 +-
 docs/videos/notebook.ipynb                 | 83 +++++++++++-----------
 requirements-dev.txt                       |  2 +-
 13 files changed, 97 insertions(+), 89 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5602a59..c5df4da 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,7 +8,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -48,4 +48,8 @@ jobs:
           coverage report -m --fail-under 100
       - name: Run notebooks
         run: |
-          for FILE in docs/*/*.ipynb; do papermill $FILE output.json -k python3; done
+          for FILE in docs/*/*.ipynb; do
+            BASE=$(basename $FILE)
+            cp $FILE .
+            jupyter nbconvert --to notebook $BASE --execute
+          done
diff --git a/docs/remove_metadata.py b/docs/remove_metadata.py
index 4150ba8..b522919 100644
--- a/docs/remove_metadata.py
+++ b/docs/remove_metadata.py
@@ -27,17 +27,16 @@ def remove_spark_warnings(cell):
                     continue
                 if "WARN Utils: Service 'SparkUI' could not bind on port" in output.text:
                     continue
+                if (
+                    "FutureWarning: is_datetime64tz_dtype is deprecated and will be removed in a future version."  # noqa: E501
+                    in output.text
+                ):
+                    continue
             outputs.append(output)
 
         cell.outputs = outputs
 
 
-def remove_papermill_metadata(nb):
-    """Removes the papermill metadata from a notebook."""
-    if "papermill" in nb.metadata.keys():
-        nb.metadata.pop("papermill")
-
-
 if __name__ == "__main__":
     FILENAME = sys.argv[1]
     nb = nbformat.read(FILENAME, as_version=4)
@@ -46,6 +45,4 @@ def remove_papermill_metadata(nb):
         clear_metadata(nb_cell)
         remove_spark_warnings(nb_cell)
 
-    remove_papermill_metadata(nb)
-
     nbformat.write(nb, FILENAME)
diff --git a/docs/run_notebooks.sh b/docs/run_notebooks.sh
index f6d7a5d..18eb9fa 100644
--- a/docs/run_notebooks.sh
+++ b/docs/run_notebooks.sh
@@ -1,4 +1,11 @@
-for FILE in docs/*/*.ipynb; do 
-    papermill $FILE $FILE; 
-    python docs/remove_metadata.py $FILE;
+for FILE in docs/*/*.ipynb; do
+    echo "Running $FILE"
+    DIR=$(dirname $FILE)
+    BASE=$(basename $FILE)
+    mv $FILE .
+
+    jupyter nbconvert --to notebook $BASE --execute --inplace
+    python docs/remove_metadata.py $BASE;
+
+    mv $BASE $DIR
 done
diff --git a/docs/source/column_ambiguity.ipynb b/docs/source/column_ambiguity.ipynb
index c06dd37..0cb59cc 100644
--- a/docs/source/column_ambiguity.ipynb
+++ b/docs/source/column_ambiguity.ipynb
@@ -87,9 +87,9 @@
       "+---+----+----+---+------+\n",
       "| id|name| age| id|salary|\n",
       "+---+----+----+---+------+\n",
-      "|  1|null|null|  1|  null|\n",
-      "|  2|null|null|  2|  null|\n",
-      "|  3|null|null|  3|  null|\n",
+      "|  1|NULL|NULL|  1|  NULL|\n",
+      "|  2|NULL|NULL|  2|  NULL|\n",
+      "|  3|NULL|NULL|  3|  NULL|\n",
       "+---+----+----+---+------+\n",
       "\n"
      ]
@@ -195,9 +195,9 @@
       "+---+----+----+------+\n",
       "| id|name| age|salary|\n",
       "+---+----+----+------+\n",
-      "|  1|null|null|  null|\n",
-      "|  2|null|null|  null|\n",
-      "|  3|null|null|  null|\n",
+      "|  1|NULL|NULL|  NULL|\n",
+      "|  2|NULL|NULL|  NULL|\n",
+      "|  3|NULL|NULL|  NULL|\n",
       "+---+----+----+------+\n",
       "\n"
      ]
diff --git a/docs/source/create_empty_datasets.ipynb b/docs/source/create_empty_datasets.ipynb
index 9361080..0a91358 100644
--- a/docs/source/create_empty_datasets.ipynb
+++ b/docs/source/create_empty_datasets.ipynb
@@ -41,9 +41,9 @@
       "+----+----+----+\n",
       "|  id|name| age|\n",
       "+----+----+----+\n",
-      "|null|null|null|\n",
-      "|null|null|null|\n",
-      "|null|null|null|\n",
+      "|NULL|NULL|NULL|\n",
+      "|NULL|NULL|NULL|\n",
+      "|NULL|NULL|NULL|\n",
       "+----+----+----+\n",
       "\n"
      ]
@@ -77,9 +77,9 @@
       "+---+----+----+\n",
       "| id|name| age|\n",
       "+---+----+----+\n",
-      "|  1|John|null|\n",
-      "|  2|Jane|null|\n",
-      "|  3|Jack|null|\n",
+      "|  1|John|NULL|\n",
+      "|  2|Jane|NULL|\n",
+      "|  3|Jack|NULL|\n",
       "+---+----+----+\n",
       "\n"
      ]
@@ -121,13 +121,13 @@
       "+----+-------+---+\n",
       "|  id|   name|age|\n",
       "+----+-------+---+\n",
-      "|null|  Alice| 20|\n",
-      "|null|    Bob| 30|\n",
-      "|null|Charlie| 40|\n",
-      "|null|   Dave| 50|\n",
-      "|null|    Eve| 60|\n",
-      "|null|  Frank| 70|\n",
-      "|null|  Grace| 80|\n",
+      "|NULL|  Alice| 20|\n",
+      "|NULL|    Bob| 30|\n",
+      "|NULL|Charlie| 40|\n",
+      "|NULL|   Dave| 50|\n",
+      "|NULL|    Eve| 60|\n",
+      "|NULL|  Frank| 70|\n",
+      "|NULL|  Grace| 80|\n",
       "+----+-------+---+\n",
       "\n"
      ]
@@ -229,7 +229,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/source/generating_schemas.ipynb b/docs/source/generating_schemas.ipynb
index 43132c1..7c00dd3 100644
--- a/docs/source/generating_schemas.ipynb
+++ b/docs/source/generating_schemas.ipynb
@@ -247,7 +247,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/source/structtype_columns.ipynb b/docs/source/structtype_columns.ipynb
index 9441510..f50cf66 100644
--- a/docs/source/structtype_columns.ipynb
+++ b/docs/source/structtype_columns.ipynb
@@ -157,9 +157,9 @@
       "+-----------+\n",
       "|consequeces|\n",
       "+-----------+\n",
-      "|  {null, 1}|\n",
-      "|  {null, 2}|\n",
-      "|  {null, 3}|\n",
+      "|  {NULL, 1}|\n",
+      "|  {NULL, 2}|\n",
+      "|  {NULL, 3}|\n",
       "+-----------+\n",
       "\n"
      ]
diff --git a/docs/source/structtypes_in_notebooks.ipynb b/docs/source/structtypes_in_notebooks.ipynb
index bc811ef..64b45fc 100644
--- a/docs/source/structtypes_in_notebooks.ipynb
+++ b/docs/source/structtypes_in_notebooks.ipynb
@@ -156,8 +156,8 @@
       "+---------+\n",
       "|   values|\n",
       "+---------+\n",
-      "|{2, null}|\n",
-      "|{3, null}|\n",
+      "|{2, NULL}|\n",
+      "|{3, NULL}|\n",
       "+---------+\n",
       "\n"
      ]
diff --git a/docs/source/transforming_datasets.ipynb b/docs/source/transforming_datasets.ipynb
index 524d825..da7ffc8 100644
--- a/docs/source/transforming_datasets.ipynb
+++ b/docs/source/transforming_datasets.ipynb
@@ -127,7 +127,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Cannot convert column into bool: please use '&' for 'and', '|' for 'or', '~' for 'not' when building DataFrame boolean expressions.\n"
+      "[CANNOT_CONVERT_COLUMN_INTO_BOOL] Cannot convert column into bool: please use '&' for 'and', '|' for 'or', '~' for 'not' when building DataFrame boolean expressions.\n"
      ]
     }
    ],
@@ -171,9 +171,9 @@
       "+----+--------+-----------+\n",
       "|  id|function|hourly_rate|\n",
       "+----+--------+-----------+\n",
-      "|null|    null|         26|\n",
-      "|null|    null|         46|\n",
-      "|null|    null|         66|\n",
+      "|NULL|    NULL|         26|\n",
+      "|NULL|    NULL|         46|\n",
+      "|NULL|    NULL|         66|\n",
       "+----+--------+-----------+\n",
       "\n"
      ]
@@ -213,7 +213,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/source/type_checking.ipynb b/docs/source/type_checking.ipynb
index 1620f64..a04c776 100644
--- a/docs/source/type_checking.ipynb
+++ b/docs/source/type_checking.ipynb
@@ -284,7 +284,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/videos/ide.ipynb b/docs/videos/ide.ipynb
index cd1dec3..b099128 100644
--- a/docs/videos/ide.ipynb
+++ b/docs/videos/ide.ipynb
@@ -183,8 +183,7 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.11.2"
-  },
-  "orig_nbformat": 4
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 2
diff --git a/docs/videos/notebook.ipynb b/docs/videos/notebook.ipynb
index 333c8b4..e991457 100644
--- a/docs/videos/notebook.ipynb
+++ b/docs/videos/notebook.ipynb
@@ -13,7 +13,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -34,7 +34,7 @@
        "DataFrame[]"
       ]
      },
-     "execution_count": 22,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -48,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -108,10 +108,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
+    "date = datetime(2023, 10, 2)\n",
+    "\n",
     "create_partially_filled_dataset(\n",
     "    spark,\n",
     "    Vaccinations,\n",
@@ -128,22 +130,22 @@
     "            \"influenza\",\n",
     "        ],\n",
     "        Vaccinations.next_due_date: [\n",
-    "            datetime.now() + timedelta(days=32),\n",
-    "            datetime.now() + timedelta(days=6),\n",
-    "            datetime.now() + timedelta(days=12),\n",
-    "            datetime.now() + timedelta(days=15),\n",
-    "            datetime.now() + timedelta(days=2),\n",
-    "            datetime.now() + timedelta(days=1),\n",
-    "            datetime.now() + timedelta(days=3),\n",
+    "            date + timedelta(days=32),\n",
+    "            date + timedelta(days=6),\n",
+    "            date + timedelta(days=12),\n",
+    "            date + timedelta(days=15),\n",
+    "            date + timedelta(days=2),\n",
+    "            date + timedelta(days=1),\n",
+    "            date + timedelta(days=3),\n",
     "        ],\n",
     "        Vaccinations.vaccine_date: [\n",
-    "            datetime.now() + timedelta(days=32) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=6) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=12) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=15) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=2) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=1) - timedelta(days=365),\n",
-    "            datetime.now() + timedelta(days=3) - timedelta(days=365),\n",
+    "            date + timedelta(days=32) - timedelta(days=365),\n",
+    "            date + timedelta(days=6) - timedelta(days=365),\n",
+    "            date + timedelta(days=12) - timedelta(days=365),\n",
+    "            date + timedelta(days=15) - timedelta(days=365),\n",
+    "            date + timedelta(days=2) - timedelta(days=365),\n",
+    "            date + timedelta(days=1) - timedelta(days=365),\n",
+    "            date + timedelta(days=3) - timedelta(days=365),\n",
     "        ],\n",
     "    },\n",
     ").write.saveAsTable(\"vet.vaccinations\", format=\"parquet\", mode=\"overwrite\")"
@@ -158,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -171,7 +173,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -180,7 +182,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -189,7 +191,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -201,9 +203,9 @@
       "+--------------+------+------------+------------+-------------+\n",
       "|             6|     2|   influenza|  2022-10-03|   2023-10-03|\n",
       "|             7|     3|   influenza|  2022-10-05|   2023-10-05|\n",
+      "|             1|     1|      rabies|  2022-11-03|   2023-11-03|\n",
       "|             3|     3|      rabies|  2022-10-14|   2023-10-14|\n",
       "|             2|     2|      rabies|  2022-10-08|   2023-10-08|\n",
-      "|             1|     1|      rabies|  2022-11-03|   2023-11-03|\n",
       "|             5|     3|        lyme|  2022-10-04|   2023-10-04|\n",
       "|             4|     1|        lyme|  2022-10-17|   2023-10-17|\n",
       "+--------------+------+------------+------------+-------------+\n",
@@ -217,7 +219,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -237,7 +239,7 @@
        "    next_due_date: Column[DateType]"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -248,7 +250,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -258,9 +260,9 @@
       "+--------------+------+------------+------------+-------------+\n",
       "|vaccination_id|pet_id|vaccine_name|vaccine_date|next_due_date|\n",
       "+--------------+------+------------+------------+-------------+\n",
+      "|             1|     1|      rabies|  2022-11-03|   2023-11-03|\n",
       "|             3|     3|      rabies|  2022-10-14|   2023-10-14|\n",
       "|             2|     2|      rabies|  2022-10-08|   2023-10-08|\n",
-      "|             1|     1|      rabies|  2022-11-03|   2023-11-03|\n",
       "+--------------+------+------------+------------+-------------+\n",
       "\n"
      ]
@@ -272,7 +274,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -285,7 +287,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -295,9 +297,9 @@
       "+------+----------+----------+----------+\n",
       "|pet_id| influenza|      lyme|    rabies|\n",
       "+------+----------+----------+----------+\n",
-      "|     2|2023-10-03|      null|2023-10-08|\n",
+      "|     2|2023-10-03|      NULL|2023-10-08|\n",
       "|     3|2023-10-05|2023-10-04|2023-10-14|\n",
-      "|     1|      null|2023-10-17|2023-11-03|\n",
+      "|     1|      NULL|2023-10-17|2023-11-03|\n",
       "+------+----------+----------+----------+\n",
       "\n"
      ]
@@ -309,7 +311,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -318,7 +320,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -337,7 +339,7 @@
        "    rabies: Column[DateType]"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -348,7 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -358,7 +360,7 @@
       "+------+----------+----------+----------+\n",
       "|pet_id| influenza|      lyme|    rabies|\n",
       "+------+----------+----------+----------+\n",
-      "|     2|2023-10-03|      null|2023-10-08|\n",
+      "|     2|2023-10-03|      NULL|2023-10-08|\n",
       "|     3|2023-10-05|2023-10-04|2023-10-14|\n",
       "+------+----------+----------+----------+\n",
       "\n"
@@ -378,7 +380,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -387,7 +389,7 @@
        "DataFrame[]"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -421,8 +423,7 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.11.2"
-  },
-  "orig_nbformat": 4
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 2
diff --git a/requirements-dev.txt b/requirements-dev.txt
index bb14bb2..5a12c57 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -20,7 +20,7 @@ pandas==2.1.1
 setuptools==68.2.2
 chispa==0.9.4
 # notebooks
-papermill==2.4.0
+nbconvert==7.9.2
 jupyter==1.0.0
 nbformat==5.9.2
 # readthedocs