diff --git a/docs/videos/ide.ipynb b/docs/videos/ide.ipynb index 8f34628..f43a9db 100644 --- a/docs/videos/ide.ipynb +++ b/docs/videos/ide.ipynb @@ -7,7 +7,7 @@ "# IDE demo\n", "\n", "This notebook contains the code accompanying the IDE demo. You can find the video [here](\n", - "https://github.com/kaiko-ai/typedspark/assets/47976799/986256c2-0438-430e-bfb0-72f117593d2c)." + "https://github.com/kaiko-ai/typedspark/assets/47976799/e6f7fa9c-6d14-4f68-baba-fe3c22f75b67)." ] }, { @@ -45,7 +45,7 @@ " last_name: Column[StringType]\n", " email: Column[StringType]\n", " phone_number: Column[StringType]\n", - " address: Column[StringType]" + " address: Column[StringType]\n" ] }, { @@ -55,7 +55,7 @@ "outputs": [], "source": [ "def get_dogs(pets: DataSet[Pets]) -> DataSet[Pets]:\n", - " return pets.filter(Pets.species == \"dog\")\n" + " return pets.filter(Pets.species == \"dog\")" ] }, { @@ -94,7 +94,7 @@ " expected,\n", " ignore_row_order=True,\n", " ignore_nullable=True,\n", - " )\n" + " )" ] }, { @@ -155,7 +155,7 @@ " Reminder.vaccine: _vaccinations.vaccine_name,\n", " Reminder.due: _vaccinations.next_due_date,\n", " },\n", - " )" + " )\n" ] }, { diff --git a/docs/videos/notebook.ipynb b/docs/videos/notebook.ipynb index fb83956..7e1c58c 100644 --- a/docs/videos/notebook.ipynb +++ b/docs/videos/notebook.ipynb @@ -6,7 +6,7 @@ "source": [ "# Notebook demo\n", "\n", - "This notebook contains the code accompanying the notebook demo. You can find the video [here](https://github.com/kaiko-ai/typedspark/assets/47976799/5ba89ef6-e79f-4a7b-bfe7-1da652ae5da8).\n", + "This notebook contains the code accompanying the notebook demo. You can find the video [here](https://github.com/kaiko-ai/typedspark/assets/47976799/39e157c3-6db0-436a-9e72-44b2062df808).\n", "\n", "## Set-up tables" ] @@ -20,7 +20,7 @@ "from pyspark.sql import SparkSession\n", "from pyspark.sql.types import DateType, LongType, StringType\n", "from typedspark import Column, Schema, create_empty_dataset, create_partially_filled_dataset\n", - "from datetime import datetime, timedelta\n" + "from datetime import datetime, timedelta" ] }, { @@ -43,7 +43,7 @@ "spark = SparkSession.builder.getOrCreate()\n", "spark.sql(\"CREATE DATABASE IF NOT EXISTS vet\")\n", "spark.sql(\"CREATE DATABASE IF NOT EXISTS library\")\n", - "spark.sql(\"CREATE DATABASE IF NOT EXISTS store\")" + "spark.sql(\"CREATE DATABASE IF NOT EXISTS store\")\n" ] }, { @@ -103,7 +103,7 @@ "create_empty_dataset(spark, Pets).write.saveAsTable(\"vet.pets\", format=\"parquet\", mode=\"overwrite\")\n", "create_empty_dataset(spark, Appointments).write.saveAsTable(\n", " \"vet.appointments\", format=\"parquet\", mode=\"overwrite\"\n", - ")" + ")\n" ] }, { @@ -146,7 +146,7 @@ " datetime.now() + timedelta(days=3) - timedelta(days=365),\n", " ],\n", " },\n", - ").write.saveAsTable(\"vet.vaccinations\", format=\"parquet\", mode=\"overwrite\")\n" + ").write.saveAsTable(\"vet.vaccinations\", format=\"parquet\", mode=\"overwrite\")" ] }, { @@ -166,7 +166,7 @@ "from pyspark.sql import SparkSession\n", "import pyspark.sql.functions as F\n", "\n", - "spark = SparkSession.builder.getOrCreate()" + "spark = SparkSession.builder.getOrCreate()\n" ] }, { @@ -175,7 +175,7 @@ "metadata": {}, "outputs": [], "source": [ - "db = Catalogs(spark)\n" + "db = Catalogs(spark)" ] }, { @@ -184,7 +184,7 @@ "metadata": {}, "outputs": [], "source": [ - "vaccinations, Vaccinations = db.spark_catalog.vet.vaccinations()" + "vaccinations, Vaccinations = db.spark_catalog.vet.vaccinations()\n" ] }, { @@ -212,7 +212,7 @@ } ], "source": [ - "vaccinations.show()" + "vaccinations.show()\n" ] }, { @@ -243,7 +243,7 @@ } ], "source": [ - "Vaccinations" + "Vaccinations\n" ] }, { @@ -267,7 +267,7 @@ } ], "source": [ - "vaccinations.filter(Vaccinations.vaccine_name == \"rabies\").show()" + "vaccinations.filter(Vaccinations.vaccine_name == \"rabies\").show()\n" ] }, { @@ -280,7 +280,7 @@ " vaccinations.groupby(Vaccinations.pet_id)\n", " .pivot(Vaccinations.vaccine_name.str)\n", " .agg(F.first(Vaccinations.next_due_date))\n", - ")\n" + ")" ] }, { @@ -304,7 +304,7 @@ } ], "source": [ - "pivot.show()" + "pivot.show()\n" ] }, { @@ -313,7 +313,7 @@ "metadata": {}, "outputs": [], "source": [ - "pivot, Pivot = create_schema(pivot)" + "pivot, Pivot = create_schema(pivot)\n" ] }, { @@ -343,7 +343,7 @@ } ], "source": [ - "Pivot" + "Pivot\n" ] }, { @@ -366,7 +366,7 @@ } ], "source": [ - "pivot.filter(Pivot.influenza.isNotNull()).show()" + "pivot.filter(Pivot.influenza.isNotNull()).show()\n" ] }, { @@ -395,7 +395,7 @@ "source": [ "spark.sql(\"DROP DATABASE vet CASCADE\")\n", "spark.sql(\"DROP DATABASE library CASCADE\")\n", - "spark.sql(\"DROP DATABASE store CASCADE\")" + "spark.sql(\"DROP DATABASE store CASCADE\")\n" ] }, {