PySpark_Intro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "application/vnd.databricks.v1+notebook": {
      "notebookName": "Spark_Intro",
      "dashboards": [],
      "notebookMetadata": {
        "pythonIndentUnit": 2
      },
      "language": "python",
      "widgets": {},
      "notebookOrigID": 2882351874606163
    },
    "colab": {
      "name": "PySpark_Intro.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "0vEkEby03TsJ",
        "xxlxvUhw3Tsf",
        "cuX6bcsG3Tsq",
        "pz1s-i3y3TtF",
        "R5rWyA4C3TtM",
        "w3n7Mgv93TtT",
        "63Umq0Ml3Ttb"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "language_info": {
      "name": "python"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pksX01/PySpark_Tutorials/blob/main/PySpark_Intro.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MbHhtsZGCZoF"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0ZIcPDsCZki"
      },
      "source": [
        "!wget -q https://www-us.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop2.7.tgz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s1YGzFy3CZgw"
      },
      "source": [
        "!tar xf spark-3.1.2-bin-hadoop2.7.tgz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-2BOjrbSCZU0"
      },
      "source": [
        "!pip install -q findspark"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QA9-mHpmCgcg"
      },
      "source": [
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.2-bin-hadoop2.7\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lj7yuY9nCi1G"
      },
      "source": [
        "import findspark\n",
        "findspark.init()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "f62ff29c-0e81-4522-a304-760c6b408644"
        },
        "id": "-Sel8YFS3Tro",
        "outputId": "0ad631ab-847f-490d-931b-973fda33cad8"
      },
      "source": [
        "import pyspark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "f8b20029-d53e-4ccd-b778-07205934407c"
        },
        "id": "r3rctA3T3Tr0"
      },
      "source": [
        "from pyspark import SparkContext\n",
        "from pyspark.sql import SQLContext\n",
        "from pyspark.sql import SparkSession"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c8e9f929-cfac-411e-a846-37f1544197f0"
        },
        "id": "HrenSPbn3Tr2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "outputId": "5d8cafd0-52f8-4ecd-db97-5367accb4091"
      },
      "source": [
        "sc = SparkContext()\n",
        "sc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://8eed2c2a98f2:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        "
            ],
            "text/plain": [
              "<SparkContext master=local[*] appName=pyspark-shell>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "bfcf08cb-05eb-444a-b878-c5c013a2e2bd"
        },
        "id": "7EbEjwsZ3Tr3"
      },
      "source": [
        "spark = SparkSession(sc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "cd5206d2-080e-4cf2-949f-c9a90e8b533f"
        },
        "id": "eu6OKAv_3Tr4",
        "outputId": "e8fcb899-0874-4837-c297-f56d9397a8e5"
      },
      "source": [
        "sc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[5]: </div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[5]: </div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"/?o=4211239421640757#setting/sparkui/0626-152828-bop61/driver-771700060611842037\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>spark://driver-proxy:7077</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Databricks Shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        "
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "\n        <div>\n            <p><b>SparkContext</b></p>\n\n            <p><a href=\"/?o=4211239421640757#setting/sparkui/0626-152828-bop61/driver-771700060611842037\">Spark UI</a></p>\n\n            <dl>\n              <dt>Version</dt>\n                <dd><code>v3.1.1</code></dd>\n              <dt>Master</dt>\n                <dd><code>spark://driver-proxy:7077</code></dd>\n              <dt>AppName</dt>\n                <dd><code>Databricks Shell</code></dd>\n            </dl>\n        </div>\n        ",
              "textData": null,
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "htmlSandbox",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d7595ba1-941b-4e73-b4a1-41c3526a2b65"
        },
        "id": "WopkQznp3Tr6",
        "outputId": "593cc090-c753-4118-fcbc-d46bd62b025b"
      },
      "source": [
        "spark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[6]: </div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[6]: </div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - hive</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"/?o=4211239421640757#setting/sparkui/0626-152828-bop61/driver-771700060611842037\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>spark://driver-proxy:7077</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Databricks Shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "\n            <div>\n                <p><b>SparkSession - hive</b></p>\n                \n        <div>\n            <p><b>SparkContext</b></p>\n\n            <p><a href=\"/?o=4211239421640757#setting/sparkui/0626-152828-bop61/driver-771700060611842037\">Spark UI</a></p>\n\n            <dl>\n              <dt>Version</dt>\n                <dd><code>v3.1.1</code></dd>\n              <dt>Master</dt>\n                <dd><code>spark://driver-proxy:7077</code></dd>\n              <dt>AppName</dt>\n                <dd><code>Databricks Shell</code></dd>\n            </dl>\n        </div>\n        \n            </div>\n        ",
              "textData": null,
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "htmlSandbox",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "8f24d3a5-2834-41ed-b8a8-4b62536f42b5"
        },
        "id": "B3nAQxVY3Tr7"
      },
      "source": [
        "#RDD and DataFrame"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "379c83ea-7c62-4fd7-bd8f-e5e05777d836"
        },
        "id": "HiHQnuQ03Tr8"
      },
      "source": [
        "##Creating a RDD using a 1D list and converting that to a DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "7e16ce12-445e-42f5-ae6b-1d186c926f42"
        },
        "id": "k4bZwpzR3Tr-",
        "outputId": "8e858469-3285-4602-a126-54d1f0c83b84"
      },
      "source": [
        "rdd_lst = sc.parallelize(['Pavan', 'ASE', 23])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "7db8a295-f7b0-446d-8c3e-8624997a4470"
        },
        "id": "_lh5ft3k3TsA",
        "outputId": "9c5be084-8679-4411-abc0-166ca25203d5"
      },
      "source": [
        "rdd_lst"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[9]: ParallelCollectionRDD[0] at readRDDFromInputStream at PythonRDD.scala:413</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[9]: ParallelCollectionRDD[0] at readRDDFromInputStream at PythonRDD.scala:413</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "55c6fed1-d635-495a-9cbb-2ee10a13f34c"
        },
        "id": "OnQ_g6OG3TsC",
        "outputId": "da1b361d-4394-483d-f204-bae89a1b0723"
      },
      "source": [
        "rdd_lst.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[10]: 3</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[10]: 3</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d69715c0-3c2c-4048-b474-7ba7597b531d"
        },
        "id": "EQ03fgNd3TsD",
        "outputId": "af355750-8573-4ba3-e454-12b494a5c363"
      },
      "source": [
        "rdd_lst.first()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[13]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[13]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "b1909e55-a52e-4032-928d-85fd5fa05e49"
        },
        "id": "b34m5uKB3TsF",
        "outputId": "828d62f4-766f-4345-9bb1-2ca2c54334ae"
      },
      "source": [
        "rdd_lst.take(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[12]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[12]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "006f461d-21ca-4ee2-86db-8e065b5209ae"
        },
        "id": "vLwczvjO3TsG",
        "outputId": "bc4fe69a-e174-4813-fde6-c4c2e2b76131"
      },
      "source": [
        "rdd_lst.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[14]: [&#39;Pavan&#39;, &#39;ASE&#39;, 23]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[14]: [&#39;Pavan&#39;, &#39;ASE&#39;, 23]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "f8c048ce-2b88-4e1c-a3df-898aeac8a58f"
        },
        "id": "mnOmjXm53TsI",
        "outputId": "a60a385f-5f94-4a1b-fd73-2613bbf02d33"
      },
      "source": [
        "df_lst = rdd_lst.toDF() #We can't convert 1D RDD to DF. We need atleast 2 dimensions."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n",
              "<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n",
              "<span class=\"ansi-green-fg\">&lt;command-2059119320414523&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n",
              "<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>df_lst <span class=\"ansi-blue-fg\">=</span> rdd_lst<span class=\"ansi-blue-fg\">.</span>toDF<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">toDF</span><span class=\"ansi-blue-fg\">(self, schema, sampleRatio)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">     65</span>         <span class=\"ansi-blue-fg\">[</span>Row<span class=\"ansi-blue-fg\">(</span>name<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-blue-fg\">&#39;Alice&#39;</span><span class=\"ansi-blue-fg\">,</span> age<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">]</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">     66</span>         &#34;&#34;&#34;\n",
              "<span class=\"ansi-green-fg\">---&gt; 67</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> sparkSession<span class=\"ansi-blue-fg\">.</span>createDataFrame<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> sampleRatio<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">     68</span> \n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">     69</span>     RDD<span class=\"ansi-blue-fg\">.</span>toDF <span class=\"ansi-blue-fg\">=</span> toDF\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">createDataFrame</span><span class=\"ansi-blue-fg\">(self, data, schema, samplingRatio, verifySchema)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    720</span>             return super(SparkSession, self).createDataFrame(\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    721</span>                 data, schema, samplingRatio, verifySchema)\n",
              "<span class=\"ansi-green-fg\">--&gt; 722</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>_create_dataframe<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> verifySchema<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    723</span> \n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    724</span>     <span class=\"ansi-green-fg\">def</span> _create_dataframe<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> data<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> verifySchema<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_create_dataframe</span><span class=\"ansi-blue-fg\">(self, data, schema, samplingRatio, verifySchema)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    750</span>         <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    751</span>             <span class=\"ansi-green-fg\">if</span> isinstance<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">,</span> RDD<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-fg\">--&gt; 752</span><span class=\"ansi-red-fg\">                 </span>rdd<span class=\"ansi-blue-fg\">,</span> schema <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_createFromRDD<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">.</span>map<span class=\"ansi-blue-fg\">(</span>prepare<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    753</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    754</span>                 rdd<span class=\"ansi-blue-fg\">,</span> schema <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_createFromLocal<span class=\"ansi-blue-fg\">(</span>map<span class=\"ansi-blue-fg\">(</span>prepare<span class=\"ansi-blue-fg\">,</span> data<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">)</span>\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_createFromRDD</span><span class=\"ansi-blue-fg\">(self, rdd, schema, samplingRatio)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    485</span>         &#34;&#34;&#34;\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    486</span>         <span class=\"ansi-green-fg\">if</span> schema <span class=\"ansi-green-fg\">is</span> <span class=\"ansi-green-fg\">None</span> <span class=\"ansi-green-fg\">or</span> isinstance<span class=\"ansi-blue-fg\">(</span>schema<span class=\"ansi-blue-fg\">,</span> <span class=\"ansi-blue-fg\">(</span>list<span class=\"ansi-blue-fg\">,</span> tuple<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-fg\">--&gt; 487</span><span class=\"ansi-red-fg\">             </span>struct <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_inferSchema<span class=\"ansi-blue-fg\">(</span>rdd<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> names<span class=\"ansi-blue-fg\">=</span>schema<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    488</span>             converter <span class=\"ansi-blue-fg\">=</span> _create_converter<span class=\"ansi-blue-fg\">(</span>struct<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    489</span>             rdd <span class=\"ansi-blue-fg\">=</span> rdd<span class=\"ansi-blue-fg\">.</span>map<span class=\"ansi-blue-fg\">(</span>converter<span class=\"ansi-blue-fg\">)</span>\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_inferSchema</span><span class=\"ansi-blue-fg\">(self, rdd, samplingRatio, names)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    465</span> \n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    466</span>         <span class=\"ansi-green-fg\">if</span> samplingRatio <span class=\"ansi-green-fg\">is</span> <span class=\"ansi-green-fg\">None</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-fg\">--&gt; 467</span><span class=\"ansi-red-fg\">             </span>schema <span class=\"ansi-blue-fg\">=</span> _infer_schema<span class=\"ansi-blue-fg\">(</span>first<span class=\"ansi-blue-fg\">,</span> names<span class=\"ansi-blue-fg\">=</span>names<span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    468</span>             <span class=\"ansi-green-fg\">if</span> _has_nulltype<span class=\"ansi-blue-fg\">(</span>schema<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">    469</span>                 <span class=\"ansi-green-fg\">for</span> row <span class=\"ansi-green-fg\">in</span> rdd<span class=\"ansi-blue-fg\">.</span>take<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-cyan-fg\">100</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">[</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">:</span><span class=\"ansi-blue-fg\">]</span><span class=\"ansi-blue-fg\">:</span>\n",
              "\n",
              "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/types.py</span> in <span class=\"ansi-cyan-fg\">_infer_schema</span><span class=\"ansi-blue-fg\">(row, names)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">   1063</span> \n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">   1064</span>     <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
              "<span class=\"ansi-green-fg\">-&gt; 1065</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">raise</span> TypeError<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;Can not infer schema for type: %s&#34;</span> <span class=\"ansi-blue-fg\">%</span> type<span class=\"ansi-blue-fg\">(</span>row<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span>\n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">   1066</span> \n",
              "<span class=\"ansi-green-intense-fg ansi-bold\">   1067</span>     fields <span class=\"ansi-blue-fg\">=</span> <span class=\"ansi-blue-fg\">[</span><span class=\"ansi-blue-fg\">]</span>\n",
              "\n",
              "<span class=\"ansi-red-fg\">TypeError</span>: Can not infer schema for type: &lt;class &#39;str&#39;&gt;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "data": "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n<span class=\"ansi-green-fg\">&lt;command-2059119320414523&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>df_lst <span class=\"ansi-blue-fg\">=</span> rdd_lst<span class=\"ansi-blue-fg\">.</span>toDF<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">toDF</span><span class=\"ansi-blue-fg\">(self, schema, sampleRatio)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     65</span>         <span class=\"ansi-blue-fg\">[</span>Row<span class=\"ansi-blue-fg\">(</span>name<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-blue-fg\">&#39;Alice&#39;</span><span class=\"ansi-blue-fg\">,</span> age<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">]</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     66</span>         &#34;&#34;&#34;\n<span class=\"ansi-green-fg\">---&gt; 67</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> sparkSession<span class=\"ansi-blue-fg\">.</span>createDataFrame<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> sampleRatio<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     68</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">     69</span>     RDD<span class=\"ansi-blue-fg\">.</span>toDF <span class=\"ansi-blue-fg\">=</span> toDF\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">createDataFrame</span><span class=\"ansi-blue-fg\">(self, data, schema, samplingRatio, verifySchema)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    720</span>             return super(SparkSession, self).createDataFrame(\n<span class=\"ansi-green-intense-fg ansi-bold\">    721</span>                 data, schema, samplingRatio, verifySchema)\n<span class=\"ansi-green-fg\">--&gt; 722</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>_create_dataframe<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> verifySchema<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    723</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">    724</span>     <span class=\"ansi-green-fg\">def</span> _create_dataframe<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> data<span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> verifySchema<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_create_dataframe</span><span class=\"ansi-blue-fg\">(self, data, schema, samplingRatio, verifySchema)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    750</span>         <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    751</span>             <span class=\"ansi-green-fg\">if</span> isinstance<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">,</span> RDD<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">--&gt; 752</span><span class=\"ansi-red-fg\">                 </span>rdd<span class=\"ansi-blue-fg\">,</span> schema <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_createFromRDD<span class=\"ansi-blue-fg\">(</span>data<span class=\"ansi-blue-fg\">.</span>map<span class=\"ansi-blue-fg\">(</span>prepare<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    753</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    754</span>                 rdd<span class=\"ansi-blue-fg\">,</span> schema <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_createFromLocal<span class=\"ansi-blue-fg\">(</span>map<span class=\"ansi-blue-fg\">(</span>prepare<span class=\"ansi-blue-fg\">,</span> data<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> schema<span class=\"ansi-blue-fg\">)</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_createFromRDD</span><span class=\"ansi-blue-fg\">(self, rdd, schema, samplingRatio)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    485</span>         &#34;&#34;&#34;\n<span class=\"ansi-green-intense-fg ansi-bold\">    486</span>         <span class=\"ansi-green-fg\">if</span> schema <span class=\"ansi-green-fg\">is</span> <span class=\"ansi-green-fg\">None</span> <span class=\"ansi-green-fg\">or</span> isinstance<span class=\"ansi-blue-fg\">(</span>schema<span class=\"ansi-blue-fg\">,</span> <span class=\"ansi-blue-fg\">(</span>list<span class=\"ansi-blue-fg\">,</span> tuple<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">--&gt; 487</span><span class=\"ansi-red-fg\">             </span>struct <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>_inferSchema<span class=\"ansi-blue-fg\">(</span>rdd<span class=\"ansi-blue-fg\">,</span> samplingRatio<span class=\"ansi-blue-fg\">,</span> names<span class=\"ansi-blue-fg\">=</span>schema<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    488</span>             converter <span class=\"ansi-blue-fg\">=</span> _create_converter<span class=\"ansi-blue-fg\">(</span>struct<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    489</span>             rdd <span class=\"ansi-blue-fg\">=</span> rdd<span class=\"ansi-blue-fg\">.</span>map<span class=\"ansi-blue-fg\">(</span>converter<span class=\"ansi-blue-fg\">)</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansi-cyan-fg\">_inferSchema</span><span class=\"ansi-blue-fg\">(self, rdd, samplingRatio, names)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    465</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">    466</span>         <span class=\"ansi-green-fg\">if</span> samplingRatio <span class=\"ansi-green-fg\">is</span> <span class=\"ansi-green-fg\">None</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">--&gt; 467</span><span class=\"ansi-red-fg\">             </span>schema <span class=\"ansi-blue-fg\">=</span> _infer_schema<span class=\"ansi-blue-fg\">(</span>first<span class=\"ansi-blue-fg\">,</span> names<span class=\"ansi-blue-fg\">=</span>names<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    468</span>             <span class=\"ansi-green-fg\">if</span> _has_nulltype<span class=\"ansi-blue-fg\">(</span>schema<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    469</span>                 <span class=\"ansi-green-fg\">for</span> row <span class=\"ansi-green-fg\">in</span> rdd<span class=\"ansi-blue-fg\">.</span>take<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-cyan-fg\">100</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">[</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">:</span><span class=\"ansi-blue-fg\">]</span><span class=\"ansi-blue-fg\">:</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/types.py</span> in <span class=\"ansi-cyan-fg\">_infer_schema</span><span class=\"ansi-blue-fg\">(row, names)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">   1063</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">   1064</span>     <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">-&gt; 1065</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">raise</span> TypeError<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;Can not infer schema for type: %s&#34;</span> <span class=\"ansi-blue-fg\">%</span> type<span class=\"ansi-blue-fg\">(</span>row<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">   1066</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">   1067</span>     fields <span class=\"ansi-blue-fg\">=</span> <span class=\"ansi-blue-fg\">[</span><span class=\"ansi-blue-fg\">]</span>\n\n<span class=\"ansi-red-fg\">TypeError</span>: Can not infer schema for type: &lt;class &#39;str&#39;&gt;</div>",
              "errorSummary": "<span class=\"ansi-red-fg\">TypeError</span>: Can not infer schema for type: &lt;class &#39;str&#39;&gt;",
              "metadata": {},
              "errorTraceType": "html",
              "type": "ipynbError",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "429b811e-b885-47c2-86d8-3e4b5b043c90"
        },
        "id": "0vEkEby03TsJ"
      },
      "source": [
        "##Creating a RDD using a 2D list and converting that to a DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "adfe1b79-44e3-4f2e-8caf-5e13e918e460"
        },
        "id": "cslcFYrO3TsK",
        "outputId": "89bac3aa-90b6-46dd-8c0d-213d671a3fb9"
      },
      "source": [
        "rdd_2d_lst = sc.parallelize([['Pavan', 'ASE'],\n",
        "                            ['Priyanka', 'MBA'],\n",
        "                            ['Mithra', 'SE']])\n",
        "rdd_2d_lst"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[7]: ParallelCollectionRDD[11] at readRDDFromInputStream at PythonRDD.scala:413</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[7]: ParallelCollectionRDD[11] at readRDDFromInputStream at PythonRDD.scala:413</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "0d8b9710-4817-4114-8862-74becc017d75"
        },
        "id": "ZtmS5GRy3TsL",
        "outputId": "61b73ef6-5e66-498d-f8a4-f15547b962e6"
      },
      "source": [
        "rdd_2d_lst.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[8]: 3</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[8]: 3</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "6a832d6d-4dce-4689-8f0f-de2672a18f2c"
        },
        "id": "OTfnctBd3TsM",
        "outputId": "b85ce3d3-cc35-4fcd-bd9b-13360fc5c5b5"
      },
      "source": [
        "rdd_2d_lst.first()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[9]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[9]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "302b3b79-75a7-45e7-8f42-de91b5d711f4"
        },
        "id": "GW-LwQbf3TsN",
        "outputId": "a2f87632-6780-4b49-aafe-10688f62adc8"
      },
      "source": [
        "rdd_2d_lst.first()[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[10]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[10]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "2e274785-bb39-4de4-af39-1ae75791d5da"
        },
        "id": "1LyL2OuD3TsP",
        "outputId": "7aa36870-841c-438d-b5cb-e9af4a31a623"
      },
      "source": [
        "rdd_2d_lst.take(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[11]: [[&#39;Pavan&#39;, &#39;ASE&#39;], [&#39;Priyanka&#39;, &#39;MBA&#39;]]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[11]: [[&#39;Pavan&#39;, &#39;ASE&#39;], [&#39;Priyanka&#39;, &#39;MBA&#39;]]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "809e3b84-1dd9-4a07-9718-d9e2c55f6841"
        },
        "id": "qx3r1gW-3TsQ",
        "outputId": "a67ffb5b-c012-416f-96c6-37f43be5cc43"
      },
      "source": [
        "rdd_2d_lst.take(2)[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[12]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[12]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "4542943d-44d1-49e3-b849-0eb6a1415ca7"
        },
        "id": "3IIuGY6I3TsQ",
        "outputId": "26840d01-297f-4286-aac9-d568f2fa5725"
      },
      "source": [
        "rdd_2d_lst.take(2)[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[13]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[13]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "e25e83f6-089c-428a-b237-567a809b52e4"
        },
        "id": "aGxQ_62Q3TsR",
        "outputId": "bb2817d6-e2c0-4010-bcc9-488896302588"
      },
      "source": [
        "rdd_2d_lst.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[14]: [[&#39;Pavan&#39;, &#39;ASE&#39;], [&#39;Priyanka&#39;, &#39;MBA&#39;], [&#39;Mithra&#39;, &#39;SE&#39;]]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[14]: [[&#39;Pavan&#39;, &#39;ASE&#39;], [&#39;Priyanka&#39;, &#39;MBA&#39;], [&#39;Mithra&#39;, &#39;SE&#39;]]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "588c98b8-c083-4d15-a5f0-55da2d40f6a8"
        },
        "id": "NPW9m82K3TsS",
        "outputId": "4d250f10-00a1-404a-b7cb-f8f2c6e6cc3d"
      },
      "source": [
        "rdd_2d_lst.collect()[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[15]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[15]: [&#39;Pavan&#39;, &#39;ASE&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "650790a9-6164-44aa-9d3c-cd787ba77f49"
        },
        "id": "RS_yHM6O3TsT",
        "outputId": "e0108135-3346-4d75-daaf-b1e9e8333427"
      },
      "source": [
        "rdd_2d_lst.collect()[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[16]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[16]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "0c6378fd-8c41-422a-92cc-e9584e1d5bfe"
        },
        "id": "aD8FRGbF3TsT",
        "outputId": "3db76f0f-8ad3-4e2f-d124-5bbb54f6d500"
      },
      "source": [
        "df = rdd_2d_lst.toDF()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "0481d017-0589-4607-a7e2-72dd8c253bba"
        },
        "id": "-SVRDM543TsU",
        "outputId": "b4bcb445-31e6-4a73-bc3d-116d10e88be1"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[19]: DataFrame[_1: string, _2: string]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[19]: DataFrame[_1: string, _2: string]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "386cda19-be1d-45fc-abdf-8adb86a0c500"
        },
        "id": "LBagb8wQ3TsV",
        "outputId": "f046ea2f-8c8d-471b-f9d5-f623de841e21"
      },
      "source": [
        "type(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[20]: pyspark.sql.dataframe.DataFrame</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[20]: pyspark.sql.dataframe.DataFrame</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "3219f8fb-7b3f-4fb8-b6ba-9527a4c3e08f"
        },
        "id": "TfIf-Vp53TsW",
        "outputId": "e3c0ae6e-07c7-48fd-f98c-8af572fdae5c"
      },
      "source": [
        "df.show() #displays data in tabular form"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------+---+\n",
              "      _1| _2|\n",
              "+--------+---+\n",
              "   Pavan|ASE|\n",
              "Priyanka|MBA|\n",
              "  Mithra| SE|\n",
              "+--------+---+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------+---+\n|      _1| _2|\n+--------+---+\n|   Pavan|ASE|\n|Priyanka|MBA|\n|  Mithra| SE|\n+--------+---+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ad32e835-c03c-4a68-be9d-145b13ad988f"
        },
        "id": "aXYHwsUn3TsX",
        "outputId": "0a3f40e6-e7ce-41b0-8fcc-d32789038aa1"
      },
      "source": [
        "df.show(1) #display only first row of dataframe"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-----+---+\n",
              "   _1| _2|\n",
              "+-----+---+\n",
              "Pavan|ASE|\n",
              "+-----+---+\n",
              "only showing top 1 row\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-----+---+\n|   _1| _2|\n+-----+---+\n|Pavan|ASE|\n+-----+---+\nonly showing top 1 row\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "245860fc-fbc7-4dad-a990-7e9ec61243e1"
        },
        "id": "bn4Eoyuk3TsY",
        "outputId": "28188be0-58e7-45ff-a755-f4fbff7a5e34"
      },
      "source": [
        "df.show(1)[0] #it is not subscriptable."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-----+---+\n",
              "   _1| _2|\n",
              "+-----+---+\n",
              "Pavan|ASE|\n",
              "+-----+---+\n",
              "only showing top 1 row\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-----+---+\n|   _1| _2|\n+-----+---+\n|Pavan|ASE|\n+-----+---+\nonly showing top 1 row\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n",
              "<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n",
              "<span class=\"ansi-green-fg\">&lt;command-679493542736813&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n",
              "<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>df<span class=\"ansi-blue-fg\">.</span>show<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">[</span><span class=\"ansi-cyan-fg\">0</span><span class=\"ansi-blue-fg\">]</span> <span class=\"ansi-red-fg\">#</span>\n",
              "\n",
              "<span class=\"ansi-red-fg\">TypeError</span>: &#39;NoneType&#39; object is not subscriptable</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "data": "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n<span class=\"ansi-green-fg\">&lt;command-679493542736813&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>df<span class=\"ansi-blue-fg\">.</span>show<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-cyan-fg\">1</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">[</span><span class=\"ansi-cyan-fg\">0</span><span class=\"ansi-blue-fg\">]</span> <span class=\"ansi-red-fg\">#</span>\n\n<span class=\"ansi-red-fg\">TypeError</span>: &#39;NoneType&#39; object is not subscriptable</div>",
              "errorSummary": "<span class=\"ansi-red-fg\">TypeError</span>: &#39;NoneType&#39; object is not subscriptable",
              "metadata": {},
              "errorTraceType": "html",
              "type": "ipynbError",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "6adee685-7917-4eaa-b233-26c2f1e24b0d"
        },
        "id": "P4b6Gkp63TsZ",
        "outputId": "fbf264b2-d7b3-4215-8913-23c7da4ba3b0"
      },
      "source": [
        "df.first() #returns data as Row function"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[24]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[24]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c1827ff2-1322-4de6-bd82-9244e12c5eb9"
        },
        "id": "JmJUeq3l3Tsa",
        "outputId": "6cf15fc2-2db5-48db-8e5d-87bc44e253fe"
      },
      "source": [
        "df.first()[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[28]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[28]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "506f2858-e998-45c2-a118-b1d7052e156e"
        },
        "id": "iTJFZo0Q3Tsb",
        "outputId": "0201df8e-dea9-4267-df9b-4c60f133db91"
      },
      "source": [
        "df.take(2) #returns data as Row function"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[25]: [Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;), Row(_1=&#39;Priyanka&#39;, _2=&#39;MBA&#39;)]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[25]: [Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;), Row(_1=&#39;Priyanka&#39;, _2=&#39;MBA&#39;)]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "86c544ce-7269-482d-856d-a98164ffd49a"
        },
        "id": "JJWs-EV13Tsc",
        "outputId": "e76e5a09-b41b-4a75-ad7b-6566e61bf234"
      },
      "source": [
        "df.take(2)[0] # returns first row as Row function"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[27]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[27]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "0477d0d9-b23a-4df1-956c-5bbd104d85bc"
        },
        "id": "B5Uq_08U3Tsc",
        "outputId": "0bcace0b-5830-447a-ef45-2d195c26d0f6"
      },
      "source": [
        "df.take(2)[0][0] #returns element in 1st row and 1st co"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[29]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[29]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "1b35c586-2967-4a13-8aa3-9f780c66ee21"
        },
        "id": "djQvQ5d03Tsd",
        "outputId": "8a6972fa-e8dc-44e8-c7fb-9d0dc0164513"
      },
      "source": [
        "df.collect() #Returns all rows and columns but as Row function"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[31]: [Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;),\n",
              " Row(_1=&#39;Priyanka&#39;, _2=&#39;MBA&#39;),\n",
              " Row(_1=&#39;Mithra&#39;, _2=&#39;SE&#39;)]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[31]: [Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;),\n Row(_1=&#39;Priyanka&#39;, _2=&#39;MBA&#39;),\n Row(_1=&#39;Mithra&#39;, _2=&#39;SE&#39;)]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "657ee83c-dddb-4b57-bf9d-4864e87d1a27"
        },
        "id": "5SNZcWPE3Tse",
        "outputId": "61f17ada-1097-4960-80ca-e570e1997493"
      },
      "source": [
        "df.collect()[0] # returns first row as Row function"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[32]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[32]: Row(_1=&#39;Pavan&#39;, _2=&#39;ASE&#39;)</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ab60b5bc-8cc8-4afc-a1b6-8b50cd8ac923"
        },
        "id": "kSoXtRPF3Tsf",
        "outputId": "920e176e-4591-4225-827c-5bb2531a23e8"
      },
      "source": [
        "df.collect()[0][0] #returns element in 1st row and 1st column"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[33]: &#39;Pavan&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[33]: &#39;Pavan&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "50ed5cba-e29a-4ffb-89a3-9315c659ee8c"
        },
        "id": "xxlxvUhw3Tsf"
      },
      "source": [
        "##Creating a RDD from Row and converting that RDD to DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "41086234-7687-4d96-a278-33c21b081957"
        },
        "id": "j68nYcZd3Tsg"
      },
      "source": [
        "from pyspark.sql import Row"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d71702e9-09de-4fd6-958c-3232446ebc0d"
        },
        "id": "VDgUdKWo3Tsh",
        "outputId": "f06fed89-3462-4955-a288-c6ee95fffcfa"
      },
      "source": [
        "rdd_row = sc.parallelize([Row(Name = \"Pavan\",\n",
        "                             Designation = \"ASE\"\n",
        "                             ),\n",
        "                         Row(Name = \"Priyanka\",\n",
        "                            Designation = \"SE\")]\n",
        "                        )\n",
        "rdd_row"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[4]: ParallelCollectionRDD[0] at readRDDFromInputStream at PythonRDD.scala:413</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[4]: ParallelCollectionRDD[0] at readRDDFromInputStream at PythonRDD.scala:413</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "87b99d4a-ac32-49da-bcbd-b966b3e458ad"
        },
        "id": "2jm2t-hx3Tsi",
        "outputId": "07356016-3292-4c80-9294-c071a0e18bf5"
      },
      "source": [
        "rdd_row.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[11]: 2</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[11]: 2</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "41bf69cb-784b-4efb-ba25-c3fd0f0e25fc"
        },
        "id": "lQsltFue3Tsj",
        "outputId": "681a94a7-5072-4dbb-8798-e7688613ea72"
      },
      "source": [
        "rdd_row.first()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[5]: Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;)</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[5]: Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;)</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ceace043-d4f9-4349-9a8f-72ab80a4f8b5"
        },
        "id": "uFIZ1fAO3Tsk",
        "outputId": "636249f0-86a1-4283-821f-98ceb4ad07d9"
      },
      "source": [
        "rdd_row.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[6]: [Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;), Row(Name=&#39;Priyanka&#39;, Designation=&#39;SE&#39;)]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[6]: [Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;), Row(Name=&#39;Priyanka&#39;, Designation=&#39;SE&#39;)]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "da7f4d22-763d-4be1-8652-4211ec10f811"
        },
        "id": "z2T_AB1P3Tsl",
        "outputId": "26b6bead-50b0-473b-f3ec-6f5beab2ef31"
      },
      "source": [
        "df1 = rdd_row.toDF()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "423c5716-c961-437c-ab27-eb6522857f5b"
        },
        "id": "daSyn_bM3Tsm",
        "outputId": "c1496b37-1eef-4fea-aaf6-6e1403ed2475"
      },
      "source": [
        "df1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[8]: DataFrame[Name: string, Designation: string]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[8]: DataFrame[Name: string, Designation: string]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "80b03098-4b7d-407e-8693-ee775c4e3af6"
        },
        "id": "j9HXQWuw3Tsn",
        "outputId": "d51dfb6a-e7b5-47a7-98be-7b74e30918d5"
      },
      "source": [
        "type(df1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[10]: pyspark.sql.dataframe.DataFrame</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[10]: pyspark.sql.dataframe.DataFrame</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "7e1b3ad0-a3f4-4e26-a114-db1dd48ab31a"
        },
        "id": "bLIFvyS83Tso",
        "outputId": "a4d72df5-32ca-4b20-d575-e142d038683f"
      },
      "source": [
        "df1.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[12]: 2</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[12]: 2</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "b4b653fe-40ad-4ea6-b812-9cf37e5ef0f9"
        },
        "id": "lVu5gQUb3Tsp",
        "outputId": "19c171af-efbc-4823-9962-c2bd18c572c1"
      },
      "source": [
        "df1.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------+-----------+\n",
              "    Name|Designation|\n",
              "+--------+-----------+\n",
              "   Pavan|        ASE|\n",
              "Priyanka|         SE|\n",
              "+--------+-----------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------+-----------+\n|    Name|Designation|\n+--------+-----------+\n|   Pavan|        ASE|\n|Priyanka|         SE|\n+--------+-----------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "5ffcb6a9-e7d9-4f4f-b4e0-f5c42aa9c5f2"
        },
        "id": "cuX6bcsG3Tsq"
      },
      "source": [
        "##RDD using Complex Data Types and converting that RDD to DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "cde67303-a650-4fa8-a500-25e60f53b87a"
        },
        "id": "B6d7JpQR3Tsr"
      },
      "source": [
        "from datetime import datetime"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "123ecdfa-74ec-47b4-8c1b-0ccaba3e0eb3"
        },
        "id": "t6sQOt9M3Tss",
        "outputId": "f43be99f-a1c2-4d59-aca3-dffa10f6d09b"
      },
      "source": [
        "rdd_complex = sc.parallelize([\n",
        "  Row(col_int = 1,\n",
        "      col_double = 1.5,\n",
        "      col_bool = True,\n",
        "      col_str = 'abc',\n",
        "      col_lst = [1, 2, 3],\n",
        "      col_tuple = (1, 2, 3),\n",
        "      col_row = Row(x=1, y=2, z=3),\n",
        "      col_dict = {1:'One', 2:'Two'},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 34, 10)\n",
        "  ),\n",
        "  Row(col_int = 2,\n",
        "      col_double = 2.5,\n",
        "      col_bool = True,\n",
        "      col_str = 'efg',\n",
        "      col_lst = [1, 2.5, 3],\n",
        "      col_tuple = (1.5, 2, 3),\n",
        "      col_row = Row(x=1, y=2, z=3.5),\n",
        "      col_dict = {1.1:'One', 2:'Two'},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 42, 10)\n",
        "  ),\n",
        "  Row(col_int = 3,\n",
        "      col_double = 3.5,\n",
        "      col_bool = False,\n",
        "      col_str = 'hij',\n",
        "      col_lst = [1, 'Two', 3.5],\n",
        "      col_tuple = ('One', 2.5, 3),\n",
        "      col_row = Row(x=1.5, y='Two', z=3),\n",
        "      col_dict = {1:'One', 'Two':2},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 44, 10)\n",
        "  )\n",
        "])\n",
        "\n",
        "rdd_complex"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[31]: ParallelCollectionRDD[37] at readRDDFromInputStream at PythonRDD.scala:413</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[31]: ParallelCollectionRDD[37] at readRDDFromInputStream at PythonRDD.scala:413</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "9cbe818c-16e6-4ca2-b3b1-a0abfc2e94d5"
        },
        "id": "kXOx7Oen3Tsu",
        "outputId": "b2e7a83f-e269-4e96-dd58-1320b605a5ec"
      },
      "source": [
        "rdd_complex.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[32]: 3</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[32]: 3</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d9feff8c-2d1f-45c7-b118-78c4f8849a7d"
        },
        "id": "0R8KSFS03Tsu",
        "outputId": "c8d3eee8-919e-447b-c396-9f74bb4c5abc"
      },
      "source": [
        "rdd_complex.first()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[33]: Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10))</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[33]: Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10))</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "7d48db9f-d8d7-4f45-8d21-6a1bf0c5d990"
        },
        "id": "koF0o99G3Tsv",
        "outputId": "bb4c2035-6562-4cb4-dacd-dce7357807e7"
      },
      "source": [
        "rdd_complex.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[34]: [Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10)),\n",
              " Row(col_int=2, col_double=2.5, col_bool=True, col_str=&#39;efg&#39;, col_lst=[1, 2.5, 3], col_tuple=(1.5, 2, 3), col_row=Row(x=1, y=2, z=3.5), col_dict={1.1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 42, 10)),\n",
              " Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, &#39;Two&#39;, 3.5], col_tuple=(&#39;One&#39;, 2.5, 3), col_row=Row(x=1.5, y=&#39;Two&#39;, z=3), col_dict={1: &#39;One&#39;, &#39;Two&#39;: 2}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10))]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[34]: [Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10)),\n Row(col_int=2, col_double=2.5, col_bool=True, col_str=&#39;efg&#39;, col_lst=[1, 2.5, 3], col_tuple=(1.5, 2, 3), col_row=Row(x=1, y=2, z=3.5), col_dict={1.1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 42, 10)),\n Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, &#39;Two&#39;, 3.5], col_tuple=(&#39;One&#39;, 2.5, 3), col_row=Row(x=1.5, y=&#39;Two&#39;, z=3), col_dict={1: &#39;One&#39;, &#39;Two&#39;: 2}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10))]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d11e9806-8018-4c37-8896-d016870377c1"
        },
        "id": "0Va_eJ753Tsw",
        "outputId": "5b244be3-1d45-465a-856c-0605742a2475"
      },
      "source": [
        "type(rdd_complex)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[35]: pyspark.rdd.RDD</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[35]: pyspark.rdd.RDD</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c50be2e5-68d9-4f74-afad-ceeeab4d4178"
        },
        "id": "CXji7i9c3Tsx",
        "outputId": "0bbd647e-4899-4dc3-ede5-8089e5514456"
      },
      "source": [
        "df_complex = rdd_complex.toDF()\n",
        "df_complex"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[36]: DataFrame[col_int: bigint, col_double: double, col_bool: boolean, col_str: string, col_lst: array&lt;bigint&gt;, col_tuple: struct&lt;_1:bigint,_2:bigint,_3:bigint&gt;, col_row: struct&lt;x:bigint,y:bigint,z:bigint&gt;, col_dict: map&lt;bigint,string&gt;, col_datetime: timestamp]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[36]: DataFrame[col_int: bigint, col_double: double, col_bool: boolean, col_str: string, col_lst: array&lt;bigint&gt;, col_tuple: struct&lt;_1:bigint,_2:bigint,_3:bigint&gt;, col_row: struct&lt;x:bigint,y:bigint,z:bigint&gt;, col_dict: map&lt;bigint,string&gt;, col_datetime: timestamp]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "17c93780-78f7-41f6-ab78-228a3379d753"
        },
        "id": "PqRMRrFt3Tsy",
        "outputId": "1c984b06-e8ea-4e9f-cb8a-5927e2aa7074"
      },
      "source": [
        "df_complex.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|      col_tuple|        col_row|            col_dict|       col_datetime|\n",
              "+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n",
              "      1|       1.5|    true|    abc|      [1, 2, 3]|      {1, 2, 3}|      {1, 2, 3}|{1 -&gt; One, 2 -&gt; Two}|2021-06-29 08:34:10|\n",
              "      2|       2.5|    true|    efg|   [1, null, 3]|   {null, 2, 3}|   {1, 2, null}|{2 -&gt; Two, 0 -&gt; One}|2021-06-29 08:42:10|\n",
              "      3|       3.5|   false|    hij|[1, null, null]|{null, null, 3}|{null, null, 3}|  {1 -&gt; One, 0 -&gt; 2}|2021-06-29 08:44:10|\n",
              "+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n|col_int|col_double|col_bool|col_str|        col_lst|      col_tuple|        col_row|            col_dict|       col_datetime|\n+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n|      1|       1.5|    true|    abc|      [1, 2, 3]|      {1, 2, 3}|      {1, 2, 3}|{1 -&gt; One, 2 -&gt; Two}|2021-06-29 08:34:10|\n|      2|       2.5|    true|    efg|   [1, null, 3]|   {null, 2, 3}|   {1, 2, null}|{2 -&gt; Two, 0 -&gt; One}|2021-06-29 08:42:10|\n|      3|       3.5|   false|    hij|[1, null, null]|{null, null, 3}|{null, null, 3}|  {1 -&gt; One, 0 -&gt; 2}|2021-06-29 08:44:10|\n+-------+----------+--------+-------+---------------+---------------+---------------+--------------------+-------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "87c8105b-5976-4327-8bd3-b6849fbf68ad"
        },
        "id": "W1NqRPsO3Tsy"
      },
      "source": [
        "We can see that if all entries in list, tuple, dictionary (keys in case of dictionary), row data structures in RDD are of same data type then only all values will be added in DataFrame. If any value is of different data type then that value will be stored as null.\n",
        "DataFrame will consider data type of any column on the basis of data structure of first entry in that column."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "1b79b92d-a2e0-4342-8fe4-138b52f9e7a5"
        },
        "id": "axU6l4hT3Tsz",
        "outputId": "0eb533df-5f1e-4921-9b5a-c66112aa4d3f"
      },
      "source": [
        "rdd_complex1 = sc.parallelize([\n",
        "   Row(col_int = 3,\n",
        "      col_double = 3.5,\n",
        "      col_bool = False,\n",
        "      col_str = 'hij',\n",
        "      col_lst = [1, 'Two', 3.5],\n",
        "      col_tuple = ('One', 2.5, 3),\n",
        "      col_row = Row(x=1.5, y='Two', z=3),\n",
        "      col_dict = {1:'One', 'Two':2},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 44, 10)\n",
        "  ),\n",
        "  Row(col_int = 1,\n",
        "      col_double = 1.5,\n",
        "      col_bool = True,\n",
        "      col_str = 'abc',\n",
        "      col_lst = [1, 2, 3],\n",
        "      col_tuple = (1, 2, 3),\n",
        "      col_row = Row(x=1, y=2, z=3),\n",
        "      col_dict = {1:'One', 2:'Two'},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 34, 10)\n",
        "  ),\n",
        "  Row(col_int = 2,\n",
        "      col_double = 2.5,\n",
        "      col_bool = True,\n",
        "      col_str = 'efg',\n",
        "      col_lst = [1, 2.5, 3],\n",
        "      col_tuple = (1.5, 2, 3),\n",
        "      col_row = Row(x=1, y=2, z=3.5),\n",
        "      col_dict = {1.1:'One', 2:'Two'},\n",
        "      col_datetime = datetime(2021, 6, 29, 8, 42, 10)\n",
        "  )\n",
        "])\n",
        "\n",
        "rdd_complex1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[38]: ParallelCollectionRDD[48] at readRDDFromInputStream at PythonRDD.scala:413</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[38]: ParallelCollectionRDD[48] at readRDDFromInputStream at PythonRDD.scala:413</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "6357ef90-ba06-4a5d-8c80-f5372b9bc01e"
        },
        "id": "67cex3UO3Ts2",
        "outputId": "da90628f-556e-4e4c-cdef-04be48dd492d"
      },
      "source": [
        "rdd_complex1.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[39]: [Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, &#39;Two&#39;, 3.5], col_tuple=(&#39;One&#39;, 2.5, 3), col_row=Row(x=1.5, y=&#39;Two&#39;, z=3), col_dict={1: &#39;One&#39;, &#39;Two&#39;: 2}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10)),\n",
              " Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10)),\n",
              " Row(col_int=2, col_double=2.5, col_bool=True, col_str=&#39;efg&#39;, col_lst=[1, 2.5, 3], col_tuple=(1.5, 2, 3), col_row=Row(x=1, y=2, z=3.5), col_dict={1.1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 42, 10))]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[39]: [Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, &#39;Two&#39;, 3.5], col_tuple=(&#39;One&#39;, 2.5, 3), col_row=Row(x=1.5, y=&#39;Two&#39;, z=3), col_dict={1: &#39;One&#39;, &#39;Two&#39;: 2}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10)),\n Row(col_int=1, col_double=1.5, col_bool=True, col_str=&#39;abc&#39;, col_lst=[1, 2, 3], col_tuple=(1, 2, 3), col_row=Row(x=1, y=2, z=3), col_dict={1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 34, 10)),\n Row(col_int=2, col_double=2.5, col_bool=True, col_str=&#39;efg&#39;, col_lst=[1, 2.5, 3], col_tuple=(1.5, 2, 3), col_row=Row(x=1, y=2, z=3.5), col_dict={1.1: &#39;One&#39;, 2: &#39;Two&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 42, 10))]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "4464219c-451e-46e4-bd0e-06d3786dcfbf"
        },
        "id": "wMPtP5gr3Ts7",
        "outputId": "74019467-b2c1-4da0-e5b9-4fedd64fd08b"
      },
      "source": [
        "df_complex1 = rdd_complex1.toDF()\n",
        "df_complex1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[40]: DataFrame[col_int: bigint, col_double: double, col_bool: boolean, col_str: string, col_lst: array&lt;bigint&gt;, col_tuple: struct&lt;_1:string,_2:double,_3:bigint&gt;, col_row: struct&lt;x:double,y:string,z:bigint&gt;, col_dict: map&lt;bigint,string&gt;, col_datetime: timestamp]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[40]: DataFrame[col_int: bigint, col_double: double, col_bool: boolean, col_str: string, col_lst: array&lt;bigint&gt;, col_tuple: struct&lt;_1:string,_2:double,_3:bigint&gt;, col_row: struct&lt;x:double,y:string,z:bigint&gt;, col_dict: map&lt;bigint,string&gt;, col_datetime: timestamp]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "bf553859-2cb4-4b3e-b35a-697c3edc2028"
        },
        "id": "_uui8oR03Ts8",
        "outputId": "23839bdb-b788-42fe-92af-c8a6ba9d811c"
      },
      "source": [
        "df_complex1.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|     col_tuple|        col_row|            col_dict|       col_datetime|\n",
              "+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n",
              "      3|       3.5|   false|    hij|[1, null, null]| {One, 2.5, 3}|  {1.5, Two, 3}|  {1 -&gt; One, 0 -&gt; 2}|2021-06-29 08:44:10|\n",
              "      1|       1.5|    true|    abc|      [1, 2, 3]|  {1, null, 3}|   {null, 2, 3}|{1 -&gt; One, 2 -&gt; Two}|2021-06-29 08:34:10|\n",
              "      2|       2.5|    true|    efg|   [1, null, 3]|{1.5, null, 3}|{null, 2, null}|{2 -&gt; Two, 0 -&gt; One}|2021-06-29 08:42:10|\n",
              "+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n|col_int|col_double|col_bool|col_str|        col_lst|     col_tuple|        col_row|            col_dict|       col_datetime|\n+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n|      3|       3.5|   false|    hij|[1, null, null]| {One, 2.5, 3}|  {1.5, Two, 3}|  {1 -&gt; One, 0 -&gt; 2}|2021-06-29 08:44:10|\n|      1|       1.5|    true|    abc|      [1, 2, 3]|  {1, null, 3}|   {null, 2, 3}|{1 -&gt; One, 2 -&gt; Two}|2021-06-29 08:34:10|\n|      2|       2.5|    true|    efg|   [1, null, 3]|{1.5, null, 3}|{null, 2, null}|{2 -&gt; Two, 0 -&gt; One}|2021-06-29 08:42:10|\n+-------+----------+--------+-------+---------------+--------------+---------------+--------------------+-------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "75bdfca5-d4f6-4a25-9f31-b031bc15305c"
        },
        "id": "7HY7ccnR3Ts9"
      },
      "source": [
        "##Creating DataFrame using SQLContext/SparkSession"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "d22d47ab-8231-43c3-90ee-14008496fb79"
        },
        "id": "NzKTgD4F3Ts9"
      },
      "source": [
        "sqlContext = SQLContext(sc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "1c52d40d-5d23-4e84-9476-ae916e07e53f"
        },
        "id": "Z0UUxfN23Ts-",
        "outputId": "fec5a44c-dc43-4531-b231-eacef983d773"
      },
      "source": [
        "sqlContext"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[5]: &lt;pyspark.sql.context.SQLContext at 0x7f9e800349a0&gt;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[5]: &lt;pyspark.sql.context.SQLContext at 0x7f9e800349a0&gt;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "9af91035-be79-4905-94b2-c532d0859bc9"
        },
        "id": "1Ih6qJA83Ts_",
        "outputId": "d266e51a-114b-4cf6-a7b6-146c1ab4f458"
      },
      "source": [
        "df_sqlCon = sqlContext.range(4) #generate a sequence of 4 consecutive numbers, create a column named 'id' and assign those numbers to id column of dataframe"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "4a3267a9-db16-40d5-a377-9c17b6c12c03"
        },
        "id": "lXcgmc6o3Ts_",
        "outputId": "5f6304b0-18f7-4c81-a119-9bf20abbf628"
      },
      "source": [
        "df_sqlCon"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[8]: DataFrame[id: bigint]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[8]: DataFrame[id: bigint]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "6febb498-9d98-4c25-97c6-1c6e53ba8905"
        },
        "id": "OpJwd_Ey3TtA",
        "outputId": "b37db281-31aa-4314-8f2f-38655a298181"
      },
      "source": [
        "df_sqlCon.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+---+\n",
              " id|\n",
              "+---+\n",
              "  0|\n",
              "  1|\n",
              "  2|\n",
              "  3|\n",
              "+---+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+---+\n| id|\n+---+\n|  0|\n|  1|\n|  2|\n|  3|\n+---+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "7e6f25e0-b721-4730-be48-31c9b37876c9"
        },
        "id": "YOxjL0cB3TtB"
      },
      "source": [
        "complex_data = [\n",
        "                 (3,\n",
        "                  3.5,\n",
        "                  False,\n",
        "                  'hij',\n",
        "                  [1, 'Two', 3.5],\n",
        "                  (1, 2.5, 3),\n",
        "                  Row(x=1, y='Two', z=3.6),\n",
        "                  {1.3:'One', 'Two':2},\n",
        "                  datetime(2021, 6, 29, 8, 44, 10)),\n",
        "                 (1,\n",
        "                  1.5,\n",
        "                  True,\n",
        "                  'abc',\n",
        "                  [1, 2, 3],\n",
        "                  (1, 2.5, 3),\n",
        "                  Row(x=1, y='Two', z=3.1),\n",
        "                  {1.8:'One', 2:'Two'},\n",
        "                  datetime(2021, 6, 29, 8, 34, 10)),\n",
        "                 (2,\n",
        "                  2.5,\n",
        "                  True,\n",
        "                  'efg',\n",
        "                  [1, 2.5, 3],\n",
        "                  (1, 2.5, 3),\n",
        "                  Row(x=1, y='2', z=3.5),\n",
        "                  {1.1:'One', 2:3},\n",
        "                  datetime(2021, 6, 29, 8, 42, 10))\n",
        "            ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "dc3140bb-77c6-4ded-8cde-8a6e48477d39"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3NpL2KB3TtC",
        "outputId": "474ad9a6-1c45-4f51-d9d7-692529f28e4d"
      },
      "source": [
        "#Here we can also use SparkSession instead of SQLContext object\n",
        "df_sqlCon_complex1 = sqlContext.createDataFrame(complex_data, ['col_int', 'col_double', 'col_bool', 'col_str', 'col_lst', 'col_tuple', 'col_row',\n",
        "                                                               'col_dict', 'col_datetime'])\n",
        "\n",
        "df_sqlCon_complex1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[col_int: bigint, col_double: double, col_bool: boolean, col_str: string, col_lst: array<bigint>, col_tuple: struct<_1:bigint,_2:double,_3:bigint>, col_row: struct<x:bigint,y:string,z:double>, col_dict: map<double,string>, col_datetime: timestamp]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "2b29d2b1-5293-4fd4-8a90-94ec488f3f2f"
        },
        "id": "NgTuLm-Y3TtD",
        "outputId": "e49ca37f-79b5-4267-e698-bbe1b001df70"
      },
      "source": [
        "df_sqlCon_complex1.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n",
              "      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10|\n",
              "      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n|      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10|\n|      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "9fcd9cb5-4b6c-4798-9db2-c629c1c66a08"
        },
        "id": "iNh_uGhO3TtE"
      },
      "source": [
        "**Here we can see that data type of elements at same index across all rows of dataframe for same column should be same, else it will throw error while creating dataframe. For example, element 1 of col_tuple should be of same data type across all the rows **"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "806f368c-e268-45f5-a16d-0c813be1847c"
        },
        "id": "pz1s-i3y3TtF"
      },
      "source": [
        "###Accessing the contents of DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "f6b473c2-0279-4ae5-a157-8873fa6c558a"
        },
        "id": "k9c0QCCE3TtG",
        "outputId": "b00b89b5-7ad7-48ee-a26a-8f7690f96b9f"
      },
      "source": [
        "df_sqlCon_complex1.first() #returns first row of dataframe in the form of row data structure"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[11]: Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, None, None], col_tuple=Row(_1=1, _2=2.5, _3=3), col_row=Row(x=1, y=&#39;Two&#39;, z=3.6), col_dict={0.0: &#39;2&#39;, 1.3: &#39;One&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10))</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[11]: Row(col_int=3, col_double=3.5, col_bool=False, col_str=&#39;hij&#39;, col_lst=[1, None, None], col_tuple=Row(_1=1, _2=2.5, _3=3), col_row=Row(x=1, y=&#39;Two&#39;, z=3.6), col_dict={0.0: &#39;2&#39;, 1.3: &#39;One&#39;}, col_datetime=datetime.datetime(2021, 6, 29, 8, 44, 10))</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "f7cc70fe-b142-4b29-8f70-222c2e7c309c"
        },
        "id": "gU5wzMz63TtH",
        "outputId": "05674d0e-4074-470b-97bd-7b993c9b1e44"
      },
      "source": [
        "df_sqlCon_complex1.collect()[0][7] #returns element present in first row in col_dict column"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[12]: {0.0: &#39;2&#39;, 1.3: &#39;One&#39;}</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[12]: {0.0: &#39;2&#39;, 1.3: &#39;One&#39;}</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "e0cd598d-721e-41c0-97b8-17b775feb2eb"
        },
        "id": "JPf7HnOZ3TtI",
        "outputId": "f52bc13b-6f84-4f08-fa5b-cb17a4102281"
      },
      "source": [
        "df_sqlCon_complex1.collect()[0][7][1.3] #returns value present in dictionary (for key = 1.3) from first row of col_dict column"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[23]: &#39;One&#39;</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[23]: &#39;One&#39;</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "8975f4b6-0100-42a0-9e5d-aa2d6f155959"
        },
        "id": "dOq8M7h33TtJ",
        "outputId": "fd4d042b-8e4b-440c-e332-3fd7eafd1300"
      },
      "source": [
        "df_sqlCon_complex1.show(1) #returns first row"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "only showing top 1 row\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\nonly showing top 1 row\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "12f7b803-81cc-43b4-8422-9e357e5b1945"
        },
        "id": "DkNldZG33TtK",
        "outputId": "0fc21d67-175c-4264-b85f-41c86e6de5d8"
      },
      "source": [
        "df_sqlCon_complex1.rdd #returns the rdd underlying the dataframe"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[25]: MapPartitionsRDD[23] at javaToPython at NativeMethodAccessorImpl.java:0</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[25]: MapPartitionsRDD[23] at javaToPython at NativeMethodAccessorImpl.java:0</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "4c00c454-dd7d-4d5f-abab-8dc9ca005131"
        },
        "id": "Ur5BR_-_3TtK",
        "outputId": "fdf7535c-5b73-47b3-891f-9efe62e03390"
      },
      "source": [
        "df_sqlCon_complex1.rdd.map(lambda df: (df.col_lst, df.col_tuple, df.col_dict)).collect() #returns selected columns from dataframe using rdd and map"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[28]: [([1, None, None], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;2&#39;, 1.3: &#39;One&#39;}),\n",
              " ([1, 2, 3], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;Two&#39;, 1.8: &#39;One&#39;}),\n",
              " ([1, None, 3], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;3&#39;, 1.1: &#39;One&#39;})]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[28]: [([1, None, None], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;2&#39;, 1.3: &#39;One&#39;}),\n ([1, 2, 3], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;Two&#39;, 1.8: &#39;One&#39;}),\n ([1, None, 3], Row(_1=1, _2=2.5, _3=3), {0.0: &#39;3&#39;, 1.1: &#39;One&#39;})]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c9f59cc8-dfe7-441c-b3e5-c663b39ba765"
        },
        "id": "7I3CLTY_3TtL",
        "outputId": "473a1ee2-2922-499f-b8f2-f5d0ac72218d"
      },
      "source": [
        "df_sqlCon_complex1.select('col_lst', 'col_tuple', 'col_dict').show() #returns selected columns from dataframe"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+---------------+-----------+--------------------+\n",
              "        col_lst|  col_tuple|            col_dict|\n",
              "+---------------+-----------+--------------------+\n",
              "[1, null, null]|{1, 2.5, 3}|{1.3 -&gt; One, 0.0 ...|\n",
              "      [1, 2, 3]|{1, 2.5, 3}|{1.8 -&gt; One, 0.0 ...|\n",
              "   [1, null, 3]|{1, 2.5, 3}|{0.0 -&gt; 3, 1.1 -&gt;...|\n",
              "+---------------+-----------+--------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+---------------+-----------+--------------------+\n|        col_lst|  col_tuple|            col_dict|\n+---------------+-----------+--------------------+\n|[1, null, null]|{1, 2.5, 3}|{1.3 -&gt; One, 0.0 ...|\n|      [1, 2, 3]|{1, 2.5, 3}|{1.8 -&gt; One, 0.0 ...|\n|   [1, null, 3]|{1, 2.5, 3}|{0.0 -&gt; 3, 1.1 -&gt;...|\n+---------------+-----------+--------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "181cac80-7b94-43e5-a07a-63b093c940f2"
        },
        "id": "R5rWyA4C3TtM"
      },
      "source": [
        "###Transformations on DataFrame"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "60c02563-2609-4bf3-b828-e6a0a8c1c4ec"
        },
        "id": "3B0KhFJO3TtM"
      },
      "source": [
        "**Concat first 4 columns of existing dataframe with ':' between each two of them and add them to dataframe**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "242577d9-f960-43de-bf4b-0562a341e7bf"
        },
        "id": "2xwzW86q3TtN"
      },
      "source": [
        "Approach 1: Using RDD and map (incomplete)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "07528e1e-811d-47a8-9f09-852ee5b59d68"
        },
        "id": "9PfLhl4L3TtN",
        "outputId": "4b5e1786-aa99-42d9-a04b-758fcb0b7423"
      },
      "source": [
        "df_sqlCon_complex1.rdd.map(lambda x: str(x[0]) + ':' + str(x[1])).collect() #incomplete"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[39]: [&#39;3:3.5&#39;, &#39;1:1.5&#39;, &#39;2:2.5&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[39]: [&#39;3:3.5&#39;, &#39;1:1.5&#39;, &#39;2:2.5&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c036d441-6400-41f4-9d4a-55147b5e1a2a"
        },
        "id": "UKvxeMOQ3TtO"
      },
      "source": [
        "Approach 2: Using withColumn method"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "2cff3980-0c9d-455a-9c2a-888a81c7ee7c"
        },
        "id": "JdwAFNZ23TtO",
        "outputId": "487af978-c92c-456b-f5f8-10cd12d912ab"
      },
      "source": [
        "from pyspark.sql.functions import concat_ws, col"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "697448be-7138-4ee0-bb4d-a995e4057324"
        },
        "id": "I-clZ2xJ3TtP",
        "outputId": "c7057f87-6a0b-4be5-b9d1-9aeae60daa8c"
      },
      "source": [
        "concat_columns = df_sqlCon_complex1.columns #returns list of column names of dataframe\n",
        "concat_columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[53]: [&#39;col_int&#39;,\n",
              " &#39;col_double&#39;,\n",
              " &#39;col_bool&#39;,\n",
              " &#39;col_str&#39;,\n",
              " &#39;col_lst&#39;,\n",
              " &#39;col_tuple&#39;,\n",
              " &#39;col_row&#39;,\n",
              " &#39;col_dict&#39;,\n",
              " &#39;col_datetime&#39;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[53]: [&#39;col_int&#39;,\n &#39;col_double&#39;,\n &#39;col_bool&#39;,\n &#39;col_str&#39;,\n &#39;col_lst&#39;,\n &#39;col_tuple&#39;,\n &#39;col_row&#39;,\n &#39;col_dict&#39;,\n &#39;col_datetime&#39;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "caaadbcb-0066-45e3-8ce1-063ca32ee05d"
        },
        "id": "4aDm2f7G3TtQ",
        "outputId": "0c418b2c-8f3f-43de-f973-ae0645a8a689"
      },
      "source": [
        "df_sqlCon_complex1.withColumn('transformed_col', concat_ws(':', *[col(x) for x in concat_columns[:4]])).show()\n",
        "# '*' unpacks the list or tuple into positional arguments and '**' unpacks dictionary into keyword arguments"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|transformed_col|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n",
              "      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|3:3.5:false:hij|\n",
              "      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10| 1:1.5:true:abc|\n",
              "      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10| 2:2.5:true:efg|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n|col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|            col_dict|       col_datetime|transformed_col|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n|      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|3:3.5:false:hij|\n|      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10| 1:1.5:true:abc|\n|      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10| 2:2.5:true:efg|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+---------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "add67098-7e06-4b49-80d5-874b492228b2"
        },
        "id": "FsZvpKJX3TtR"
      },
      "source": [
        "Renaming the existing column of a dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "a79aa380-7d55-4db5-a760-772bb3495acd"
        },
        "id": "ZEiUW4E73TtR",
        "outputId": "f2c9ce91-53a0-4985-a64f-803f79bec349"
      },
      "source": [
        "#1st method\n",
        "df_sqlCon_complex1.withColumnRenamed(\"col_dict\", \"col_dictionary\").show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|      col_dictionary|       col_datetime|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n",
              "      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10|\n",
              "      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10|\n",
              "+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|col_int|col_double|col_bool|col_str|        col_lst|  col_tuple|      col_row|      col_dictionary|       col_datetime|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n|      3|       3.5|   false|    hij|[1, null, null]|{1, 2.5, 3}|{1, Two, 3.6}|{1.3 -&gt; One, 0.0 ...|2021-06-29 08:44:10|\n|      1|       1.5|    true|    abc|      [1, 2, 3]|{1, 2.5, 3}|{1, Two, 3.1}|{1.8 -&gt; One, 0.0 ...|2021-06-29 08:34:10|\n|      2|       2.5|    true|    efg|   [1, null, 3]|{1, 2.5, 3}|  {1, 2, 3.5}|{0.0 -&gt; 3, 1.1 -&gt;...|2021-06-29 08:42:10|\n+-------+----------+--------+-------+---------------+-----------+-------------+--------------------+-------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "2dbf94a0-e535-45fa-a8c4-af3bc0bad6ab"
        },
        "id": "0QBOPiyG3TtS",
        "outputId": "17076ed5-c942-438e-833d-50538f2ef338"
      },
      "source": [
        "#2nd method\n",
        "df_sqlCon_complex1.select(df_sqlCon_complex1.col_dict.alias(\"col_dictionary\")).show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------------------+\n",
              "      col_dictionary|\n",
              "+--------------------+\n",
              "{1.3 -&gt; One, 0.0 ...|\n",
              "{1.8 -&gt; One, 0.0 ...|\n",
              "{0.0 -&gt; 3, 1.1 -&gt;...|\n",
              "+--------------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------------------+\n|      col_dictionary|\n+--------------------+\n|{1.3 -&gt; One, 0.0 ...|\n|{1.8 -&gt; One, 0.0 ...|\n|{0.0 -&gt; 3, 1.1 -&gt;...|\n+--------------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "48ba7299-4977-4532-ba2d-dde88f3edce1"
        },
        "id": "w3n7Mgv93TtT"
      },
      "source": [
        "##Mapping header to an existing RDD and then converting that RDD to DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "8333e1ad-00de-4f51-880b-b5fc3194f3ad"
        },
        "id": "DwX1cdy53TtU",
        "outputId": "f29468b1-b128-496b-c731-c26c63138f94"
      },
      "source": [
        "data_rdd = sc.parallelize([Row(\"Pavan\", 'ASE'),\n",
        "                           Row(\"Priyanka\", \"SE\")\n",
        "                          ])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "c725e2ee-c41a-411a-a273-36cb25740894"
        },
        "id": "9rCgsAb23TtU",
        "outputId": "7d673817-1e10-4fa0-e010-3d2b5f8c5dba"
      },
      "source": [
        "data_rdd.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[16]: [&lt;Row(&#39;Pavan&#39;, &#39;ASE&#39;)&gt;, &lt;Row(&#39;Priyanka&#39;, &#39;SE&#39;)&gt;]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[16]: [&lt;Row(&#39;Pavan&#39;, &#39;ASE&#39;)&gt;, &lt;Row(&#39;Priyanka&#39;, &#39;SE&#39;)&gt;]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "b801ca76-6c69-4495-98b5-461a7d871055"
        },
        "id": "0aHx-37t3TtV",
        "outputId": "885a74ff-1012-4ce2-930c-c826c0fb3e41"
      },
      "source": [
        "column_names = Row(\"Name\", \"Designation\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ce7a63b2-0aa4-41f6-bd24-ce8b684f2be8"
        },
        "id": "2inI_LSO3TtW",
        "outputId": "b32208c2-c800-43e3-fac8-1b0bf4cf46e1"
      },
      "source": [
        "data_rdd = data_rdd.map(lambda row: column_names(*row))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ad9afb9f-167e-4945-b20a-fbb8d4d12bad"
        },
        "id": "rrRlYvye3TtX",
        "outputId": "868a325e-ebcd-46d9-bcf7-ca5dd4d78c75"
      },
      "source": [
        "data_rdd.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[51]: [Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;), Row(Name=&#39;Priyanka&#39;, Designation=&#39;SE&#39;)]</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[51]: [Row(Name=&#39;Pavan&#39;, Designation=&#39;ASE&#39;), Row(Name=&#39;Priyanka&#39;, Designation=&#39;SE&#39;)]</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "5590206c-7cf7-4559-8504-7a408aa0eca3"
        },
        "id": "mzhobeIE3TtY",
        "outputId": "f063196d-6fdc-4618-de1b-e32808dffb4d"
      },
      "source": [
        "data_rdd.toDF().show() #Converting RDD to DatFrame and then printing the content of DataFrame"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------+-----------+\n",
              "    Name|Designation|\n",
              "+--------+-----------+\n",
              "   Pavan|        ASE|\n",
              "Priyanka|         SE|\n",
              "+--------+-----------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------+-----------+\n|    Name|Designation|\n+--------+-----------+\n|   Pavan|        ASE|\n|Priyanka|         SE|\n+--------+-----------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "dc66c832-b83e-4898-8ebf-8ac3a3b228bd"
        },
        "id": "OeJrt7y03TtY",
        "outputId": "1f524e1c-3e3d-4705-aa7c-6650adb7afc4"
      },
      "source": [
        "df_1 = sqlContext.createDataFrame(data_rdd)#Creating DataFrame from RDD using SQLContext and then printing the content of DataFrame\n",
        "df_1.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------+-----------+\n",
              "    Name|Designation|\n",
              "+--------+-----------+\n",
              "   Pavan|        ASE|\n",
              "Priyanka|         SE|\n",
              "+--------+-----------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------+-----------+\n|    Name|Designation|\n+--------+-----------+\n|   Pavan|        ASE|\n|Priyanka|         SE|\n+--------+-----------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "6c7fcbc4-9754-4770-bf32-7024a63966ef"
        },
        "id": "ME20mFtP3TtZ",
        "outputId": "3e05e6ed-ddec-4f78-ac1f-eefca2185e61"
      },
      "source": [
        "data_rdd1 = data_rdd.map(lambda row: column_names(row)) #if we will not pass elements of rdd as positional arguments to column_names then it will just assign single header "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ab3f962e-6995-410d-9966-cbf4e8be0866"
        },
        "id": "fpb9uf2T3Tta",
        "outputId": "1c67bddb-528d-4fa7-9f31-1803ed9ca12c"
      },
      "source": [
        "data_rdd1.toDF().show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------------+\n",
              "          Name|\n",
              "+--------------+\n",
              "  {Pavan, ASE}|\n",
              "{Priyanka, SE}|\n",
              "+--------------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------------+\n|          Name|\n+--------------+\n|  {Pavan, ASE}|\n|{Priyanka, SE}|\n+--------------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "1124326a-782b-4eeb-a546-de59c8039be6"
        },
        "id": "63Umq0Ml3Ttb"
      },
      "source": [
        "##Creating Pandas DataFrame from Spark DataFrame and vice-versa"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "bf73f883-d068-4b34-977e-fb3855c89a3f"
        },
        "id": "pceXytL43Ttc",
        "outputId": "a7c25570-2ca1-4968-9fe6-ceee3ad93631"
      },
      "source": [
        "import pandas"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\"></div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\"></div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "898c1935-4927-4939-8db5-78cdda16cf30"
        },
        "id": "LguTHBPJ3Ttd",
        "outputId": "70932bb5-dc60-416f-a591-9e918f8f7d68"
      },
      "source": [
        "pandas_df = df_1.toPandas()\n",
        "pandas_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">Out[25]: </div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">Out[25]: </div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Designation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Pavan</td>\n",
              "      <td>ASE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Priyanka</td>\n",
              "      <td>SE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Name</th>\n      <th>Designation</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Pavan</td>\n      <td>ASE</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Priyanka</td>\n      <td>SE</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
              "textData": null,
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "htmlSandbox",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "33a81663-8de8-412a-a8dd-d5a92a979f8b"
        },
        "id": "4ACfCbkV3Ttd",
        "outputId": "1a0f0631-e920-4666-886b-9489b4d8a924"
      },
      "source": [
        "spark_df = sqlContext.createDataFrame(pandas_df)\n",
        "spark_df.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>\n",
              "<div class=\"ansiout\">+--------+-----------+\n",
              "    Name|Designation|\n",
              "+--------+-----------+\n",
              "   Pavan|        ASE|\n",
              "Priyanka|         SE|\n",
              "+--------+-----------+\n",
              "\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "datasetInfos": [],
              "data": "<div class=\"ansiout\">+--------+-----------+\n|    Name|Designation|\n+--------+-----------+\n|   Pavan|        ASE|\n|Priyanka|         SE|\n+--------+-----------+\n\n</div>",
              "removedWidgets": [],
              "addedWidgets": {},
              "metadata": {},
              "type": "html",
              "arguments": {}
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "title": "",
          "showTitle": false,
          "inputWidgets": {},
          "nuid": "ebeb4f0c-e032-4db2-afcd-c2d6f8e5f4c0"
        },
        "id": "uFGXCDq53Tte",
        "outputId": "eb1b5ca5-5253-4dd4-b24f-cd7d11c4dc2c"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n",
              "  .ansiout {\n",
              "    display: block;\n",
              "    unicode-bidi: embed;\n",
              "    white-space: pre-wrap;\n",
              "    word-wrap: break-word;\n",
              "    word-break: break-all;\n",
              "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
              "    font-size: 13px;\n",
              "    color: #555;\n",
              "    margin-left: 4px;\n",
              "    line-height: 19px;\n",
              "  }\n",
              "</style>"
            ]
          },
          "metadata": {
            "tags": [],
            "application/vnd.databricks.v1+output": {
              "data": "",
              "errorSummary": "",
              "metadata": {},
              "errorTraceType": null,
              "type": "ipynbError",
              "arguments": {}
            }
          }
        }
      ]
    }
  ]
}