data_insight.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO7pRT2WfuhAyoZejLhvPTS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Saherpathan/FinTech/blob/master/data_insightipynb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k3c3YMDLsEXO",
        "outputId": "219cc698-b44d-4292-92d5-7d1559875146"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: sec-edgar-downloader in /usr/local/lib/python3.10/dist-packages (5.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from sec-edgar-downloader) (2.31.0)\n",
            "Requirement already satisfied: pyrate-limiter>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from sec-edgar-downloader) (3.6.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->sec-edgar-downloader) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->sec-edgar-downloader) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->sec-edgar-downloader) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->sec-edgar-downloader) (2024.2.2)\n"
          ]
        }
      ],
      "source": [
        "pip install sec-edgar-downloader\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sec_edgar_downloader import Downloader\n",
        "import os\n"
      ],
      "metadata": {
        "id": "35b7EA3JsIb9"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def download_10k_filings(companies):\n",
        "    # Initialize downloader\n",
        "    dl = Downloader(company_name=\"Company Name\", email_address=\"email@example.com\")\n",
        "\n",
        "    for company in companies:\n",
        "        # Define the directory to save filings\n",
        "        save_dir = f\"./{company}_10k_filings\"\n",
        "\n",
        "        # Create directory if it doesn't exist\n",
        "        if not os.path.exists(save_dir):\n",
        "            os.makedirs(save_dir)\n",
        "\n",
        "        # Download filings for each year from 1995 through 2023\n",
        "        for year in range(1995, 2024):\n",
        "            try:\n",
        "                dl.get(\"10-K\", company)\n",
        "                print(f\"Downloaded 10-K filing for {company} for the year {year}\")\n",
        "            except Exception as e:\n",
        "                print(f\"Failed to download filing for {company} for the year {year}: {e}\")\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    companies = [\"AAPL\", \"GOOGL\", \"MSFT\"]  # Example companies\n",
        "    download_10k_filings(companies)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1CEl0QyVsLIw",
        "outputId": "381b458d-cd1e-4d82-d666-56644c51507f"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloaded 10-K filing for AAPL for the year 1995\n",
            "Downloaded 10-K filing for AAPL for the year 1996\n",
            "Downloaded 10-K filing for AAPL for the year 1997\n",
            "Downloaded 10-K filing for AAPL for the year 1998\n",
            "Downloaded 10-K filing for AAPL for the year 1999\n",
            "Downloaded 10-K filing for AAPL for the year 2000\n",
            "Downloaded 10-K filing for AAPL for the year 2001\n",
            "Downloaded 10-K filing for AAPL for the year 2002\n",
            "Downloaded 10-K filing for AAPL for the year 2003\n",
            "Downloaded 10-K filing for AAPL for the year 2004\n",
            "Downloaded 10-K filing for AAPL for the year 2005\n",
            "Downloaded 10-K filing for AAPL for the year 2006\n",
            "Downloaded 10-K filing for AAPL for the year 2007\n",
            "Downloaded 10-K filing for AAPL for the year 2008\n",
            "Downloaded 10-K filing for AAPL for the year 2009\n",
            "Downloaded 10-K filing for AAPL for the year 2010\n",
            "Downloaded 10-K filing for AAPL for the year 2011\n",
            "Downloaded 10-K filing for AAPL for the year 2012\n",
            "Downloaded 10-K filing for AAPL for the year 2013\n",
            "Downloaded 10-K filing for AAPL for the year 2014\n",
            "Downloaded 10-K filing for AAPL for the year 2015\n",
            "Downloaded 10-K filing for AAPL for the year 2016\n",
            "Downloaded 10-K filing for AAPL for the year 2017\n",
            "Downloaded 10-K filing for AAPL for the year 2018\n",
            "Downloaded 10-K filing for AAPL for the year 2019\n",
            "Downloaded 10-K filing for AAPL for the year 2020\n",
            "Downloaded 10-K filing for AAPL for the year 2021\n",
            "Downloaded 10-K filing for AAPL for the year 2022\n",
            "Downloaded 10-K filing for AAPL for the year 2023\n",
            "Downloaded 10-K filing for GOOGL for the year 1995\n",
            "Downloaded 10-K filing for GOOGL for the year 1996\n",
            "Downloaded 10-K filing for GOOGL for the year 1997\n",
            "Downloaded 10-K filing for GOOGL for the year 1998\n",
            "Downloaded 10-K filing for GOOGL for the year 1999\n",
            "Downloaded 10-K filing for GOOGL for the year 2000\n",
            "Downloaded 10-K filing for GOOGL for the year 2001\n",
            "Downloaded 10-K filing for GOOGL for the year 2002\n",
            "Downloaded 10-K filing for GOOGL for the year 2003\n",
            "Downloaded 10-K filing for GOOGL for the year 2004\n",
            "Downloaded 10-K filing for GOOGL for the year 2005\n",
            "Downloaded 10-K filing for GOOGL for the year 2006\n",
            "Downloaded 10-K filing for GOOGL for the year 2007\n",
            "Downloaded 10-K filing for GOOGL for the year 2008\n",
            "Downloaded 10-K filing for GOOGL for the year 2009\n",
            "Downloaded 10-K filing for GOOGL for the year 2010\n",
            "Downloaded 10-K filing for GOOGL for the year 2011\n",
            "Downloaded 10-K filing for GOOGL for the year 2012\n",
            "Downloaded 10-K filing for GOOGL for the year 2013\n",
            "Downloaded 10-K filing for GOOGL for the year 2014\n",
            "Downloaded 10-K filing for GOOGL for the year 2015\n",
            "Downloaded 10-K filing for GOOGL for the year 2016\n",
            "Downloaded 10-K filing for GOOGL for the year 2017\n",
            "Downloaded 10-K filing for GOOGL for the year 2018\n",
            "Downloaded 10-K filing for GOOGL for the year 2019\n",
            "Downloaded 10-K filing for GOOGL for the year 2020\n",
            "Downloaded 10-K filing for GOOGL for the year 2021\n",
            "Downloaded 10-K filing for GOOGL for the year 2022\n",
            "Downloaded 10-K filing for GOOGL for the year 2023\n",
            "Downloaded 10-K filing for MSFT for the year 1995\n",
            "Downloaded 10-K filing for MSFT for the year 1996\n",
            "Downloaded 10-K filing for MSFT for the year 1997\n",
            "Downloaded 10-K filing for MSFT for the year 1998\n",
            "Downloaded 10-K filing for MSFT for the year 1999\n",
            "Downloaded 10-K filing for MSFT for the year 2000\n",
            "Downloaded 10-K filing for MSFT for the year 2001\n",
            "Downloaded 10-K filing for MSFT for the year 2002\n",
            "Downloaded 10-K filing for MSFT for the year 2003\n",
            "Downloaded 10-K filing for MSFT for the year 2004\n",
            "Downloaded 10-K filing for MSFT for the year 2005\n",
            "Downloaded 10-K filing for MSFT for the year 2006\n",
            "Downloaded 10-K filing for MSFT for the year 2007\n",
            "Downloaded 10-K filing for MSFT for the year 2008\n",
            "Downloaded 10-K filing for MSFT for the year 2009\n",
            "Downloaded 10-K filing for MSFT for the year 2010\n",
            "Downloaded 10-K filing for MSFT for the year 2011\n",
            "Downloaded 10-K filing for MSFT for the year 2012\n",
            "Downloaded 10-K filing for MSFT for the year 2013\n",
            "Downloaded 10-K filing for MSFT for the year 2014\n",
            "Downloaded 10-K filing for MSFT for the year 2015\n",
            "Downloaded 10-K filing for MSFT for the year 2016\n",
            "Downloaded 10-K filing for MSFT for the year 2017\n",
            "Downloaded 10-K filing for MSFT for the year 2018\n",
            "Downloaded 10-K filing for MSFT for the year 2019\n",
            "Downloaded 10-K filing for MSFT for the year 2020\n",
            "Downloaded 10-K filing for MSFT for the year 2021\n",
            "Downloaded 10-K filing for MSFT for the year 2022\n",
            "Downloaded 10-K filing for MSFT for the year 2023\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install google-generativeai"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0nZSilqQXG9f",
        "outputId": "ffb209e2-d821-4847-b321-50de09e137db"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: google-generativeai in /usr/local/lib/python3.10/dist-packages (0.5.2)\n",
            "Requirement already satisfied: google-ai-generativelanguage==0.6.2 in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (0.6.2)\n",
            "Requirement already satisfied: google-api-core in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (2.11.1)\n",
            "Requirement already satisfied: google-api-python-client in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (2.84.0)\n",
            "Requirement already satisfied: google-auth>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (2.27.0)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (3.20.3)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (2.7.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from google-generativeai) (4.11.0)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /usr/local/lib/python3.10/dist-packages (from google-ai-generativelanguage==0.6.2->google-generativeai) (1.23.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=2.15.0->google-generativeai) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=2.15.0->google-generativeai) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=2.15.0->google-generativeai) (4.9)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core->google-generativeai) (1.63.0)\n",
            "Requirement already satisfied: requests<3.0.0.dev0,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-api-core->google-generativeai) (2.31.0)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client->google-generativeai) (0.22.0)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client->google-generativeai) (0.1.1)\n",
            "Requirement already satisfied: uritemplate<5,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client->google-generativeai) (4.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic->google-generativeai) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic->google-generativeai) (2.18.2)\n",
            "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core->google-generativeai) (1.63.0)\n",
            "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core->google-generativeai) (1.48.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2<1dev,>=0.15.0->google-api-python-client->google-generativeai) (3.1.2)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=2.15.0->google-generativeai) (0.6.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core->google-generativeai) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core->google-generativeai) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core->google-generativeai) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core->google-generativeai) (2024.2.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install python-dotenv\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pf006M7dXLf_",
        "outputId": "a139f344-58ff-43f7-a108-f8cf81e04e63"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (1.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import requests\n",
        "from dotenv import load_dotenv\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.probability import FreqDist\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Download NLTK resources\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "\n",
        "# Load environment variables\n",
        "load_dotenv()\n",
        "\n",
        "# Load Google API key from environment variable\n",
        "GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')\n",
        "\n",
        "# Configure Google LLM API\n",
        "import google.generativeai as genai\n",
        "genai.configure(api_key=GOOGLE_API_KEY)\n",
        "\n",
        "# Function to read text file\n",
        "def read_text_file(file_path):\n",
        "    with open(file_path, 'r', encoding='utf-8') as file:\n",
        "        text_data = file.read()\n",
        "    return text_data\n",
        "\n",
        "# Function to perform text analysis\n",
        "def text_analysis(data):\n",
        "    # Tokenize text\n",
        "    tokens = word_tokenize(data)\n",
        "\n",
        "    # Remove stopwords\n",
        "    stop_words = set(stopwords.words(\"english\"))\n",
        "    filtered_tokens = [word for word in tokens if word.lower() not in stop_words]\n",
        "\n",
        "    # Calculate word frequencies\n",
        "    freq_dist = FreqDist(filtered_tokens)\n",
        "    top_words = freq_dist.most_common(10)  # Get top 10 words\n",
        "\n",
        "    return top_words\n",
        "\n",
        "# Function to generate visualizations\n",
        "def visualize_data(insights):\n",
        "    if insights:\n",
        "        # Create a bar chart of top words\n",
        "        words, frequencies = zip(*insights)\n",
        "        plt.bar(words, frequencies)\n",
        "        plt.xlabel(\"Words\")\n",
        "        plt.ylabel(\"Frequency\")\n",
        "        plt.title(\"Top Words in 10-K Filings\")\n",
        "        plt.savefig('fig.png')\n",
        "        plt.show()\n",
        "    else:\n",
        "        print(\"No insights found.\")\n",
        "\n",
        "# Define the path to the directory containing 10-K filings\n",
        "directory_path = \"./sec-edgar-filings/AAPL/10-K\"\n",
        "\n",
        "# Read all 10-K filings and perform text analysis\n",
        "insights = []\n",
        "for filename in os.listdir(directory_path):\n",
        "    if filename.endswith(\".txt\"):\n",
        "        file_path = os.path.join(directory_path, filename)\n",
        "        text_data = read_text_file(file_path)\n",
        "        insights += text_analysis(text_data)\n",
        "\n",
        "# Generate visualizations\n",
        "visualize_data(insights)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lWwxb4XOWxj6",
        "outputId": "56fa82f8-6811-479a-d712-6cfad4083c2b"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No insights found.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ]
    }
  ]
}