Merolagani_rotating_proxy.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Welcome To Colaboratory",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/satishchaudhary382/algorithmic-trading-with-python/blob/master/Merolagani_rotating_proxy.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O0N-HhqXlqJK"
      },
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from random import choice\n",
        "# url = \"http://httpbin.org/ip\"\n",
        "\n",
        "\n",
        "# proxies = {\n",
        "#     'https':\"92.204.129.161:80\",\n",
        "#     'http':'92.204.129.161:80'\n",
        "# # }\n",
        "# resp = requests.get(url,proxies = proxies)\n",
        "# print(resp.json)\n",
        "# print(resp.text)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IAUbpu6QoxMY"
      },
      "source": [
        "def get_proxy():\n",
        "  url = 'https://www.sslproxies.org/'\n",
        "  r = requests.get(url)\n",
        "  soup = BeautifulSoup(r.content, 'html5lib')\n",
        "  return {\"https\" : choice(list(map(lambda x: x[0]+\":\"+x[1],list(zip(map(lambda x: x.text,(soup.findAll(\"td\")[::8])),\n",
        "         map(lambda x: x.text,(soup.findAll(\"td\")[1::8])))))))}\n",
        "\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "poKXqYd_qqYt"
      },
      "source": [
        "def request_proxy(request_type, url, **kwrags):\n",
        "  while 1:\n",
        "    try:\n",
        "      proxy = get_proxy()\n",
        "      r = requests.request(request_type,url, proxies=proxy, timeout =5,**kwargs)\n",
        "      break\n",
        "    except:\n",
        "      pass\n",
        "  return r\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "xfmpCcMW6_vY"
      },
      "source": [
        "#@title Libraries\n",
        "import time\n",
        "import datetime\n",
        "import pandas as pd\n",
        "import numpy as np \n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "from fake_useragent import UserAgent\n",
        "import requests\n",
        "\n",
        "from selenium import webdriver\n",
        "from market_profile import MarketProfile\n",
        "import random\n",
        "import pandas_ta as ta\n",
        "useragent = UserAgent()\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.pyplot as plt\n",
        "from bokeh.plotting import figure, output_file, show\n",
        "import bokeh\n",
        "from sklearn.preprocessing import StandardScaler,MinMaxScaler\n",
        "\n",
        "import altair as alt\n",
        "from math import pi\n",
        "from bokeh.io import output_notebook\n",
        "from bokeh.models import Label\n",
        "from datetime import datetime as dt\n",
        "# Call once to configure Bokeh to display plots inline in the notebook.\n",
        "output_notebook()\n",
        "%matplotlib inline\n",
        "from math import pi\n",
        "from bokeh.models import Span\n",
        "from mpl_finance import candlestick_ohlc\n",
        "import matplotlib.dates as mpl_dates\n",
        "import pandas_datareader as pdr\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.common.exceptions import TimeoutException"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "zohuvhftwHXf",
        "outputId": "dc9fb7b9-fb5c-4aa9-91d3-3830de9170ef",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#@title Finding IP address\n",
        "from selenium import webdriver\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.common.exceptions import TimeoutException\n",
        "\n",
        "# options = webdriver.ChromeOptions()\n",
        "# options.add_argument(\"start-maximized\")\n",
        "# options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
        "# options.add_experimental_option('useAutomationExtension', False)\n",
        "# driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\\WebDrivers\\chromedriver.exe')\n",
        "\n",
        "chrome_options = webdriver.ChromeOptions()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "chrome_options.add_argument(\"start-maximized\")\n",
        "chrome_options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
        "chrome_options.add_experimental_option('useAutomationExtension', False)\n",
        "wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "driver =webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "\n",
        "\n",
        "driver.get(\"https://sslproxies.org/\")\n",
        "driver.execute_script(\"return arguments[0].scrollIntoView(true);\", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, \"//table[@class='table table-striped table-bordered dataTable']//th[contains(., 'IP Address')]\"))))\n",
        "ips = [my_elem.get_attribute(\"innerHTML\") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, \"//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 1]\")))]\n",
        "ports = [my_elem.get_attribute(\"innerHTML\") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, \"//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 2]\")))]\n",
        "driver.quit()\n",
        "proxies = []\n",
        "for i in range(0, len(ips)):\n",
        "    proxies.append(ips[i]+':'+ports[i])\n",
        "print(proxies)\n",
        "# for i in range(0, len(proxies)):\n",
        "#     try:\n",
        "#         print(\"Proxy selected: {}\".format(proxies[i]))\n",
        "#         options = webdriver.ChromeOptions()\n",
        "#         options.add_argument('--proxy-server={}'.format(proxies[i]))\n",
        "#         driver = webdriver.Chrome(options=options, executable_path=r'C:\\WebDrivers\\chromedriver.exe')\n",
        "#         driver.get(\"https://www.whatismyip.com/proxy-check/?iref=home\")\n",
        "#         if \"Proxy Type\" in WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.CSS_SELECTOR, \"p.card-text\"))):\n",
        "#             break\n",
        "#     except Exception:\n",
        "#         driver.quit()\n",
        "# print(\"Proxy Invoked\")"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['167.172.123.221:9300', '193.168.153.165:8080', '103.154.241.252:8080', '167.172.171.115:40061', '43.231.21.176:36415', '219.92.3.149:8080', '51.222.67.208:32768', '185.18.215.109:7890', '36.89.18.217:8080', '103.103.175.253:3128', '213.230.90.106:3128', '144.217.101.245:3129', '81.95.230.211:3128', '20.195.17.90:3128', '202.61.51.204:3128', '132.248.196.2:8080', '168.119.137.56:3128', '160.19.232.85:3128', '1.4.157.35:46944', '158.51.109.34:26019']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uJ-d5DiJysMX",
        "outputId": "f864eb60-dff7-42b9-8026-ed66d48e04ff",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "p = []\n",
        "for proxy in proxies:\n",
        "  try:\n",
        "      requests.request('get',\n",
        "          \"http://httpbin.org/ip\",\n",
        "          proxies={'http':f'{proxy}'}\n",
        "      )\n",
        "      break\n",
        "  except IOError:\n",
        "      print( \"Connection error! (Check proxy)\")\n",
        "      p.append(proxy)\n",
        "      print(\"proxy available: \", proxy)\n",
        "\n",
        "      chrome_options = webdriver.ChromeOptions()\n",
        "      chrome_options.add_argument('--headless')\n",
        "      chrome_options.add_argument('--no-sandbox')\n",
        "      chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "      chrome_options.add_argument(f\"user-agent = {useragent.random}\")\n",
        "      chrome_options.add_argument(f'--proxy-server={p[0]}')\n",
        "      wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "      driver =webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "\n",
        "      delay = 10\n",
        "\n",
        "      url = \"https://merolagani.com/CompanyDetail.aspx?symbol=stc\"\n",
        "      wait = WebDriverWait(driver, 10)\n",
        "      driver.get(url)\n",
        "\n",
        "      price = wait.until(EC.element_to_be_clickable((By.XPATH,\"//a[@title='Price History']\")))\n",
        "      price.click()\n",
        "      time.sleep(5)\n",
        "\n",
        "\n",
        "      tables = driver.find_element_by_xpath(\"//table[@class='table table-bordered table-striped table-hover']\")\n",
        "      driver.execute_script(\"window.open(arguments[0], '_blank');\", tables)\n",
        "\n",
        "      data = []\n",
        "      data1 = []\n",
        "      data2 = []\n",
        "      a = 1\n",
        "      delay = 5\n",
        "\n",
        "      l = [range(2,102),range(2,99)]\n",
        "      while a<3:\n",
        "        for i in l:\n",
        "          # frame = driver.find_element_by_xpath('//*[@id=\"ctl00_ContentPlaceHolder1_CompanyDetail1_divDataPrice\"]/div[2]')\n",
        "          # driver.switch_to.frame(frame)\n",
        "          try:\n",
        "            for j in i:\n",
        "              WebDriverWait(driver, 10).until(EC.visibility_of_all_elements_located((By.ID, 'ctl00_ContentPlaceHolder1_CompanyDetail1_divDataPrice')))\n",
        "              # c = len(driver.find_element_by_xpath(f'//*[@id=\"ctl00_ContentPlaceHolder1_CompanyDetail1_divDataPrice\"]/div[2]/table/tbody/tr[{j}]'))\n",
        "              d = driver.find_element_by_xpath(f'//*[@id=\"ctl00_ContentPlaceHolder1_CompanyDetail1_divDataPrice\"]/div[2]/table/tbody/tr[{j}]')\n",
        "              data2.append(d.text)\n",
        "            driver.find_elements_by_xpath('//*[@title=\"Next Page\"]')[0].click()\n",
        "            # next.click()\n",
        "            time.sleep(5)\n",
        "            a+=1\n",
        "          except:\n",
        "            pass\n",
        "        \n",
        "        # driver.find_elements_by_xpath('//*[@title=\"Next Page\"]')[0].click()\n",
        "        #   # next.click()\n",
        "        # time.sleep(5)\n",
        "        # a+=1\n",
        "\n",
        "\n",
        "      driver.quit()\n",
        "\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Connection error! (Check proxy)\n",
            "proxy available:  167.172.123.221:9300\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5DCIVUYQ4-hs",
        "outputId": "4dbf0ca6-3268-42ec-901e-2943c72405b0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "data2"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['1 2021/05/06 10,000.00 0 10,199.00 9,850.00 10,199.00 1,297 13,006,350.00',\n",
              " '2 2021/05/05 10,000.00 -1.43 10,300.00 10,000.00 10,289.00 2,445 24,615,235.00',\n",
              " '3 2021/05/04 10,145.00 1.24 10,221.00 9,900.00 10,221.00 1,675 16,831,693.00',\n",
              " '4 2021/05/03 10,021.00 -1.85 10,414.00 10,005.00 10,400.00 2,202 22,516,830.00',\n",
              " '5 2021/05/02 10,210.00 1.09 10,500.00 10,100.00 10,302.00 4,014 40,974,928.00',\n",
              " '6 2021/04/29 10,100.00 2.8 10,194.00 9,850.00 10,020.00 1,105 11,054,810.00',\n",
              " '7 2021/04/28 9,825.00 -0.81 10,300.00 9,611.00 10,100.00 3,361 33,723,100.00',\n",
              " '8 2021/04/27 9,905.00 5.09 9,950.00 9,359.00 9,550.00 2,542 24,817,713.00',\n",
              " '9 2021/04/26 9,425.00 -4.8 9,999.00 9,220.00 9,812.00 3,780 36,189,620.00',\n",
              " '10 2021/04/25 9,900.00 -3.88 10,300.00 9,712.00 10,300.00 4,242 42,056,647.00',\n",
              " '11 2021/04/22 10,300.00 0.1 10,402.00 10,100.00 10,100.00 1,598 16,390,381.00',\n",
              " '12 2021/04/21 10,290.00 -0.05 10,444.00 10,160.00 10,301.00 1,130 11,604,300.00',\n",
              " '13 2021/04/20 10,295.00 1.02 10,612.00 10,000.00 10,000.00 4,419 45,710,705.00',\n",
              " '14 2021/04/19 10,191.00 -3.11 10,500.00 10,191.00 10,308.00 3,494 35,950,748.00',\n",
              " '15 2021/04/18 10,518.00 -3.49 11,110.00 10,477.00 11,110.00 3,621 38,365,187.00',\n",
              " '16 2021/04/15 10,898.00 -2.7 11,350.00 10,801.00 11,350.00 3,023 33,022,951.00',\n",
              " '17 2021/04/13 11,200.00 1.91 11,445.00 10,586.00 10,900.00 3,122 33,942,236.00',\n",
              " '18 2021/04/12 10,990.00 -1.88 11,345.00 10,944.00 11,344.00 1,886 21,200,861.00',\n",
              " '19 2021/04/08 11,201.00 -0.86 11,505.00 11,150.00 11,200.00 3,436 39,975,711.00',\n",
              " '20 2021/04/07 11,298.00 -4.19 11,590.00 10,765.00 11,203.00 9,152 103,860,136.00',\n",
              " '21 2021/04/06 11,792.00 -10 12,447.00 11,792.00 12,447.00 4,263 50,762,491.00',\n",
              " '22 2021/04/05 13,102.00 3.17 13,450.00 12,446.00 12,950.00 8,435 109,983,730.00',\n",
              " '23 2021/04/04 12,700.00 -4.63 13,320.00 12,500.00 13,320.00 5,803 74,250,019.00',\n",
              " '24 2021/04/01 13,317.00 9.16 13,317.00 12,000.00 12,200.00 5,962 73,935,100.00',\n",
              " '25 2021/03/31 12,200.00 -2.4 12,561.00 11,662.00 12,561.00 4,536 55,056,656.00',\n",
              " '26 2021/03/30 12,500.00 -0.71 12,830.00 12,360.00 12,400.00 2,546 32,046,835.00',\n",
              " '27 2021/03/29 12,590.00 -4.87 13,300.00 12,500.00 13,300.00 3,041 38,568,322.00',\n",
              " '28 2021/03/25 13,235.00 4.17 13,780.00 12,501.00 12,959.00 6,690 88,527,145.00',\n",
              " '29 2021/03/24 12,705.00 10 12,705.00 12,127.00 12,127.00 10,179 128,423,196.00',\n",
              " '30 2021/03/23 11,550.00 10 11,550.00 10,500.00 10,600.00 11,318 125,899,549.00',\n",
              " '31 2021/03/22 10,500.00 -0.47 10,710.00 10,430.00 10,699.00 3,587 37,800,274.00',\n",
              " '32 2021/03/21 10,550.00 0.38 10,719.00 10,434.00 10,697.00 3,318 35,044,496.00',\n",
              " '33 2021/03/18 10,510.00 -1.13 10,800.00 10,486.00 10,651.00 2,449 25,966,421.00',\n",
              " '34 2021/03/17 10,630.00 1.24 11,000.00 10,425.00 10,425.00 2,524 27,026,383.00',\n",
              " '35 2021/03/16 10,500.00 -2.05 10,900.00 10,500.00 10,506.00 2,139 22,725,567.00',\n",
              " '36 2021/03/15 10,720.00 1.58 10,900.00 10,193.00 10,764.00 2,819 30,061,791.00',\n",
              " '37 2021/03/14 10,553.00 -5.77 11,422.00 10,495.00 11,400.00 1,723 18,468,305.00',\n",
              " '38 2021/03/10 11,199.00 8.74 11,328.00 9,961.00 10,101.00 5,783 61,218,688.00',\n",
              " '39 2021/03/09 10,299.00 -1.44 10,313.00 10,050.00 10,241.00 1,917 19,528,064.00',\n",
              " '40 2021/03/07 10,450.00 -5 10,800.00 10,153.00 10,780.00 5,216 54,351,979.00',\n",
              " '41 2021/03/04 11,000.00 7.61 11,244.00 10,426.00 10,426.00 6,431 70,477,068.00',\n",
              " '42 2021/03/03 10,222.00 10 10,222.00 9,101.00 9,293.00 9,310 93,989,967.00',\n",
              " '43 2021/03/02 9,293.00 9.99 9,293.00 8,399.00 8,555.00 15,215 138,491,661.00',\n",
              " '44 2021/03/01 8,449.00 3.62 8,561.00 8,202.00 8,561.00 4,105 34,248,468.00',\n",
              " '45 2021/02/28 8,154.00 -0.42 8,415.00 8,154.00 8,190.00 2,602 21,439,814.00',\n",
              " '46 2021/02/25 8,188.00 -2 8,693.00 8,188.00 8,500.00 3,747 31,543,587.00',\n",
              " '47 2021/02/24 8,355.00 0.06 8,590.00 8,301.00 8,450.00 1,593 13,423,829.00',\n",
              " '48 2021/02/23 8,350.00 0.98 8,350.00 7,944.00 8,176.00 3,292 26,706,283.00',\n",
              " '49 2021/02/22 8,269.00 -3.29 8,500.00 8,150.00 8,500.00 1,609 13,265,072.00',\n",
              " '50 2021/02/21 8,550.00 0.59 8,660.00 8,470.00 8,660.00 1,354 11,581,611.00',\n",
              " '51 2021/02/18 8,500.00 2.67 8,612.00 8,145.00 8,444.00 4,666 39,328,603.00',\n",
              " '52 2021/02/17 8,279.00 2.21 8,279.00 8,074.00 8,260.00 2,288 18,677,335.00',\n",
              " '53 2021/02/16 8,100.00 -1.4 8,377.00 8,100.00 8,377.00 983 8,048,755.00',\n",
              " '54 2021/02/15 8,215.00 2.71 8,320.00 8,157.00 8,157.00 1,897 15,623,663.00',\n",
              " '55 2021/02/14 7,998.00 -0.89 8,105.00 7,851.00 8,100.00 7,193 56,736,027.00',\n",
              " '56 2021/02/11 8,070.00 -1.5 8,399.00 8,030.00 8,105.00 1,086 8,859,143.00',\n",
              " '57 2021/02/10 8,193.00 -3.49 8,485.00 8,193.00 8,327.00 3,800 31,442,402.00',\n",
              " '58 2021/02/09 8,530.00 -1.39 8,823.00 8,374.00 8,819.00 1,118 13,715,833.00',\n",
              " '59 2021/02/08 8,650.00 -0.71 8,900.00 8,562.00 8,712.00 879 7,920,444.00',\n",
              " '60 2021/02/07 8,712.00 2.51 9,015.00 8,499.00 8,499.00 2,993 26,299,675.00',\n",
              " '61 2021/02/04 8,499.00 4.93 8,525.00 8,061.00 8,100.00 6,874 57,426,488.00',\n",
              " '62 2021/02/03 8,150.00 -1.81 8,305.00 8,100.00 8,200.00 2,351 20,656,686.00',\n",
              " '63 2021/02/02 8,225.00 0.3 8,350.00 8,071.00 8,036.00 2,814 23,402,996.00',\n",
              " '64 2021/02/01 8,200.00 1.86 8,210.00 8,050.00 8,070.00 2,008 16,384,940.00',\n",
              " '65 2021/01/31 8,050.00 -0.62 8,200.00 8,000.00 8,200.00 2,022 17,442,442.00',\n",
              " '66 2021/01/28 8,100.00 0 8,427.00 7,950.00 8,100.00 1,515 12,407,145.00',\n",
              " '67 2021/01/27 8,100.00 1.31 8,200.00 7,995.00 7,995.00 1,471 11,898,590.00',\n",
              " '68 2021/01/26 7,995.00 -3.09 8,400.00 7,994.00 8,400.00 900 7,355,722.00',\n",
              " '69 2021/01/25 8,250.00 3.12 8,800.00 8,000.00 8,000.00 1,507 12,565,800.00',\n",
              " '70 2021/01/24 8,000.00 -2.43 8,213.00 7,900.00 8,150.00 1,779 14,289,335.00',\n",
              " '71 2021/01/21 8,199.00 0.23 8,200.00 8,050.00 8,199.00 1,171 9,535,380.00',\n",
              " '72 2021/01/20 8,180.00 0.07 8,200.00 7,938.00 8,050.00 2,213 17,855,126.00',\n",
              " '73 2021/01/19 8,174.00 0.91 8,500.00 7,575.00 8,262.00 3,659 28,996,859.00',\n",
              " '74 2021/01/18 8,100.00 6.26 8,385.00 7,775.00 7,775.00 2,903 23,690,900.00',\n",
              " '75 2021/01/17 7,623.00 10 7,623.00 7,068.00 7,068.00 3,197 23,589,247.00',\n",
              " '76 2021/01/13 6,930.00 0.43 6,949.00 6,801.00 6,801.00 842 5,808,915.00',\n",
              " '77 2021/01/12 6,900.00 -0.72 6,989.00 6,815.00 6,900.00 948 6,502,390.00',\n",
              " '78 2021/01/11 6,950.00 -0.54 7,000.00 6,850.00 7,000.00 1,795 12,438,708.00',\n",
              " '79 2021/01/10 6,988.00 3.68 7,151.00 6,874.00 6,874.00 4,839 33,844,865.00',\n",
              " '80 2021/01/07 6,740.00 5.31 7,000.00 6,411.00 6,528.00 2,589 17,322,637.00',\n",
              " '81 2021/01/06 6,400.00 -1.08 6,848.00 6,290.00 6,350.00 1,680 10,805,135.00',\n",
              " '82 2021/01/05 6,470.00 -6.16 6,887.00 6,210.00 6,758.00 3,071 20,131,645.00',\n",
              " '83 2021/01/04 6,895.00 -0.2 7,254.00 6,777.00 7,254.00 535 3,766,785.00',\n",
              " '84 2021/01/03 6,909.00 10 6,909.00 6,406.00 6,406.00 3,202 22,019,048.00',\n",
              " '85 2020/12/31 6,281.00 10 6,281.00 5,700.00 5,995.00 6,518 40,233,176.00',\n",
              " '86 2020/12/30 5,710.00 10 5,710.00 5,250.00 5,250.00 7,215 40,871,842.00',\n",
              " '87 2020/12/29 5,191.00 0.02 5,293.00 5,185.00 5,292.00 6,040 31,451,420.00',\n",
              " '88 2020/12/28 5,190.00 -2.99 5,295.00 5,139.00 5,243.00 976 5,080,636.00',\n",
              " '89 2020/12/27 5,350.00 0.87 5,454.00 5,150.00 5,220.00 3,478 18,340,670.00',\n",
              " '90 2020/12/24 5,304.00 1.03 5,304.00 5,065.00 5,150.00 370 1,903,920.00',\n",
              " '91 2020/12/23 5,250.00 -0.38 5,355.00 5,190.00 5,270.00 729 3,851,340.00',\n",
              " '92 2020/12/22 5,270.00 4.15 5,320.00 5,161.00 5,161.00 1,510 7,905,660.00',\n",
              " '93 2020/12/21 5,060.00 -2.69 5,196.00 4,799.00 5,096.00 1,937 9,719,285.00',\n",
              " '94 2020/12/20 5,200.00 -4.11 5,420.00 5,110.00 5,315.00 1,533 8,091,777.00',\n",
              " '95 2020/12/17 5,423.00 -2.73 5,686.00 5,355.00 5,686.00 700 3,845,960.00',\n",
              " '96 2020/12/16 5,575.00 1.36 5,775.00 5,451.00 5,775.00 1,816 10,184,300.00',\n",
              " '97 2020/12/15 5,500.00 -2.48 6,103.00 5,500.00 5,640.00 1,944 11,149,425.00',\n",
              " '98 2020/12/14 5,640.00 9.51 5,665.00 5,150.00 5,150.00 2,502 14,047,285.00',\n",
              " '99 2020/12/13 5,150.00 -1.72 5,300.00 5,105.00 5,300.00 693 3,566,113.00',\n",
              " '100 2020/12/10 5,240.00 2.04 5,240.00 5,100.00 5,033.00 887 4,578,885.00',\n",
              " '101 2020/12/09 5,135.00 -4.91 5,340.00 5,100.00 5,292.00 1,333 6,913,290.00',\n",
              " '102 2020/12/08 5,400.00 -3.83 5,505.00 5,350.00 5,504.00 895 4,842,465.00',\n",
              " '103 2020/12/07 5,615.00 6.1 5,619.00 4,996.00 5,300.00 2,238 11,944,039.00',\n",
              " '104 2020/12/06 5,292.00 -7.16 5,700.00 5,262.00 5,700.00 1,732 9,461,563.00',\n",
              " '105 2020/12/03 5,700.00 -6.94 6,200.00 5,700.00 6,200.00 1,479 8,668,480.00',\n",
              " '106 2020/12/02 6,125.00 -2.08 6,600.00 6,027.00 6,150.00 2,003 12,517,660.00',\n",
              " '107 2020/12/01 6,255.00 -4.5 6,800.00 6,003.00 6,681.00 1,759 11,233,927.00',\n",
              " '108 2020/11/30 6,550.00 9.17 6,600.00 6,300.00 6,300.00 2,879 18,908,900.00',\n",
              " '109 2020/11/29 7,500.00 -0.08 7,809.00 7,400.00 7,656.00 4,532 34,483,461.00',\n",
              " '110 2020/11/26 7,506.00 -6.17 7,996.00 7,407.00 7,840.00 2,131 16,408,960.00',\n",
              " '111 2020/11/25 7,965.00 6.91 8,189.00 7,505.00 7,505.00 1,425 14,768,065.00',\n",
              " '112 2020/11/24 7,450.00 7.97 7,590.00 6,900.00 6,900.00 4,075 30,411,488.00',\n",
              " '113 2020/11/23 6,900.00 1.46 7,400.00 6,200.00 6,670.00 1,728 11,375,000.00',\n",
              " '114 2020/11/22 6,801.00 -6.09 7,966.00 6,518.00 7,386.00 2,398 17,367,766.00',\n",
              " '115 2020/11/19 7,242.00 9.94 7,245.00 6,916.00 6,916.00 2,787 20,018,093.00',\n",
              " '116 2020/11/18 6,587.00 9.98 6,587.00 6,108.00 6,108.00 4,067 26,743,994.00',\n",
              " '117 2020/11/12 5,989.00 9.99 5,989.00 5,553.00 5,553.00 3,423 21,171,542.00',\n",
              " '118 2020/11/11 5,445.00 10 5,445.00 5,049.00 5,049.00 2,680 14,580,910.00',\n",
              " '119 2020/11/10 4,950.00 10 4,950.00 4,500.00 4,500.00 2,478 11,762,819.00',\n",
              " '120 2020/11/09 4,500.00 5.26 4,701.00 4,273.00 4,360.00 1,670 7,610,420.00',\n",
              " '121 2020/11/08 4,275.00 9.95 4,276.00 3,888.00 3,888.00 5,058 21,082,871.00',\n",
              " '122 2020/11/05 3,888.00 -0.05 3,890.00 3,850.00 3,890.00 1,080 4,193,210.00',\n",
              " '123 2020/11/04 3,890.00 0.54 3,900.00 3,869.00 3,869.00 435 1,692,050.00',\n",
              " '124 2020/11/03 3,869.00 3.87 3,869.00 3,799.00 3,799.00 1,044 3,993,776.00',\n",
              " '125 2020/11/02 3,725.00 1.89 3,845.00 3,660.00 3,660.00 1,750 6,584,345.00',\n",
              " '126 2020/11/01 3,656.00 0.99 3,668.00 3,650.00 3,660.00 655 2,394,430.00',\n",
              " '127 2020/10/29 3,620.00 1.12 3,640.00 3,590.00 3,590.00 876 3,161,870.00',\n",
              " '128 2020/10/28 3,580.00 -1.21 3,645.00 3,580.00 3,645.00 30 108,050.00',\n",
              " '129 2020/10/22 3,624.00 0.78 3,641.00 3,600.00 3,615.00 800 2,898,720.00',\n",
              " '130 2020/10/21 3,596.00 2.74 3,600.00 3,475.00 3,475.00 630 2,262,590.00',\n",
              " '131 2020/10/20 3,500.00 0 3,570.00 3,475.00 3,500.00 1,172 4,142,159.00',\n",
              " '132 2020/10/19 3,500.00 0 3,500.00 3,465.00 3,500.00 320 1,119,300.00',\n",
              " '133 2020/10/18 3,500.00 1.45 3,500.00 3,445.00 3,450.00 535 1,862,650.00',\n",
              " '134 2020/10/15 3,450.00 0.41 3,470.00 3,405.00 3,469.00 1,250 4,304,930.00',\n",
              " '135 2020/10/14 3,436.00 -0.26 3,445.00 3,430.00 3,445.00 80 274,850.00',\n",
              " '136 2020/10/13 3,445.00 0.15 3,445.00 3,380.00 3,382.00 200 679,350.00',\n",
              " '137 2020/10/12 3,440.00 0.15 3,450.00 3,440.00 3,450.00 110 379,400.00',\n",
              " '138 2020/10/11 3,435.00 -1.86 3,457.00 3,435.00 3,457.00 565 1,946,655.00',\n",
              " '139 2020/10/07 3,500.00 0.14 3,524.00 3,450.00 3,490.00 470 1,642,180.00',\n",
              " '140 2020/10/06 3,495.00 0.58 3,500.00 3,450.00 3,500.00 438 1,520,680.00',\n",
              " '141 2020/10/05 3,475.00 -1.05 3,500.00 3,475.00 3,500.00 290 1,009,720.00',\n",
              " '142 2020/10/04 3,550.00 2.01 3,550.00 3,520.00 3,548.00 423 2,025,965.00',\n",
              " '143 2020/10/01 3,480.00 2.17 3,543.00 3,466.00 3,470.00 370 1,287,500.00',\n",
              " '144 2020/09/30 3,406.00 -1.22 3,450.00 3,405.00 3,448.00 632 2,160,825.00',\n",
              " '145 2020/09/29 3,448.00 -2.74 3,480.00 3,412.00 3,475.00 429 1,473,719.00',\n",
              " '146 2020/09/28 3,545.00 -1.53 3,567.00 3,396.00 3,557.00 660 2,316,000.00',\n",
              " '147 2020/09/27 3,600.00 0 3,650.00 3,530.00 3,650.00 639 2,299,060.00',\n",
              " '148 2020/09/24 3,600.00 8.11 3,640.00 3,350.00 3,370.00 5,435 18,622,470.00',\n",
              " '149 2020/09/23 3,330.00 0.91 3,366.00 3,257.00 3,257.00 1,332 4,408,261.00',\n",
              " '150 2020/09/22 3,300.00 1.26 3,390.00 3,265.00 3,324.00 2,615 8,673,337.00',\n",
              " '151 2020/09/21 3,259.00 -1.24 3,280.00 3,235.00 3,280.00 768 2,490,768.00',\n",
              " '152 2020/09/17 3,300.00 0.15 3,320.00 3,290.00 3,295.00 570 1,880,650.00',\n",
              " '153 2020/09/16 3,295.00 2.17 3,354.00 3,289.00 3,289.00 655 2,177,165.00',\n",
              " '154 2020/09/15 3,225.00 -1.41 3,336.00 3,225.00 3,336.00 1,054 5,221,985.00',\n",
              " '155 2020/09/14 3,271.00 -0.12 3,340.00 3,210.00 3,211.00 915 3,065,870.00',\n",
              " '156 2020/09/13 3,275.00 -2.82 3,350.00 3,240.00 3,350.00 1,185 3,868,835.00',\n",
              " '157 2020/09/10 3,370.00 -3.58 3,514.00 3,362.00 3,500.00 600 2,047,580.00',\n",
              " '158 2020/09/09 3,495.00 3.1 3,540.00 3,392.00 3,457.00 1,828 6,371,955.00',\n",
              " '159 2020/09/08 3,390.00 9.99 3,390.00 3,100.00 3,100.00 2,080 6,690,834.00',\n",
              " '160 2020/09/07 3,082.00 2.9 3,111.00 2,936.00 2,940.00 1,919 5,783,225.00',\n",
              " '161 2020/09/06 2,995.00 3.28 3,000.00 2,898.00 2,925.00 2,190 6,528,840.00',\n",
              " '162 2020/09/03 2,900.00 -1.36 2,998.00 2,900.00 2,998.00 180 527,040.00',\n",
              " '163 2020/09/02 2,940.00 2.08 3,009.00 2,870.00 2,900.00 1,586 4,719,442.00',\n",
              " '164 2020/08/31 2,880.00 2.49 2,902.00 2,820.00 2,820.00 1,230 3,528,330.00',\n",
              " '165 2020/08/27 2,755.00 -1.22 2,756.00 2,750.00 2,756.00 170 468,170.00',\n",
              " '166 2020/08/25 2,800.00 -0.07 2,820.00 2,750.00 2,820.00 250 693,010.00',\n",
              " '167 2020/08/24 2,802.00 1.89 2,861.00 2,753.00 2,805.00 835 2,360,845.00',\n",
              " '168 2020/08/23 2,750.00 -0.36 2,760.00 2,750.00 2,760.00 90 247,670.00',\n",
              " '169 2020/08/20 2,760.00 1.81 2,760.00 2,760.00 2,760.00 20 55,200.00',\n",
              " '170 2020/08/19 2,711.00 -2.13 2,760.00 2,705.00 2,760.00 395 1,075,435.00',\n",
              " '171 2020/08/18 2,770.00 1.47 2,770.00 2,705.00 2,758.00 990 2,707,640.00',\n",
              " '172 2020/08/17 2,730.00 1 2,740.00 2,720.00 2,735.00 670 1,832,250.00',\n",
              " '173 2020/08/16 2,703.00 1.69 2,703.00 2,650.00 2,659.00 660 1,764,545.00',\n",
              " '174 2020/08/13 2,658.00 0.08 2,660.00 2,658.00 2,660.00 195 518,530.00',\n",
              " '175 2020/08/12 2,656.00 -0.71 2,680.00 2,637.00 2,637.00 130 343,810.00',\n",
              " '176 2020/08/11 2,675.00 -2.37 2,700.00 2,675.00 2,700.00 1,044 2,802,655.00',\n",
              " '177 2020/08/10 2,740.00 1.48 2,740.00 2,700.00 2,700.00 440 546,100.00',\n",
              " '178 2020/08/09 2,700.00 1.69 2,700.00 2,650.00 2,655.00 740 1,985,795.00',\n",
              " '179 2020/08/06 2,655.00 -1.3 2,690.00 2,655.00 2,690.00 135 358,870.00',\n",
              " '180 2020/08/05 2,700.00 -0.04 2,701.00 2,655.00 2,701.00 660 2,307,640.00',\n",
              " '181 2020/08/03 2,701.00 1.12 2,725.00 2,671.00 2,719.00 1,318 3,546,370.00',\n",
              " '182 2020/08/02 2,671.00 -1.98 2,700.00 2,671.00 2,675.00 577 1,550,066.00',\n",
              " '183 2020/07/30 2,725.00 0.52 2,725.00 2,710.00 2,711.00 150 408,140.00',\n",
              " '184 2020/07/29 2,711.00 -1.2 2,744.00 2,710.00 2,730.00 1,693 4,634,160.00',\n",
              " '185 2020/07/28 2,744.00 -2 2,755.00 2,744.00 2,755.00 240 659,870.00',\n",
              " '186 2020/07/27 2,798.00 -1.13 2,830.00 2,744.00 2,825.00 1,795 5,354,470.00',\n",
              " '187 2020/07/23 2,800.00 1.45 2,802.00 2,780.00 2,780.00 690 1,931,350.00',\n",
              " '188 2020/07/22 2,760.00 -1.43 2,800.00 2,745.00 2,745.00 1,014 2,812,176.00',\n",
              " '189 2020/07/21 2,800.00 3.44 2,816.00 2,720.00 2,755.00 670 1,853,084.00',\n",
              " '190 2020/07/20 2,707.00 -5.35 2,917.00 2,707.00 2,917.00 645 1,793,875.00',\n",
              " '191 2020/07/16 2,860.00 1.6 2,860.00 2,760.00 2,760.00 875 2,438,330.00',\n",
              " '192 2020/07/15 2,815.00 1.99 2,815.00 2,799.00 2,799.00 499 1,402,646.00',\n",
              " '193 2020/07/14 2,760.00 -1.29 2,850.00 2,760.00 2,800.00 286 808,470.00',\n",
              " '194 2020/07/13 2,796.00 1.9 2,796.00 2,701.00 2,701.00 120 325,610.00',\n",
              " '195 2020/07/12 2,744.00 0.15 2,800.00 2,686.00 2,686.00 2,113 5,779,830.00',\n",
              " '196 2020/07/09 2,740.00 -2.14 2,745.00 2,690.00 2,745.00 1,262 3,418,526.00',\n",
              " '197 2020/07/08 2,800.00 1.82 2,800.00 2,695.00 2,695.00 1,360 3,705,110.00']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oFu0YbFZ3Lou"
      },
      "source": [
        "Date = [value.split(\" \")[1] for value in data2]\n",
        "Close = [value.split(\" \")[2] for value in data2]\n",
        "High = [value.split(\" \")[4] for value in data2]\n",
        "Low = [value.split(\" \")[5] for value in data2]\n",
        "Open = [value.split(\" \")[6] for value in data2]\n",
        "Qty = [value.split(\" \")[7] for value in data2]\n",
        "Volume = [Qty[i].replace(',', '') for i in range(len(Qty))]\n",
        "\n",
        "df = pd.DataFrame({\"Date\":Date,\n",
        "              \"Close\":Close,\n",
        "              \"High\":High,\n",
        "              \"Low\":Low,\n",
        "              \"Open\":Open,\n",
        "              \"Volume\":Volume})\n",
        "df['Date'] = pd.to_datetime(df['Date'])\n",
        "df['Close'] = pd.to_numeric(df['Close'].apply(lambda x: x.replace(\",\",\"\")))\n",
        "df['High'] = pd.to_numeric(df['High'].apply(lambda x: x.replace(\",\",\"\")))\n",
        "df['Low'] = pd.to_numeric(df['Low'].apply(lambda x: x.replace(\",\",\"\")))\n",
        "df['Open'] = pd.to_numeric(df['Open'].apply(lambda x: x.replace(\",\",\"\")))\n",
        "df['Volume'] = pd.to_numeric(df['Volume'].apply(lambda x: x.replace(\",\",\"\")))\n"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pa13jS6CTeA"
      },
      "source": [
        "df = df[['Date','Open','High','Low','Close','Volume']]"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gN9pCigoC8Ih",
        "outputId": "3cb8e044-5063-4e8c-d85b-d0de879394e7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 197 entries, 0 to 196\n",
            "Data columns (total 6 columns):\n",
            " #   Column  Non-Null Count  Dtype         \n",
            "---  ------  --------------  -----         \n",
            " 0   Date    197 non-null    datetime64[ns]\n",
            " 1   Open    197 non-null    float64       \n",
            " 2   High    197 non-null    float64       \n",
            " 3   Low     197 non-null    float64       \n",
            " 4   Close   197 non-null    float64       \n",
            " 5   Volume  197 non-null    int64         \n",
            "dtypes: datetime64[ns](1), float64(4), int64(1)\n",
            "memory usage: 9.4 KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4vhSNqKAszu"
      },
      "source": [
        "\n",
        "df.sort_values(by = \"Date\", ascending = True, inplace = True)\n",
        "df.to_csv(\"stc.csv\", index= False)"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mibpV7YqDozA"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}