Googlenews_bs4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "標題: []\n",
      "描述: []\n",
      "連結: []\n",
      "ok fine. it is talking about Taiwan news\n"
     ]
    }
   ],
   "source": [
    "# 當時解析 Google 新聞版本，目前已失效\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "user_agent = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'\n",
    "}\n",
    "\n",
    "url = 'https://news.google.com/topics/CAAqJQgKIh9DQkFTRVFvSUwyMHZNRFptTXpJU0JYcG9MVlJYS0FBUAE'\n",
    "response = requests.get(url, headers=user_agent)\n",
    "web = response.text\n",
    "\n",
    "soup = BeautifulSoup(web, 'html.parser')\n",
    "\n",
    "titles = []\n",
    "dscps = []\n",
    "links = []\n",
    "\n",
    "title_elements = soup.find_all('div', class_='DY5T1d')\n",
    "\n",
    "for title_element in title_elements:\n",
    "    title = title_element.find('a', class_=\"DY5T1d\")\n",
    "    if title:\n",
    "        titles.append(title.text)\n",
    "    desc = title_element.find('span')\n",
    "    if desc:\n",
    "        dscps.append(desc.text)\n",
    "    link = title_element.find('a', href=True)\n",
    "    if link:\n",
    "        full_link = requests.get(link['href'].replace('.', 'https://news.google.com/')).url\n",
    "        links.append(full_link)\n",
    "\n",
    "print('標題:', titles)\n",
    "print('描述:', dscps)\n",
    "print('連結:', links)\n",
    "\n",
    "print('ok fine. it is talking about Taiwan news')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "標題: 更多 - 日本鳥島近海發生規模5.9地震 「這兩地」發海嘯警報\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiygJBVV95cUxPWmcxVmRtcG1SZXRxNFVBRTJzX3g1Nng2Zy1GaW9yeDJzSWU2UDJNLVExbVZZUzJOUDAweDZlQS1JdUZhQVdtVGw2NHJwYlh0eHVtX0VWZnp1bEhod3FBd3BNeFFzaC10cEs5MmdPb0tZSzM0M3F0M1JOd3JmMXUwRk1pZmdxa3VnV3dlNU9Oam41UVA0RFhXU0xqWEtXckdvMk84dFB5ajRkeTdjbXo3OXl1RWY1b1JDY2xpcUJRNURnemVOczZKSEpkRjBFTkJHNXAtQTVrZ0xlWFhCOUtPWkFrNElsOVNRcFFYS2hRb2lIQ25vOVlEX2RIalRLOHFQZFg2VU1TUTd1b0lDeUtfcmlKcWFlZWVhZVJlYWV6U0FQMU1oQ29VbUY0WHhweGs2NVVkWHRoMlJCVjlyeWpzOVhLWnh0eE1zaVE?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 日本外海爆規模5.9地震深度10公里第一波海嘯抵達- 政治圈\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMibkFVX3lxTFBWQmc4YVR2UDVybGdGeXpHdW04WWl5dzhkS3FRekxoOUNhTTJNelk4bC03c3dEdURibFQ0Q1REM2hSYmRPQVJMQkJMVDZBblByYkc2a3lmdXlFY29TZmR0NG4ycGtQT0t4ZS1BZlZn?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 日本伊豆群島觀測到50公分高海嘯 無引發災害疑慮\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMi0wJBVV95cUxQX2tWeU5uU3ljNGJlNThxcFNKdHprN0Ewdi1jMFNVRGl4WnFmR1pMVXFGSlJGQ1VmWmNQQnZkR3BqOVQyS241aC1kM0E3WGJJMVVRcVg1UTMxNERXLWJsNl8ySUwtTTlxcldfRVl0RklOQ1FrODRuYnlRV0dGbjRMaUZXOWtCR200NlcyTVlva3FCWnZQYmFLNEJlZzIxWXJ0NjIwTDV3aGNGd0FDVEVWVlRqVVF6ajBDT3NQSHlKVjhEZmJ1ZWxpMmRram5zMmVVdDRGVFJWTE5FcFBiNUJweFJrYnV1MkNza1dVSDBoeWs2UF9qZURWSmE1TG1XbzBENkNHRHBSOXFMVGgxMWVVVjJVTWFwMks5cTZud1NMaDFFXzlpRWhGTTlTYm91UlVLbzJhVE5KSGJVWTJ6b1pzeGVpUUZybC12S2p6ZE1OWXhxXzg?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 快訊／日本近海5.9地震 伊豆群島八丈島出現50cm海嘯\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiXkFVX3lxTE9hQmVDaUpPQU84N0ZEUzE5Y2dtVXE5elIxOXRIZW10aWozMktRRWJTS1NNd0g5eUdWUzQtZVB0eTdVejhyZ2tNbU1kZThYaHQ3NnQzWWNFQVZqWnBoSlHSAWhBVV95cUxOQUNZTWtWYzhMc3Y4NUtld0V2UVQ5TF9TVXlIMk80NW5RUGFxU0dUcWF1MmR0ZmM0anJpTnpiT2x0SU9uSXZuN2QzeFJPTE9GQzNnR0ZzeS1NRVlMZDVUeVM4emtqUWNpQw?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 以軍狂轟黎巴嫩釀492死1640傷 瞄準真主黨領袖、美增軍中東\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiUEFVX3lxTE8xY212Nm1UbXI4bVY1cVI1aWx0NWFiakNSNEQ0YkM4TGxfU0dHVWFDV2Rzd1JkaFJfRlFuQnIzdGRneXpGZUhzbGRQaWNhVm9N?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 以色列24小時空襲逾1300目標 黎巴嫩：356死1246傷\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiX0FVX3lxTE5oVkY4UXRrRFJEdmJNV1RCM0NwTnpTd2lTdDU4TUpNSWJfNk1mUnR5RXUwZC1UaUhmcno0bFliOENkREl1RzhNSXR6QXhUam5kb2w4SmlxUGhWTmpUdml3?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 捲BB Call爆炸案！金阿波羅2員工現身當證人 超狂學歷曝光\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMi3AJBVV95cUxOdkpTVGZIVHJFY01rRU1tbEJERmdpRDhHamNQNGtHVmJIU09zaFRaRVloQlh2Tmd1UFN5ZU5KSkFkRk5wdnJuOVVueDg2R3owSXA5X0tIa2N3RTV5UnpaLUUwOWk5TzdVNnpVT3ZrcXBDQVJkM2FQQV9icW85czhnTm5pZGlZWklfdFhMUENGTHI1RVhLZEM5Vy1WaUozSENoNlA2MVZISFcycTRIR19ZdjloV3VGRDdSaU5sYVoxNkw2a1Fkd09aTDRieDhIVUxrRGUwMmhiRUc1aGpMNkRQTUh4NzlVd0hyZy1kQllQS0tNNUsxYkJIQVlmSll4R3pZMWo4eUR6eERiREd5ZC13cnBZMW5lbTlUNVBqT3hsc3JtTXpRdnFYY1Zmc3Q1MWJXc2R5ZXo1TWlkNmxQQmVkaEl1SUZod001VVk5Um5MZmFFemwxQm5BSnJWaTU?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 以色列猛烈轟炸黎巴嫩 釀492死 2006年以來最慘 | Anue鉅亨 - 國際政經\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiT0FVX3lxTE5heVNWeERFYm52N0lxbzhmT3dlOHM2SkJ5SWFBaGVQSjlCeUVpRDlRVTlqZlBLVHgzd2dfVmdXTnhoMzRrSDkxMjQ3T3pESGs?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 台獨網軍\"匿名者64\"慘了?! 蔡正元預言中共下一步｜國軍駭客試圖癱瘓\"陸機構\"? 楊永明:恐成\"軍事攻擊\"\n",
      "描述: \n",
      "連結: https://news.google.com./read/CCAiC2hWQVJ4RW94TUVnmAEB?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n",
      "標題: 更多 - 巨業客運司機撞死女大生籌不到錢 交保金額「20萬元降為1萬元」\n",
      "描述: \n",
      "連結: https://news.google.com./read/CBMiXkFVX3lxTE1wMnJuUTQxSHRldUlmdUc2emdYMGNvcnBvY2wwRnpqSjcyVU9SZVdsdTBwVFNYWnBmTEtlbUFyanFyM01UWU5qX0d4NHk0M2RfUTQ4MTA5SkN6Wkk3cmfSAWhBVV95cUxPamJLbHVtMGhaOVNQOEx1OGhORVlPbGVRaFZzT2hPQ21CN1djeVVkYXdwZEpFTm5mV0gzQ0YyZ3pES3RkUDZHYVR1WlAxc0NzN3N2akkwcjV4Q0dmakRud21uMVRRdlY4cg?hl=zh-TW&gl=TW&ceid=TW%3Azh-Hant\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 此部分基於上述程式架構，觀察目前(2024/0924) Google 新聞網頁內容結構，修改後的版本，網址要另外處理，沒有找到 Googlenew 套件回傳的 desc\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "user_agent = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'\n",
    "}\n",
    "\n",
    "url = 'https://news.google.com/home?hl=zh-TW&gl=TW&ceid=TW:zh-Hant'\n",
    "response = requests.get(url, headers=user_agent)\n",
    "web = response.text\n",
    "\n",
    "soup = BeautifulSoup(web, 'html.parser')\n",
    "\n",
    "data = []\n",
    "\n",
    "article_elements = soup.find_all('article')\n",
    "\n",
    "for article_element in article_elements:\n",
    "    title_element = article_element.find('button', class_='VfPpkd-Bz112c-LgbsSe')\n",
    "    if title_element:\n",
    "        title = title_element['aria-label']\n",
    "        desc = \"\"\n",
    "        link = \"\"\n",
    "        desc_element = article_element.find('p')\n",
    "        if desc_element:\n",
    "            desc = desc_element.text\n",
    "        link_element = article_element.find('a', href=True)\n",
    "        if link_element:\n",
    "            link = f'https://news.google.com{link_element[\"href\"]}'\n",
    "        data.append({\n",
    "            '標題': title,\n",
    "            '描述': desc,\n",
    "            '連結': link\n",
    "        })\n",
    "\n",
    "for item in data:\n",
    "    print('標題:', item['標題'])\n",
    "    print('描述:', item['描述'])\n",
    "    print('連結:', item['連結'])\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">目前得到的是 google news 的網址\n",
    "\n",
    ">點擊後才會跳轉到正確的新聞網址\n",
    "\n",
    ">所以要得到真正新聞的內文要再處理\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}