feat: able to get data from tab/latest search tab

helmisatria · Oct 25, 2023 · 87ce06b · 87ce06b
1 parent a44809f
commit 87ce06b
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@
 
 ### Added
 
+- Add `SEARCH_TAB` or `--search-tab` or `--tab` option to specify the tab to search for tweets. Default is `LATEST` tab. The options are `LATEST` and `TOP`.
+
+## 2.3.0 (2023-10-25)
+
+### Added
+
 - Implemented optional exponential backoff for rate limit handling. The wait time between retries will now be calculated dynamically based on the number of attempts made, resulting in fewer requests during the rate-limit window. This should help to reduce the risk of account bans. To utilize this feature, set the `ENABLE_EXPONENTIAL_BACKOFF` environment variable to true.
 
 ### Changed

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "tweet-harvest",
   "description": "A Twitter crawler helper with auth",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "license": "MIT",
   "author": "Helmi Satria",
   "publishConfig": {

diff --git a/src/bin.ts b/src/bin.ts
@@ -58,6 +58,12 @@ async function run() {
         describe: "Output filename",
         type: "string",
       },
+      search_tab: {
+        alias: "tab",
+        describe: "Search tab (TOP or LATEST)",
+        default: "LATEST",
+        choices: ["TOP", "LATEST"],
+      },
     })
     .help()
     .alias("help", "h").argv;
@@ -145,6 +151,7 @@ async function run() {
       TARGET_TWEET_COUNT: argv.limit,
       DELAY_EACH_TWEET_SECONDS: argv.delay_each_tweet,
       OUTPUT_FILENAME: argv.output_filename,
+      SEARCH_TAB: String(argv.search_tab).toUpperCase() as "TOP" | "LATEST",
     });
   } catch (err) {
     console.error("Error running script:", err);

diff --git a/src/constants.ts b/src/constants.ts
@@ -0,0 +1,4 @@
+export const TWITTER_SEARCH_ADVANCED_URL = {
+  TOP: "https://twitter.com/search-advanced",
+  LATEST: "https://twitter.com/search-advanced",
+};
diff --git a/src/crawl.ts b/src/crawl.ts
@@ -10,6 +10,7 @@ import { inputKeywords } from "./features/input-keywords";
 import { listenNetworkRequests } from "./features/listen-network-requests";
 import { calculateForRateLimit } from "./features/exponential-backoff";
 import { HEADLESS_MODE } from "./env";
+import { TWITTER_SEARCH_ADVANCED_URL } from "./constants";
 
 chromium.use(stealth());
 
@@ -57,6 +58,7 @@ export async function crawl({
   DELAY_EVERY_100_TWEETS_SECONDS = 10,
   DEBUG_MODE,
   OUTPUT_FILENAME,
+  SEARCH_TAB = "LATEST",
 }: {
   ACCESS_TOKEN: string;
   SEARCH_KEYWORDS?: string;
@@ -68,8 +70,11 @@ export async function crawl({
   DEBUG_MODE?: boolean;
   OUTPUT_FILENAME?: string;
   TWEET_THREAD_URL?: string;
+  SEARCH_TAB?: "LATEST" | "TOP";
 }) {
   const CRAWL_MODE = TWEET_THREAD_URL ? "DETAIL" : "SEARCH";
+  const SWITCHED_SEARCH_TAB = SEARCH_TAB === "TOP" ? "LATEST" : "TOP";
+
   const IS_DETAIL_MODE = CRAWL_MODE === "DETAIL";
   const IS_SEARCH_MODE = CRAWL_MODE === "SEARCH";
   const TIMEOUT_LIMIT = 4;
@@ -94,7 +99,7 @@ export async function crawl({
     fs.renameSync(FILE_NAME, FILE_NAME.replace(".csv", ".old.csv"));
   }
 
-  let TWEETS_NOT_FOUND_ON_LIVE_TAB = false;
+  let TWEETS_NOT_FOUND_ON_CURRENT_TAB = false;
 
   const browser = await chromium.launch({ headless: HEADLESS_MODE });
 
@@ -123,7 +128,7 @@ export async function crawl({
   listenNetworkRequests(page);
 
   async function startCrawlTwitter({
-    twitterSearchUrl = "https://twitter.com/search-advanced?f=live",
+    twitterSearchUrl = TWITTER_SEARCH_ADVANCED_URL[SEARCH_TAB],
   }: StartCrawlTwitterParams = {}) {
     if (IS_DETAIL_MODE) {
       await page.goto(TWEET_THREAD_URL);
@@ -214,7 +219,7 @@ export async function crawl({
           if (!tweets.length) {
             // found text "not found" on the page
             if (await page.getByText("No results for").count()) {
-              TWEETS_NOT_FOUND_ON_LIVE_TAB = true;
+              TWEETS_NOT_FOUND_ON_CURRENT_TAB = true;
               console.info("No tweets found for the search criteria");
               break;
             }
@@ -354,11 +359,11 @@ export async function crawl({
   try {
     await startCrawlTwitter();
 
-    if (TWEETS_NOT_FOUND_ON_LIVE_TAB && (SEARCH_FROM_DATE || SEARCH_TO_DATE)) {
-      console.info('No tweets found on "Latest" tab, trying "Top" tab...');
+    if (TWEETS_NOT_FOUND_ON_CURRENT_TAB && (SEARCH_FROM_DATE || SEARCH_TO_DATE)) {
+      console.info(`No tweets found on "${SEARCH_TAB}" tab, trying "${SWITCHED_SEARCH_TAB}" tab...`);
 
       await startCrawlTwitter({
-        twitterSearchUrl: "https://twitter.com/search-advanced",
+        twitterSearchUrl: TWITTER_SEARCH_ADVANCED_URL[SWITCHED_SEARCH_TAB],
       });
     }
   } catch (error) {

diff --git a/src/dev.ts b/src/dev.ts
@@ -9,4 +9,5 @@ crawl({
   OUTPUT_FILENAME: "gibran.csv",
   DELAY_EACH_TWEET_SECONDS: 0.1,
   DELAY_EVERY_100_TWEETS_SECONDS: 0,
+  SEARCH_TAB: "TOP",
 });