diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 95f68812..bfa2a9b2 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -2,8 +2,8 @@ name: Documentation run-name: Building the documentation for the release on: - push: - branches: release + release: + types: [released] jobs: # Builds and packages the documentation diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dfd005bd..512e8dd3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ run-name: Publishing the package to NPM on: release: - types: [created] + types: [released] jobs: # Packages and publishes the package to NPM diff --git a/README.md b/README.md index f10e303c..0887dba4 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ So far, the following operations are supported: - [Retweeting/reposting a tweet](https://rishikant181.github.io/Rettiwt-API/classes/TweetService.html#retweet) - [Getting the list of users who retweeted/reposted a given tweet](https://rishikant181.github.io/Rettiwt-API/classes/TweetService.html#retweeters) - [Searching for the list of tweets that match a given filter](https://rishikant181.github.io/Rettiwt-API/classes/TweetService.html#search) +- [Streaming filtered tweets in pseudo-realtime](https://rishikant181.github.io/Rettiwt-API/classes/TweetService.html#stream) - [Posting a new tweet](https://rishikant181.github.io/Rettiwt-API/classes/TweetService.html#tweet) ### Users diff --git a/package.json b/package.json index cbb7a38a..9763ff83 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rettiwt-api", - "version": "2.5.3", + "version": "2.6.0", "main": "dist/index.js", "types": "dist/index.d.ts", "description": "An API for fetching data from TwitterAPI, without any rate limits!", @@ -34,7 +34,7 @@ "commander": "11.1.0", "https-proxy-agent": "7.0.2", "rettiwt-auth": "2.1.0", - "rettiwt-core": "3.3.2" + "rettiwt-core": "3.4.0" }, "devDependencies": { "@types/node": "20.4.1", diff --git a/src/commands/Tweet.ts b/src/commands/Tweet.ts index 7d2d1928..50815d9a 100644 --- a/src/commands/Tweet.ts +++ b/src/commands/Tweet.ts @@ -32,22 +32,22 @@ function createTweetCommand(rettiwt: Rettiwt): Command { .description('Fetch the list of tweets that match the given filter options') .argument('[count]', 'The number of tweets to fetch') .argument('[cursor]', 'The cursor to the batch of tweets to fetch') - .option('-f, --from ', "Matches the tweets made by list of given users, separated by ';'") - .option('-t, --to ', "Matches the tweets made to the list of given users, separated by ';'") - .option('-w, --words ', "Matches the tweets containing the given list of words, separated by ';'") + .option('-f, --from ', 'Matches the tweets made by the comma-separated list of given users') + .option('-t, --to ', 'Matches the tweets made to the comma-separated list of given users') + .option('-w, --words ', 'Matches the tweets containing the given comma-separated list of words') .option('-p, --phrase ', 'Matches the tweets containing the exact phrase') .option( '--optional-words ', - "Matches the tweets containing any of the given list of words, separated by ';'", + 'Matches the tweets containing any of the given comma-separated list of words', ) .option( '--exclude-words ', - "Matches the tweets that do not contain any of the give list of words, separated by ';'", + 'Matches the tweets that do not contain any of the give comma-separated list of words', ) - .option('-h, --hashtags ', "Matches the tweets containing the given list of hashtags, separated by ';'") + .option('-h, --hashtags ', 'Matches the tweets containing the given comma-separated list of hashtags') .option( '-m, --mentions ', - "Matches the tweets that mention the give list of usernames, separated by ';'", + 'Matches the tweets that mention the given comma-separated list of usernames', ) .option('-r, --min-replies ', 'Matches the tweets that have a minimum of given number of replies') .option('-l, --min-likes ', 'Matches the tweets that have a minimum of given number of likes') @@ -57,13 +57,27 @@ function createTweetCommand(rettiwt: Rettiwt): Command { .option('--exclude-replies', 'Matches the tweets that are not replies') .option('-s, --start ', 'Matches the tweets made since the given date (valid date string)') .option('-e, --end ', 'Matches the tweets made upto the given date (valid date string)') + .option('--stream', 'Stream the filtered tweets in pseudo-realtime') + .option('-i, --interval ', 'The polling interval (in ms) to use for streaming. Default is 60000') .action(async (count?: string, cursor?: string, options?: TweetSearchOptions) => { - const tweets = await rettiwt.tweet.search( - new TweetSearchOptions(options).toTweetFilter(), - count ? parseInt(count) : undefined, - cursor, - ); - output(tweets); + // If search results are to be streamed + if (options?.stream) { + for await (const tweet of rettiwt.tweet.stream( + new TweetSearchOptions(options).toTweetFilter(), + options?.interval, + )) { + output(tweet); + } + } + // If a normal search is to be done + else { + const tweets = await rettiwt.tweet.search( + new TweetSearchOptions(options).toTweetFilter(), + count ? parseInt(count) : undefined, + cursor, + ); + output(tweets); + } }); // List @@ -107,11 +121,16 @@ function createTweetCommand(rettiwt: Rettiwt): Command { .command('post') .description('Post a tweet (text only)') .argument('', 'The text to post as a tweet') - .option('-m, --media [string]', "The path to the media item(s) to be posted, separated by ';'") - .action(async (text: string, options?: { media?: string }) => { + .option('-m, --media [string]', 'Comma-separated list of path(s) to the media item(s) to be posted') + .option( + '-r, --reply [string]', + 'The id of the tweet to which the reply is to be made, if the tweet is to be a reply', + ) + .action(async (text: string, options?: { media?: string; reply?: string }) => { const result = await rettiwt.tweet.tweet( text, - options?.media ? options?.media.split(';').map((item) => ({ path: item })) : undefined, + options?.media ? options?.media.split(',').map((item) => ({ path: item })) : undefined, + options?.reply, ); output(result); }); @@ -161,6 +180,8 @@ class TweetSearchOptions { public excludeReplies?: boolean = false; public start?: string; public end?: string; + public stream?: boolean; + public interval?: number; /** * Initializes a new object from the given options. @@ -184,6 +205,8 @@ class TweetSearchOptions { this.excludeReplies = options?.excludeReplies; this.start = options?.start; this.end = options?.end; + this.stream = options?.stream; + this.interval = options?.interval; } /** @@ -193,14 +216,14 @@ class TweetSearchOptions { */ public toTweetFilter(): TweetFilter { return new TweetFilter({ - fromUsers: this.from ? this.from.split(';') : undefined, - toUsers: this.to ? this.to.split(';') : undefined, - includeWords: this.words ? this.words.split(';') : undefined, + fromUsers: this.from ? this.from.split(',') : undefined, + toUsers: this.to ? this.to.split(',') : undefined, + includeWords: this.words ? this.words.split(',') : undefined, includePhrase: this.phrase, - optionalWords: this.optionalWords ? this.optionalWords.split(';') : undefined, - excludeWords: this.excludeWords ? this.excludeWords.split(';') : undefined, - hashtags: this.hashtags ? this.hashtags.split(';') : undefined, - mentions: this.mentions ? this.mentions.split(';') : undefined, + optionalWords: this.optionalWords ? this.optionalWords.split(',') : undefined, + excludeWords: this.excludeWords ? this.excludeWords.split(',') : undefined, + hashtags: this.hashtags ? this.hashtags.split(',') : undefined, + mentions: this.mentions ? this.mentions.split(',') : undefined, minReplies: this.minReplies, minLikes: this.minLikes, minRetweets: this.minRetweets, diff --git a/src/helper/JsonUtils.ts b/src/helper/JsonUtils.ts index 03ece586..b26d11a9 100644 --- a/src/helper/JsonUtils.ts +++ b/src/helper/JsonUtils.ts @@ -46,24 +46,6 @@ export function findByFilter(data: NonNullable, key: string, value: return res; } -/** - * @param text - The text to be normalized. - * @returns The text after being formatted to remove unnecessary characters. - * - * @internal - */ -export function normalizeText(text: string): string { - let normalizedText: string = ''; // To store the normalized text - - // Removing unnecessary full stops, and other characters - normalizedText = text.replace(/\n/g, '.').replace(/[.]+[\s+.\s+]+/g, '. '); - - // Adding full-stop to the end if does not exist already - normalizedText = normalizedText.endsWith('.') ? normalizedText : normalizedText + '.'; - - return normalizedText; -} - /** * Searches for the key which has the given value in the given object. * diff --git a/src/models/data/Tweet.ts b/src/models/data/Tweet.ts index e3c26382..45ffbfb3 100644 --- a/src/models/data/Tweet.ts +++ b/src/models/data/Tweet.ts @@ -9,9 +9,6 @@ import { // MODELS import { User } from './User'; -// PARSERS -import { normalizeText } from '../../helper/JsonUtils'; - /** * The details of a single Tweet. * @@ -39,7 +36,7 @@ export class Tweet { /** The full text content of the tweet. */ public fullText: string; - /** The rest id of the user to which the tweet is a reply. */ + /** The rest id of the tweet to which the tweet is a reply. */ public replyTo: string; /** The language in which the tweet is written. */ @@ -75,9 +72,7 @@ export class Tweet { this.entities = new TweetEntities(tweet.legacy.entities); this.media = tweet.legacy.extended_entities?.media?.map((media) => new TweetMedia(media)); this.quoted = tweet.legacy.quoted_status_id_str; - this.fullText = tweet.note_tweet - ? tweet.note_tweet.note_tweet_results.result.text - : normalizeText(tweet.legacy.full_text); + this.fullText = tweet.note_tweet ? tweet.note_tweet.note_tweet_results.result.text : tweet.legacy.full_text; this.replyTo = tweet.legacy.in_reply_to_status_id_str; this.lang = tweet.legacy.lang; this.quoteCount = tweet.legacy.quote_count; diff --git a/src/services/public/TweetService.ts b/src/services/public/TweetService.ts index ef14a832..3ae35827 100644 --- a/src/services/public/TweetService.ts +++ b/src/services/public/TweetService.ts @@ -103,6 +103,71 @@ export class TweetService extends FetcherService { return data; } + /** + * Stream tweets in pseudo real-time using a filter. + * + * @param filter - The filter to be used for searching the tweets. + * @param pollingIntervalMs - The interval in milliseconds to poll for new tweets. Default interval is 60000 ms. + * @returns An async generator that yields matching tweets as they are found. + * + * @example + * ``` + * import { Rettiwt } from 'rettiwt-api'; + * + * // Creating a new Rettiwt instance using the given 'API_KEY' + * const rettiwt = new Rettiwt({ apiKey: API_KEY }); + * + * // Streaming all upcoming tweets from user 'user1' + * async () => { + * try { + * for await (const tweet of rettiwt.tweet.stream({ fromUsers: ['user1'] }, 1000)) { + * console.log(tweet.fullText); + * } + * } + * catch (err) { + * console.log(err); + * } + * }(); + * ``` + * + * @public + */ + public async *stream(filter: TweetFilter, pollingInterval: number = 60000): AsyncGenerator { + const startDate = new Date(); + + let cursor: string | undefined = undefined; + let sinceId: string | undefined = undefined; + let nextSinceId: string | undefined = undefined; + + while (true) { + // Pause execution for the specified polling interval before proceeding to the next iteration + await new Promise((resolve) => setTimeout(resolve, pollingInterval)); + + // Search for tweets + const tweets = await this.search({ ...filter, startDate: startDate, sinceId: sinceId }, undefined, cursor); + + // Yield the matching tweets + for (const tweet of tweets.list) { + yield tweet; + } + + // Store the most recent tweet ID from this batch + if (tweets.list.length > 0 && cursor === undefined) { + nextSinceId = tweets.list[0].id; + } + + // If there are more tweets to fetch, adjust the cursor value + if (tweets.list.length > 0 && tweets.next) { + cursor = tweets.next.value; + } + // Else, start the next iteration from this batch's most recent tweet + else { + sinceId = nextSinceId; + cursor = undefined; + } + } + } + /** * Get the tweets from the tweet list with the given id. * @@ -261,9 +326,26 @@ export class TweetService extends FetcherService { * }); * ``` * + * @example Posting a reply to a tweet + * ``` + * import { Rettiwt } from 'rettiwt-api'; + * + * // Creating a new Rettiwt instance using the given 'API_KEY' + * const rettiwt = new Rettiwt({ apiKey: API_KEY }); + * + * // Posting a simple text reply, to a tweet with id "1234567890" + * rettiwt.tweet.tweet('Hello!', undefined, "1234567890") + * .then(res => { + * console.log(res); + * }) + * .catch(err => { + * console.log(err); + * }); + * ``` + * * @public */ - public async tweet(text: string, media?: TweetMediaArgs[]): Promise { + public async tweet(text: string, media?: TweetMediaArgs[], replyTo?: string): Promise { // Converting JSON args to object const tweet: TweetArgs = new TweetArgs({ text: text, media: media }); @@ -282,7 +364,9 @@ export class TweetService extends FetcherService { } // Posting the tweet - const data = await this.post(EResourceType.CREATE_TWEET, { tweet: { text: text, media: uploadedMedia } }); + const data = await this.post(EResourceType.CREATE_TWEET, { + tweet: { text: text, media: uploadedMedia, replyTo: replyTo }, + }); return data; } diff --git a/yarn.lock b/yarn.lock index 8efa8df0..3342ed1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1090,10 +1090,10 @@ rettiwt-auth@2.1.0: cookiejar "2.1.4" https-proxy-agent "7.0.2" -rettiwt-core@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/rettiwt-core/-/rettiwt-core-3.3.2.tgz#9f47d80579646d8d4d94551ecd92795854e466c2" - integrity sha512-STrDXsZ5dTRAEI071pCw8QPpHWh7WXqRDexJ0tLOEtutttlMvANLRumaRaw8rI0U/AnVr4w7kYohax39Ig8ing== +rettiwt-core@3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/rettiwt-core/-/rettiwt-core-3.4.0.tgz#515aa168a5a64effa6cb2340debafebf5d4ee8a1" + integrity sha512-7N2wV+AB5fJVMzyE0s6FS9jZ/4z/eXIM1DlhOqeWous36PZSDPprINIcdPM8LVmfMEpq4aanplHkqCheNGLXBw== dependencies: axios "1.6.3" class-validator "0.14.1"