diff --git a/src/lib/query-ids.json b/src/lib/query-ids.json index 9b76a72..882c6ac 100644 --- a/src/lib/query-ids.json +++ b/src/lib/query-ids.json @@ -1,7 +1,7 @@ { - "CreateTweet": "TAJw1rBsjAtdNgTdlo2oeg", - "CreateRetweet": "ojPdsZsimiJrUGLR1sjUtA", - "FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A", - "TweetDetail": "nBS-WpgA6ZG0CyNHD517JQ", - "SearchTimeline": "Tp1sewRU1AsZpBWhqCZicQ" + "CreateTweet": "2BA2DIWTRVxHgKc4Xh2uKQ", + "CreateRetweet": "Uf3io9zVp1DsYxrmL5FJ7g", + "FavoriteTweet": "kuFUYP9eV1FPoEy4N-pi7w", + "TweetDetail": "aFvUsJm2c-oDkJV75blV6g", + "SearchTimeline": "M1jEez78PEfVfbQLvlWMvQ" } diff --git a/src/lib/twitter-client.ts b/src/lib/twitter-client.ts index 1a79de9..2c314f1 100644 --- a/src/lib/twitter-client.ts +++ b/src/lib/twitter-client.ts @@ -15,7 +15,8 @@ const FALLBACK_QUERY_IDS = { CreateRetweet: 'ojPdsZsimiJrUGLR1sjUtA', FavoriteTweet: 'lI07N6Otwv1PhnEgXILM7A', TweetDetail: 'nBS-WpgA6ZG0CyNHD517JQ', - SearchTimeline: 'Tp1sewRU1AsZpBWhqCZicQ', + SearchTimeline: 'M1jEez78PEfVfbQLvlWMvQ', + UserArticlesTweets: '8zBy9h4L90aDL02RsBcCFg', } as const; type OperationName = keyof typeof FALLBACK_QUERY_IDS; @@ -25,6 +26,10 @@ const QUERY_IDS: Record = { ...(queryIds as Partial>), }; +const SEARCH_TIMELINE_QUERY_IDS = Array.from( + new Set([QUERY_IDS.SearchTimeline, 'M1jEez78PEfVfbQLvlWMvQ', '5h0kNbk3ii97rmfY6CdgAA', 'Tp1sewRU1AsZpBWhqCZicQ']), +); + type GraphqlTweetResult = { rest_id?: string; legacy?: { @@ -39,13 +44,137 @@ type GraphqlTweetResult = { core?: { user_results?: { result?: { + rest_id?: string; + id?: string; legacy?: { screen_name?: string; name?: string; }; + core?: { + screen_name?: string; + name?: string; + }; }; }; }; + note_tweet?: { + note_tweet_results?: { + result?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + content?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + }; + }; + }; + }; + article?: { + article_results?: { + result?: { + title?: string; + plain_text?: string; + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + body?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + }; + content?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + items?: Array<{ + text?: string; + content?: { + text?: string; + richtext?: { text?: string }; + rich_text?: { text?: string }; + }; + }>; + }; + sections?: Array<{ + items?: Array<{ + text?: string; + content?: { + text?: string; + richtext?: { text?: string }; + rich_text?: { text?: string }; + }; + }>; + }>; + }; + }; + title?: string; + plain_text?: string; + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + body?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + }; + content?: { + text?: string; + richtext?: { + text?: string; + }; + rich_text?: { + text?: string; + }; + items?: Array<{ + text?: string; + content?: { + text?: string; + richtext?: { text?: string }; + rich_text?: { text?: string }; + }; + }>; + }; + sections?: Array<{ + items?: Array<{ + text?: string; + content?: { + text?: string; + richtext?: { text?: string }; + rich_text?: { text?: string }; + }; + }>; + }>; + }; }; export interface TweetResult { @@ -61,6 +190,7 @@ export interface TweetData { username: string; name: string; }; + authorId?: string; createdAt?: string; replyCount?: number; retweetCount?: number; @@ -94,6 +224,7 @@ export interface CurrentUserResult { export interface TwitterClientOptions { cookies: TwitterCookies; userAgent?: string; + timeoutMs?: number; } interface CreateTweetResponse { @@ -116,6 +247,7 @@ export class TwitterClient { private authToken: string; private ct0: string; private userAgent: string; + private timeoutMs?: number; constructor(options: TwitterClientOptions) { if (!options.cookies.authToken || !options.cookies.ct0) { @@ -126,6 +258,21 @@ export class TwitterClient { this.userAgent = options.userAgent || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; + this.timeoutMs = options.timeoutMs; + } + + private async fetchWithTimeout(url: string, init: RequestInit): Promise { + if (!this.timeoutMs || this.timeoutMs <= 0) { + return fetch(url, init); + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs); + try { + return await fetch(url, { ...init, signal: controller.signal }); + } finally { + clearTimeout(timeoutId); + } } private findTweetInInstructions( @@ -135,26 +282,7 @@ export class TwitterClient { content?: { itemContent?: { tweet_results?: { - result?: { - rest_id?: string; - legacy?: { - full_text?: string; - created_at?: string; - reply_count?: number; - retweet_count?: number; - favorite_count?: number; - }; - core?: { - user_results?: { - result?: { - legacy?: { - screen_name?: string; - name?: string; - }; - }; - }; - }; - }; + result?: GraphqlTweetResult; }; }; }; @@ -193,24 +321,221 @@ export class TwitterClient { }; } + private firstText(...values: Array): string | undefined { + for (const value of values) { + if (typeof value === 'string') { + const trimmed = value.trim(); + if (trimmed) return trimmed; + } + } + return undefined; + } + + private collectTextFields(value: unknown, keys: Set, output: string[]): void { + if (!value) return; + if (typeof value === 'string') return; + + if (Array.isArray(value)) { + for (const item of value) { + this.collectTextFields(item, keys, output); + } + return; + } + + if (typeof value === 'object') { + for (const [key, nested] of Object.entries(value as Record)) { + if (keys.has(key)) { + if (typeof nested === 'string') { + const trimmed = nested.trim(); + if (trimmed) output.push(trimmed); + continue; + } + } + this.collectTextFields(nested, keys, output); + } + } + } + + private uniqueOrdered(values: string[]): string[] { + const seen = new Set(); + const result: string[] = []; + for (const value of values) { + if (seen.has(value)) continue; + seen.add(value); + result.push(value); + } + return result; + } + + private extractArticleText(result: GraphqlTweetResult | undefined): string | undefined { + const article = result?.article; + if (!article) return undefined; + + const articleResult = article.article_results?.result ?? article; + if (process.env.BIRD_DEBUG_ARTICLE === '1') { + console.error( + '[bird][debug][article] payload:', + JSON.stringify( + { + rest_id: result?.rest_id, + article: articleResult, + note_tweet: result?.note_tweet?.note_tweet_results?.result ?? null, + }, + null, + 2, + ), + ); + } + const title = this.firstText(articleResult.title, article.title); + let body = this.firstText( + articleResult.plain_text, + article.plain_text, + articleResult.body?.text, + articleResult.body?.richtext?.text, + articleResult.body?.rich_text?.text, + articleResult.content?.text, + articleResult.content?.richtext?.text, + articleResult.content?.rich_text?.text, + articleResult.text, + articleResult.richtext?.text, + articleResult.rich_text?.text, + article.body?.text, + article.body?.richtext?.text, + article.body?.rich_text?.text, + article.content?.text, + article.content?.richtext?.text, + article.content?.rich_text?.text, + article.text, + article.richtext?.text, + article.rich_text?.text, + ); + + if (body && title && body.trim() === title.trim()) { + body = undefined; + } + + if (!body) { + const collected: string[] = []; + this.collectTextFields(articleResult, new Set(['text', 'title']), collected); + this.collectTextFields(article, new Set(['text', 'title']), collected); + const unique = this.uniqueOrdered(collected); + const filtered = title ? unique.filter((value) => value !== title) : unique; + if (filtered.length > 0) { + body = filtered.join('\n\n'); + } + } + + if (title && body && !body.startsWith(title)) { + return `${title}\n\n${body}`; + } + + return body ?? title; + } + + private extractNoteTweetText(result: GraphqlTweetResult | undefined): string | undefined { + const note = result?.note_tweet?.note_tweet_results?.result; + if (!note) return undefined; + + return this.firstText( + note.text, + note.richtext?.text, + note.rich_text?.text, + note.content?.text, + note.content?.richtext?.text, + note.content?.rich_text?.text, + ); + } + + private extractTweetText(result: GraphqlTweetResult | undefined): string | undefined { + return ( + this.extractArticleText(result) ?? this.extractNoteTweetText(result) ?? this.firstText(result?.legacy?.full_text) + ); + } + private mapTweetResult(result: GraphqlTweetResult | undefined): TweetData | undefined { - if (!result?.legacy || !result.core?.user_results?.result?.legacy?.screen_name) return undefined; + const userResult = result?.core?.user_results?.result; + const userLegacy = userResult?.legacy; + const userCore = userResult?.core; + const username = userLegacy?.screen_name ?? userCore?.screen_name; + const name = userLegacy?.name ?? userCore?.name ?? username; + const userId = userResult?.rest_id; + if (!result?.rest_id || !username) return undefined; + + const text = this.extractTweetText(result); + if (!text) return undefined; + return { - id: result.rest_id || '', - text: result.legacy.full_text || '', - createdAt: result.legacy.created_at, - replyCount: result.legacy.reply_count, - retweetCount: result.legacy.retweet_count, - likeCount: result.legacy.favorite_count, - conversationId: result.legacy.conversation_id_str, - inReplyToStatusId: result.legacy.in_reply_to_status_id_str ?? undefined, + id: result.rest_id, + text, + createdAt: result.legacy?.created_at, + replyCount: result.legacy?.reply_count, + retweetCount: result.legacy?.retweet_count, + likeCount: result.legacy?.favorite_count, + conversationId: result.legacy?.conversation_id_str, + inReplyToStatusId: result.legacy?.in_reply_to_status_id_str ?? undefined, author: { - username: result.core.user_results.result.legacy.screen_name, - name: result.core.user_results.result.legacy.name || result.core.user_results.result.legacy.screen_name, + username, + name: name || username, }, + authorId: userId, }; } + private collectTweetResultsFromEntry(entry: { + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + item?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + items?: Array<{ + item?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + }>; + }; + }): GraphqlTweetResult[] { + const results: GraphqlTweetResult[] = []; + const pushResult = (result?: GraphqlTweetResult) => { + if (result?.rest_id) results.push(result); + }; + + const content = entry.content; + pushResult(content?.itemContent?.tweet_results?.result); + pushResult(content?.item?.itemContent?.tweet_results?.result); + + for (const item of content?.items ?? []) { + pushResult(item?.item?.itemContent?.tweet_results?.result); + pushResult(item?.itemContent?.tweet_results?.result); + pushResult(item?.content?.itemContent?.tweet_results?.result); + } + + return results; + } + private parseTweetsFromInstructions( instructions: | Array<{ @@ -221,22 +546,224 @@ export class TwitterClient { result?: GraphqlTweetResult; }; }; + item?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + items?: Array<{ + item?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + }>; }; }>; }> | undefined, ): TweetData[] { const tweets: TweetData[] = []; + const seen = new Set(); + for (const instruction of instructions ?? []) { for (const entry of instruction.entries ?? []) { - const result = entry.content?.itemContent?.tweet_results?.result; - const mapped = this.mapTweetResult(result); - if (mapped) tweets.push(mapped); + const results = this.collectTweetResultsFromEntry(entry); + for (const result of results) { + const mapped = this.mapTweetResult(result); + if (!mapped || seen.has(mapped.id)) continue; + seen.add(mapped.id); + tweets.push(mapped); + } } } + return tweets; } + private buildArticleFeatures(): Record { + return { + rweb_video_screen_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: false, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: true, + creator_subscriptions_quote_tweet_preview_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, + responsive_web_enhance_cards_enabled: false, + }; + } + + private buildArticleFieldToggles(): Record { + return { + withPayments: false, + withAuxiliaryUserLabels: false, + withArticleRichContentState: true, + withArticlePlainText: true, + withGrokAnalyze: false, + withDisallowedReplyControls: false, + }; + } + + private buildSearchFeatures(): Record { + return { + rweb_video_screen_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: false, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: true, + creator_subscriptions_quote_tweet_preview_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + rweb_video_timestamps_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, + articles_preview_enabled: true, + responsive_web_enhance_cards_enabled: false, + }; + } + + private async fetchUserArticlePlainText( + userId: string, + tweetId: string, + ): Promise<{ title?: string; plainText?: string }> { + const variables = { + userId, + count: 20, + includePromotedContent: true, + withVoice: true, + withQuickPromoteEligibilityTweetFields: true, + withBirdwatchNotes: true, + withCommunity: true, + withSafetyModeUserFields: true, + withSuperFollowsUserFields: true, + withDownvotePerspective: false, + withReactionsMetadata: false, + withReactionsPerspective: false, + withSuperFollowsTweetFields: true, + withSuperFollowsReplyCount: false, + withClientEventToken: false, + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(this.buildArticleFeatures()), + fieldToggles: JSON.stringify(this.buildArticleFieldToggles()), + }); + + const url = `${TWITTER_API_BASE}/${QUERY_IDS.UserArticlesTweets}/UserArticlesTweets?${params}`; + + try { + const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders() }); + if (!response.ok) { + return {}; + } + + const data = (await response.json()) as { + data?: { + user?: { + result?: { + timeline?: { + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { result?: GraphqlTweetResult }; + }; + }; + }>; + }>; + }; + }; + }; + }; + }; + }; + + const instructions = data.data?.user?.result?.timeline?.timeline?.instructions ?? []; + for (const instruction of instructions) { + for (const entry of instruction.entries ?? []) { + const result = entry.content?.itemContent?.tweet_results?.result; + if (result?.rest_id !== tweetId) continue; + const articleResult = result.article?.article_results?.result; + const title = this.firstText(articleResult?.title, result.article?.title); + const plainText = this.firstText(articleResult?.plain_text, result.article?.plain_text); + return { title, plainText }; + } + } + } catch { + return {}; + } + + return {}; + } + private async fetchTweetDetail(tweetId: string): Promise< | { success: true; @@ -280,11 +807,15 @@ export class TwitterClient { communities_web_enable_tweet_community_results_fetch: true, c9s_tweet_anatomy_moderator_badge_enabled: true, articles_preview_enabled: true, + articles_rest_api_enabled: true, responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true, responsive_web_twitter_article_tweet_consumption_enabled: true, + responsive_web_twitter_article_plain_text_enabled: true, + responsive_web_twitter_article_seed_tweet_detail_enabled: true, + responsive_web_twitter_article_seed_tweet_summary_enabled: true, tweet_awards_web_tipping_enabled: false, creator_subscriptions_quote_tweet_preview_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true, @@ -304,7 +835,7 @@ export class TwitterClient { const url = `${TWITTER_API_BASE}/${QUERY_IDS.TweetDetail}/TweetDetail?${params}`; try { - const response = await fetch(url, { + const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders(), }); @@ -374,6 +905,19 @@ export class TwitterClient { const mapped = this.mapTweetResult(tweetResult); if (mapped) { + if (tweetResult?.article) { + const title = this.firstText(tweetResult.article.article_results?.result?.title, tweetResult.article.title); + const articleText = this.extractArticleText(tweetResult); + if (title && (!articleText || articleText.trim() === title.trim())) { + const userId = tweetResult.core?.user_results?.result?.rest_id; + if (userId) { + const fallback = await this.fetchUserArticlePlainText(userId, tweetId); + if (fallback.plainText) { + mapped.text = fallback.title ? `${fallback.title}\n\n${fallback.plainText}` : fallback.plainText; + } + } + } + } return { success: true, tweet: mapped }; } return { success: false, error: 'Tweet not found in response' }; @@ -500,7 +1044,7 @@ export class TwitterClient { }); try { - const response = await fetch(url, { + const response = await this.fetchWithTimeout(url, { method: 'POST', headers: this.getHeaders(), body, @@ -554,79 +1098,60 @@ export class TwitterClient { product: 'Latest', }; - const features = { - rweb_tipjar_consumption_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, - verified_phone_label_enabled: false, - creator_subscriptions_tweet_preview_api_enabled: true, - responsive_web_graphql_timeline_navigation_enabled: true, - responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, - communities_web_enable_tweet_community_results_fetch: true, - c9s_tweet_anatomy_moderator_badge_enabled: true, - articles_preview_enabled: true, - responsive_web_edit_tweet_api_enabled: true, - graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, - view_counts_everywhere_api_enabled: true, - longform_notetweets_consumption_enabled: true, - responsive_web_twitter_article_tweet_consumption_enabled: true, - tweet_awards_web_tipping_enabled: false, - creator_subscriptions_quote_tweet_preview_enabled: false, - freedom_of_speech_not_reach_fetch_enabled: true, - standardized_nudges_misinfo: true, - tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - rweb_video_timestamps_enabled: true, - longform_notetweets_rich_text_read_enabled: true, - longform_notetweets_inline_media_enabled: true, - responsive_web_enhance_cards_enabled: false, - }; + const features = this.buildSearchFeatures(); - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); + let lastError: string | undefined; - const url = `${TWITTER_API_BASE}/${QUERY_IDS.SearchTimeline}/SearchTimeline?${params}`; - - try { - const response = await fetch(url, { - method: 'GET', - headers: this.getHeaders(), + for (const queryId of SEARCH_TIMELINE_QUERY_IDS) { + const params = new URLSearchParams({ + variables: JSON.stringify(variables), }); + const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params}`; - if (!response.ok) { - const text = await response.text(); - return { - success: false, - error: `HTTP ${response.status}: ${text.slice(0, 200)}`, - }; - } + try { + const response = await this.fetchWithTimeout(url, { + method: 'POST', + headers: this.getHeaders(), + body: JSON.stringify({ features, queryId }), + }); - const data = (await response.json()) as { - data?: { - search_by_raw_query?: { - search_timeline?: { - timeline?: { - instructions?: Array<{ - entries?: Array<{ - content?: { - itemContent?: { - tweet_results?: { - result?: { - rest_id?: string; - legacy?: { - full_text?: string; - created_at?: string; - reply_count?: number; - retweet_count?: number; - favorite_count?: number; - in_reply_to_status_id_str?: string; - }; - core?: { - user_results?: { - result?: { - legacy?: { - screen_name?: string; - name?: string; + if (response.status === 404) { + lastError = `HTTP ${response.status}`; + continue; + } + + if (!response.ok) { + const text = await response.text(); + return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}` }; + } + + const data = (await response.json()) as { + data?: { + search_by_raw_query?: { + search_timeline?: { + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: { + rest_id?: string; + legacy?: { + full_text?: string; + created_at?: string; + reply_count?: number; + retweet_count?: number; + favorite_count?: number; + in_reply_to_status_id_str?: string; + }; + core?: { + user_results?: { + result?: { + legacy?: { + screen_name?: string; + name?: string; + }; }; }; }; @@ -634,61 +1159,29 @@ export class TwitterClient { }; }; }; - }; + }>; }>; - }>; + }; }; }; }; + errors?: Array<{ message: string }>; }; - errors?: Array<{ message: string }>; - }; - if (data.errors && data.errors.length > 0) { - return { - success: false, - error: data.errors.map((e) => e.message).join(', '), - }; - } - - const tweets: TweetData[] = []; - const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || []; - - for (const instruction of instructions) { - for (const entry of instruction.entries || []) { - const result = entry.content?.itemContent?.tweet_results?.result; - if (!result) continue; - - const legacy = result.legacy; - const userLegacy = result.core?.user_results?.result?.legacy; - - if (legacy?.full_text && userLegacy?.screen_name) { - tweets.push({ - id: result.rest_id || '', - text: legacy.full_text, - author: { - username: userLegacy.screen_name, - name: userLegacy.name || userLegacy.screen_name, - }, - createdAt: legacy.created_at, - replyCount: legacy.reply_count, - retweetCount: legacy.retweet_count, - likeCount: legacy.favorite_count, - }); - } + if (data.errors && data.errors.length > 0) { + return { success: false, error: data.errors.map((e) => e.message).join(', ') }; } - } - return { - success: true, - tweets, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : String(error), - }; + const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; + const tweets = this.parseTweetsFromInstructions(instructions); + + return { success: true, tweets }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } } + + return { success: false, error: lastError ?? 'Unknown error fetching search results' }; } /** @@ -706,7 +1199,7 @@ export class TwitterClient { for (const url of candidateUrls) { try { - const response = await fetch(url, { + const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders(), }); @@ -772,7 +1265,7 @@ export class TwitterClient { const profilePages = ['https://x.com/settings/account', 'https://twitter.com/settings/account']; for (const page of profilePages) { try { - const response = await fetch(page, { + const response = await this.fetchWithTimeout(page, { headers: { cookie: `auth_token=${this.authToken}; ct0=${this.ct0}`, 'user-agent': this.userAgent, diff --git a/tests/twitter-client.test.ts b/tests/twitter-client.test.ts index d7dad16..5179c09 100644 --- a/tests/twitter-client.test.ts +++ b/tests/twitter-client.test.ts @@ -295,6 +295,185 @@ describe('TwitterClient', () => { expect(result.success).toBe(false); expect(result.error).toContain('HTTP 404'); }); + + it('should return article text when present', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + tweetResult: { + result: { + rest_id: 'article123', + legacy: { + full_text: '', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + }, + article: { + article_results: { + result: { + title: '2025 LLM Year in Review', + sections: [ + { + items: [ + { text: 'Intro paragraph of the article.' }, + { content: { text: 'Second paragraph.' } }, + ], + }, + ], + }, + }, + }, + core: { + user_results: { + result: { + legacy: { + screen_name: 'author', + name: 'Article Author', + }, + }, + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getTweet('article123'); + + expect(result.success).toBe(true); + expect(result.tweet?.text).toBe( + '2025 LLM Year in Review\n\nIntro paragraph of the article.\n\nSecond paragraph.', + ); + }); + + it('should fall back to user article timeline for plain text', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + tweetResult: { + result: { + rest_id: 'article123', + legacy: { + full_text: '', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + }, + article: { + article_results: { + result: { + title: '2025 LLM Year in Review', + }, + }, + }, + core: { + user_results: { + result: { + rest_id: '33836629', + legacy: { + screen_name: 'author', + name: 'Article Author', + }, + }, + }, + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'article123', + article: { + article_results: { + result: { + title: '2025 LLM Year in Review', + plain_text: 'Full article body.', + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getTweet('article123'); + + expect(result.success).toBe(true); + expect(result.tweet?.text).toBe('2025 LLM Year in Review\n\nFull article body.'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('should return note tweet text when present', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + tweetResult: { + result: { + rest_id: 'note123', + legacy: { + full_text: '', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + }, + note_tweet: { + note_tweet_results: { + result: { + text: 'Long form note content.', + }, + }, + }, + core: { + user_results: { + result: { + legacy: { + screen_name: 'noter', + name: 'Note Author', + }, + }, + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getTweet('note123'); + + expect(result.success).toBe(true); + expect(result.tweet?.text).toBe('Long form note content.'); + }); }); describe('getCurrentUser', () => { @@ -375,6 +554,85 @@ describe('TwitterClient', () => { }); }); + describe('search', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch as unknown as typeof fetch; + }); + + it('retries on 404 and posts search payload', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 404, + text: async () => '', + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'found', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { + user_results: { + result: { legacy: { screen_name: 'root', name: 'Root' } }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.search('needle', 1); + + expect(result.success).toBe(true); + expect(result.tweets?.[0].id).toBe('1'); + expect(mockFetch).toHaveBeenCalledTimes(2); + + const [url, options] = mockFetch.mock.calls[1]; + expect(options.method).toBe('POST'); + const body = JSON.parse(options.body); + expect(body.features).toBeDefined(); + expect(body.queryId).toBeDefined(); + const urlVars = new URL(url as string).searchParams.get('variables'); + expect(urlVars).toBeTruthy(); + const parsed = JSON.parse(urlVars as string) as { rawQuery?: string }; + expect(parsed.rawQuery).toBe('needle'); + }); + }); + describe('conversation helpers', () => { let mockFetch: ReturnType; @@ -455,9 +713,12 @@ describe('TwitterClient', () => { it('getThread returns sorted thread by createdAt', async () => { const payload = makeConversationPayload(); // swap dates to verify sorting - payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.[0]?.content?.itemContent?.tweet_results?.result?.legacy && - ((payload.data.threaded_conversation_with_injections_v2.instructions[0]!.entries![0]!.content!.itemContent! - .tweet_results!.result!.legacy!.created_at = '2024-01-03T00:00:00Z')); + const legacy = + payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.[0]?.content?.itemContent + ?.tweet_results?.result?.legacy; + if (legacy) { + legacy.created_at = '2024-01-03T00:00:00Z'; + } mockFetch.mockResolvedValue({ ok: true, @@ -472,6 +733,51 @@ describe('TwitterClient', () => { expect(result.tweets?.map((t) => t.id)).toEqual(['2', '1']); // sorted by createdAt asc }); + it('getThread includes tweets from timeline module items', async () => { + const payload = makeConversationPayload(); + payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.push({ + content: { + items: [ + { + item: { + itemContent: { + tweet_results: { + result: { + rest_id: '3', + legacy: { + full_text: 'nested reply', + created_at: '2024-01-04T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + in_reply_to_status_id_str: '1', + }, + core: { + user_results: { result: { legacy: { screen_name: 'nested', name: 'Nested' } } }, + }, + }, + }, + }, + }, + }, + ], + }, + }); + + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => payload, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThread('1'); + + expect(result.success).toBe(true); + expect(result.tweets?.map((t) => t.id)).toEqual(['1', '2', '3']); + }); + it('propagates fetchTweetDetail errors', async () => { mockFetch.mockResolvedValue({ ok: false,