feat(twitter): improve graphql search and threads
This commit is contained in:
Peter Steinberger
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"CreateTweet": "TAJw1rBsjAtdNgTdlo2oeg",
|
||||
"CreateRetweet": "ojPdsZsimiJrUGLR1sjUtA",
|
||||
"FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A",
|
||||
"TweetDetail": "nBS-WpgA6ZG0CyNHD517JQ",
|
||||
"SearchTimeline": "Tp1sewRU1AsZpBWhqCZicQ"
|
||||
"CreateTweet": "2BA2DIWTRVxHgKc4Xh2uKQ",
|
||||
"CreateRetweet": "Uf3io9zVp1DsYxrmL5FJ7g",
|
||||
"FavoriteTweet": "kuFUYP9eV1FPoEy4N-pi7w",
|
||||
"TweetDetail": "aFvUsJm2c-oDkJV75blV6g",
|
||||
"SearchTimeline": "M1jEez78PEfVfbQLvlWMvQ"
|
||||
}
|
||||
|
||||
+607
-114
@@ -15,7 +15,8 @@ const FALLBACK_QUERY_IDS = {
|
||||
CreateRetweet: 'ojPdsZsimiJrUGLR1sjUtA',
|
||||
FavoriteTweet: 'lI07N6Otwv1PhnEgXILM7A',
|
||||
TweetDetail: 'nBS-WpgA6ZG0CyNHD517JQ',
|
||||
SearchTimeline: 'Tp1sewRU1AsZpBWhqCZicQ',
|
||||
SearchTimeline: 'M1jEez78PEfVfbQLvlWMvQ',
|
||||
UserArticlesTweets: '8zBy9h4L90aDL02RsBcCFg',
|
||||
} as const;
|
||||
|
||||
type OperationName = keyof typeof FALLBACK_QUERY_IDS;
|
||||
@@ -25,6 +26,10 @@ const QUERY_IDS: Record<OperationName, string> = {
|
||||
...(queryIds as Partial<Record<OperationName, string>>),
|
||||
};
|
||||
|
||||
const SEARCH_TIMELINE_QUERY_IDS = Array.from(
|
||||
new Set([QUERY_IDS.SearchTimeline, 'M1jEez78PEfVfbQLvlWMvQ', '5h0kNbk3ii97rmfY6CdgAA', 'Tp1sewRU1AsZpBWhqCZicQ']),
|
||||
);
|
||||
|
||||
type GraphqlTweetResult = {
|
||||
rest_id?: string;
|
||||
legacy?: {
|
||||
@@ -39,13 +44,137 @@ type GraphqlTweetResult = {
|
||||
core?: {
|
||||
user_results?: {
|
||||
result?: {
|
||||
rest_id?: string;
|
||||
id?: string;
|
||||
legacy?: {
|
||||
screen_name?: string;
|
||||
name?: string;
|
||||
};
|
||||
core?: {
|
||||
screen_name?: string;
|
||||
name?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
note_tweet?: {
|
||||
note_tweet_results?: {
|
||||
result?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
article?: {
|
||||
article_results?: {
|
||||
result?: {
|
||||
title?: string;
|
||||
plain_text?: string;
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
body?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
};
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
items?: Array<{
|
||||
text?: string;
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: { text?: string };
|
||||
rich_text?: { text?: string };
|
||||
};
|
||||
}>;
|
||||
};
|
||||
sections?: Array<{
|
||||
items?: Array<{
|
||||
text?: string;
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: { text?: string };
|
||||
rich_text?: { text?: string };
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
title?: string;
|
||||
plain_text?: string;
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
body?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
};
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: {
|
||||
text?: string;
|
||||
};
|
||||
rich_text?: {
|
||||
text?: string;
|
||||
};
|
||||
items?: Array<{
|
||||
text?: string;
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: { text?: string };
|
||||
rich_text?: { text?: string };
|
||||
};
|
||||
}>;
|
||||
};
|
||||
sections?: Array<{
|
||||
items?: Array<{
|
||||
text?: string;
|
||||
content?: {
|
||||
text?: string;
|
||||
richtext?: { text?: string };
|
||||
rich_text?: { text?: string };
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
|
||||
export interface TweetResult {
|
||||
@@ -61,6 +190,7 @@ export interface TweetData {
|
||||
username: string;
|
||||
name: string;
|
||||
};
|
||||
authorId?: string;
|
||||
createdAt?: string;
|
||||
replyCount?: number;
|
||||
retweetCount?: number;
|
||||
@@ -94,6 +224,7 @@ export interface CurrentUserResult {
|
||||
export interface TwitterClientOptions {
|
||||
cookies: TwitterCookies;
|
||||
userAgent?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface CreateTweetResponse {
|
||||
@@ -116,6 +247,7 @@ export class TwitterClient {
|
||||
private authToken: string;
|
||||
private ct0: string;
|
||||
private userAgent: string;
|
||||
private timeoutMs?: number;
|
||||
|
||||
constructor(options: TwitterClientOptions) {
|
||||
if (!options.cookies.authToken || !options.cookies.ct0) {
|
||||
@@ -126,6 +258,21 @@ export class TwitterClient {
|
||||
this.userAgent =
|
||||
options.userAgent ||
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
||||
this.timeoutMs = options.timeoutMs;
|
||||
}
|
||||
|
||||
private async fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
||||
if (!this.timeoutMs || this.timeoutMs <= 0) {
|
||||
return fetch(url, init);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
try {
|
||||
return await fetch(url, { ...init, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
private findTweetInInstructions(
|
||||
@@ -135,26 +282,7 @@ export class TwitterClient {
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: {
|
||||
rest_id?: string;
|
||||
legacy?: {
|
||||
full_text?: string;
|
||||
created_at?: string;
|
||||
reply_count?: number;
|
||||
retweet_count?: number;
|
||||
favorite_count?: number;
|
||||
};
|
||||
core?: {
|
||||
user_results?: {
|
||||
result?: {
|
||||
legacy?: {
|
||||
screen_name?: string;
|
||||
name?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
@@ -193,24 +321,221 @@ export class TwitterClient {
|
||||
};
|
||||
}
|
||||
|
||||
private mapTweetResult(result: GraphqlTweetResult | undefined): TweetData | undefined {
|
||||
if (!result?.legacy || !result.core?.user_results?.result?.legacy?.screen_name) return undefined;
|
||||
return {
|
||||
id: result.rest_id || '',
|
||||
text: result.legacy.full_text || '',
|
||||
createdAt: result.legacy.created_at,
|
||||
replyCount: result.legacy.reply_count,
|
||||
retweetCount: result.legacy.retweet_count,
|
||||
likeCount: result.legacy.favorite_count,
|
||||
conversationId: result.legacy.conversation_id_str,
|
||||
inReplyToStatusId: result.legacy.in_reply_to_status_id_str ?? undefined,
|
||||
author: {
|
||||
username: result.core.user_results.result.legacy.screen_name,
|
||||
name: result.core.user_results.result.legacy.name || result.core.user_results.result.legacy.screen_name,
|
||||
private firstText(...values: Array<string | undefined | null>): string | undefined {
|
||||
for (const value of values) {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private collectTextFields(value: unknown, keys: Set<string>, output: string[]): void {
|
||||
if (!value) return;
|
||||
if (typeof value === 'string') return;
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
this.collectTextFields(item, keys, output);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof value === 'object') {
|
||||
for (const [key, nested] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (keys.has(key)) {
|
||||
if (typeof nested === 'string') {
|
||||
const trimmed = nested.trim();
|
||||
if (trimmed) output.push(trimmed);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
this.collectTextFields(nested, keys, output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private uniqueOrdered(values: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (const value of values) {
|
||||
if (seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
result.push(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private extractArticleText(result: GraphqlTweetResult | undefined): string | undefined {
|
||||
const article = result?.article;
|
||||
if (!article) return undefined;
|
||||
|
||||
const articleResult = article.article_results?.result ?? article;
|
||||
if (process.env.BIRD_DEBUG_ARTICLE === '1') {
|
||||
console.error(
|
||||
'[bird][debug][article] payload:',
|
||||
JSON.stringify(
|
||||
{
|
||||
rest_id: result?.rest_id,
|
||||
article: articleResult,
|
||||
note_tweet: result?.note_tweet?.note_tweet_results?.result ?? null,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
}
|
||||
const title = this.firstText(articleResult.title, article.title);
|
||||
let body = this.firstText(
|
||||
articleResult.plain_text,
|
||||
article.plain_text,
|
||||
articleResult.body?.text,
|
||||
articleResult.body?.richtext?.text,
|
||||
articleResult.body?.rich_text?.text,
|
||||
articleResult.content?.text,
|
||||
articleResult.content?.richtext?.text,
|
||||
articleResult.content?.rich_text?.text,
|
||||
articleResult.text,
|
||||
articleResult.richtext?.text,
|
||||
articleResult.rich_text?.text,
|
||||
article.body?.text,
|
||||
article.body?.richtext?.text,
|
||||
article.body?.rich_text?.text,
|
||||
article.content?.text,
|
||||
article.content?.richtext?.text,
|
||||
article.content?.rich_text?.text,
|
||||
article.text,
|
||||
article.richtext?.text,
|
||||
article.rich_text?.text,
|
||||
);
|
||||
|
||||
if (body && title && body.trim() === title.trim()) {
|
||||
body = undefined;
|
||||
}
|
||||
|
||||
if (!body) {
|
||||
const collected: string[] = [];
|
||||
this.collectTextFields(articleResult, new Set(['text', 'title']), collected);
|
||||
this.collectTextFields(article, new Set(['text', 'title']), collected);
|
||||
const unique = this.uniqueOrdered(collected);
|
||||
const filtered = title ? unique.filter((value) => value !== title) : unique;
|
||||
if (filtered.length > 0) {
|
||||
body = filtered.join('\n\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (title && body && !body.startsWith(title)) {
|
||||
return `${title}\n\n${body}`;
|
||||
}
|
||||
|
||||
return body ?? title;
|
||||
}
|
||||
|
||||
private extractNoteTweetText(result: GraphqlTweetResult | undefined): string | undefined {
|
||||
const note = result?.note_tweet?.note_tweet_results?.result;
|
||||
if (!note) return undefined;
|
||||
|
||||
return this.firstText(
|
||||
note.text,
|
||||
note.richtext?.text,
|
||||
note.rich_text?.text,
|
||||
note.content?.text,
|
||||
note.content?.richtext?.text,
|
||||
note.content?.rich_text?.text,
|
||||
);
|
||||
}
|
||||
|
||||
private extractTweetText(result: GraphqlTweetResult | undefined): string | undefined {
|
||||
return (
|
||||
this.extractArticleText(result) ?? this.extractNoteTweetText(result) ?? this.firstText(result?.legacy?.full_text)
|
||||
);
|
||||
}
|
||||
|
||||
private mapTweetResult(result: GraphqlTweetResult | undefined): TweetData | undefined {
|
||||
const userResult = result?.core?.user_results?.result;
|
||||
const userLegacy = userResult?.legacy;
|
||||
const userCore = userResult?.core;
|
||||
const username = userLegacy?.screen_name ?? userCore?.screen_name;
|
||||
const name = userLegacy?.name ?? userCore?.name ?? username;
|
||||
const userId = userResult?.rest_id;
|
||||
if (!result?.rest_id || !username) return undefined;
|
||||
|
||||
const text = this.extractTweetText(result);
|
||||
if (!text) return undefined;
|
||||
|
||||
return {
|
||||
id: result.rest_id,
|
||||
text,
|
||||
createdAt: result.legacy?.created_at,
|
||||
replyCount: result.legacy?.reply_count,
|
||||
retweetCount: result.legacy?.retweet_count,
|
||||
likeCount: result.legacy?.favorite_count,
|
||||
conversationId: result.legacy?.conversation_id_str,
|
||||
inReplyToStatusId: result.legacy?.in_reply_to_status_id_str ?? undefined,
|
||||
author: {
|
||||
username,
|
||||
name: name || username,
|
||||
},
|
||||
authorId: userId,
|
||||
};
|
||||
}
|
||||
|
||||
private collectTweetResultsFromEntry(entry: {
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
items?: Array<{
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}): GraphqlTweetResult[] {
|
||||
const results: GraphqlTweetResult[] = [];
|
||||
const pushResult = (result?: GraphqlTweetResult) => {
|
||||
if (result?.rest_id) results.push(result);
|
||||
};
|
||||
|
||||
const content = entry.content;
|
||||
pushResult(content?.itemContent?.tweet_results?.result);
|
||||
pushResult(content?.item?.itemContent?.tweet_results?.result);
|
||||
|
||||
for (const item of content?.items ?? []) {
|
||||
pushResult(item?.item?.itemContent?.tweet_results?.result);
|
||||
pushResult(item?.itemContent?.tweet_results?.result);
|
||||
pushResult(item?.content?.itemContent?.tweet_results?.result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private parseTweetsFromInstructions(
|
||||
instructions:
|
||||
| Array<{
|
||||
@@ -221,22 +546,224 @@ export class TwitterClient {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
items?: Array<{
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
}>
|
||||
| undefined,
|
||||
): TweetData[] {
|
||||
const tweets: TweetData[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const instruction of instructions ?? []) {
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const result = entry.content?.itemContent?.tweet_results?.result;
|
||||
const results = this.collectTweetResultsFromEntry(entry);
|
||||
for (const result of results) {
|
||||
const mapped = this.mapTweetResult(result);
|
||||
if (mapped) tweets.push(mapped);
|
||||
if (!mapped || seen.has(mapped.id)) continue;
|
||||
seen.add(mapped.id);
|
||||
tweets.push(mapped);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tweets;
|
||||
}
|
||||
|
||||
private buildArticleFeatures(): Record<string, boolean> {
|
||||
return {
|
||||
rweb_video_screen_enabled: true,
|
||||
profile_label_improvements_pcf_label_in_post_enabled: true,
|
||||
responsive_web_profile_redirect_enabled: true,
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
premium_content_api_read_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
responsive_web_grok_analyze_button_fetch_trends_enabled: false,
|
||||
responsive_web_grok_analyze_post_followups_enabled: false,
|
||||
responsive_web_jetfuel_frame: true,
|
||||
responsive_web_grok_share_attachment_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
responsive_web_grok_show_grok_translated_post: false,
|
||||
responsive_web_grok_analysis_button_from_backend: true,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_grok_image_annotation_enabled: true,
|
||||
responsive_web_grok_imagine_annotation_enabled: true,
|
||||
responsive_web_grok_community_note_auto_translation_is_enabled: false,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
private buildArticleFieldToggles(): Record<string, boolean> {
|
||||
return {
|
||||
withPayments: false,
|
||||
withAuxiliaryUserLabels: false,
|
||||
withArticleRichContentState: true,
|
||||
withArticlePlainText: true,
|
||||
withGrokAnalyze: false,
|
||||
withDisallowedReplyControls: false,
|
||||
};
|
||||
}
|
||||
|
||||
private buildSearchFeatures(): Record<string, boolean> {
|
||||
return {
|
||||
rweb_video_screen_enabled: true,
|
||||
profile_label_improvements_pcf_label_in_post_enabled: true,
|
||||
responsive_web_profile_redirect_enabled: true,
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_exclude_directive_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
premium_content_api_read_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
responsive_web_grok_analyze_button_fetch_trends_enabled: false,
|
||||
responsive_web_grok_analyze_post_followups_enabled: false,
|
||||
responsive_web_jetfuel_frame: true,
|
||||
responsive_web_grok_share_attachment_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
responsive_web_grok_show_grok_translated_post: false,
|
||||
responsive_web_grok_analysis_button_from_backend: true,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
rweb_video_timestamps_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_grok_image_annotation_enabled: true,
|
||||
responsive_web_grok_imagine_annotation_enabled: true,
|
||||
responsive_web_grok_community_note_auto_translation_is_enabled: false,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
private async fetchUserArticlePlainText(
|
||||
userId: string,
|
||||
tweetId: string,
|
||||
): Promise<{ title?: string; plainText?: string }> {
|
||||
const variables = {
|
||||
userId,
|
||||
count: 20,
|
||||
includePromotedContent: true,
|
||||
withVoice: true,
|
||||
withQuickPromoteEligibilityTweetFields: true,
|
||||
withBirdwatchNotes: true,
|
||||
withCommunity: true,
|
||||
withSafetyModeUserFields: true,
|
||||
withSuperFollowsUserFields: true,
|
||||
withDownvotePerspective: false,
|
||||
withReactionsMetadata: false,
|
||||
withReactionsPerspective: false,
|
||||
withSuperFollowsTweetFields: true,
|
||||
withSuperFollowsReplyCount: false,
|
||||
withClientEventToken: false,
|
||||
};
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(this.buildArticleFeatures()),
|
||||
fieldToggles: JSON.stringify(this.buildArticleFieldToggles()),
|
||||
});
|
||||
|
||||
const url = `${TWITTER_API_BASE}/${QUERY_IDS.UserArticlesTweets}/UserArticlesTweets?${params}`;
|
||||
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders() });
|
||||
if (!response.ok) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
data?: {
|
||||
user?: {
|
||||
result?: {
|
||||
timeline?: {
|
||||
timeline?: {
|
||||
instructions?: Array<{
|
||||
entries?: Array<{
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: { result?: GraphqlTweetResult };
|
||||
};
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const instructions = data.data?.user?.result?.timeline?.timeline?.instructions ?? [];
|
||||
for (const instruction of instructions) {
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const result = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (result?.rest_id !== tweetId) continue;
|
||||
const articleResult = result.article?.article_results?.result;
|
||||
const title = this.firstText(articleResult?.title, result.article?.title);
|
||||
const plainText = this.firstText(articleResult?.plain_text, result.article?.plain_text);
|
||||
return { title, plainText };
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
private async fetchTweetDetail(tweetId: string): Promise<
|
||||
| {
|
||||
success: true;
|
||||
@@ -280,11 +807,15 @@ export class TwitterClient {
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
articles_rest_api_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
responsive_web_twitter_article_plain_text_enabled: true,
|
||||
responsive_web_twitter_article_seed_tweet_detail_enabled: true,
|
||||
responsive_web_twitter_article_seed_tweet_summary_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
@@ -304,7 +835,7 @@ export class TwitterClient {
|
||||
const url = `${TWITTER_API_BASE}/${QUERY_IDS.TweetDetail}/TweetDetail?${params}`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
@@ -374,6 +905,19 @@ export class TwitterClient {
|
||||
|
||||
const mapped = this.mapTweetResult(tweetResult);
|
||||
if (mapped) {
|
||||
if (tweetResult?.article) {
|
||||
const title = this.firstText(tweetResult.article.article_results?.result?.title, tweetResult.article.title);
|
||||
const articleText = this.extractArticleText(tweetResult);
|
||||
if (title && (!articleText || articleText.trim() === title.trim())) {
|
||||
const userId = tweetResult.core?.user_results?.result?.rest_id;
|
||||
if (userId) {
|
||||
const fallback = await this.fetchUserArticlePlainText(userId, tweetId);
|
||||
if (fallback.plainText) {
|
||||
mapped.text = fallback.title ? `${fallback.title}\n\n${fallback.plainText}` : fallback.plainText;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { success: true, tweet: mapped };
|
||||
}
|
||||
return { success: false, error: 'Tweet not found in response' };
|
||||
@@ -500,7 +1044,7 @@ export class TwitterClient {
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
body,
|
||||
@@ -554,51 +1098,31 @@ export class TwitterClient {
|
||||
product: 'Latest',
|
||||
};
|
||||
|
||||
const features = {
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
responsive_web_graphql_exclude_directive_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
rweb_video_timestamps_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
};
|
||||
const features = this.buildSearchFeatures();
|
||||
|
||||
let lastError: string | undefined;
|
||||
|
||||
for (const queryId of SEARCH_TIMELINE_QUERY_IDS) {
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
const url = `${TWITTER_API_BASE}/${QUERY_IDS.SearchTimeline}/SearchTimeline?${params}`;
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params}`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'POST',
|
||||
headers: this.getHeaders(),
|
||||
body: JSON.stringify({ features, queryId }),
|
||||
});
|
||||
|
||||
if (response.status === 404) {
|
||||
lastError = `HTTP ${response.status}`;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
return {
|
||||
success: false,
|
||||
error: `HTTP ${response.status}: ${text.slice(0, 200)}`,
|
||||
};
|
||||
return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}` };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
@@ -645,52 +1169,21 @@ export class TwitterClient {
|
||||
};
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.errors.map((e) => e.message).join(', '),
|
||||
};
|
||||
return { success: false, error: data.errors.map((e) => e.message).join(', ') };
|
||||
}
|
||||
|
||||
const tweets: TweetData[] = [];
|
||||
const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
|
||||
const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions;
|
||||
const tweets = this.parseTweetsFromInstructions(instructions);
|
||||
|
||||
for (const instruction of instructions) {
|
||||
for (const entry of instruction.entries || []) {
|
||||
const result = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (!result) continue;
|
||||
|
||||
const legacy = result.legacy;
|
||||
const userLegacy = result.core?.user_results?.result?.legacy;
|
||||
|
||||
if (legacy?.full_text && userLegacy?.screen_name) {
|
||||
tweets.push({
|
||||
id: result.rest_id || '',
|
||||
text: legacy.full_text,
|
||||
author: {
|
||||
username: userLegacy.screen_name,
|
||||
name: userLegacy.name || userLegacy.screen_name,
|
||||
},
|
||||
createdAt: legacy.created_at,
|
||||
replyCount: legacy.reply_count,
|
||||
retweetCount: legacy.retweet_count,
|
||||
likeCount: legacy.favorite_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
tweets,
|
||||
};
|
||||
return { success: true, tweets };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false, error: lastError ?? 'Unknown error fetching search results' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the account associated with the current cookies
|
||||
*/
|
||||
@@ -706,7 +1199,7 @@ export class TwitterClient {
|
||||
|
||||
for (const url of candidateUrls) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
@@ -772,7 +1265,7 @@ export class TwitterClient {
|
||||
const profilePages = ['https://x.com/settings/account', 'https://twitter.com/settings/account'];
|
||||
for (const page of profilePages) {
|
||||
try {
|
||||
const response = await fetch(page, {
|
||||
const response = await this.fetchWithTimeout(page, {
|
||||
headers: {
|
||||
cookie: `auth_token=${this.authToken}; ct0=${this.ct0}`,
|
||||
'user-agent': this.userAgent,
|
||||
|
||||
@@ -295,6 +295,185 @@ describe('TwitterClient', () => {
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('HTTP 404');
|
||||
});
|
||||
|
||||
it('should return article text when present', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: 'article123',
|
||||
legacy: {
|
||||
full_text: '',
|
||||
created_at: 'Mon Jan 01 00:00:00 +0000 2024',
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: '2025 LLM Year in Review',
|
||||
sections: [
|
||||
{
|
||||
items: [
|
||||
{ text: 'Intro paragraph of the article.' },
|
||||
{ content: { text: 'Second paragraph.' } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
screen_name: 'author',
|
||||
name: 'Article Author',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getTweet('article123');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tweet?.text).toBe(
|
||||
'2025 LLM Year in Review\n\nIntro paragraph of the article.\n\nSecond paragraph.',
|
||||
);
|
||||
});
|
||||
|
||||
it('should fall back to user article timeline for plain text', async () => {
|
||||
mockFetch
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: 'article123',
|
||||
legacy: {
|
||||
full_text: '',
|
||||
created_at: 'Mon Jan 01 00:00:00 +0000 2024',
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: '2025 LLM Year in Review',
|
||||
},
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
rest_id: '33836629',
|
||||
legacy: {
|
||||
screen_name: 'author',
|
||||
name: 'Article Author',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: {
|
||||
user: {
|
||||
result: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
entries: [
|
||||
{
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
rest_id: 'article123',
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: '2025 LLM Year in Review',
|
||||
plain_text: 'Full article body.',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getTweet('article123');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tweet?.text).toBe('2025 LLM Year in Review\n\nFull article body.');
|
||||
expect(mockFetch).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should return note tweet text when present', async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: 'note123',
|
||||
legacy: {
|
||||
full_text: '',
|
||||
created_at: 'Mon Jan 01 00:00:00 +0000 2024',
|
||||
},
|
||||
note_tweet: {
|
||||
note_tweet_results: {
|
||||
result: {
|
||||
text: 'Long form note content.',
|
||||
},
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
screen_name: 'noter',
|
||||
name: 'Note Author',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getTweet('note123');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tweet?.text).toBe('Long form note content.');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCurrentUser', () => {
|
||||
@@ -375,6 +554,85 @@ describe('TwitterClient', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
let mockFetch: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch = vi.fn();
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
});
|
||||
|
||||
it('retries on 404 and posts search payload', async () => {
|
||||
mockFetch
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 404,
|
||||
text: async () => '',
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
data: {
|
||||
search_by_raw_query: {
|
||||
search_timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
entries: [
|
||||
{
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
rest_id: '1',
|
||||
legacy: {
|
||||
full_text: 'found',
|
||||
created_at: '2024-01-01T00:00:00Z',
|
||||
reply_count: 0,
|
||||
retweet_count: 0,
|
||||
favorite_count: 0,
|
||||
conversation_id_str: '1',
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: { legacy: { screen_name: 'root', name: 'Root' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.search('needle', 1);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tweets?.[0].id).toBe('1');
|
||||
expect(mockFetch).toHaveBeenCalledTimes(2);
|
||||
|
||||
const [url, options] = mockFetch.mock.calls[1];
|
||||
expect(options.method).toBe('POST');
|
||||
const body = JSON.parse(options.body);
|
||||
expect(body.features).toBeDefined();
|
||||
expect(body.queryId).toBeDefined();
|
||||
const urlVars = new URL(url as string).searchParams.get('variables');
|
||||
expect(urlVars).toBeTruthy();
|
||||
const parsed = JSON.parse(urlVars as string) as { rawQuery?: string };
|
||||
expect(parsed.rawQuery).toBe('needle');
|
||||
});
|
||||
});
|
||||
|
||||
describe('conversation helpers', () => {
|
||||
let mockFetch: ReturnType<typeof vi.fn>;
|
||||
|
||||
@@ -455,9 +713,12 @@ describe('TwitterClient', () => {
|
||||
it('getThread returns sorted thread by createdAt', async () => {
|
||||
const payload = makeConversationPayload();
|
||||
// swap dates to verify sorting
|
||||
payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.[0]?.content?.itemContent?.tweet_results?.result?.legacy &&
|
||||
((payload.data.threaded_conversation_with_injections_v2.instructions[0]!.entries![0]!.content!.itemContent!
|
||||
.tweet_results!.result!.legacy!.created_at = '2024-01-03T00:00:00Z'));
|
||||
const legacy =
|
||||
payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.[0]?.content?.itemContent
|
||||
?.tweet_results?.result?.legacy;
|
||||
if (legacy) {
|
||||
legacy.created_at = '2024-01-03T00:00:00Z';
|
||||
}
|
||||
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
@@ -472,6 +733,51 @@ describe('TwitterClient', () => {
|
||||
expect(result.tweets?.map((t) => t.id)).toEqual(['2', '1']); // sorted by createdAt asc
|
||||
});
|
||||
|
||||
it('getThread includes tweets from timeline module items', async () => {
|
||||
const payload = makeConversationPayload();
|
||||
payload.data.threaded_conversation_with_injections_v2.instructions[0]?.entries?.push({
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
item: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
rest_id: '3',
|
||||
legacy: {
|
||||
full_text: 'nested reply',
|
||||
created_at: '2024-01-04T00:00:00Z',
|
||||
reply_count: 0,
|
||||
retweet_count: 0,
|
||||
favorite_count: 0,
|
||||
conversation_id_str: '1',
|
||||
in_reply_to_status_id_str: '1',
|
||||
},
|
||||
core: {
|
||||
user_results: { result: { legacy: { screen_name: 'nested', name: 'Nested' } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => payload,
|
||||
});
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getThread('1');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tweets?.map((t) => t.id)).toEqual(['1', '2', '3']);
|
||||
});
|
||||
|
||||
it('propagates fetchTweetDetail errors', async () => {
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: false,
|
||||
|
||||
Reference in New Issue
Block a user