diff --git a/README.md b/README.md index b1b4e3b..5a0006a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,12 @@ import { TwitterClient, resolveCredentials } from '@steipete/bird'; const { cookies } = await resolveCredentials({ cookieSource: 'safari' }); const client = new TwitterClient({ cookies }); -const result = await client.search('from:steipete', 50); + +// Search for tweets +const searchResult = await client.search('from:steipete', 50); + +// Fetch news and trending topics +const newsResult = await client.getNews(10, { aiOnly: true, withTweets: true }); ``` ## Commands @@ -110,6 +115,8 @@ const result = await client.search('from:steipete', 50); - `bird bookmarks [-n count] [--folder-id id] [--all] [--max-pages n] [--json]` — list your bookmarked tweets (or a specific bookmark folder); `--max-pages` requires `--all`. - `bird unbookmark ` — remove one or more bookmarks by tweet ID or URL. - `bird likes [-n count] [--json]` — list your liked tweets. +- `bird news [-n count] [--ai-only] [--with-tweets] [--tweets-per-item n] [--for-you] [--news-only] [--sports] [--entertainment] [--trending-only] [--json]` — fetch news and trending topics from X's Explore tabs. +- `bird trending` — alias for `news` command. - `bird list-timeline [-n count] [--all] [--max-pages n] [--cursor string] [--json]` — get tweets from a list timeline; `--max-pages` implies `--all`. - `bird following [--user ] [-n count] [--json]` — list users that you (or another user) follow. - `bird followers [--user ] [-n count] [--json]` — list users that follow you (or another user). @@ -212,8 +219,23 @@ When using `--json` with `following`/`followers`, user objects include: | `followersCount` | number? | Followers count | | `followingCount` | number? | Following count | | `isBlueVerified` | boolean? | Blue verified flag | -| `profileImageUrl` | string? | Profile image URL | -| `createdAt` | string? | Account creation timestamp | + | `profileImageUrl` | string? | Profile image URL | + | `createdAt` | string? | Account creation timestamp | + +When using `--json` with `news`/`trending`, news objects include: + +| Field | Type | Description | +|-------|------|-------------| +| `id` | string | Unique identifier for the news item | +| `headline` | string | News headline or trend title | +| `category` | string? | Category (e.g., "AI · Technology", "Trending", "News") | +| `timeAgo` | string? | Relative time (e.g., "2h ago") | +| `postCount` | number? | Number of posts | +| `description` | string? | Item description | +| `url` | string? | URL to the trend or news article | +| `tweets` | array? | Related tweets (only when `--with-tweets` is used) | +| `_raw` | object? | Raw API response (only when `--json-full` is used) | + ## Query IDs (GraphQL) diff --git a/scripts/update-query-ids.ts b/scripts/update-query-ids.ts index baab967..975d8d6 100644 --- a/scripts/update-query-ids.ts +++ b/scripts/update-query-ids.ts @@ -19,6 +19,9 @@ const TARGET_OPERATIONS = [ 'Following', 'Followers', 'Likes', + 'ExploreSidebar', + 'ExplorePage', + 'TrendHistory', ] as const; type OperationName = (typeof TARGET_OPERATIONS)[number]; diff --git a/src/cli/program.ts b/src/cli/program.ts index 37e0e1d..39d9501 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -4,6 +4,7 @@ import { registerCheckCommand } from '../commands/check.js'; import { registerHelpCommand } from '../commands/help.js'; import { registerHomeCommand } from '../commands/home.js'; import { registerListsCommand } from '../commands/lists.js'; +import { registerNewsCommand } from '../commands/news.js'; import { registerPostCommands } from '../commands/post.js'; import { registerQueryIdsCommand } from '../commands/query-ids.js'; import { registerReadCommands } from '../commands/read.js'; @@ -32,6 +33,8 @@ export const KNOWN_COMMANDS = new Set([ 'list-timeline', 'home', 'user-tweets', + 'news', + 'trending', 'help', 'whoami', 'check', @@ -138,6 +141,7 @@ export function createProgram(ctx: CliContext): Command { registerHomeCommand(program, ctx); registerUserCommands(program, ctx); registerUserTweetsCommand(program, ctx); + registerNewsCommand(program, ctx); registerCheckCommand(program, ctx); return program; diff --git a/src/commands/news.ts b/src/commands/news.ts new file mode 100644 index 0000000..7f351c0 --- /dev/null +++ b/src/commands/news.ts @@ -0,0 +1,133 @@ +import type { Command } from 'commander'; +import type { CliContext } from '../cli/shared.js'; +import { TwitterClient } from '../lib/twitter-client.js'; +import type { NewsItem } from '../lib/twitter-client-news.js'; + +function formatPostCount(count: number): string { + if (count >= 1_000_000) { + return `${(count / 1_000_000).toFixed(1)}M`; + } + if (count >= 1_000) { + return `${(count / 1_000).toFixed(1)}K`; + } + return String(count); +} + +function printNewsItems( + items: NewsItem[], + ctx: CliContext, + opts: { json?: boolean; emptyMessage?: string } = {}, +): void { + if (opts.json) { + console.log(JSON.stringify(items, null, 2)); + return; + } + + if (items.length === 0) { + console.log(opts.emptyMessage ?? 'No news items found.'); + return; + } + + for (const item of items) { + const categoryLabel = item.category ? `[${item.category}]` : ''; + console.log(`\n${ctx.colors.accent(categoryLabel)} ${ctx.colors.command(item.headline)}`); + + if (item.description) { + console.log(` ${ctx.colors.muted(item.description)}`); + } + + const meta: string[] = []; + if (item.timeAgo) { + meta.push(item.timeAgo); + } + if (item.postCount) { + meta.push(`${formatPostCount(item.postCount)} posts`); + } + if (meta.length > 0) { + console.log(` ${ctx.colors.muted(meta.join(' | '))}`); + } + + if (item.url) { + console.log(` ${ctx.l('url')}${item.url}`); + } + + // Print related tweets if available + if (item.tweets && item.tweets.length > 0) { + console.log(` ${ctx.colors.section('Related tweets:')}`); + for (const tweet of item.tweets.slice(0, 3)) { + console.log( + ` @${tweet.author.username}: ${tweet.text.slice(0, 100)}${tweet.text.length > 100 ? '...' : ''}`, + ); + } + } + + console.log(ctx.colors.muted('─'.repeat(50))); + } +} + +export function registerNewsCommand(program: Command, ctx: CliContext): void { + program + .command('news') + .alias('trending') + .description("Fetch today's news and trending topics") + .option('-n, --count ', 'Number of items to fetch', '10') + .option('--ai-only', 'Show only AI-curated news items') + .option('--with-tweets', 'Also fetch related tweets for each news item') + .option('--tweets-per-item ', 'Number of tweets to fetch per news item (default: 5)', '5') + .option('--json', 'Output as JSON') + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action( + async (cmdOpts: { + count?: string; + aiOnly?: boolean; + withTweets?: boolean; + tweetsPerItem?: string; + json?: boolean; + jsonFull?: boolean; + }) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); + const count = Number.parseInt(cmdOpts.count || '10', 10); + const tweetsPerItem = Number.parseInt(cmdOpts.tweetsPerItem || '5', 10); + + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); + + for (const warning of warnings) { + console.error(`${ctx.p('warn')}${warning}`); + } + + if (Number.isNaN(count) || count < 1) { + console.error(`${ctx.p('err')}--count must be a positive number`); + process.exit(1); + } + + if (Number.isNaN(tweetsPerItem) || tweetsPerItem < 1) { + console.error(`${ctx.p('err')}--tweets-per-item must be a positive number`); + process.exit(1); + } + + if (!cookies.authToken || !cookies.ct0) { + console.error(`${ctx.p('err')}Missing required credentials`); + process.exit(1); + } + + const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); + const includeRaw = cmdOpts.jsonFull ?? false; + const withTweets = cmdOpts.withTweets ?? false; + const aiOnly = cmdOpts.aiOnly ?? false; + + const result = await client.getNews(count, { includeRaw, withTweets, tweetsPerItem, aiOnly }); + + if (result.success) { + printNewsItems(result.items, ctx, { + json: cmdOpts.json || cmdOpts.jsonFull, + emptyMessage: 'No news items found.', + }); + } else { + console.error(`${ctx.p('err')}Failed to fetch news: ${result.error}`); + process.exit(1); + } + }, + ); +} diff --git a/src/lib/index.ts b/src/lib/index.ts index fd9018f..eca545f 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -19,6 +19,7 @@ export { type TwitterUser, } from './twitter-client.js'; export type { HomeTimelineFetchOptions } from './twitter-client-home.js'; +export type { NewsFetchOptions, NewsItem, NewsResult } from './twitter-client-news.js'; export type { SearchFetchOptions } from './twitter-client-search.js'; export type { TimelineFetchOptions } from './twitter-client-timelines.js'; export type { TweetFetchOptions } from './twitter-client-tweet-detail.js'; diff --git a/src/lib/query-ids.json b/src/lib/query-ids.json index a6f3e2c..d0823b6 100644 --- a/src/lib/query-ids.json +++ b/src/lib/query-ids.json @@ -1,17 +1,16 @@ { - "CreateTweet": "Uf3io9zVp1DsYxrmL5FJ7g", + "CreateTweet": "nmdAQXJDxw6-0KKF2on7eA", "CreateRetweet": "LFho5rIi4xcKO90p9jwG7A", "FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A", "DeleteBookmark": "Wlmlj2-xzyS1GN3a6cj-mQ", - "TweetDetail": "97JF30KziU00483E_8elBA", - "SearchTimeline": "M1jEez78PEfVfbQLvlWMvQ", + "TweetDetail": "_NvJCnIjOW__EP5-RF197A", + "SearchTimeline": "6AAys3t42mosm_yTI_QENg", "Bookmarks": "RV1g3b8n_SGOHwkqKYSCFw", "BookmarkFolderTimeline": "KJIQpsvxrTfRIlbaRIySHQ", - "Following": "BEkNpEt5pNETESoqMsTEGA", - "Followers": "kuFUYP9eV1FPoEy4N-pi7w", - "Likes": "JR2gceKucIKcVNB_9JkhsA", - "ListOwnerships": "wQcOSjSQ8NtgxIwvYl1lMg", - "ListMemberships": "BlEXXdARdSeL_0KyKHHvvg", - "ListLatestTweetsTimeline": "2TemLyqrMpTeAmysdbnVqw", - "ListByRestId": "wXzyA5vM_aVkBL9G8Vp3kw" + "Following": "mWYeougg_ocJS2Vr1Vt28w", + "Followers": "SFYY3WsgwjlXSLlfnEUE4A", + "Likes": "ETJflBunfqNa1uE1mBPCaw", + "ExploreSidebar": "lpSN4M6qpimkF4nRFPE3nQ", + "ExplorePage": "kheAINB_4pzRDqkzG3K-ng", + "TrendHistory": "Sj4T-jSB9pr0Mxtsc1UKZQ" } diff --git a/src/lib/twitter-client-constants.ts b/src/lib/twitter-client-constants.ts index 98276b1..5365b07 100644 --- a/src/lib/twitter-client-constants.ts +++ b/src/lib/twitter-client-constants.ts @@ -33,6 +33,10 @@ export const FALLBACK_QUERY_IDS = { ListByRestId: 'wXzyA5vM_aVkBL9G8Vp3kw', HomeTimeline: 'edseUwk9sP5Phz__9TIRnA', HomeLatestTimeline: 'iOEZpOdfekFsxSlPQCQtPg', + ExploreSidebar: 'lpSN4M6qpimkF4nRFPE3nQ', + ExplorePage: 'kheAINB_4pzRDqkzG3K-ng', + GenericTimelineById: 'uGSr7alSjR9v6QJAIaqSKQ', + TrendHistory: 'Sj4T-jSB9pr0Mxtsc1UKZQ', } as const; export type OperationName = keyof typeof FALLBACK_QUERY_IDS; diff --git a/src/lib/twitter-client-features.ts b/src/lib/twitter-client-features.ts index 64849cc..c8525c8 100644 --- a/src/lib/twitter-client-features.ts +++ b/src/lib/twitter-client-features.ts @@ -313,3 +313,47 @@ export function buildFollowingFeatures(): Record { responsive_web_enhance_cards_enabled: false, }); } + +export function buildExploreFeatures(): Record { + return applyFeatureOverrides('explore', { + rweb_video_screen_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: true, + responsive_web_grok_analyze_post_followups_enabled: true, + responsive_web_grok_annotations_enabled: true, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + responsive_web_grok_show_grok_translated_post: true, + responsive_web_grok_analysis_button_from_backend: true, + creator_subscriptions_quote_tweet_preview_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: true, + responsive_web_enhance_cards_enabled: false, + // Additional features required for ExploreSidebar + post_ctas_fetch_enabled: true, + rweb_video_timestamps_enabled: true, + }); +} diff --git a/src/lib/twitter-client-news.ts b/src/lib/twitter-client-news.ts new file mode 100644 index 0000000..3697c75 --- /dev/null +++ b/src/lib/twitter-client-news.ts @@ -0,0 +1,486 @@ +import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js'; +import { TWITTER_API_BASE } from './twitter-client-constants.js'; +import { buildExploreFeatures } from './twitter-client-features.js'; +import type { SearchResult, TweetData } from './twitter-client-types.js'; + +const POST_COUNT_REGEX = /[\d.]+[KMB]?\s*posts?/i; +const POST_COUNT_MATCH_REGEX = /([\d.]+)([KMB]?)\s*posts?/i; + +/** Options for news fetch methods */ +export interface NewsFetchOptions { + /** Include raw GraphQL response in `_raw` field */ + includeRaw?: boolean; + /** Also fetch related tweets for each news item */ + withTweets?: boolean; + /** Number of tweets to fetch per news item (default: 5) */ + tweetsPerItem?: number; + /** Filter to show only AI-curated news items */ + aiOnly?: boolean; +} + +export interface NewsItem { + id: string; + headline: string; + category?: string; + timeAgo?: string; + postCount?: number; + description?: string; + url?: string; + tweets?: TweetData[]; + // biome-ignore lint/suspicious/noExplicitAny: Raw API response can have any structure + _raw?: any; +} + +export type NewsResult = + | { + success: true; + items: NewsItem[]; + } + | { + success: false; + error: string; + }; + +export interface TwitterClientNewsMethods { + getNews(count?: number, options?: NewsFetchOptions): Promise; +} + +export function withNews>( + Base: TBase, +): Mixin { + abstract class TwitterClientNews extends Base { + // biome-ignore lint/complexity/noUselessConstructor lint/suspicious/noExplicitAny: TS mixin constructor requirement. + constructor(...args: any[]) { + super(...args); + } + + /** + * Fetch news and trending topics from Twitter's Explore page + */ + async getNews(count = 10, options: NewsFetchOptions = {}): Promise { + const { includeRaw = false, withTweets = false, tweetsPerItem = 5, aiOnly = false } = options; + + const debug = process.env.BIRD_DEBUG === '1'; + + // Try ExplorePage first - this has AI headlines in initialTimeline + if (debug) { + console.error('[getNews] Fetching from ExplorePage (has AI headlines)...'); + } + const queryId = await this.getQueryId('ExplorePage'); + const features = buildExploreFeatures(); + + const variables = { + includePromotedContent: true, + withBirdwatchNotes: false, + withCommunity: true, + withSuperFollowsUserFields: true, + withDownvotePerspective: false, + withReactionsMetadata: false, + withReactionsPerspective: false, + withSuperFollowsTweetFields: true, + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); + + const url = `${TWITTER_API_BASE}/${queryId}/ExplorePage?${params.toString()}`; + + try { + const response = await this.fetchWithTimeout(url, { + method: 'GET', + headers: this.getHeaders(), + }); + + if (!response.ok) { + const text = await response.text(); + return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}` }; + } + + const data = (await response.json()) as { + // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex + data?: any; + // biome-ignore lint/suspicious/noExplicitAny: API errors can have any structure + errors?: Array<{ message: string; code?: number; [key: string]: any }>; + }; + + // Debug: save response if BIRD_DEBUG_JSON is set + if (process.env.BIRD_DEBUG_JSON) { + const fs = await import('node:fs/promises'); + const debugPath = process.env.BIRD_DEBUG_JSON.replace('.json', '-explorepage.json'); + await fs.writeFile(debugPath, JSON.stringify(data, null, 2)).catch(() => {}); + if (debug) { + console.error(`[ExplorePage] Saved response to ${debugPath}`); + } + } + + if (data.errors && data.errors.length > 0) { + return { success: false, error: data.errors.map((e) => e.message).join('; ') }; + } + + const items = this.parseNewsItems(data, count, aiOnly, includeRaw); + + if (items.length === 0) { + return { success: false, error: 'No news items found' }; + } + + if (withTweets) { + await this.enrichWithTweets(items, tweetsPerItem, includeRaw); + } + + return { success: true, items }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { success: false, error: `Failed to fetch news: ${errorMessage}` }; + } + } + + // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex + private parseNewsItems(data: any, maxCount: number, aiOnly: boolean, includeRaw: boolean): NewsItem[] { + const allItems: NewsItem[] = []; + const seenHeadlines = new Set(); + const debug = process.env.BIRD_DEBUG === '1'; + + if (debug) { + console.error('[ExplorePage] Processing explore_page data...'); + } + + if (!data.data?.explore_page) { + return []; + } + + const explorePage = data.data.explore_page; + + if (debug) { + console.error('[ExplorePage] Available sections:', Object.keys(explorePage.body || {})); + } + + const body = explorePage.body || {}; + + // Check the timelines array first (For You, News tabs, etc.) + // biome-ignore lint/suspicious/noExplicitAny: exploring API structure + const timelines = (body as any).timelines || []; + if (debug) { + console.error(`[ExplorePage] Found ${timelines.length} timelines`); + } + + for (const timelineObj of timelines) { + if (debug) { + console.error(`[ExplorePage] Timeline ID: ${timelineObj.id}, Label: ${timelineObj.labelText}`); + console.error(`[ExplorePage] Timeline keys:`, Object.keys(timelineObj)); + } + + const timeline = timelineObj.timeline; + if (timeline) { + const instructions = timeline.timeline?.instructions ?? timeline.instructions ?? []; + + if (debug) { + console.error(`[ExplorePage] Timeline ${timelineObj.labelText} has ${instructions.length} instructions`); + } + + const itemsFromTimeline = this.extractNewsItemsFromInstructions( + instructions, + timelineObj.__typename || 'timeline', + seenHeadlines, + maxCount, + aiOnly, + includeRaw, + ); + + if (debug) { + console.error(`[ExplorePage] Timeline found ${itemsFromTimeline.length} AI news items`); + } + + allItems.push(...itemsFromTimeline); + + if (allItems.length >= maxCount) { + break; + } + } + } + + // Also check initialTimeline as fallback + if (allItems.length < maxCount && body.initialTimeline) { + const timeline = body.initialTimeline.timeline; + if (timeline) { + const instructions = timeline.timeline?.instructions ?? []; + + const itemsFromInitial = this.extractNewsItemsFromInstructions( + instructions, + 'initialTimeline', + seenHeadlines, + maxCount - allItems.length, + aiOnly, + includeRaw, + ); + + allItems.push(...itemsFromInitial); + } + } + + return allItems; + } + + private extractNewsItemsFromInstructions( + // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex + instructions: any[], + source: string, + seenHeadlines: Set, + maxCount: number, + aiOnly: boolean, + includeRaw: boolean, + ): NewsItem[] { + const items: NewsItem[] = []; + const debug = process.env.BIRD_DEBUG === '1'; + + for (const instruction of instructions) { + if (instruction.type !== 'TimelineAddEntries') { + continue; + } + + const entries = instruction.entries ?? []; + + if (debug) { + console.error(`[${source}] Processing ${entries.length} entries`); + } + + for (const entry of entries) { + if (items.length >= maxCount) { + break; + } + + if (debug) { + console.error( + `[${source}] Entry ID: ${entry.entryId}, content type: ${entry.content?.__typename || entry.content?.entryType || 'unknown'}`, + ); + + // Check if this is a "Today's News" or news section header + if (entry.content?.header || entry.content?.displayType === 'VerticalConversation') { + console.error(`[${source}] Found potential news section:`, entry.content?.header); + } + } + + const content = entry.content; + if (!content) { + continue; + } + + // Handle TimelineTimelineItem (single trend item) + if (content.itemContent && items.length < maxCount) { + if (debug && content.itemContent.is_ai_trend) { + console.error( + `[ExplorePage] Found AI trend in ${entry.entryId}:`, + JSON.stringify(content.itemContent, null, 2).substring(0, 500), + ); + } + + const newsItem = this.parseNewsItemFromContent( + content.itemContent, + entry.entryId, + source, + seenHeadlines, + aiOnly, + includeRaw, + ); + + if (newsItem) { + items.push(newsItem); + } + } + + // Handle TimelineTimelineModule (multiple items) + const itemsArray = content?.items ?? []; + + if (debug && itemsArray.length > 0) { + console.error(`[${source}] Module has ${itemsArray.length} items`); + } + + for (const data of itemsArray) { + if (items.length >= maxCount) { + break; + } + + // Structure can be data.itemContent OR data.item.itemContent + const itemContent = data?.itemContent || data?.item?.itemContent; + if (!itemContent) { + continue; + } + + if (debug) { + console.error( + `[${source}] Module item type: ${itemContent.__typename}, name: ${itemContent.name}, is_ai: ${itemContent.is_ai_trend}`, + ); + } + + if (debug && itemContent.is_ai_trend) { + console.error( + `[ExplorePage] Found AI trend in module ${entry.entryId}:`, + JSON.stringify(itemContent, null, 2).substring(0, 500), + ); + } + + const newsItem = this.parseNewsItemFromContent( + itemContent, + entry.entryId, + source, + seenHeadlines, + aiOnly, + includeRaw, + ); + + if (newsItem) { + items.push(newsItem); + } + } + } + } + + return items; + } + + private parseNewsItemFromContent( + // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex + itemContent: any, + entryId: string, + source: string, + seenHeadlines: Set, + aiOnly: boolean, + includeRaw: boolean, + ): NewsItem | null { + const headline = itemContent.name || itemContent.title; + + if (!headline) { + return null; + } + + // Detect AI news by characteristics: + // 1. Full sentence headlines (contains spaces and is longer) + // 2. Has social_context with "News" category + // 3. Or explicitly marked as is_ai_trend + const socialContext = itemContent?.social_context?.text || ''; + const hasNewsCategory = socialContext.includes('News') || socialContext.includes('hours ago'); + const isFullSentence = headline.split(' ').length >= 5; // AI news are full sentences + const isExplicitlyAiTrend = itemContent.is_ai_trend === true; + + const isAiNews = isExplicitlyAiTrend || (isFullSentence && hasNewsCategory); + + // Filter AI trends if aiOnly is enabled + if (aiOnly && !isAiNews) { + return null; + } + + if (seenHeadlines.has(headline)) { + return null; + } + + seenHeadlines.add(headline); + + let postCount: number | undefined; + let timeAgo: string | undefined; + let category = 'Trending'; + + // Parse social context for metadata + const socialCtx = itemContent?.social_context; + if (socialCtx?.text) { + const socialContextText = socialCtx.text; + const parts = socialContextText.split('·').map((s: string) => s.trim()); + + for (const part of parts) { + if (part.includes('ago')) { + timeAgo = part; + } else if (part.match(POST_COUNT_REGEX)) { + const match = part.match(POST_COUNT_MATCH_REGEX); + if (match) { + let num = Number.parseFloat(match[1]); + const suffix = match[2]?.toUpperCase(); + + if (suffix === 'K') { + num *= 1000; + } else if (suffix === 'M') { + num *= 1_000_000; + } else if (suffix === 'B') { + num *= 1_000_000_000; + } + + postCount = Math.round(num); + } + } else { + category = part; + } + } + } + + // Parse trend metadata + const trendMetadata = itemContent?.trend_metadata; + if (trendMetadata?.meta_description) { + const metaDesc = trendMetadata.meta_description; + const postMatch = metaDesc.match(POST_COUNT_MATCH_REGEX); + if (postMatch) { + let num = Number.parseFloat(postMatch[1]); + const suffix = postMatch[2]?.toUpperCase(); + + if (suffix === 'K') { + num *= 1000; + } else if (suffix === 'M') { + num *= 1_000_000; + } else if (suffix === 'B') { + num *= 1_000_000_000; + } + + postCount = Math.round(num); + } + } + + if (trendMetadata?.domain_context && (category === 'Trending' || category === 'News')) { + category = trendMetadata.domain_context; + } + + const item: NewsItem = { + id: entryId || `${source}-${headline}`, + headline, + category: isAiNews ? `AI · ${category}` : category, + timeAgo, + postCount, + description: itemContent.description, + url: itemContent.trend_url?.url || trendMetadata?.url?.url, + }; + + if (includeRaw) { + item._raw = itemContent; + } + + return item; + } + + private async enrichWithTweets(items: NewsItem[], tweetsPerItem: number, includeRaw: boolean): Promise { + const debug = process.env.BIRD_DEBUG === '1'; + + for (const item of items) { + try { + const searchQuery = item.headline; + if (!searchQuery) { + continue; + } + + // Use the search method if available (requires search mixin) + if ('search' in this && typeof (this as { search?: unknown }).search === 'function') { + const result = (await ( + this as { search: (q: string, c: number, o: { includeRaw: boolean }) => Promise } + ).search(searchQuery, tweetsPerItem, { includeRaw })) as SearchResult; + + if (result.success && result.tweets) { + item.tweets = result.tweets; + } + } + } catch { + if (debug) { + console.error('[getNews] Failed to enrich item with tweets:', item.headline); + } + } + } + } + } + + return TwitterClientNews; +} diff --git a/src/lib/twitter-client.ts b/src/lib/twitter-client.ts index 4924097..c79a26b 100644 --- a/src/lib/twitter-client.ts +++ b/src/lib/twitter-client.ts @@ -4,6 +4,7 @@ import { type TwitterClientBookmarkMethods, withBookmarks } from './twitter-clie import { type TwitterClientHomeMethods, withHome } from './twitter-client-home.js'; import { type TwitterClientListMethods, withLists } from './twitter-client-lists.js'; import { type TwitterClientMediaMethods, withMedia } from './twitter-client-media.js'; +import { type TwitterClientNewsMethods, withNews } from './twitter-client-news.js'; import { type TwitterClientPostingMethods, withPosting } from './twitter-client-posting.js'; import { type TwitterClientSearchMethods, withSearch } from './twitter-client-search.js'; import { type TwitterClientTimelineMethods, withTimelines } from './twitter-client-timelines.js'; @@ -17,6 +18,7 @@ type TwitterClientInstance = TwitterClientBase & TwitterClientHomeMethods & TwitterClientListMethods & TwitterClientMediaMethods & + TwitterClientNewsMethods & TwitterClientPostingMethods & TwitterClientSearchMethods & TwitterClientTimelineMethods & @@ -25,11 +27,16 @@ type TwitterClientInstance = TwitterClientBase & TwitterClientUserLookupMethods & TwitterClientUserTweetsMethods; -const MixedTwitterClient = withUserTweets( - withUserLookup( - withUsers( - withLists( - withHome(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase))))))), +// News mixin wraps search because it depends on the search() method +const MixedTwitterClient = withNews( + withUserTweets( + withUserLookup( + withUsers( + withLists( + withHome( + withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase)))))), + ), + ), ), ), ), @@ -37,6 +44,7 @@ const MixedTwitterClient = withUserTweets( export class TwitterClient extends MixedTwitterClient {} +export type { NewsFetchOptions, NewsItem, NewsResult } from './twitter-client-news.js'; export type { BookmarkMutationResult, CurrentUserResult, diff --git a/tests/commands.news.test.ts b/tests/commands.news.test.ts new file mode 100644 index 0000000..9ea9bf3 --- /dev/null +++ b/tests/commands.news.test.ts @@ -0,0 +1,154 @@ +import { Command } from 'commander'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { CliContext } from '../src/cli/shared.js'; +import { registerNewsCommand } from '../src/commands/news.js'; + +describe('news command', () => { + let program: Command; + let mockContext: Partial; + + beforeEach(() => { + program = new Command(); + mockContext = { + resolveTimeoutFromOptions: () => 30000, + resolveQuoteDepthFromOptions: () => undefined, + resolveCredentialsFromOptions: async () => ({ + cookies: { + authToken: 'auth', + ct0: 'ct0', + cookieHeader: 'auth=auth; ct0=ct0', + }, + warnings: [], + }), + p: (type: string) => `[${type}] `, + colors: { + accent: (text: string) => text, + command: (text: string) => text, + muted: (text: string) => text, + section: (text: string) => text, + }, + l: (key: string) => key, + }; + }); + + it('requires positive count value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--count', '0'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('rejects negative count value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--count', '-5'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('rejects non-numeric count value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--count', 'abc'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('requires positive tweets-per-item value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', '0'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('rejects negative tweets-per-item value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', '-3'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('rejects non-numeric tweets-per-item value', async () => { + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', 'xyz'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('requires both authToken and ct0 credentials', async () => { + mockContext.resolveCredentialsFromOptions = async () => ({ + cookies: { + authToken: '', + ct0: '', + cookieHeader: '', + }, + warnings: [], + }); + + registerNewsCommand(program, mockContext as CliContext); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'news'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Missing required credentials')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); +}); diff --git a/tests/twitter-client-coverage.news.test.ts b/tests/twitter-client-coverage.news.test.ts new file mode 100644 index 0000000..93fc0e1 --- /dev/null +++ b/tests/twitter-client-coverage.news.test.ts @@ -0,0 +1,360 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { TwitterClient } from '../src/lib/twitter-client.js'; + +const validCookies = { + authToken: 'test_auth_token', + ct0: 'test_ct0_token', + cookieHeader: 'auth_token=test_auth_token; ct0=test_ct0_token', + source: 'test', +}; + +type ResponseLike = { + ok: boolean; + status: number; + json: () => Promise; + text: () => Promise; +}; + +const makeResponse = (overrides: Partial = {}): ResponseLike => ({ + ok: true, + status: 200, + json: async (): Promise => ({}), + text: async (): Promise => '', + ...overrides, +}); + +describe('TwitterClient news API coverage', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + describe('getNews', () => { + it('returns news items from ExplorePage', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + data: { + explore_page: { + body: { + initialTimeline: { + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'test-entry-1', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'AI Breakthrough in Machine Learning', + social_context: { + text: 'AI · 2h ago · 15.5K posts', + }, + trend_url: { + url: 'https://x.com/hashtag/AI', + }, + }, + }, + ], + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(1); + + expect(result.success).toBe(true); + expect(result.items).toBeDefined(); + expect(result.items?.length).toBe(1); + expect(result.items?.[0].headline).toBe('AI Breakthrough in Machine Learning'); + expect(result.items?.[0].category).toBe('AI · AI'); + expect(result.items?.[0].timeAgo).toBe('2h ago'); + expect(result.items?.[0].postCount).toBe(15500); + expect(result.items?.[0].url).toBe('https://x.com/hashtag/AI'); + }); + + it('filters to AI-only items when aiOnly is true', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + data: { + explore_page: { + body: { + initialTimeline: { + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'test-entry-1', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'AI News', + }, + }, + ], + }, + }, + { + entryId: 'test-entry-2', + content: { + items: [ + { + itemContent: { + is_ai_trend: false, + name: 'Regular News', + }, + }, + ], + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(10, { aiOnly: true }); + + expect(result.success).toBe(true); + expect(result.items?.length).toBe(1); + expect(result.items?.[0].headline).toBe('AI News'); + }); + + it('returns error for non-ok responses', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce(makeResponse({ ok: false, status: 500, text: async () => 'Server error' })); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(10); + + expect(result.success).toBe(false); + expect(result.error).toContain('HTTP 500'); + }); + + it('returns error when API returns errors', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + errors: [{ message: 'Rate limited' }, { message: 'Too many requests' }], + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(10); + + expect(result.success).toBe(false); + expect(result.error).toContain('Rate limited'); + expect(result.error).toContain('Too many requests'); + }); + + it('returns error when no news items found', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + data: { + explore_page: { + body: { + initialTimeline: { + timeline: { + timeline: { + instructions: [], + }, + }, + }, + }, + }, + }, + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(10); + + expect(result.success).toBe(false); + expect(result.error).toContain('No news items found'); + }); + + it('deduplicates headlines', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + data: { + explore_page: { + body: { + initialTimeline: { + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'test-entry-1', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'Duplicate Headline', + }, + }, + ], + }, + }, + { + entryId: 'test-entry-2', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'Duplicate Headline', + }, + }, + ], + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(10); + + expect(result.success).toBe(true); + expect(result.items?.length).toBe(1); + expect(result.items?.[0].headline).toBe('Duplicate Headline'); + }); + + it('respects count parameter', async () => { + const mockFetch = vi.fn().mockResolvedValueOnce( + makeResponse({ + json: async () => ({ + data: { + explore_page: { + body: { + initialTimeline: { + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'test-entry-1', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'News 1', + }, + }, + ], + }, + }, + { + entryId: 'test-entry-2', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'News 2', + }, + }, + ], + }, + }, + { + entryId: 'test-entry-3', + content: { + items: [ + { + itemContent: { + is_ai_trend: true, + name: 'News 3', + }, + }, + ], + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, + }), + }), + ); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(2); + + expect(result.success).toBe(true); + expect(result.items?.length).toBe(2); + }); + }); +});