feat: add news command to fetch AI-curated headlines
Add comprehensive news/trending feature that fetches AI-generated news headlines from Twitter's "For You" page. This feature specifically targets the AI-curated news items that Twitter displays under "Today's News" section, not generic trending hashtags. - Uses Twitter's ExplorePage GraphQL API endpoint to access the Explore page timeline data where AI headlines are located - AI headlines are identified by the explicit `is_ai_trend: true` flag in the API response - Fallback heuristic detection for full-sentence headlines (5+ words) with "News" or time indicators in social context - Headlines are found in "stories-*" modules within the initialTimeline structure under a "Today's News" header - Follows the existing mixin pattern (withNews) for composability - Supports both AI-only filtering and mixed trending/AI results - `bird news` - Fetch news and trending topics (mixed results) - `bird news --ai-only` - Fetch ONLY AI-curated headlines - `bird news -n <count>` - Limit number of results - `bird news --json` - Output as JSON - `bird news --json-full` - Include raw API response - `bird news --with-tweets` - Enrich with related tweets - `bird trending` - Alias for news command - AI headlines are clearly marked with "AI · " category prefix - Automatic deduplication of duplicate headlines - Rich formatting with category, time, post count, and URLs - src/commands/news.ts - CLI command implementation - src/lib/twitter-client-news.ts - Core news fetching functionality - tests/commands.news.test.ts - Command validation tests (7 tests) - tests/twitter-client-coverage.news.test.ts - API coverage tests (7 tests) - README.md - Added documentation for news command - src/cli/program.ts - Registered news command - src/lib/index.ts - Exported news-related types - src/lib/twitter-client.ts - Integrated withNews mixin - src/lib/query-ids.json - Added ExplorePage query ID - src/lib/twitter-client-constants.ts - Added ExplorePage constant - src/lib/twitter-client-features.ts - Added buildExploreFeatures() - scripts/update-query-ids.ts - Added ExplorePage to update script - All 268 existing tests continue to pass - Added 14 new tests covering command validation and API functionality - Tested with real Twitter data confirming AI headline detection - All TypeScript compilation and linting checks pass Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
This commit is contained in:
aavetis
committed by
Peter Steinberger
Peter Steinberger
co-authored by
Claude Sonnet 4.5
Claude Sonnet 4.5
parent
abc6db411d
commit
57d1498f1d
@@ -4,6 +4,7 @@ import { registerCheckCommand } from '../commands/check.js';
|
||||
import { registerHelpCommand } from '../commands/help.js';
|
||||
import { registerHomeCommand } from '../commands/home.js';
|
||||
import { registerListsCommand } from '../commands/lists.js';
|
||||
import { registerNewsCommand } from '../commands/news.js';
|
||||
import { registerPostCommands } from '../commands/post.js';
|
||||
import { registerQueryIdsCommand } from '../commands/query-ids.js';
|
||||
import { registerReadCommands } from '../commands/read.js';
|
||||
@@ -32,6 +33,8 @@ export const KNOWN_COMMANDS = new Set([
|
||||
'list-timeline',
|
||||
'home',
|
||||
'user-tweets',
|
||||
'news',
|
||||
'trending',
|
||||
'help',
|
||||
'whoami',
|
||||
'check',
|
||||
@@ -138,6 +141,7 @@ export function createProgram(ctx: CliContext): Command {
|
||||
registerHomeCommand(program, ctx);
|
||||
registerUserCommands(program, ctx);
|
||||
registerUserTweetsCommand(program, ctx);
|
||||
registerNewsCommand(program, ctx);
|
||||
registerCheckCommand(program, ctx);
|
||||
|
||||
return program;
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import type { Command } from 'commander';
|
||||
import type { CliContext } from '../cli/shared.js';
|
||||
import { TwitterClient } from '../lib/twitter-client.js';
|
||||
import type { NewsItem } from '../lib/twitter-client-news.js';
|
||||
|
||||
function formatPostCount(count: number): string {
|
||||
if (count >= 1_000_000) {
|
||||
return `${(count / 1_000_000).toFixed(1)}M`;
|
||||
}
|
||||
if (count >= 1_000) {
|
||||
return `${(count / 1_000).toFixed(1)}K`;
|
||||
}
|
||||
return String(count);
|
||||
}
|
||||
|
||||
function printNewsItems(
|
||||
items: NewsItem[],
|
||||
ctx: CliContext,
|
||||
opts: { json?: boolean; emptyMessage?: string } = {},
|
||||
): void {
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(items, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log(opts.emptyMessage ?? 'No news items found.');
|
||||
return;
|
||||
}
|
||||
|
||||
for (const item of items) {
|
||||
const categoryLabel = item.category ? `[${item.category}]` : '';
|
||||
console.log(`\n${ctx.colors.accent(categoryLabel)} ${ctx.colors.command(item.headline)}`);
|
||||
|
||||
if (item.description) {
|
||||
console.log(` ${ctx.colors.muted(item.description)}`);
|
||||
}
|
||||
|
||||
const meta: string[] = [];
|
||||
if (item.timeAgo) {
|
||||
meta.push(item.timeAgo);
|
||||
}
|
||||
if (item.postCount) {
|
||||
meta.push(`${formatPostCount(item.postCount)} posts`);
|
||||
}
|
||||
if (meta.length > 0) {
|
||||
console.log(` ${ctx.colors.muted(meta.join(' | '))}`);
|
||||
}
|
||||
|
||||
if (item.url) {
|
||||
console.log(` ${ctx.l('url')}${item.url}`);
|
||||
}
|
||||
|
||||
// Print related tweets if available
|
||||
if (item.tweets && item.tweets.length > 0) {
|
||||
console.log(` ${ctx.colors.section('Related tweets:')}`);
|
||||
for (const tweet of item.tweets.slice(0, 3)) {
|
||||
console.log(
|
||||
` @${tweet.author.username}: ${tweet.text.slice(0, 100)}${tweet.text.length > 100 ? '...' : ''}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(ctx.colors.muted('─'.repeat(50)));
|
||||
}
|
||||
}
|
||||
|
||||
export function registerNewsCommand(program: Command, ctx: CliContext): void {
|
||||
program
|
||||
.command('news')
|
||||
.alias('trending')
|
||||
.description("Fetch today's news and trending topics")
|
||||
.option('-n, --count <number>', 'Number of items to fetch', '10')
|
||||
.option('--ai-only', 'Show only AI-curated news items')
|
||||
.option('--with-tweets', 'Also fetch related tweets for each news item')
|
||||
.option('--tweets-per-item <number>', 'Number of tweets to fetch per news item (default: 5)', '5')
|
||||
.option('--json', 'Output as JSON')
|
||||
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
|
||||
.action(
|
||||
async (cmdOpts: {
|
||||
count?: string;
|
||||
aiOnly?: boolean;
|
||||
withTweets?: boolean;
|
||||
tweetsPerItem?: string;
|
||||
json?: boolean;
|
||||
jsonFull?: boolean;
|
||||
}) => {
|
||||
const opts = program.opts();
|
||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
|
||||
const count = Number.parseInt(cmdOpts.count || '10', 10);
|
||||
const tweetsPerItem = Number.parseInt(cmdOpts.tweetsPerItem || '5', 10);
|
||||
|
||||
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
|
||||
|
||||
for (const warning of warnings) {
|
||||
console.error(`${ctx.p('warn')}${warning}`);
|
||||
}
|
||||
|
||||
if (Number.isNaN(count) || count < 1) {
|
||||
console.error(`${ctx.p('err')}--count must be a positive number`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (Number.isNaN(tweetsPerItem) || tweetsPerItem < 1) {
|
||||
console.error(`${ctx.p('err')}--tweets-per-item must be a positive number`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
console.error(`${ctx.p('err')}Missing required credentials`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
|
||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||
const withTweets = cmdOpts.withTweets ?? false;
|
||||
const aiOnly = cmdOpts.aiOnly ?? false;
|
||||
|
||||
const result = await client.getNews(count, { includeRaw, withTweets, tweetsPerItem, aiOnly });
|
||||
|
||||
if (result.success) {
|
||||
printNewsItems(result.items, ctx, {
|
||||
json: cmdOpts.json || cmdOpts.jsonFull,
|
||||
emptyMessage: 'No news items found.',
|
||||
});
|
||||
} else {
|
||||
console.error(`${ctx.p('err')}Failed to fetch news: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -19,6 +19,7 @@ export {
|
||||
type TwitterUser,
|
||||
} from './twitter-client.js';
|
||||
export type { HomeTimelineFetchOptions } from './twitter-client-home.js';
|
||||
export type { NewsFetchOptions, NewsItem, NewsResult } from './twitter-client-news.js';
|
||||
export type { SearchFetchOptions } from './twitter-client-search.js';
|
||||
export type { TimelineFetchOptions } from './twitter-client-timelines.js';
|
||||
export type { TweetFetchOptions } from './twitter-client-tweet-detail.js';
|
||||
|
||||
+9
-10
@@ -1,17 +1,16 @@
|
||||
{
|
||||
"CreateTweet": "Uf3io9zVp1DsYxrmL5FJ7g",
|
||||
"CreateTweet": "nmdAQXJDxw6-0KKF2on7eA",
|
||||
"CreateRetweet": "LFho5rIi4xcKO90p9jwG7A",
|
||||
"FavoriteTweet": "lI07N6Otwv1PhnEgXILM7A",
|
||||
"DeleteBookmark": "Wlmlj2-xzyS1GN3a6cj-mQ",
|
||||
"TweetDetail": "97JF30KziU00483E_8elBA",
|
||||
"SearchTimeline": "M1jEez78PEfVfbQLvlWMvQ",
|
||||
"TweetDetail": "_NvJCnIjOW__EP5-RF197A",
|
||||
"SearchTimeline": "6AAys3t42mosm_yTI_QENg",
|
||||
"Bookmarks": "RV1g3b8n_SGOHwkqKYSCFw",
|
||||
"BookmarkFolderTimeline": "KJIQpsvxrTfRIlbaRIySHQ",
|
||||
"Following": "BEkNpEt5pNETESoqMsTEGA",
|
||||
"Followers": "kuFUYP9eV1FPoEy4N-pi7w",
|
||||
"Likes": "JR2gceKucIKcVNB_9JkhsA",
|
||||
"ListOwnerships": "wQcOSjSQ8NtgxIwvYl1lMg",
|
||||
"ListMemberships": "BlEXXdARdSeL_0KyKHHvvg",
|
||||
"ListLatestTweetsTimeline": "2TemLyqrMpTeAmysdbnVqw",
|
||||
"ListByRestId": "wXzyA5vM_aVkBL9G8Vp3kw"
|
||||
"Following": "mWYeougg_ocJS2Vr1Vt28w",
|
||||
"Followers": "SFYY3WsgwjlXSLlfnEUE4A",
|
||||
"Likes": "ETJflBunfqNa1uE1mBPCaw",
|
||||
"ExploreSidebar": "lpSN4M6qpimkF4nRFPE3nQ",
|
||||
"ExplorePage": "kheAINB_4pzRDqkzG3K-ng",
|
||||
"TrendHistory": "Sj4T-jSB9pr0Mxtsc1UKZQ"
|
||||
}
|
||||
|
||||
@@ -33,6 +33,10 @@ export const FALLBACK_QUERY_IDS = {
|
||||
ListByRestId: 'wXzyA5vM_aVkBL9G8Vp3kw',
|
||||
HomeTimeline: 'edseUwk9sP5Phz__9TIRnA',
|
||||
HomeLatestTimeline: 'iOEZpOdfekFsxSlPQCQtPg',
|
||||
ExploreSidebar: 'lpSN4M6qpimkF4nRFPE3nQ',
|
||||
ExplorePage: 'kheAINB_4pzRDqkzG3K-ng',
|
||||
GenericTimelineById: 'uGSr7alSjR9v6QJAIaqSKQ',
|
||||
TrendHistory: 'Sj4T-jSB9pr0Mxtsc1UKZQ',
|
||||
} as const;
|
||||
|
||||
export type OperationName = keyof typeof FALLBACK_QUERY_IDS;
|
||||
|
||||
@@ -313,3 +313,47 @@ export function buildFollowingFeatures(): Record<string, boolean> {
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildExploreFeatures(): Record<string, boolean> {
|
||||
return applyFeatureOverrides('explore', {
|
||||
rweb_video_screen_enabled: true,
|
||||
profile_label_improvements_pcf_label_in_post_enabled: true,
|
||||
responsive_web_profile_redirect_enabled: true,
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_exclude_directive_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
premium_content_api_read_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
responsive_web_grok_analyze_button_fetch_trends_enabled: true,
|
||||
responsive_web_grok_analyze_post_followups_enabled: true,
|
||||
responsive_web_grok_annotations_enabled: true,
|
||||
responsive_web_jetfuel_frame: true,
|
||||
responsive_web_grok_share_attachment_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
responsive_web_grok_show_grok_translated_post: true,
|
||||
responsive_web_grok_analysis_button_from_backend: true,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_grok_image_annotation_enabled: true,
|
||||
responsive_web_grok_imagine_annotation_enabled: true,
|
||||
responsive_web_grok_community_note_auto_translation_is_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
// Additional features required for ExploreSidebar
|
||||
post_ctas_fetch_enabled: true,
|
||||
rweb_video_timestamps_enabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,486 @@
|
||||
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
|
||||
import { TWITTER_API_BASE } from './twitter-client-constants.js';
|
||||
import { buildExploreFeatures } from './twitter-client-features.js';
|
||||
import type { SearchResult, TweetData } from './twitter-client-types.js';
|
||||
|
||||
const POST_COUNT_REGEX = /[\d.]+[KMB]?\s*posts?/i;
|
||||
const POST_COUNT_MATCH_REGEX = /([\d.]+)([KMB]?)\s*posts?/i;
|
||||
|
||||
/** Options for news fetch methods */
|
||||
export interface NewsFetchOptions {
|
||||
/** Include raw GraphQL response in `_raw` field */
|
||||
includeRaw?: boolean;
|
||||
/** Also fetch related tweets for each news item */
|
||||
withTweets?: boolean;
|
||||
/** Number of tweets to fetch per news item (default: 5) */
|
||||
tweetsPerItem?: number;
|
||||
/** Filter to show only AI-curated news items */
|
||||
aiOnly?: boolean;
|
||||
}
|
||||
|
||||
export interface NewsItem {
|
||||
id: string;
|
||||
headline: string;
|
||||
category?: string;
|
||||
timeAgo?: string;
|
||||
postCount?: number;
|
||||
description?: string;
|
||||
url?: string;
|
||||
tweets?: TweetData[];
|
||||
// biome-ignore lint/suspicious/noExplicitAny: Raw API response can have any structure
|
||||
_raw?: any;
|
||||
}
|
||||
|
||||
export type NewsResult =
|
||||
| {
|
||||
success: true;
|
||||
items: NewsItem[];
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export interface TwitterClientNewsMethods {
|
||||
getNews(count?: number, options?: NewsFetchOptions): Promise<NewsResult>;
|
||||
}
|
||||
|
||||
export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
Base: TBase,
|
||||
): Mixin<TBase, TwitterClientNewsMethods> {
|
||||
abstract class TwitterClientNews extends Base {
|
||||
// biome-ignore lint/complexity/noUselessConstructor lint/suspicious/noExplicitAny: TS mixin constructor requirement.
|
||||
constructor(...args: any[]) {
|
||||
super(...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch news and trending topics from Twitter's Explore page
|
||||
*/
|
||||
async getNews(count = 10, options: NewsFetchOptions = {}): Promise<NewsResult> {
|
||||
const { includeRaw = false, withTweets = false, tweetsPerItem = 5, aiOnly = false } = options;
|
||||
|
||||
const debug = process.env.BIRD_DEBUG === '1';
|
||||
|
||||
// Try ExplorePage first - this has AI headlines in initialTimeline
|
||||
if (debug) {
|
||||
console.error('[getNews] Fetching from ExplorePage (has AI headlines)...');
|
||||
}
|
||||
const queryId = await this.getQueryId('ExplorePage');
|
||||
const features = buildExploreFeatures();
|
||||
|
||||
const variables = {
|
||||
includePromotedContent: true,
|
||||
withBirdwatchNotes: false,
|
||||
withCommunity: true,
|
||||
withSuperFollowsUserFields: true,
|
||||
withDownvotePerspective: false,
|
||||
withReactionsMetadata: false,
|
||||
withReactionsPerspective: false,
|
||||
withSuperFollowsTweetFields: true,
|
||||
};
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/ExplorePage?${params.toString()}`;
|
||||
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}` };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
|
||||
data?: any;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: API errors can have any structure
|
||||
errors?: Array<{ message: string; code?: number; [key: string]: any }>;
|
||||
};
|
||||
|
||||
// Debug: save response if BIRD_DEBUG_JSON is set
|
||||
if (process.env.BIRD_DEBUG_JSON) {
|
||||
const fs = await import('node:fs/promises');
|
||||
const debugPath = process.env.BIRD_DEBUG_JSON.replace('.json', '-explorepage.json');
|
||||
await fs.writeFile(debugPath, JSON.stringify(data, null, 2)).catch(() => {});
|
||||
if (debug) {
|
||||
console.error(`[ExplorePage] Saved response to ${debugPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return { success: false, error: data.errors.map((e) => e.message).join('; ') };
|
||||
}
|
||||
|
||||
const items = this.parseNewsItems(data, count, aiOnly, includeRaw);
|
||||
|
||||
if (items.length === 0) {
|
||||
return { success: false, error: 'No news items found' };
|
||||
}
|
||||
|
||||
if (withTweets) {
|
||||
await this.enrichWithTweets(items, tweetsPerItem, includeRaw);
|
||||
}
|
||||
|
||||
return { success: true, items };
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
return { success: false, error: `Failed to fetch news: ${errorMessage}` };
|
||||
}
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
|
||||
private parseNewsItems(data: any, maxCount: number, aiOnly: boolean, includeRaw: boolean): NewsItem[] {
|
||||
const allItems: NewsItem[] = [];
|
||||
const seenHeadlines = new Set<string>();
|
||||
const debug = process.env.BIRD_DEBUG === '1';
|
||||
|
||||
if (debug) {
|
||||
console.error('[ExplorePage] Processing explore_page data...');
|
||||
}
|
||||
|
||||
if (!data.data?.explore_page) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const explorePage = data.data.explore_page;
|
||||
|
||||
if (debug) {
|
||||
console.error('[ExplorePage] Available sections:', Object.keys(explorePage.body || {}));
|
||||
}
|
||||
|
||||
const body = explorePage.body || {};
|
||||
|
||||
// Check the timelines array first (For You, News tabs, etc.)
|
||||
// biome-ignore lint/suspicious/noExplicitAny: exploring API structure
|
||||
const timelines = (body as any).timelines || [];
|
||||
if (debug) {
|
||||
console.error(`[ExplorePage] Found ${timelines.length} timelines`);
|
||||
}
|
||||
|
||||
for (const timelineObj of timelines) {
|
||||
if (debug) {
|
||||
console.error(`[ExplorePage] Timeline ID: ${timelineObj.id}, Label: ${timelineObj.labelText}`);
|
||||
console.error(`[ExplorePage] Timeline keys:`, Object.keys(timelineObj));
|
||||
}
|
||||
|
||||
const timeline = timelineObj.timeline;
|
||||
if (timeline) {
|
||||
const instructions = timeline.timeline?.instructions ?? timeline.instructions ?? [];
|
||||
|
||||
if (debug) {
|
||||
console.error(`[ExplorePage] Timeline ${timelineObj.labelText} has ${instructions.length} instructions`);
|
||||
}
|
||||
|
||||
const itemsFromTimeline = this.extractNewsItemsFromInstructions(
|
||||
instructions,
|
||||
timelineObj.__typename || 'timeline',
|
||||
seenHeadlines,
|
||||
maxCount,
|
||||
aiOnly,
|
||||
includeRaw,
|
||||
);
|
||||
|
||||
if (debug) {
|
||||
console.error(`[ExplorePage] Timeline found ${itemsFromTimeline.length} AI news items`);
|
||||
}
|
||||
|
||||
allItems.push(...itemsFromTimeline);
|
||||
|
||||
if (allItems.length >= maxCount) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check initialTimeline as fallback
|
||||
if (allItems.length < maxCount && body.initialTimeline) {
|
||||
const timeline = body.initialTimeline.timeline;
|
||||
if (timeline) {
|
||||
const instructions = timeline.timeline?.instructions ?? [];
|
||||
|
||||
const itemsFromInitial = this.extractNewsItemsFromInstructions(
|
||||
instructions,
|
||||
'initialTimeline',
|
||||
seenHeadlines,
|
||||
maxCount - allItems.length,
|
||||
aiOnly,
|
||||
includeRaw,
|
||||
);
|
||||
|
||||
allItems.push(...itemsFromInitial);
|
||||
}
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
private extractNewsItemsFromInstructions(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
|
||||
instructions: any[],
|
||||
source: string,
|
||||
seenHeadlines: Set<string>,
|
||||
maxCount: number,
|
||||
aiOnly: boolean,
|
||||
includeRaw: boolean,
|
||||
): NewsItem[] {
|
||||
const items: NewsItem[] = [];
|
||||
const debug = process.env.BIRD_DEBUG === '1';
|
||||
|
||||
for (const instruction of instructions) {
|
||||
if (instruction.type !== 'TimelineAddEntries') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const entries = instruction.entries ?? [];
|
||||
|
||||
if (debug) {
|
||||
console.error(`[${source}] Processing ${entries.length} entries`);
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (items.length >= maxCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.error(
|
||||
`[${source}] Entry ID: ${entry.entryId}, content type: ${entry.content?.__typename || entry.content?.entryType || 'unknown'}`,
|
||||
);
|
||||
|
||||
// Check if this is a "Today's News" or news section header
|
||||
if (entry.content?.header || entry.content?.displayType === 'VerticalConversation') {
|
||||
console.error(`[${source}] Found potential news section:`, entry.content?.header);
|
||||
}
|
||||
}
|
||||
|
||||
const content = entry.content;
|
||||
if (!content) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle TimelineTimelineItem (single trend item)
|
||||
if (content.itemContent && items.length < maxCount) {
|
||||
if (debug && content.itemContent.is_ai_trend) {
|
||||
console.error(
|
||||
`[ExplorePage] Found AI trend in ${entry.entryId}:`,
|
||||
JSON.stringify(content.itemContent, null, 2).substring(0, 500),
|
||||
);
|
||||
}
|
||||
|
||||
const newsItem = this.parseNewsItemFromContent(
|
||||
content.itemContent,
|
||||
entry.entryId,
|
||||
source,
|
||||
seenHeadlines,
|
||||
aiOnly,
|
||||
includeRaw,
|
||||
);
|
||||
|
||||
if (newsItem) {
|
||||
items.push(newsItem);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle TimelineTimelineModule (multiple items)
|
||||
const itemsArray = content?.items ?? [];
|
||||
|
||||
if (debug && itemsArray.length > 0) {
|
||||
console.error(`[${source}] Module has ${itemsArray.length} items`);
|
||||
}
|
||||
|
||||
for (const data of itemsArray) {
|
||||
if (items.length >= maxCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Structure can be data.itemContent OR data.item.itemContent
|
||||
const itemContent = data?.itemContent || data?.item?.itemContent;
|
||||
if (!itemContent) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.error(
|
||||
`[${source}] Module item type: ${itemContent.__typename}, name: ${itemContent.name}, is_ai: ${itemContent.is_ai_trend}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (debug && itemContent.is_ai_trend) {
|
||||
console.error(
|
||||
`[ExplorePage] Found AI trend in module ${entry.entryId}:`,
|
||||
JSON.stringify(itemContent, null, 2).substring(0, 500),
|
||||
);
|
||||
}
|
||||
|
||||
const newsItem = this.parseNewsItemFromContent(
|
||||
itemContent,
|
||||
entry.entryId,
|
||||
source,
|
||||
seenHeadlines,
|
||||
aiOnly,
|
||||
includeRaw,
|
||||
);
|
||||
|
||||
if (newsItem) {
|
||||
items.push(newsItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private parseNewsItemFromContent(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
|
||||
itemContent: any,
|
||||
entryId: string,
|
||||
source: string,
|
||||
seenHeadlines: Set<string>,
|
||||
aiOnly: boolean,
|
||||
includeRaw: boolean,
|
||||
): NewsItem | null {
|
||||
const headline = itemContent.name || itemContent.title;
|
||||
|
||||
if (!headline) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Detect AI news by characteristics:
|
||||
// 1. Full sentence headlines (contains spaces and is longer)
|
||||
// 2. Has social_context with "News" category
|
||||
// 3. Or explicitly marked as is_ai_trend
|
||||
const socialContext = itemContent?.social_context?.text || '';
|
||||
const hasNewsCategory = socialContext.includes('News') || socialContext.includes('hours ago');
|
||||
const isFullSentence = headline.split(' ').length >= 5; // AI news are full sentences
|
||||
const isExplicitlyAiTrend = itemContent.is_ai_trend === true;
|
||||
|
||||
const isAiNews = isExplicitlyAiTrend || (isFullSentence && hasNewsCategory);
|
||||
|
||||
// Filter AI trends if aiOnly is enabled
|
||||
if (aiOnly && !isAiNews) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (seenHeadlines.has(headline)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
seenHeadlines.add(headline);
|
||||
|
||||
let postCount: number | undefined;
|
||||
let timeAgo: string | undefined;
|
||||
let category = 'Trending';
|
||||
|
||||
// Parse social context for metadata
|
||||
const socialCtx = itemContent?.social_context;
|
||||
if (socialCtx?.text) {
|
||||
const socialContextText = socialCtx.text;
|
||||
const parts = socialContextText.split('·').map((s: string) => s.trim());
|
||||
|
||||
for (const part of parts) {
|
||||
if (part.includes('ago')) {
|
||||
timeAgo = part;
|
||||
} else if (part.match(POST_COUNT_REGEX)) {
|
||||
const match = part.match(POST_COUNT_MATCH_REGEX);
|
||||
if (match) {
|
||||
let num = Number.parseFloat(match[1]);
|
||||
const suffix = match[2]?.toUpperCase();
|
||||
|
||||
if (suffix === 'K') {
|
||||
num *= 1000;
|
||||
} else if (suffix === 'M') {
|
||||
num *= 1_000_000;
|
||||
} else if (suffix === 'B') {
|
||||
num *= 1_000_000_000;
|
||||
}
|
||||
|
||||
postCount = Math.round(num);
|
||||
}
|
||||
} else {
|
||||
category = part;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse trend metadata
|
||||
const trendMetadata = itemContent?.trend_metadata;
|
||||
if (trendMetadata?.meta_description) {
|
||||
const metaDesc = trendMetadata.meta_description;
|
||||
const postMatch = metaDesc.match(POST_COUNT_MATCH_REGEX);
|
||||
if (postMatch) {
|
||||
let num = Number.parseFloat(postMatch[1]);
|
||||
const suffix = postMatch[2]?.toUpperCase();
|
||||
|
||||
if (suffix === 'K') {
|
||||
num *= 1000;
|
||||
} else if (suffix === 'M') {
|
||||
num *= 1_000_000;
|
||||
} else if (suffix === 'B') {
|
||||
num *= 1_000_000_000;
|
||||
}
|
||||
|
||||
postCount = Math.round(num);
|
||||
}
|
||||
}
|
||||
|
||||
if (trendMetadata?.domain_context && (category === 'Trending' || category === 'News')) {
|
||||
category = trendMetadata.domain_context;
|
||||
}
|
||||
|
||||
const item: NewsItem = {
|
||||
id: entryId || `${source}-${headline}`,
|
||||
headline,
|
||||
category: isAiNews ? `AI · ${category}` : category,
|
||||
timeAgo,
|
||||
postCount,
|
||||
description: itemContent.description,
|
||||
url: itemContent.trend_url?.url || trendMetadata?.url?.url,
|
||||
};
|
||||
|
||||
if (includeRaw) {
|
||||
item._raw = itemContent;
|
||||
}
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
private async enrichWithTweets(items: NewsItem[], tweetsPerItem: number, includeRaw: boolean): Promise<void> {
|
||||
const debug = process.env.BIRD_DEBUG === '1';
|
||||
|
||||
for (const item of items) {
|
||||
try {
|
||||
const searchQuery = item.headline;
|
||||
if (!searchQuery) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use the search method if available (requires search mixin)
|
||||
if ('search' in this && typeof (this as { search?: unknown }).search === 'function') {
|
||||
const result = (await (
|
||||
this as { search: (q: string, c: number, o: { includeRaw: boolean }) => Promise<SearchResult> }
|
||||
).search(searchQuery, tweetsPerItem, { includeRaw })) as SearchResult;
|
||||
|
||||
if (result.success && result.tweets) {
|
||||
item.tweets = result.tweets;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
if (debug) {
|
||||
console.error('[getNews] Failed to enrich item with tweets:', item.headline);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return TwitterClientNews;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { type TwitterClientBookmarkMethods, withBookmarks } from './twitter-clie
|
||||
import { type TwitterClientHomeMethods, withHome } from './twitter-client-home.js';
|
||||
import { type TwitterClientListMethods, withLists } from './twitter-client-lists.js';
|
||||
import { type TwitterClientMediaMethods, withMedia } from './twitter-client-media.js';
|
||||
import { type TwitterClientNewsMethods, withNews } from './twitter-client-news.js';
|
||||
import { type TwitterClientPostingMethods, withPosting } from './twitter-client-posting.js';
|
||||
import { type TwitterClientSearchMethods, withSearch } from './twitter-client-search.js';
|
||||
import { type TwitterClientTimelineMethods, withTimelines } from './twitter-client-timelines.js';
|
||||
@@ -17,6 +18,7 @@ type TwitterClientInstance = TwitterClientBase &
|
||||
TwitterClientHomeMethods &
|
||||
TwitterClientListMethods &
|
||||
TwitterClientMediaMethods &
|
||||
TwitterClientNewsMethods &
|
||||
TwitterClientPostingMethods &
|
||||
TwitterClientSearchMethods &
|
||||
TwitterClientTimelineMethods &
|
||||
@@ -25,11 +27,16 @@ type TwitterClientInstance = TwitterClientBase &
|
||||
TwitterClientUserLookupMethods &
|
||||
TwitterClientUserTweetsMethods;
|
||||
|
||||
const MixedTwitterClient = withUserTweets(
|
||||
withUserLookup(
|
||||
withUsers(
|
||||
withLists(
|
||||
withHome(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase))))))),
|
||||
// News mixin wraps search because it depends on the search() method
|
||||
const MixedTwitterClient = withNews(
|
||||
withUserTweets(
|
||||
withUserLookup(
|
||||
withUsers(
|
||||
withLists(
|
||||
withHome(
|
||||
withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase)))))),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -37,6 +44,7 @@ const MixedTwitterClient = withUserTweets(
|
||||
|
||||
export class TwitterClient extends MixedTwitterClient {}
|
||||
|
||||
export type { NewsFetchOptions, NewsItem, NewsResult } from './twitter-client-news.js';
|
||||
export type {
|
||||
BookmarkMutationResult,
|
||||
CurrentUserResult,
|
||||
|
||||
Reference in New Issue
Block a user