From c96ce5238f4832611a8ee99e47a520841867d441 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 12 Jan 2026 05:37:44 +0000 Subject: [PATCH] fix: tighten news output and parsing (#39) (thanks @aavetis) --- CHANGELOG.md | 2 + src/commands/news.ts | 28 +++++-- src/lib/twitter-client-news.ts | 22 +++--- tests/commands.news.test.ts | 33 +++++++++ tests/twitter-client-coverage.news.test.ts | 86 +++++++++++++++++++++- 5 files changed, 146 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c02efc..ce74d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - `user-tweets` command to fetch a user's profile timeline (#34) — thanks @crcatala. - `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala. - Long-form article tweets now render rich Draft.js content blocks/entities (#36) — thanks @crcatala. +- `news`/`trending` command for Explore tabs with AI-curated headlines (#39) — thanks @aavetis. ### Changed - Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`). @@ -20,6 +21,7 @@ - Lists GraphQL feature flags updated to prevent 400s (#27) — thanks @zheli. - Likes pagination now follows cursors and avoids stalling on duplicate pages (#12) — thanks @titouv. - `pnpm run build:dist` now succeeds after tightening JSON/pagination option typing in tweet output commands. +- News output now respects `--tweets-per-item`, keeps unique IDs, and parses non-add entry instructions (#39) — thanks @aavetis. ### Tests - Following: split following/likes tests + cover cursor handling (#33) — thanks @VACInc. diff --git a/src/commands/news.ts b/src/commands/news.ts index 6d7715d..853180a 100644 --- a/src/commands/news.ts +++ b/src/commands/news.ts @@ -16,7 +16,7 @@ function formatPostCount(count: number): string { function printNewsItems( items: NewsItem[], ctx: CliContext, - opts: { json?: boolean; emptyMessage?: string } = {}, + opts: { json?: boolean; emptyMessage?: string; tweetLimit?: number } = {}, ): void { if (opts.json) { console.log(JSON.stringify(items, null, 2)); @@ -54,7 +54,8 @@ function printNewsItems( // Print related tweets if available if (item.tweets && item.tweets.length > 0) { console.log(` ${ctx.colors.section('Related tweets:')}`); - for (const tweet of item.tweets.slice(0, 3)) { + const tweetLimit = opts.tweetLimit ?? item.tweets.length; + for (const tweet of item.tweets.slice(0, tweetLimit)) { console.log( ` @${tweet.author.username}: ${tweet.text.slice(0, 100)}${tweet.text.length > 100 ? '...' : ''}`, ); @@ -69,7 +70,7 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void { program .command('news') .alias('trending') - .description("Fetch AI-curated news and trending topics from Explore tabs") + .description('Fetch AI-curated news and trending topics from Explore tabs') .option('-n, --count ', 'Number of items to fetch', '10') .option('--ai-only', 'Show only AI-curated news items') .option('--with-tweets', 'Also fetch related tweets for each news item') @@ -124,11 +125,21 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void { // Determine which tabs to fetch from const tabs: ExploreTab[] = []; - if (cmdOpts.forYou) tabs.push('forYou'); - if (cmdOpts.newsOnly) tabs.push('news'); - if (cmdOpts.sports) tabs.push('sports'); - if (cmdOpts.entertainment) tabs.push('entertainment'); - if (cmdOpts.trendingOnly) tabs.push('trending'); + if (cmdOpts.forYou) { + tabs.push('forYou'); + } + if (cmdOpts.newsOnly) { + tabs.push('news'); + } + if (cmdOpts.sports) { + tabs.push('sports'); + } + if (cmdOpts.entertainment) { + tabs.push('entertainment'); + } + if (cmdOpts.trendingOnly) { + tabs.push('trending'); + } // If no specific tabs selected, use defaults (all tabs except trending) const tabsToFetch = tabs.length > 0 ? tabs : undefined; @@ -150,6 +161,7 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void { printNewsItems(result.items, ctx, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No news items found.', + tweetLimit: withTweets ? tweetsPerItem : undefined, }); } else { console.error(`${ctx.p('err')}Failed to fetch news: ${result.error}`); diff --git a/src/lib/twitter-client-news.ts b/src/lib/twitter-client-news.ts index 9a22858..b82c5f1 100644 --- a/src/lib/twitter-client-news.ts +++ b/src/lib/twitter-client-news.ts @@ -96,7 +96,7 @@ export function withNews>( } try { - const tabItems = await this.fetchTimelineTab(tab, timelineId, count, aiOnly, includeRaw, debug); + const tabItems = await this.fetchTimelineTab(tab, timelineId, count, aiOnly, includeRaw); // Deduplicate across tabs for (const item of tabItems) { @@ -107,9 +107,7 @@ export function withNews>( } if (debug) { - console.error( - `[getNews] Tab ${tab}: found ${tabItems.length} items, total unique: ${allItems.length}`, - ); + console.error(`[getNews] Tab ${tab}: found ${tabItems.length} items, total unique: ${allItems.length}`); } // Stop early if we have enough @@ -147,7 +145,6 @@ export function withNews>( maxCount: number, aiOnly: boolean, includeRaw: boolean, - debug: boolean, ): Promise { const queryId = await this.getQueryId('GenericTimelineById'); const features = buildExploreFeatures(); @@ -220,12 +217,11 @@ export function withNews>( const instructions = timeline.instructions || []; for (const instruction of instructions) { - if (instruction.type !== 'TimelineAddEntries') { + const entries = instruction.entries ?? (instruction.entry ? [instruction.entry] : []); + if (!entries || entries.length === 0) { continue; } - const entries = instruction.entries || []; - for (const entry of entries) { if (items.length >= maxCount) { break; @@ -285,8 +281,6 @@ export function withNews>( return items; } - // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex - private parseNewsItemFromContent( // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex itemContent: any, @@ -302,6 +296,9 @@ export function withNews>( return null; } + const trendMetadata = itemContent?.trend_metadata; + const trendUrl = itemContent.trend_url?.url || trendMetadata?.url?.url; + // Detect AI news by characteristics: // 1. Full sentence headlines (contains spaces and is longer) // 2. Has social_context with "News" category @@ -360,7 +357,6 @@ export function withNews>( } // Parse trend metadata - const trendMetadata = itemContent?.trend_metadata; if (trendMetadata?.meta_description) { const metaDesc = trendMetadata.meta_description; const postMatch = metaDesc.match(POST_COUNT_MATCH_REGEX); @@ -385,13 +381,13 @@ export function withNews>( } const item: NewsItem = { - id: entryId || `${source}-${headline}`, + id: trendUrl ?? (entryId ? `${entryId}-${headline}` : `${source}-${headline}`), headline, category: isAiNews ? `AI · ${category}` : category, timeAgo, postCount, description: itemContent.description, - url: itemContent.trend_url?.url || trendMetadata?.url?.url, + url: trendUrl, }; if (includeRaw) { diff --git a/tests/commands.news.test.ts b/tests/commands.news.test.ts index 9ea9bf3..bbaff3e 100644 --- a/tests/commands.news.test.ts +++ b/tests/commands.news.test.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { CliContext } from '../src/cli/shared.js'; import { registerNewsCommand } from '../src/commands/news.js'; +import { TwitterClient } from '../src/lib/twitter-client.js'; describe('news command', () => { let program: Command; @@ -151,4 +152,36 @@ describe('news command', () => { errorSpy.mockRestore(); } }); + + it('limits related tweets to tweets-per-item in CLI output', async () => { + registerNewsCommand(program, mockContext as CliContext); + const getNewsSpy = vi.spyOn(TwitterClient.prototype, 'getNews').mockResolvedValue({ + success: true, + items: [ + { + id: 'item-1', + headline: 'News headline', + category: 'News', + tweets: [ + { id: 't1', text: 'first tweet', author: { username: 'a', name: 'A' } }, + { id: 't2', text: 'second tweet', author: { username: 'b', name: 'B' } }, + { id: 't3', text: 'third tweet', author: { username: 'c', name: 'C' } }, + ], + }, + ], + }); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + try { + await program.parseAsync(['node', 'bird', 'news', '--with-tweets', '--tweets-per-item', '2', '--count', '1']); + expect(getNewsSpy).toHaveBeenCalledWith(1, expect.objectContaining({ tweetsPerItem: 2, withTweets: true })); + const tweetLines = logSpy.mock.calls + .map((call) => String(call[0])) + .filter((line) => line.trimStart().startsWith('@')); + expect(tweetLines).toHaveLength(2); + } finally { + getNewsSpy.mockRestore(); + logSpy.mockRestore(); + } + }); }); diff --git a/tests/twitter-client-coverage.news.test.ts b/tests/twitter-client-coverage.news.test.ts index 4c6867c..cc0a859 100644 --- a/tests/twitter-client-coverage.news.test.ts +++ b/tests/twitter-client-coverage.news.test.ts @@ -24,7 +24,7 @@ const makeResponse = (overrides: Partial = {}): ResponseLike => ({ }); // Helper to create GenericTimelineById response structure -const makeTimelineResponse = (items: any[]) => ({ +const makeTimelineResponse = (items: unknown[]) => ({ data: { timeline: { timeline: { @@ -50,6 +50,21 @@ const makeTimelineResponse = (items: any[]) => ({ }, }); +const makeTimelineResponseWithInstruction = (type: string, entries: unknown[]) => ({ + data: { + timeline: { + timeline: { + instructions: [ + { + type, + entries, + }, + ], + }, + }, + }, +}); + describe('TwitterClient news API coverage', () => { const originalFetch = global.fetch; @@ -162,9 +177,7 @@ describe('TwitterClient news API coverage', () => { }); it('returns error when no news items found', async () => { - const mockFetch = vi - .fn() - .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) })); + const mockFetch = vi.fn().mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) })); global.fetch = mockFetch as unknown as typeof fetch; @@ -244,5 +257,70 @@ describe('TwitterClient news API coverage', () => { expect(result.success).toBe(true); expect(result.items?.length).toBe(2); }); + + it('parses entries from non-TimelineAddEntries instructions', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce( + makeResponse({ + json: async () => + makeTimelineResponseWithInstruction('TimelineReplaceEntry', [ + { + entryId: 'replace-entry-1', + content: { + itemContent: { + is_ai_trend: true, + name: 'Replacement headline', + }, + }, + }, + ]), + }), + ) + .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) })); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(1); + + expect(result.success).toBe(true); + expect(result.items?.[0].headline).toBe('Replacement headline'); + }); + + it('assigns unique ids for module entries', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce( + makeResponse({ + json: async () => + makeTimelineResponseWithInstruction('TimelineAddEntries', [ + { + entryId: 'module-entry', + content: { + items: [ + { + item: { itemContent: { is_ai_trend: true, name: 'Headline A' } }, + }, + { + item: { itemContent: { is_ai_trend: true, name: 'Headline B' } }, + }, + ], + }, + }, + ]), + }), + ) + .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) })); + + global.fetch = mockFetch as unknown as typeof fetch; + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getNews(2); + + expect(result.success).toBe(true); + const ids = result.items?.map((item) => item.id) ?? []; + expect(new Set(ids).size).toBe(ids.length); + }); }); });