fix: tighten news output and parsing (#39) (thanks @aavetis)

CHANGELOG.md

@@ -9,6 +9,7 @@
 - `user-tweets` command to fetch a user's profile timeline (#34) — thanks @crcatala.
 - `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala.
 - Long-form article tweets now render rich Draft.js content blocks/entities (#36) — thanks @crcatala.
+- `news`/`trending` command for Explore tabs with AI-curated headlines (#39) — thanks @aavetis.
 
 ### Changed
 - Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`).
@@ -20,6 +21,7 @@
 - Lists GraphQL feature flags updated to prevent 400s (#27) — thanks @zheli.
 - Likes pagination now follows cursors and avoids stalling on duplicate pages (#12) — thanks @titouv.
 - `pnpm run build:dist` now succeeds after tightening JSON/pagination option typing in tweet output commands.
+- News output now respects `--tweets-per-item`, keeps unique IDs, and parses non-add entry instructions (#39) — thanks @aavetis.
 
 ### Tests
 - Following: split following/likes tests + cover cursor handling (#33) — thanks @VACInc.

src/commands/news.ts

@@ -16,7 +16,7 @@ function formatPostCount(count: number): string {
 function printNewsItems(
   items: NewsItem[],
   ctx: CliContext,
-  opts: { json?: boolean; emptyMessage?: string } = {},
+  opts: { json?: boolean; emptyMessage?: string; tweetLimit?: number } = {},
 ): void {
   if (opts.json) {
     console.log(JSON.stringify(items, null, 2));
@@ -54,7 +54,8 @@ function printNewsItems(
     // Print related tweets if available
     if (item.tweets && item.tweets.length > 0) {
       console.log(`  ${ctx.colors.section('Related tweets:')}`);
-      for (const tweet of item.tweets.slice(0, 3)) {
+      const tweetLimit = opts.tweetLimit ?? item.tweets.length;
+      for (const tweet of item.tweets.slice(0, tweetLimit)) {
         console.log(
           `    @${tweet.author.username}: ${tweet.text.slice(0, 100)}${tweet.text.length > 100 ? '...' : ''}`,
         );
@@ -69,7 +70,7 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void {
   program
     .command('news')
     .alias('trending')
-    .description("Fetch AI-curated news and trending topics from Explore tabs")
+    .description('Fetch AI-curated news and trending topics from Explore tabs')
     .option('-n, --count <number>', 'Number of items to fetch', '10')
     .option('--ai-only', 'Show only AI-curated news items')
     .option('--with-tweets', 'Also fetch related tweets for each news item')
@@ -124,11 +125,21 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void {
 
         // Determine which tabs to fetch from
         const tabs: ExploreTab[] = [];
-        if (cmdOpts.forYou) tabs.push('forYou');
+        if (cmdOpts.forYou) {
-        if (cmdOpts.newsOnly) tabs.push('news');
+          tabs.push('forYou');
-        if (cmdOpts.sports) tabs.push('sports');
+        }
-        if (cmdOpts.entertainment) tabs.push('entertainment');
+        if (cmdOpts.newsOnly) {
-        if (cmdOpts.trendingOnly) tabs.push('trending');
+          tabs.push('news');
+        }
+        if (cmdOpts.sports) {
+          tabs.push('sports');
+        }
+        if (cmdOpts.entertainment) {
+          tabs.push('entertainment');
+        }
+        if (cmdOpts.trendingOnly) {
+          tabs.push('trending');
+        }
 
         // If no specific tabs selected, use defaults (all tabs except trending)
         const tabsToFetch = tabs.length > 0 ? tabs : undefined;
@@ -150,6 +161,7 @@ export function registerNewsCommand(program: Command, ctx: CliContext): void {
           printNewsItems(result.items, ctx, {
             json: cmdOpts.json || cmdOpts.jsonFull,
             emptyMessage: 'No news items found.',
+            tweetLimit: withTweets ? tweetsPerItem : undefined,
           });
         } else {
           console.error(`${ctx.p('err')}Failed to fetch news: ${result.error}`);

src/lib/twitter-client-news.ts

@@ -96,7 +96,7 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
         }
 
         try {
-          const tabItems = await this.fetchTimelineTab(tab, timelineId, count, aiOnly, includeRaw, debug);
+          const tabItems = await this.fetchTimelineTab(tab, timelineId, count, aiOnly, includeRaw);
 
           // Deduplicate across tabs
           for (const item of tabItems) {
@@ -107,9 +107,7 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
           }
 
           if (debug) {
-            console.error(
+            console.error(`[getNews] Tab ${tab}: found ${tabItems.length} items, total unique: ${allItems.length}`);
-              `[getNews] Tab ${tab}: found ${tabItems.length} items, total unique: ${allItems.length}`,
-            );
           }
 
           // Stop early if we have enough
@@ -147,7 +145,6 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
       maxCount: number,
       aiOnly: boolean,
       includeRaw: boolean,
-      debug: boolean,
     ): Promise<NewsItem[]> {
       const queryId = await this.getQueryId('GenericTimelineById');
       const features = buildExploreFeatures();
@@ -220,12 +217,11 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
       const instructions = timeline.instructions || [];
 
       for (const instruction of instructions) {
-        if (instruction.type !== 'TimelineAddEntries') {
+        const entries = instruction.entries ?? (instruction.entry ? [instruction.entry] : []);
+        if (!entries || entries.length === 0) {
           continue;
         }
 
-        const entries = instruction.entries || [];
-
         for (const entry of entries) {
           if (items.length >= maxCount) {
             break;
@@ -285,8 +281,6 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
       return items;
     }
 
-    // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
-
     private parseNewsItemFromContent(
       // biome-ignore lint/suspicious/noExplicitAny: API response structure is complex
       itemContent: any,
@@ -302,6 +296,9 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
         return null;
       }
 
+      const trendMetadata = itemContent?.trend_metadata;
+      const trendUrl = itemContent.trend_url?.url || trendMetadata?.url?.url;
+
       // Detect AI news by characteristics:
       // 1. Full sentence headlines (contains spaces and is longer)
       // 2. Has social_context with "News" category
@@ -360,7 +357,6 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
       }
 
       // Parse trend metadata
-      const trendMetadata = itemContent?.trend_metadata;
       if (trendMetadata?.meta_description) {
         const metaDesc = trendMetadata.meta_description;
         const postMatch = metaDesc.match(POST_COUNT_MATCH_REGEX);
@@ -385,13 +381,13 @@ export function withNews<TBase extends AbstractConstructor<TwitterClientBase>>(
       }
 
       const item: NewsItem = {
-        id: entryId || `${source}-${headline}`,
+        id: trendUrl ?? (entryId ? `${entryId}-${headline}` : `${source}-${headline}`),
         headline,
         category: isAiNews ? `AI · ${category}` : category,
         timeAgo,
         postCount,
         description: itemContent.description,
-        url: itemContent.trend_url?.url || trendMetadata?.url?.url,
+        url: trendUrl,
       };
 
       if (includeRaw) {

tests/commands.news.test.ts

@@ -2,6 +2,7 @@ import { Command } from 'commander';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import type { CliContext } from '../src/cli/shared.js';
 import { registerNewsCommand } from '../src/commands/news.js';
+import { TwitterClient } from '../src/lib/twitter-client.js';
 
 describe('news command', () => {
   let program: Command;
@@ -151,4 +152,36 @@ describe('news command', () => {
       errorSpy.mockRestore();
     }
   });
+
+  it('limits related tweets to tweets-per-item in CLI output', async () => {
+    registerNewsCommand(program, mockContext as CliContext);
+    const getNewsSpy = vi.spyOn(TwitterClient.prototype, 'getNews').mockResolvedValue({
+      success: true,
+      items: [
+        {
+          id: 'item-1',
+          headline: 'News headline',
+          category: 'News',
+          tweets: [
+            { id: 't1', text: 'first tweet', author: { username: 'a', name: 'A' } },
+            { id: 't2', text: 'second tweet', author: { username: 'b', name: 'B' } },
+            { id: 't3', text: 'third tweet', author: { username: 'c', name: 'C' } },
+          ],
+        },
+      ],
+    });
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+
+    try {
+      await program.parseAsync(['node', 'bird', 'news', '--with-tweets', '--tweets-per-item', '2', '--count', '1']);
+      expect(getNewsSpy).toHaveBeenCalledWith(1, expect.objectContaining({ tweetsPerItem: 2, withTweets: true }));
+      const tweetLines = logSpy.mock.calls
+        .map((call) => String(call[0]))
+        .filter((line) => line.trimStart().startsWith('@'));
+      expect(tweetLines).toHaveLength(2);
+    } finally {
+      getNewsSpy.mockRestore();
+      logSpy.mockRestore();
+    }
+  });
 });

tests/twitter-client-coverage.news.test.ts

@@ -24,7 +24,7 @@ const makeResponse = (overrides: Partial<ResponseLike> = {}): ResponseLike => ({
 });
 
 // Helper to create GenericTimelineById response structure
-const makeTimelineResponse = (items: any[]) => ({
+const makeTimelineResponse = (items: unknown[]) => ({
   data: {
     timeline: {
       timeline: {
@@ -50,6 +50,21 @@ const makeTimelineResponse = (items: any[]) => ({
   },
 });
 
+const makeTimelineResponseWithInstruction = (type: string, entries: unknown[]) => ({
+  data: {
+    timeline: {
+      timeline: {
+        instructions: [
+          {
+            type,
+            entries,
+          },
+        ],
+      },
+    },
+  },
+});
+
 describe('TwitterClient news API coverage', () => {
   const originalFetch = global.fetch;
 
@@ -162,9 +177,7 @@ describe('TwitterClient news API coverage', () => {
     });
 
     it('returns error when no news items found', async () => {
-      const mockFetch = vi
+      const mockFetch = vi.fn().mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) }));
-        .fn()
-        .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) }));
 
       global.fetch = mockFetch as unknown as typeof fetch;
 
@@ -244,5 +257,70 @@ describe('TwitterClient news API coverage', () => {
       expect(result.success).toBe(true);
       expect(result.items?.length).toBe(2);
     });
+
+    it('parses entries from non-TimelineAddEntries instructions', async () => {
+      const mockFetch = vi
+        .fn()
+        .mockResolvedValueOnce(
+          makeResponse({
+            json: async () =>
+              makeTimelineResponseWithInstruction('TimelineReplaceEntry', [
+                {
+                  entryId: 'replace-entry-1',
+                  content: {
+                    itemContent: {
+                      is_ai_trend: true,
+                      name: 'Replacement headline',
+                    },
+                  },
+                },
+              ]),
+          }),
+        )
+        .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) }));
+
+      global.fetch = mockFetch as unknown as typeof fetch;
+
+      const client = new TwitterClient({ cookies: validCookies });
+      const result = await client.getNews(1);
+
+      expect(result.success).toBe(true);
+      expect(result.items?.[0].headline).toBe('Replacement headline');
+    });
+
+    it('assigns unique ids for module entries', async () => {
+      const mockFetch = vi
+        .fn()
+        .mockResolvedValueOnce(
+          makeResponse({
+            json: async () =>
+              makeTimelineResponseWithInstruction('TimelineAddEntries', [
+                {
+                  entryId: 'module-entry',
+                  content: {
+                    items: [
+                      {
+                        item: { itemContent: { is_ai_trend: true, name: 'Headline A' } },
+                      },
+                      {
+                        item: { itemContent: { is_ai_trend: true, name: 'Headline B' } },
+                      },
+                    ],
+                  },
+                },
+              ]),
+          }),
+        )
+        .mockResolvedValue(makeResponse({ json: async () => makeTimelineResponse([]) }));
+
+      global.fetch = mockFetch as unknown as typeof fetch;
+
+      const client = new TwitterClient({ cookies: validCookies });
+      const result = await client.getNews(2);
+
+      expect(result.success).toBe(true);
+      const ids = result.items?.map((item) => item.id) ?? [];
+      expect(new Set(ids).size).toBe(ids.length);
+    });
   });
 });