From 803bd25b0c722906c6830b3a5ffd3b015b6d0d8f Mon Sep 17 00:00:00 2001 From: Zhe Li Date: Mon, 5 Jan 2026 15:48:16 +0000 Subject: [PATCH] feat: add pagination support to list-timeline command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds --all, --max-pages, and --cursor options to list-timeline command, enabling users to fetch all tweets from a list through automatic pagination. Follows the same pattern as bookmarks pagination for consistency. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- src/commands/lists.ts | 38 +- src/lib/twitter-client-lists.ts | 117 ++++-- tests/twitter-client.lists.test.ts | 565 +++++++++++++++++++++++++++++ 3 files changed, 686 insertions(+), 34 deletions(-) diff --git a/src/commands/lists.ts b/src/commands/lists.ts index ac2eb34..4bce345 100644 --- a/src/commands/lists.ts +++ b/src/commands/lists.ts @@ -78,13 +78,17 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { .command('list-timeline ') .description('Get tweets from a list timeline') .option('-n, --count ', 'Number of tweets to fetch', '20') + .option('--all', 'Fetch all tweets from list (paged)') + .option('--max-pages ', 'Stop after N pages when using --all') + .option('--cursor ', 'Resume pagination from a cursor') .option('--json', 'Output as JSON') .option('--json-full', 'Output as JSON with full raw API response in _raw field') - .action(async (listIdOrUrl: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => { + .action(async (listIdOrUrl: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean; all?: boolean; maxPages?: string; cursor?: string }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '20', 10); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; const listId = extractListId(listIdOrUrl); if (!listId) { @@ -92,6 +96,20 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { process.exit(2); } + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (!usePagination && (!Number.isFinite(count) || count <= 0)) { + console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); + process.exit(1); + } + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); for (const warning of warnings) { @@ -104,13 +122,21 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.getListTimeline(listId, count, { includeRaw: cmdOpts.jsonFull }); + const includeRaw = cmdOpts.jsonFull ?? false; + const timelineOptions = { includeRaw }; + const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor }; + + const result = usePagination + ? await client.getAllListTimeline(listId, paginationOptions) + : await client.getListTimeline(listId, count, timelineOptions); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { - json: cmdOpts.json || cmdOpts.jsonFull, - emptyMessage: 'No tweets found in this list.', - }); + const isJson = cmdOpts.json || cmdOpts.jsonFull; + if (isJson && usePagination) { + console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2)); + } else { + ctx.printTweets(result.tweets, { json: isJson, emptyMessage: 'No tweets found in this list.' }); + } } else { console.error(`${ctx.p('err')}Failed to fetch list timeline: ${result.error}`); process.exit(1); diff --git a/src/lib/twitter-client-lists.ts b/src/lib/twitter-client-lists.ts index 4929970..8f57c8b 100644 --- a/src/lib/twitter-client-lists.ts +++ b/src/lib/twitter-client-lists.ts @@ -4,14 +4,15 @@ import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js'; import { TWITTER_API_BASE } from './twitter-client-constants.js'; import { buildListsFeatures } from './twitter-client-features.js'; -import type { TimelineFetchOptions } from './twitter-client-timelines.js'; -import type { GraphqlTweetResult, ListsResult, SearchResult, TwitterList } from './twitter-client-types.js'; -import { parseTweetsFromInstructions } from './twitter-client-utils.js'; +import type { TimelineFetchOptions, TimelinePaginationOptions } from './twitter-client-timelines.js'; +import type { GraphqlTweetResult, ListsResult, SearchResult, TweetData, TwitterList } from './twitter-client-types.js'; +import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js'; export interface TwitterClientListMethods { getOwnedLists(count?: number): Promise; getListMemberships(count?: number): Promise; getListTimeline(listId: string, count?: number, options?: TimelineFetchOptions): Promise; + getAllListTimeline(listId: string, options?: TimelinePaginationOptions): Promise; } interface GraphqlListResult { @@ -329,25 +330,49 @@ export function withLists>( * Get tweets from a list timeline */ async getListTimeline(listId: string, count = 20, options: TimelineFetchOptions = {}): Promise { - const { includeRaw = false } = options; + return this.getListTimelinePaged(listId, count, options); + } - const variables = { - listId, - count, - }; + /** + * Get all tweets from a list timeline (paginated) + */ + async getAllListTimeline(listId: string, options?: TimelinePaginationOptions): Promise { + return this.getListTimelinePaged(listId, Number.POSITIVE_INFINITY, options); + } + /** + * Internal paginated list timeline fetcher + */ + private async getListTimelinePaged( + listId: string, + limit: number, + options: TimelinePaginationOptions = {}, + ): Promise { const features = buildListsFeatures(); + const pageSize = 20; + const seen = new Set(); + const tweets: TweetData[] = []; + let cursor: string | undefined = options.cursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + const { includeRaw = false, maxPages } = options; - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); - - const tryOnce = async () => { + const fetchPage = async (pageCount: number, pageCursor?: string) => { let lastError: string | undefined; let had404 = false; const queryIds = await this.getListTimelineQueryIds(); + const variables = { + listId, + count: pageCount, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); + for (const queryId of queryIds) { const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`; @@ -396,9 +421,10 @@ export function withLists>( } const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions; - const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const nextCursor = extractCursorFromInstructions(instructions); - return { success: true as const, tweets, had404 }; + return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); } @@ -407,21 +433,56 @@ export function withLists>( return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 }; }; - const firstAttempt = await tryOnce(); - if (firstAttempt.success) { - return { success: true, tweets: firstAttempt.tweets }; - } - - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await tryOnce(); - if (secondAttempt.success) { - return { success: true, tweets: secondAttempt.tweets }; + const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { + const firstAttempt = await fetchPage(pageCount, pageCursor); + if (firstAttempt.success) { + return firstAttempt; } - return { success: false, error: secondAttempt.error }; + if (firstAttempt.had404) { + await this.refreshQueryIds(); + const secondAttempt = await fetchPage(pageCount, pageCursor); + if (secondAttempt.success) { + return secondAttempt; + } + return { success: false as const, error: secondAttempt.error }; + } + return { success: false as const, error: firstAttempt.error }; + }; + + const unlimited = !Number.isFinite(limit); + while (unlimited || tweets.length < limit) { + const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); + const page = await fetchWithRefresh(pageCount, cursor); + if (!page.success) { + return { success: false, error: page.error }; + } + pagesFetched += 1; + + for (const tweet of page.tweets) { + if (seen.has(tweet.id)) { + continue; + } + seen.add(tweet.id); + tweets.push(tweet); + if (!unlimited && tweets.length >= limit) { + break; + } + } + + const pageCursor = page.cursor; + if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) { + nextCursor = undefined; + break; + } + if (maxPages && pagesFetched >= maxPages) { + nextCursor = pageCursor; + break; + } + cursor = pageCursor; + nextCursor = pageCursor; } - return { success: false, error: firstAttempt.error }; + return { success: true, tweets, nextCursor }; } } diff --git a/tests/twitter-client.lists.test.ts b/tests/twitter-client.lists.test.ts index e5407e5..6e1d8ec 100644 --- a/tests/twitter-client.lists.test.ts +++ b/tests/twitter-client.lists.test.ts @@ -884,4 +884,569 @@ describe('TwitterClient lists', () => { expect(mockFetch).toHaveBeenCalledTimes(2); }); }); + + describe('getAllListTimeline', () => { + it('fetches multiple pages and returns all tweets', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { + full_text: 'First tweet', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-2', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet2', + legacy: { + full_text: 'Second tweet', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet2', + }, + core: { + user_results: { + result: { + rest_id: 'u2', + legacy: { screen_name: 'user2', name: 'User 2' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getListTimelineQueryIds = async () => ['test']; + + const result = await client.getAllListTimeline('1234567890'); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(2); + expect(result.tweets?.[0].id).toBe('tweet1'); + expect(result.tweets?.[1].id).toBe('tweet2'); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(2); + + // Verify cursor was passed in second request + const secondCallUrl = (mockFetch.mock.calls[1] as [string])[0]; + expect(secondCallUrl).toContain('cursor-page-2'); + }); + + it('stops at maxPages limit and returns nextCursor', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { + full_text: 'Page 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-2', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet2', + legacy: { + full_text: 'Page 2', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet2', + }, + core: { + user_results: { + result: { + rest_id: 'u2', + legacy: { screen_name: 'user2', name: 'User 2' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-3', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getListTimelineQueryIds = async () => ['test']; + + const result = await client.getAllListTimeline('1234567890', { maxPages: 2 }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(2); + expect(result.nextCursor).toBe('cursor-page-3'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('resumes from cursor', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet-resumed', + legacy: { + full_text: 'Resumed tweet', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet-resumed', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getListTimelineQueryIds = async () => ['test']; + + const result = await client.getAllListTimeline('1234567890', { cursor: 'resume-cursor' }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0].id).toBe('tweet-resumed'); + + // Verify cursor was passed in the request + const callUrl = (mockFetch.mock.calls[0] as [string])[0]; + expect(callUrl).toContain('resume-cursor'); + }); + + it('deduplicates tweets across pages', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'duplicate-tweet', + legacy: { + full_text: 'Duplicate', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'duplicate-tweet', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-2', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'duplicate-tweet', + legacy: { + full_text: 'Duplicate', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'duplicate-tweet', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'unique-tweet', + legacy: { + full_text: 'Unique', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'unique-tweet', + }, + core: { + user_results: { + result: { + rest_id: 'u2', + legacy: { screen_name: 'user2', name: 'User 2' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getListTimelineQueryIds = async () => ['test']; + + const result = await client.getAllListTimeline('1234567890'); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(2); + expect(result.tweets?.[0].id).toBe('duplicate-tweet'); + expect(result.tweets?.[1].id).toBe('unique-tweet'); + }); + + it('stops when no cursor is returned', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { + full_text: 'Page 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'user1', name: 'User 1' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-2', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + list: { + tweets_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet2', + legacy: { + full_text: 'Last page', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: 'tweet2', + }, + core: { + user_results: { + result: { + rest_id: 'u2', + legacy: { screen_name: 'user2', name: 'User 2' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getListTimelineQueryIds = async () => ['test']; + + const result = await client.getAllListTimeline('1234567890'); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(2); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + }); });