From 90037ce880a17ae709e078876286b5f380df21e5 Mon Sep 17 00:00:00 2001 From: cc-vps Date: Tue, 6 Jan 2026 19:56:11 -0800 Subject: [PATCH] feat: add pagination support for thread and replies commands Add --all, --max-pages, --delay, and --cursor options to both thread and replies commands, enabling fetching of all conversation content with rate-limit-friendly pagination. Changes: - Modified fetchTweetDetail() to accept optional cursor parameter - Added getRepliesPaged() and getThreadPaged() methods with pagination loop - Updated CLI commands with new options (--all, --max-pages, --delay, --cursor) - Added TweetDetailPaginationOptions interface - JSON output includes nextCursor for scripting/resumption Options: - --all: Fetch all pages (no hard limit) - --max-pages : Limit number of pages when using --all - --delay : Delay between page fetches (default: 1000ms) - --cursor : Resume from a previous cursor Follows patterns established by bookmarks command and PR #34 (user-tweets). Co-authored-by: Christian Catalan --- src/commands/read.ts | 200 +++++++++--- src/lib/twitter-client-tweet-detail.ts | 154 ++++++++- tests/commands.read.test.ts | 152 +++++++++ tests/twitter-client.pagination.test.ts | 397 ++++++++++++++++++++++++ 4 files changed, 852 insertions(+), 51 deletions(-) create mode 100644 tests/commands.read.test.ts create mode 100644 tests/twitter-client.pagination.test.ts diff --git a/src/commands/read.ts b/src/commands/read.ts index 5a5ebb6..ccd61f1 100644 --- a/src/commands/read.ts +++ b/src/commands/read.ts @@ -49,72 +49,174 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { .command('replies') .description('List replies to a tweet (by ID or URL)') .argument('', 'Tweet ID or URL') + .option('--all', 'Fetch all replies (paged)') + .option('--max-pages ', 'Stop after N pages when using --all') + .option('--delay ', 'Delay in ms between page fetches (default: 1000)') + .option('--cursor ', 'Resume pagination from a cursor') .option('--json', 'Output as JSON') .option('--json-full', 'Output as JSON with full raw API response in _raw field') - .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => { - const opts = program.opts(); - const timeoutMs = ctx.resolveTimeoutFromOptions(opts); - const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); - const tweetId = ctx.extractTweetId(tweetIdOrUrl); + .action( + async ( + tweetIdOrUrl: string, + cmdOpts: { + all?: boolean; + maxPages?: string; + delay?: string; + cursor?: string; + json?: boolean; + jsonFull?: boolean; + }, + ) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); + const tweetId = ctx.extractTweetId(tweetIdOrUrl); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; + const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000; - const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); + process.exit(1); + } + if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) { + console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`); + process.exit(1); + } - for (const warning of warnings) { - console.error(`${ctx.p('warn')}${warning}`); - } + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); - if (!cookies.authToken || !cookies.ct0) { - console.error(`${ctx.p('err')}Missing required credentials`); - process.exit(1); - } + for (const warning of warnings) { + console.error(`${ctx.p('warn')}${warning}`); + } - const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const includeRaw = cmdOpts.jsonFull ?? false; - const result = await client.getReplies(tweetId, { includeRaw }); + if (!cookies.authToken || !cookies.ct0) { + console.error(`${ctx.p('err')}Missing required credentials`); + process.exit(1); + } - if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No replies found.' }); - } else { - console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`); - process.exit(1); - } - }); + const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); + const includeRaw = cmdOpts.jsonFull ?? false; + + const result = usePagination + ? await client.getRepliesPaged(tweetId, { + includeRaw, + maxPages, + cursor: cmdOpts.cursor, + pageDelayMs, + }) + : await client.getReplies(tweetId, { includeRaw }); + + if (result.success && result.tweets) { + const isJson = cmdOpts.json || cmdOpts.jsonFull; + if (isJson && usePagination) { + console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2)); + } else { + ctx.printTweets(result.tweets, { json: isJson, emptyMessage: 'No replies found.' }); + } + + // Show pagination hint if there's more + if (result.nextCursor && !isJson) { + console.error(`${ctx.p('info')}More replies available. Use --cursor "${result.nextCursor}" to continue.`); + } + } else { + console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`); + process.exit(1); + } + }, + ); program .command('thread') .description('Show the full conversation thread containing the tweet') .argument('', 'Tweet ID or URL') + .option('--all', 'Fetch all thread tweets (paged)') + .option('--max-pages ', 'Stop after N pages when using --all') + .option('--delay ', 'Delay in ms between page fetches (default: 1000)') + .option('--cursor ', 'Resume pagination from a cursor') .option('--json', 'Output as JSON') .option('--json-full', 'Output as JSON with full raw API response in _raw field') - .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => { - const opts = program.opts(); - const timeoutMs = ctx.resolveTimeoutFromOptions(opts); - const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); - const tweetId = ctx.extractTweetId(tweetIdOrUrl); + .action( + async ( + tweetIdOrUrl: string, + cmdOpts: { + all?: boolean; + maxPages?: string; + delay?: string; + cursor?: string; + json?: boolean; + jsonFull?: boolean; + }, + ) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); + const tweetId = ctx.extractTweetId(tweetIdOrUrl); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; + const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000; - const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); + process.exit(1); + } + if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) { + console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`); + process.exit(1); + } - for (const warning of warnings) { - console.error(`${ctx.p('warn')}${warning}`); - } + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); - if (!cookies.authToken || !cookies.ct0) { - console.error(`${ctx.p('err')}Missing required credentials`); - process.exit(1); - } + for (const warning of warnings) { + console.error(`${ctx.p('warn')}${warning}`); + } - const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const includeRaw = cmdOpts.jsonFull ?? false; - const result = await client.getThread(tweetId, { includeRaw }); + if (!cookies.authToken || !cookies.ct0) { + console.error(`${ctx.p('err')}Missing required credentials`); + process.exit(1); + } - if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { - json: cmdOpts.json || cmdOpts.jsonFull, - emptyMessage: 'No thread tweets found.', - }); - } else { - console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`); - process.exit(1); - } - }); + const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); + const includeRaw = cmdOpts.jsonFull ?? false; + + const result = usePagination + ? await client.getThreadPaged(tweetId, { + includeRaw, + maxPages, + cursor: cmdOpts.cursor, + pageDelayMs, + }) + : await client.getThread(tweetId, { includeRaw }); + + if (result.success && result.tweets) { + const isJson = cmdOpts.json || cmdOpts.jsonFull; + if (isJson && usePagination) { + console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2)); + } else { + ctx.printTweets(result.tweets, { + json: isJson, + emptyMessage: 'No thread tweets found.', + }); + } + + // Show pagination hint if there's more + if (result.nextCursor && !isJson) { + console.error( + `${ctx.p('info')}More thread tweets available. Use --cursor "${result.nextCursor}" to continue.`, + ); + } + } else { + console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`); + process.exit(1); + } + }, + ); } diff --git a/src/lib/twitter-client-tweet-detail.ts b/src/lib/twitter-client-tweet-detail.ts index 1a5a248..62768ab 100644 --- a/src/lib/twitter-client-tweet-detail.ts +++ b/src/lib/twitter-client-tweet-detail.ts @@ -1,9 +1,10 @@ import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js'; import { TWITTER_API_BASE } from './twitter-client-constants.js'; import { buildArticleFeatures, buildArticleFieldToggles, buildTweetDetailFeatures } from './twitter-client-features.js'; -import type { GetTweetResult, GraphqlTweetResult, SearchResult } from './twitter-client-types.js'; +import type { GetTweetResult, GraphqlTweetResult, SearchResult, TweetData } from './twitter-client-types.js'; import { extractArticleText, + extractCursorFromInstructions, findTweetInInstructions, firstText, mapTweetResult, @@ -16,10 +17,22 @@ export interface TweetFetchOptions { includeRaw?: boolean; } +/** Options for paginated tweet detail fetch */ +export interface TweetDetailPaginationOptions extends TweetFetchOptions { + /** Maximum number of pages to fetch (default: unlimited when using pagination) */ + maxPages?: number; + /** Starting cursor for pagination (resume from previous fetch) */ + cursor?: string; + /** Delay in milliseconds between page fetches (default: 1000) */ + pageDelayMs?: number; +} + export interface TwitterClientTweetDetailMethods { getTweet(tweetId: string, options?: TweetFetchOptions): Promise; getReplies(tweetId: string, options?: TweetFetchOptions): Promise; getThread(tweetId: string, options?: TweetFetchOptions): Promise; + getRepliesPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise; + getThreadPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise; } export function withTweetDetails>( @@ -110,7 +123,14 @@ export function withTweetDetails { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + private async fetchTweetDetail( + tweetId: string, + cursor?: string, + ): Promise< | { success: true; data: { @@ -141,6 +161,7 @@ export function withTweetDetails { + const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options; + const seen = new Set(); + const allReplies: TweetData[] = []; + let cursor: string | undefined = options.cursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + + while (true) { + // Add delay between pages (but not before the first page) + if (pagesFetched > 0 && pageDelayMs > 0) { + await this.sleep(pageDelayMs); + } + + const response = await this.fetchTweetDetail(tweetId, cursor); + if (!response.success) { + // If we have some replies already, return them with the error + if (allReplies.length > 0) { + return { success: true, tweets: allReplies, nextCursor: cursor, error: response.error }; + } + return response; + } + pagesFetched += 1; + + const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId); + + for (const reply of replies) { + if (seen.has(reply.id)) { + continue; + } + seen.add(reply.id); + allReplies.push(reply); + } + + const pageCursor = extractCursorFromInstructions(instructions); + if (!pageCursor || pageCursor === cursor || replies.length === 0) { + nextCursor = undefined; + break; + } + + if (maxPages && pagesFetched >= maxPages) { + nextCursor = pageCursor; + break; + } + + cursor = pageCursor; + nextCursor = pageCursor; + } + + return { success: true, tweets: allReplies, nextCursor }; + } + + /** + * Get full conversation thread with pagination support + */ + async getThreadPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise { + const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options; + const seen = new Set(); + const allTweets: TweetData[] = []; + let cursor: string | undefined = options.cursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + let rootId: string | undefined; + + while (true) { + // Add delay between pages (but not before the first page) + if (pagesFetched > 0 && pageDelayMs > 0) { + await this.sleep(pageDelayMs); + } + + const response = await this.fetchTweetDetail(tweetId, cursor); + if (!response.success) { + // If we have some tweets already, return them with the error + if (allTweets.length > 0) { + return { success: true, tweets: allTweets, nextCursor: cursor, error: response.error }; + } + return response; + } + pagesFetched += 1; + + const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + + // Determine root conversation ID from first page + if (!rootId) { + const target = tweets.find((t) => t.id === tweetId); + rootId = target?.conversationId || tweetId; + } + + const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId); + + for (const tweet of threadTweets) { + if (seen.has(tweet.id)) { + continue; + } + seen.add(tweet.id); + allTweets.push(tweet); + } + + const pageCursor = extractCursorFromInstructions(instructions); + if (!pageCursor || pageCursor === cursor || threadTweets.length === 0) { + nextCursor = undefined; + break; + } + + if (maxPages && pagesFetched >= maxPages) { + nextCursor = pageCursor; + break; + } + + cursor = pageCursor; + nextCursor = pageCursor; + } + + // Sort by creation time + allTweets.sort((a, b) => { + const aTime = a.createdAt ? Date.parse(a.createdAt) : 0; + const bTime = b.createdAt ? Date.parse(b.createdAt) : 0; + return aTime - bTime; + }); + + return { success: true, tweets: allTweets, nextCursor }; + } } return TwitterClientTweetDetails; diff --git a/tests/commands.read.test.ts b/tests/commands.read.test.ts new file mode 100644 index 0000000..9599cb7 --- /dev/null +++ b/tests/commands.read.test.ts @@ -0,0 +1,152 @@ +import { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; +import type { CliContext } from '../src/cli/shared.js'; +import { registerReadCommands } from '../src/commands/read.js'; + +describe('replies command', () => { + const createMockContext = () => + ({ + resolveTimeoutFromOptions: () => undefined, + resolveQuoteDepthFromOptions: () => 1, + extractTweetId: (input: string) => input, + resolveCredentialsFromOptions: async () => ({ + cookies: { authToken: 'auth', ct0: 'ct0', cookieHeader: 'auth=auth; ct0=ct0' }, + warnings: [], + }), + p: () => '', + printTweets: () => undefined, + }) as unknown as CliContext; + + it('requires --all or --cursor when --max-pages is provided', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'replies', '123', '--max-pages', '2'])).rejects.toThrow( + 'exit 1', + ); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('validates --max-pages is a positive integer', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect( + program.parseAsync(['node', 'bird', 'replies', '123', '--all', '--max-pages', '-1']), + ).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --max-pages')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('validates --delay is a non-negative integer', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'replies', '123', '--all', '--delay', '-100'])).rejects.toThrow( + 'exit 1', + ); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --delay')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); +}); + +describe('thread command', () => { + const createMockContext = () => + ({ + resolveTimeoutFromOptions: () => undefined, + resolveQuoteDepthFromOptions: () => 1, + extractTweetId: (input: string) => input, + resolveCredentialsFromOptions: async () => ({ + cookies: { authToken: 'auth', ct0: 'ct0', cookieHeader: 'auth=auth; ct0=ct0' }, + warnings: [], + }), + p: () => '', + printTweets: () => undefined, + }) as unknown as CliContext; + + it('requires --all or --cursor when --max-pages is provided', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--max-pages', '2'])).rejects.toThrow('exit 1'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('validates --max-pages is a positive integer', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--all', '--max-pages', '0'])).rejects.toThrow( + 'exit 1', + ); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --max-pages')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + + it('validates --delay is a non-negative integer', async () => { + const program = new Command(); + registerReadCommands(program, createMockContext()); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`); + }) as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--all', '--delay', 'abc'])).rejects.toThrow( + 'exit 1', + ); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --delay')); + } finally { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); +}); diff --git a/tests/twitter-client.pagination.test.ts b/tests/twitter-client.pagination.test.ts new file mode 100644 index 0000000..c34f7d2 --- /dev/null +++ b/tests/twitter-client.pagination.test.ts @@ -0,0 +1,397 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { TwitterClient } from '../src/lib/twitter-client.js'; +import { validCookies } from './twitter-client-fixtures.js'; + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); +}); + +describe('TwitterClient pagination for thread and replies', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch as unknown as typeof fetch; + }); + + const makeConversationPayload = (tweetIds: string[], cursor?: string, inReplyTo?: string, conversationId = '1') => ({ + data: { + threaded_conversation_with_injections_v2: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + ...tweetIds.map((id, i) => ({ + entryId: `tweet-${id}`, + content: { + itemContent: { + tweet_results: { + result: { + rest_id: id, + legacy: { + full_text: `tweet ${id}`, + created_at: `2024-01-0${i + 1}T00:00:00Z`, + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: conversationId, + ...(inReplyTo ? { in_reply_to_status_id_str: inReplyTo } : {}), + }, + core: { + user_results: { result: { legacy: { screen_name: `user${id}`, name: `User ${id}` } } }, + }, + }, + }, + }, + }, + })), + // Add cursor entry if provided + ...(cursor + ? [ + { + entryId: `cursor-bottom-${Date.now()}`, + content: { + cursorType: 'Bottom', + value: cursor, + }, + }, + ] + : []), + ], + }, + ], + }, + }, + }); + + describe('getRepliesPaged', () => { + it('returns replies from single page when no cursor in response', async () => { + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['2', '3'], undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1'); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(2); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('fetches multiple pages when cursor is present and --all is used', async () => { + // First page returns cursor, second page does not + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['4', '5'], undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1', { pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(4); + expect(result.tweets?.map((t) => t.id)).toEqual(['2', '3', '4', '5']); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('respects maxPages option', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['4', '5'], 'cursor-page-3', '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1', { maxPages: 2, pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(4); + expect(result.nextCursor).toBe('cursor-page-3'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('resumes from provided cursor', async () => { + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['4', '5'], undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1', { cursor: 'resume-cursor', pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(2); + // Verify cursor was passed in the request + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('resume-cursor'); + }); + + it('deduplicates tweets across pages', async () => { + // Second page returns a duplicate tweet + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['3', '4'], undefined, '1'), // '3' is duplicate + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1', { pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(3); // 2, 3, 4 (no duplicate) + expect(result.tweets?.map((t) => t.id)).toEqual(['2', '3', '4']); + }); + + it('returns partial results on error after first page', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'), + }) + .mockResolvedValueOnce({ + ok: false, + status: 500, + text: async () => 'Internal Server Error', + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getRepliesPaged('1', { pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(2); + expect(result.error).toContain('500'); + expect(result.nextCursor).toBe('cursor-page-2'); + }); + }); + + describe('getThreadPaged', () => { + it('returns thread tweets from single page', async () => { + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['1', '2', '3'], undefined, undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThreadPaged('1'); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(3); + expect(result.nextCursor).toBeUndefined(); + }); + + it('fetches multiple pages for thread', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['1', '2'], 'cursor-page-2', undefined, '1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['3', '4'], undefined, undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThreadPaged('1', { pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(4); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('sorts thread tweets by creation time', async () => { + // Return tweets in non-chronological order + const payload = { + data: { + threaded_conversation_with_injections_v2: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'tweet-3', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '3', + legacy: { + full_text: 'tweet 3', + created_at: '2024-01-03T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { user_results: { result: { legacy: { screen_name: 'user3', name: 'User 3' } } } }, + }, + }, + }, + }, + }, + { + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'tweet 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { user_results: { result: { legacy: { screen_name: 'user1', name: 'User 1' } } } }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }; + + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => payload, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThreadPaged('1'); + + expect(result.success).toBe(true); + expect(result.tweets?.map((t) => t.id)).toEqual(['1', '3']); // Sorted by date + }); + + it('filters by conversation ID', async () => { + // Include a tweet from a different conversation + const payload = { + data: { + threaded_conversation_with_injections_v2: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'tweet 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { user_results: { result: { legacy: { screen_name: 'user1', name: 'User 1' } } } }, + }, + }, + }, + }, + }, + { + entryId: 'tweet-99', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '99', + legacy: { + full_text: 'different conversation', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '99', // Different conversation + }, + core: { user_results: { result: { legacy: { screen_name: 'user99', name: 'User 99' } } } }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }; + + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => payload, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThreadPaged('1'); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(1); + expect(result.tweets?.[0].id).toBe('1'); + }); + + it('respects maxPages for thread', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['1', '2'], 'cursor-page-2', undefined, '1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeConversationPayload(['3', '4'], 'cursor-page-3', undefined, '1'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThreadPaged('1', { maxPages: 2, pageDelayMs: 0 }); + + expect(result.success).toBe(true); + expect(result.tweets?.length).toBe(4); + expect(result.nextCursor).toBe('cursor-page-3'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + }); +});