diff --git a/README.md b/README.md index 7a316ce..f688854 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ Fields: - `bird reply ""` — reply to a tweet using its ID or URL. - `bird help [command]` — show help (or help for a subcommand). - `bird query-ids [--fresh] [--json]` — inspect or refresh cached GraphQL query IDs. -- `bird home [-n count] [--following] [--json] [--json-full]` — fetch your home timeline (For You) or Following feed. +- `bird home [-n count] [--following] [--all] [--max-pages n] [--cursor string] [--delay ms] [--json] [--json-full]` — fetch your home timeline (For You) or Following feed; `--max-pages` implies pagination. - `bird read [--json]` — fetch tweet content as text or JSON. - `bird [--json]` — shorthand for `read` when only a URL or ID is provided. - `bird replies [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — list replies to a tweet. diff --git a/src/commands/home.ts b/src/commands/home.ts index 9ad2391..5c98975 100644 --- a/src/commands/home.ts +++ b/src/commands/home.ts @@ -1,4 +1,5 @@ import type { Command } from 'commander'; +import { parsePaginationFlags } from '../cli/pagination.js'; import type { CliContext } from '../cli/shared.js'; import { TwitterClient } from '../lib/twitter-client.js'; @@ -7,36 +8,73 @@ export function registerHomeCommand(program: Command, ctx: CliContext): void { .command('home') .description('Get your home timeline ("For You" feed)') .option('-n, --count ', 'Number of tweets to fetch', '20') + .option('--all', 'Fetch all available pages up to --count') + .option('--max-pages ', 'Stop after N pages') + .option('--cursor ', 'Resume pagination from a cursor') + .option('--delay ', 'Delay in ms between page fetches', '1000') .option('--following', 'Get "Following" feed (chronological) instead of "For You"') .option('--json', 'Output as JSON') .option('--json-full', 'Output as JSON with full raw API response in _raw field') - .action(async (cmdOpts: { count?: string; following?: boolean; json?: boolean; jsonFull?: boolean }) => { - const opts = program.opts(); - const timeoutMs = ctx.resolveTimeoutFromOptions(opts); - const count = Number.parseInt(cmdOpts.count || '20', 10); - const relayBaseUrl = ctx.resolveRelayBaseUrlFromOptions(opts); - const profileName = ctx.resolveProfileNameFromOptions(opts); + .action( + async (cmdOpts: { + count?: string; + all?: boolean; + maxPages?: string; + cursor?: string; + delay?: string; + following?: boolean; + json?: boolean; + jsonFull?: boolean; + }) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const count = Number.parseInt(cmdOpts.count || '20', 10); + const pagination = parsePaginationFlags(cmdOpts, { + maxPagesImpliesPagination: true, + includeDelay: true, + }); + if (!pagination.ok) { + console.error(`${ctx.p('err')}${pagination.error}`); + process.exit(1); + } + const relayBaseUrl = ctx.resolveRelayBaseUrlFromOptions(opts); + const profileName = ctx.resolveProfileNameFromOptions(opts); - if (!Number.isFinite(count) || count <= 0) { - console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); - process.exit(1); - } + if (!Number.isFinite(count) || count <= 0) { + console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); + process.exit(1); + } - const client = new TwitterClient({ relayBaseUrl, profileName, timeoutMs }); - const includeRaw = cmdOpts.jsonFull ?? false; + const client = new TwitterClient({ relayBaseUrl, profileName, timeoutMs }); + const includeRaw = cmdOpts.jsonFull ?? false; + const usePagination = pagination.usePagination; + const fetchOptions = { + includeRaw, + maxPages: pagination.maxPages, + cursor: pagination.cursor, + pageDelayMs: pagination.pageDelayMs, + }; - const result = cmdOpts.following - ? await client.getHomeLatestTimeline(count, { includeRaw }) - : await client.getHomeTimeline(count, { includeRaw }); + const result = cmdOpts.following + ? await client.getHomeLatestTimeline(count, fetchOptions) + : await client.getHomeTimeline(count, fetchOptions); - if (result.success) { - const feedType = cmdOpts.following ? 'Following' : 'For You'; - const emptyMessage = `No tweets found in ${feedType} timeline.`; - const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); - ctx.printTweets(result.tweets, { json: isJson, emptyMessage }); - } else { - console.error(`${ctx.p('err')}Failed to fetch home timeline: ${result.error}`); - process.exit(1); - } - }); + if (result.success) { + const feedType = cmdOpts.following ? 'Following' : 'For You'; + const emptyMessage = `No tweets found in ${feedType} timeline.`; + const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); + ctx.printTweetsResult(result, { + json: isJson, + usePagination, + emptyMessage, + }); + if (result.nextCursor && !cmdOpts.json && !cmdOpts.jsonFull) { + console.error(`${ctx.p('info')}More tweets available. Use --cursor "${result.nextCursor}" to continue.`); + } + } else { + console.error(`${ctx.p('err')}Failed to fetch home timeline: ${result.error}`); + process.exit(1); + } + }, + ); } diff --git a/src/lib/twitter-client-home.ts b/src/lib/twitter-client-home.ts index 61f77f7..f71a3f7 100644 --- a/src/lib/twitter-client-home.ts +++ b/src/lib/twitter-client-home.ts @@ -14,6 +14,12 @@ function isQueryIdMismatch(errors: Array<{ message?: string }>): boolean { export interface HomeTimelineFetchOptions { /** Include raw GraphQL response in `_raw` field */ includeRaw?: boolean; + /** Starting cursor for pagination */ + cursor?: string; + /** Maximum number of pages to fetch */ + maxPages?: number; + /** Delay in milliseconds between page fetches */ + pageDelayMs?: number; } export interface TwitterClientHomeMethods { @@ -64,7 +70,17 @@ export function withHome>( const pageSize = 20; const seen = new Set(); const tweets: TweetData[] = []; - let cursor: string | undefined; + let cursor: string | undefined = options.cursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + const effectiveMaxPages = + options.maxPages !== undefined && Number.isFinite(options.maxPages) && options.maxPages > 0 + ? Math.floor(options.maxPages) + : Number.POSITIVE_INFINITY; + const pageDelayMs = + options.pageDelayMs !== undefined && Number.isFinite(options.pageDelayMs) && options.pageDelayMs > 0 + ? options.pageDelayMs + : 0; const fetchPage = async (pageCount: number, pageCursor?: string) => { let lastError: string | undefined; @@ -168,11 +184,16 @@ export function withHome>( }; while (tweets.length < count) { + if (pagesFetched > 0 && pageDelayMs > 0) { + await this.sleep(pageDelayMs); + } + const pageCount = Math.min(pageSize, count - tweets.length); const page = await fetchWithRefresh(pageCount, cursor); if (!page.success) { return { success: false, error: page.error }; } + pagesFetched += 1; let added = 0; for (const tweet of page.tweets) { @@ -187,13 +208,20 @@ export function withHome>( } } - if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) { + const pageCursor = page.cursor; + if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { + nextCursor = undefined; break; } - cursor = page.cursor; + if (pagesFetched >= effectiveMaxPages) { + nextCursor = pageCursor; + break; + } + cursor = pageCursor; + nextCursor = pageCursor; } - return { success: true, tweets }; + return { success: true, tweets, nextCursor }; } } diff --git a/tests/commands.home.test.ts b/tests/commands.home.test.ts new file mode 100644 index 0000000..0cfaf4d --- /dev/null +++ b/tests/commands.home.test.ts @@ -0,0 +1,61 @@ +import { Command } from 'commander'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { CliContext } from '../src/cli/shared.js'; +import { registerHomeCommand } from '../src/commands/home.js'; +import { TwitterClient } from '../src/lib/twitter-client.js'; + +describe('home command', () => { + let program: Command; + let mockContext: Partial; + + beforeEach(() => { + program = new Command(); + mockContext = { + resolveTimeoutFromOptions: () => undefined, + resolveRelayBaseUrlFromOptions: () => 'https://relay.example', + resolveProfileNameFromOptions: () => undefined, + p: () => '', + printTweetsResult: vi.fn(), + }; + }); + + it('uses cursor pagination when --cursor is set', async () => { + registerHomeCommand(program, mockContext as CliContext); + const timelineSpy = vi + .spyOn(TwitterClient.prototype, 'getHomeLatestTimeline') + .mockResolvedValue({ success: true, tweets: [], nextCursor: 'next-1' }); + + try { + await program.parseAsync(['node', 'bird', 'home', '--following', '--cursor', 'cursor-1']); + expect(timelineSpy).toHaveBeenCalledWith(20, expect.objectContaining({ cursor: 'cursor-1' })); + expect(mockContext.printTweetsResult).toHaveBeenCalledWith(expect.objectContaining({ nextCursor: 'next-1' }), { + json: false, + usePagination: true, + emptyMessage: 'No tweets found in Following timeline.', + }); + } finally { + timelineSpy.mockRestore(); + } + }); + + it('uses paged home timeline when --max-pages is set', async () => { + registerHomeCommand(program, mockContext as CliContext); + const timelineSpy = vi.spyOn(TwitterClient.prototype, 'getHomeTimeline').mockResolvedValue({ + success: true, + tweets: [], + nextCursor: 'next-2', + }); + + try { + await program.parseAsync(['node', 'bird', 'home', '--max-pages', '3', '--json']); + expect(timelineSpy).toHaveBeenCalledWith(20, expect.objectContaining({ maxPages: 3 })); + expect(mockContext.printTweetsResult).toHaveBeenCalledWith(expect.objectContaining({ nextCursor: 'next-2' }), { + json: true, + usePagination: true, + emptyMessage: 'No tweets found in For You timeline.', + }); + } finally { + timelineSpy.mockRestore(); + } + }); +}); diff --git a/tests/twitter-client.home-timeline.test.ts b/tests/twitter-client.home-timeline.test.ts index c20e439..0e0a4d0 100644 --- a/tests/twitter-client.home-timeline.test.ts +++ b/tests/twitter-client.home-timeline.test.ts @@ -145,6 +145,121 @@ describe('TwitterClient home timeline', () => { }); describe('pagination', () => { + it('returns the next cursor when stopping before the end', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor123', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + const client = new TwitterClient({ relayBaseUrl: testRelayBaseUrl }); + const result = await client.getHomeTimeline(1); + + expect(result.success).toBe(true); + expect(result.nextCursor).toBe('cursor123'); + }); + + it('resumes from a provided cursor', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => makeHomeTimelineResponse('tweet2', 'Second page tweet', 'user2'), + }); + + const client = new TwitterClient({ relayBaseUrl: testRelayBaseUrl }); + const result = await client.getHomeLatestTimeline(1, { cursor: 'resume-cursor' }); + + expect(result.success).toBe(true); + const callUrl = String(mockFetch.mock.calls[0]?.[0]); + const variables = JSON.parse(new URL(callUrl).searchParams.get('variables') ?? '{}') as { cursor?: string }; + expect(variables.cursor).toBe('resume-cursor'); + }); + + it('stops at maxPages and returns the cursor to resume', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-page-2', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + const client = new TwitterClient({ relayBaseUrl: testRelayBaseUrl }); + const result = await client.getHomeTimeline(5, { maxPages: 1 }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.nextCursor).toBe('cursor-page-2'); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + it('should deduplicate tweets across pages', async () => { // First page mockFetch.mockResolvedValueOnce({