From 39a477265cd7a3f62c8700f6b459e4f86ce91f8d Mon Sep 17 00:00:00 2001 From: Micah Alpern Date: Mon, 5 Jan 2026 12:39:22 -0800 Subject: [PATCH] feat: add cursor-based pagination for following/followers commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pagination support to `bird following` and `bird followers` commands, similar to existing pagination in search/bookmarks/likes. Changes: - Add `cursor` parameter to `getFollowing()` and `getFollowers()` client methods - Return `nextCursor` in `FollowingResult` for pagination continuation - Add `--cursor ` CLI option for manual pagination - Add `--all` CLI flag to automatically fetch all pages - Add `--max-pages ` option to limit pages when using --all - Add input validation for --max-pages (requires --all or --cursor) - Add deduplication using Set to prevent duplicate users - Add 1-second delay between pages to avoid overwhelming the API - Update `-n/--count` description to clarify it's per-page - Add unit tests for cursor parameter and nextCursor response Usage examples: # Fetch first page (default 20 users) bird following # Fetch with specific page size bird following -n 50 # Use cursor for next page bird following --cursor "CURSOR_FROM_PREVIOUS" # Fetch ALL following users automatically (with rate limiting) bird following --all --json # Limit to first 5 pages bird following --all --max-pages 5 # Same options work for followers bird followers --all Note: REST API fallback does not support cursor pagination. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/commands/users.ts | 295 +++++++++++++++---- src/lib/twitter-client-types.ts | 2 + src/lib/twitter-client-users.ts | 42 ++- tests/twitter-client.following-likes.test.ts | 154 ++++++++++ 4 files changed, 424 insertions(+), 69 deletions(-) diff --git a/src/commands/users.ts b/src/commands/users.ts index 62ce548..57be392 100644 --- a/src/commands/users.ts +++ b/src/commands/users.ts @@ -1,18 +1,41 @@ import type { Command } from 'commander'; import type { CliContext } from '../cli/shared.js'; import { TwitterClient } from '../lib/twitter-client.js'; +import type { TwitterUser } from '../lib/twitter-client-types.js'; export function registerUserCommands(program: Command, ctx: CliContext): void { program .command('following') .description('Get users that you (or another user) follow') .option('--user ', 'User ID to get following for (defaults to current user)') - .option('-n, --count ', 'Number of users to fetch', '20') + .option('-n, --count ', 'Number of users to fetch per page', '20') + .option('--cursor ', 'Cursor for pagination (from previous response)') + .option('--all', 'Fetch all users (paginate automatically)') + .option('--max-pages ', 'Stop after N pages when using --all') .option('--json', 'Output as JSON') - .action(async (cmdOpts: { user?: string; count?: string; json?: boolean }) => { - const opts = program.opts(); - const timeoutMs = ctx.resolveTimeoutFromOptions(opts); - const count = Number.parseInt(cmdOpts.count || '20', 10); + .action( + async (cmdOpts: { + user?: string; + count?: string; + cursor?: string; + all?: boolean; + maxPages?: string; + json?: boolean; + }) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const count = Number.parseInt(cmdOpts.count || '20', 10); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; + + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); + process.exit(1); + } const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); @@ -37,16 +60,58 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { userId = currentUser.user.id; } - const result = await client.getFollowing(userId, count); + if (cmdOpts.all) { + // Fetch all pages + const allUsers: TwitterUser[] = []; + const seen = new Set(); + let cursor: string | undefined = cmdOpts.cursor; + let pageNum = 0; + let nextCursor: string | undefined; + + while (true) { + pageNum++; + if (!cmdOpts.json) { + console.error(`${ctx.p('info')}Fetching page ${pageNum}...`); + } + + const result = await client.getFollowing(userId, count, cursor); + + if (!result.success || !result.users) { + console.error(`${ctx.p('err')}Failed to fetch following: ${result.error}`); + process.exit(1); + } - if (result.success && result.users) { - if (cmdOpts.json) { - console.log(JSON.stringify(result.users, null, 2)); - } else { - if (result.users.length === 0) { - console.log('No users found.'); - } else { for (const user of result.users) { + if (!seen.has(user.id)) { + seen.add(user.id); + allUsers.push(user); + } + } + + if (!result.nextCursor || result.users.length === 0) { + nextCursor = undefined; + break; + } + + if (maxPages && pageNum >= maxPages) { + nextCursor = result.nextCursor; + break; + } + + cursor = result.nextCursor; + + // Rate limit: wait between pages to avoid overwhelming the API + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + if (cmdOpts.json) { + console.log(JSON.stringify(allUsers, null, 2)); + } else { + console.error(`${ctx.p('info')}Total: ${allUsers.length} users`); + if (nextCursor) { + console.error(`${ctx.p('info')}Stopped at --max-pages. Use --cursor to continue.`); + } + for (const user of allUsers) { console.log(`@${user.username} (${user.name})`); if (user.description) { console.log(` ${user.description.slice(0, 100)}${user.description.length > 100 ? '...' : ''}`); @@ -57,57 +122,148 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { console.log('──────────────────────────────────────────────────'); } } + } else { + // Single page fetch + const result = await client.getFollowing(userId, count, cmdOpts.cursor); + + if (result.success && result.users) { + if (cmdOpts.json) { + console.log(JSON.stringify({ users: result.users, nextCursor: result.nextCursor }, null, 2)); + } else { + if (result.users.length === 0) { + console.log('No users found.'); + } else { + for (const user of result.users) { + console.log(`@${user.username} (${user.name})`); + if (user.description) { + console.log(` ${user.description.slice(0, 100)}${user.description.length > 100 ? '...' : ''}`); + } + if (user.followersCount !== undefined) { + console.log(` ${ctx.p('info')}${user.followersCount.toLocaleString()} followers`); + } + console.log('──────────────────────────────────────────────────'); + } + if (result.nextCursor) { + console.error(`${ctx.p('info')}Next cursor: ${result.nextCursor}`); + } + } + } + } else { + console.error(`${ctx.p('err')}Failed to fetch following: ${result.error}`); + process.exit(1); + } } - } else { - console.error(`${ctx.p('err')}Failed to fetch following: ${result.error}`); - process.exit(1); - } - }); + }, + ); program .command('followers') .description('Get users that follow you (or another user)') .option('--user ', 'User ID to get followers for (defaults to current user)') - .option('-n, --count ', 'Number of users to fetch', '20') + .option('-n, --count ', 'Number of users to fetch per page', '20') + .option('--cursor ', 'Cursor for pagination (from previous response)') + .option('--all', 'Fetch all users (paginate automatically)') + .option('--max-pages ', 'Stop after N pages when using --all') .option('--json', 'Output as JSON') - .action(async (cmdOpts: { user?: string; count?: string; json?: boolean }) => { - const opts = program.opts(); - const timeoutMs = ctx.resolveTimeoutFromOptions(opts); - const count = Number.parseInt(cmdOpts.count || '20', 10); + .action( + async (cmdOpts: { + user?: string; + count?: string; + cursor?: string; + all?: boolean; + maxPages?: string; + json?: boolean; + }) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const count = Number.parseInt(cmdOpts.count || '20', 10); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; - const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); - - for (const warning of warnings) { - console.error(`${ctx.p('warn')}${warning}`); - } - - if (!cookies.authToken || !cookies.ct0) { - console.error(`${ctx.p('err')}Missing required credentials`); - process.exit(1); - } - - const client = new TwitterClient({ cookies, timeoutMs }); - - let userId = cmdOpts.user; - if (!userId) { - const currentUser = await client.getCurrentUser(); - if (!currentUser.success || !currentUser.user?.id) { - console.error(`${ctx.p('err')}Failed to get current user: ${currentUser.error || 'Unknown error'}`); + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); process.exit(1); } - userId = currentUser.user.id; - } - const result = await client.getFollowers(userId, count); + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); + + for (const warning of warnings) { + console.error(`${ctx.p('warn')}${warning}`); + } + + if (!cookies.authToken || !cookies.ct0) { + console.error(`${ctx.p('err')}Missing required credentials`); + process.exit(1); + } + + const client = new TwitterClient({ cookies, timeoutMs }); + + let userId = cmdOpts.user; + if (!userId) { + const currentUser = await client.getCurrentUser(); + if (!currentUser.success || !currentUser.user?.id) { + console.error(`${ctx.p('err')}Failed to get current user: ${currentUser.error || 'Unknown error'}`); + process.exit(1); + } + userId = currentUser.user.id; + } + + if (cmdOpts.all) { + // Fetch all pages + const allUsers: TwitterUser[] = []; + const seen = new Set(); + let cursor: string | undefined = cmdOpts.cursor; + let pageNum = 0; + let nextCursor: string | undefined; + + while (true) { + pageNum++; + if (!cmdOpts.json) { + console.error(`${ctx.p('info')}Fetching page ${pageNum}...`); + } + + const result = await client.getFollowers(userId, count, cursor); + + if (!result.success || !result.users) { + console.error(`${ctx.p('err')}Failed to fetch followers: ${result.error}`); + process.exit(1); + } - if (result.success && result.users) { - if (cmdOpts.json) { - console.log(JSON.stringify(result.users, null, 2)); - } else { - if (result.users.length === 0) { - console.log('No users found.'); - } else { for (const user of result.users) { + if (!seen.has(user.id)) { + seen.add(user.id); + allUsers.push(user); + } + } + + if (!result.nextCursor || result.users.length === 0) { + nextCursor = undefined; + break; + } + + if (maxPages && pageNum >= maxPages) { + nextCursor = result.nextCursor; + break; + } + + cursor = result.nextCursor; + + // Rate limit: wait between pages to avoid overwhelming the API + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + if (cmdOpts.json) { + console.log(JSON.stringify(allUsers, null, 2)); + } else { + console.error(`${ctx.p('info')}Total: ${allUsers.length} users`); + if (nextCursor) { + console.error(`${ctx.p('info')}Stopped at --max-pages. Use --cursor to continue.`); + } + for (const user of allUsers) { console.log(`@${user.username} (${user.name})`); if (user.description) { console.log(` ${user.description.slice(0, 100)}${user.description.length > 100 ? '...' : ''}`); @@ -118,12 +274,39 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { console.log('──────────────────────────────────────────────────'); } } + } else { + // Single page fetch + const result = await client.getFollowers(userId, count, cmdOpts.cursor); + + if (result.success && result.users) { + if (cmdOpts.json) { + console.log(JSON.stringify({ users: result.users, nextCursor: result.nextCursor }, null, 2)); + } else { + if (result.users.length === 0) { + console.log('No users found.'); + } else { + for (const user of result.users) { + console.log(`@${user.username} (${user.name})`); + if (user.description) { + console.log(` ${user.description.slice(0, 100)}${user.description.length > 100 ? '...' : ''}`); + } + if (user.followersCount !== undefined) { + console.log(` ${ctx.p('info')}${user.followersCount.toLocaleString()} followers`); + } + console.log('──────────────────────────────────────────────────'); + } + if (result.nextCursor) { + console.error(`${ctx.p('info')}Next cursor: ${result.nextCursor}`); + } + } + } + } else { + console.error(`${ctx.p('err')}Failed to fetch followers: ${result.error}`); + process.exit(1); + } } - } else { - console.error(`${ctx.p('err')}Failed to fetch followers: ${result.error}`); - process.exit(1); - } - }); + }, + ); program .command('likes') diff --git a/src/lib/twitter-client-types.ts b/src/lib/twitter-client-types.ts index 67d4768..b0118a7 100644 --- a/src/lib/twitter-client-types.ts +++ b/src/lib/twitter-client-types.ts @@ -280,6 +280,8 @@ export interface FollowingResult { success: boolean; users?: TwitterUser[]; error?: string; + /** Cursor for fetching the next page of results */ + nextCursor?: string; } export interface TwitterClientOptions { diff --git a/src/lib/twitter-client-users.ts b/src/lib/twitter-client-users.ts index 227404a..3736195 100644 --- a/src/lib/twitter-client-users.ts +++ b/src/lib/twitter-client-users.ts @@ -7,12 +7,12 @@ import { } from './twitter-client-constants.js'; import { buildFollowingFeatures } from './twitter-client-features.js'; import type { CurrentUserResult, FollowingResult } from './twitter-client-types.js'; -import { parseUsersFromInstructions } from './twitter-client-utils.js'; +import { extractCursorFromInstructions, parseUsersFromInstructions } from './twitter-client-utils.js'; export interface TwitterClientUserMethods { getCurrentUser(): Promise; - getFollowing(userId: string, count?: number): Promise; - getFollowers(userId: string, count?: number): Promise; + getFollowing(userId: string, count?: number, cursor?: string): Promise; + getFollowers(userId: string, count?: number, cursor?: string): Promise; } export function withUsers>( @@ -309,13 +309,17 @@ export function withUsers>( /** * Get users that a user is following */ - async getFollowing(userId: string, count = 20): Promise { - const variables = { + async getFollowing(userId: string, count = 20, cursor?: string): Promise { + const variables: Record = { userId, count, includePromotedContent: false, }; + if (cursor) { + variables.cursor = cursor; + } + const features = buildFollowingFeatures(); const params = new URLSearchParams({ @@ -369,8 +373,11 @@ export function withUsers>( const instructions = data.data?.user?.result?.timeline?.timeline?.instructions; const users = parseUsersFromInstructions(instructions); + const nextCursor = extractCursorFromInstructions( + instructions as Array<{ entries?: Array<{ content?: unknown }> }> | undefined, + ); - return { success: true as const, users, had404 }; + return { success: true as const, users, nextCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); } @@ -381,18 +388,19 @@ export function withUsers>( const firstAttempt = await tryOnce(); if (firstAttempt.success) { - return { success: true, users: firstAttempt.users }; + return { success: true, users: firstAttempt.users, nextCursor: firstAttempt.nextCursor }; } if (firstAttempt.had404) { await this.refreshQueryIds(); const secondAttempt = await tryOnce(); if (secondAttempt.success) { - return { success: true, users: secondAttempt.users }; + return { success: true, users: secondAttempt.users, nextCursor: secondAttempt.nextCursor }; } // GraphQL Following can also return 404 (queryId churn / endpoint flakiness). // Fallback to the internal v1.1 REST endpoint used by the web client (cookie-auth; no dev API key). + // Note: REST fallback does not support cursor pagination. const restAttempt = await this.getFollowingViaRest(userId, count); if (restAttempt.success) { return restAttempt; @@ -407,13 +415,17 @@ export function withUsers>( /** * Get users that follow a user */ - async getFollowers(userId: string, count = 20): Promise { - const variables = { + async getFollowers(userId: string, count = 20, cursor?: string): Promise { + const variables: Record = { userId, count, includePromotedContent: false, }; + if (cursor) { + variables.cursor = cursor; + } + const features = buildFollowingFeatures(); const params = new URLSearchParams({ @@ -467,8 +479,11 @@ export function withUsers>( const instructions = data.data?.user?.result?.timeline?.timeline?.instructions; const users = parseUsersFromInstructions(instructions); + const nextCursor = extractCursorFromInstructions( + instructions as Array<{ entries?: Array<{ content?: unknown }> }> | undefined, + ); - return { success: true as const, users, had404 }; + return { success: true as const, users, nextCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); } @@ -479,18 +494,19 @@ export function withUsers>( const firstAttempt = await tryOnce(); if (firstAttempt.success) { - return { success: true, users: firstAttempt.users }; + return { success: true, users: firstAttempt.users, nextCursor: firstAttempt.nextCursor }; } if (firstAttempt.had404) { await this.refreshQueryIds(); const secondAttempt = await tryOnce(); if (secondAttempt.success) { - return { success: true, users: secondAttempt.users }; + return { success: true, users: secondAttempt.users, nextCursor: secondAttempt.nextCursor }; } // GraphQL Followers regularly returns 404 (queryId churn / endpoint flakiness). // Fallback to the internal v1.1 REST endpoint used by the web client (cookie-auth; no dev API key). + // Note: REST fallback does not support cursor pagination. const restAttempt = await this.getFollowersViaRest(userId, count); if (restAttempt.success) { return restAttempt; diff --git a/tests/twitter-client.following-likes.test.ts b/tests/twitter-client.following-likes.test.ts index 872c649..606d47f 100644 --- a/tests/twitter-client.following-likes.test.ts +++ b/tests/twitter-client.following-likes.test.ts @@ -280,6 +280,160 @@ describe('TwitterClient following/followers', () => { expect(urls[1]).toContain('/Followers?'); expect(urls[2]).toContain('/followers/list.json?'); }); + + it('passes cursor parameter to following API and returns nextCursor', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + content: { + itemContent: { + user_results: { + result: makeUserResult('1', 'alpha', 'Alpha'), + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'next-cursor-abc123', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClient & { getFollowingQueryIds: () => Promise }; + clientPrivate.getFollowingQueryIds = async () => ['test']; + + const result = await client.getFollowing('123', 20, 'prev-cursor-xyz'); + + expect(result.success).toBe(true); + expect(result.users?.length).toBe(1); + expect(result.nextCursor).toBe('next-cursor-abc123'); + + // Verify cursor was passed in the request + const [url] = mockFetch.mock.calls[0]; + const parsedVars = JSON.parse(new URL(url as string).searchParams.get('variables') as string); + expect(parsedVars.cursor).toBe('prev-cursor-xyz'); + }); + + it('passes cursor parameter to followers API and returns nextCursor', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + user_results: { + result: makeUserResult('9', 'beta', 'Beta'), + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'followers-next-cursor', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClient & { getFollowersQueryIds: () => Promise }; + clientPrivate.getFollowersQueryIds = async () => ['test']; + + const result = await client.getFollowers('456', 50, 'my-cursor'); + + expect(result.success).toBe(true); + expect(result.users?.[0].username).toBe('beta'); + expect(result.nextCursor).toBe('followers-next-cursor'); + + // Verify cursor was passed in the request + const [url] = mockFetch.mock.calls[0]; + const parsedVars = JSON.parse(new URL(url as string).searchParams.get('variables') as string); + expect(parsedVars.cursor).toBe('my-cursor'); + }); + + it('returns undefined nextCursor when no cursor in response', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + user_results: { + result: makeUserResult('1', 'only', 'Only'), + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClient & { getFollowingQueryIds: () => Promise }; + clientPrivate.getFollowingQueryIds = async () => ['test']; + + const result = await client.getFollowing('123', 20); + + expect(result.success).toBe(true); + expect(result.nextCursor).toBeUndefined(); + }); }); describe('TwitterClient likes', () => {