diff --git a/README.md b/README.md index ef01584..704967b 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ const sportsNews = await client.getNews(10, { - `bird [--json]` — shorthand for `read` when only a URL or ID is provided. - `bird replies [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — list replies to a tweet. - `bird thread [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — show the full conversation thread. -- `bird search "" [-n count] [--json]` — search for tweets matching a query. +- `bird search "" [-n count] [--all] [--max-pages n] [--cursor string] [--json]` — search for tweets matching a query; `--max-pages` requires `--all` or `--cursor`. - `bird mentions [-n count] [--user @handle] [--json]` — find tweets mentioning a user (defaults to the authenticated user). - `bird user-tweets <@handle> [-n count] [--cursor string] [--max-pages n] [--delay ms] [--json]` — get tweets from a user's profile timeline. - `bird bookmarks [-n count] [--folder-id id] [--all] [--max-pages n] [--json]` — list your bookmarked tweets (or a specific bookmark folder); `--max-pages` requires `--all`. diff --git a/docs/testing.md b/docs/testing.md index 71d3fb5..d5f33e1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -17,6 +17,7 @@ Run: - `pnpm test:live` - `pnpm bird following --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default` - `pnpm bird list-timeline --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default` +- `pnpm bird search "from:steipete" --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default` - `pnpm bird home --count 5 --json --cookie-source chrome --chrome-profile Default` - `pnpm bird home --count 5 --following --json --cookie-source chrome --chrome-profile Default` diff --git a/src/commands/search.ts b/src/commands/search.ts index 7311a07..326bf80 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -9,13 +9,17 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void .description('Search for tweets') .argument('', 'Search query (e.g., "@clawdbot" or "from:clawdbot")') .option('-n, --count ', 'Number of tweets to fetch', '10') + .option('--all', 'Fetch all search results (paged)') + .option('--max-pages ', 'Stop after N pages when using --all') + .option('--cursor ', 'Resume pagination from a cursor') .option('--json', 'Output as JSON') .option('--json-full', 'Output as JSON with full raw API response in _raw field') - .action(async (query: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => { + .action(async (query: string, cmdOpts: { count?: string; all?: boolean; maxPages?: string; cursor?: string; json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '10', 10); + const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); @@ -28,13 +32,33 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void process.exit(1); } + const usePagination = cmdOpts.all || cmdOpts.cursor; + if (maxPages !== undefined && !usePagination) { + console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`); + process.exit(1); + } + if (!usePagination && (!Number.isFinite(count) || count <= 0)) { + console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); + process.exit(1); + } + if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { + console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); + process.exit(1); + } + const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); const includeRaw = cmdOpts.jsonFull ?? false; - const result = await client.search(query, count, { includeRaw }); + const searchOptions = { includeRaw }; + const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor }; + const result = usePagination + ? await client.getAllSearchResults(query, paginationOptions) + : await client.search(query, count, searchOptions); if (result.success) { - ctx.printTweets(result.tweets, { - json: cmdOpts.json || cmdOpts.jsonFull, + const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); + ctx.printTweetsResult(result, { + json: isJson, + usePagination: Boolean(usePagination), emptyMessage: 'No tweets found.', }); } else { diff --git a/src/lib/twitter-client-search.ts b/src/lib/twitter-client-search.ts index 118457d..6e8eb2f 100644 --- a/src/lib/twitter-client-search.ts +++ b/src/lib/twitter-client-search.ts @@ -12,8 +12,16 @@ export interface SearchFetchOptions { includeRaw?: boolean; } +/** Options for paged search methods */ +export interface SearchPaginationOptions extends SearchFetchOptions { + maxPages?: number; + /** Starting cursor for pagination (resume from previous fetch) */ + cursor?: string; +} + export interface TwitterClientSearchMethods { search(query: string, count?: number, options?: SearchFetchOptions): Promise; + getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise; } function isQueryIdMismatch(payload: string): boolean { @@ -50,12 +58,25 @@ export function withSearch> * Search for tweets matching a query */ async search(query: string, count = 20, options: SearchFetchOptions = {}): Promise { - const { includeRaw = false } = options; + return this.searchPaged(query, count, options); + } + + /** + * Get all search results (paged) + */ + async getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise { + return this.searchPaged(query, Number.POSITIVE_INFINITY, options); + } + + private async searchPaged(query: string, limit: number, options: SearchPaginationOptions = {}): Promise { const features = buildSearchFeatures(); const pageSize = 20; const seen = new Set(); const tweets: TweetData[] = []; - let cursor: string | undefined; + let cursor: string | undefined = options.cursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + const { includeRaw = false, maxPages } = options; const fetchPage = async (pageCount: number, pageCursor?: string) => { let lastError: string | undefined; @@ -184,31 +205,42 @@ export function withSearch> return { success: false as const, error: firstAttempt.error }; }; - while (tweets.length < count) { - const pageCount = Math.min(pageSize, count - tweets.length); + const unlimited = !Number.isFinite(limit); + while (unlimited || tweets.length < limit) { + const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); const page = await fetchWithRefresh(pageCount, cursor); if (!page.success) { return { success: false, error: page.error }; } + pagesFetched += 1; + let added = 0; for (const tweet of page.tweets) { if (seen.has(tweet.id)) { continue; } seen.add(tweet.id); tweets.push(tweet); - if (tweets.length >= count) { + added += 1; + if (!unlimited && tweets.length >= limit) { break; } } - if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) { + const pageCursor = page.cursor; + if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { + nextCursor = undefined; break; } - cursor = page.cursor; + if (maxPages && pagesFetched >= maxPages) { + nextCursor = pageCursor; + break; + } + cursor = pageCursor; + nextCursor = pageCursor; } - return { success: true, tweets }; + return { success: true, tweets, nextCursor }; } } diff --git a/tests/twitter-client.search-bookmarks.test.ts b/tests/twitter-client.search-bookmarks.test.ts index 07cbf3d..688b165 100644 --- a/tests/twitter-client.search-bookmarks.test.ts +++ b/tests/twitter-client.search-bookmarks.test.ts @@ -341,6 +341,141 @@ describe('TwitterClient search', () => { expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']); expect(mockFetch).toHaveBeenCalledTimes(2); }); + + it('respects maxPages when fetching all search results', async () => { + const makeSearchEntry = (id: string, text: string) => ({ + content: { + itemContent: { + tweet_results: { + result: { + rest_id: id, + legacy: { + full_text: text, + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: id, + }, + core: { + user_results: { + result: { legacy: { screen_name: 'root', name: 'Root' } }, + }, + }, + }, + }, + }, + }, + }); + + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [ + makeSearchEntry('1', 'page 1'), + { content: { cursorType: 'Bottom', value: 'cursor-1' } }, + ], + }, + ], + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [makeSearchEntry('2', 'page 2')], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getAllSearchResults('query', { maxPages: 1 }); + + expect(result.success).toBe(true); + expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']); + expect(result.nextCursor).toBe('cursor-1'); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('does not return a stale cursor when search pagination ends', async () => { + const makeSearchEntry = (id: string) => ({ + content: { + itemContent: { + tweet_results: { + result: { + rest_id: id, + legacy: { + full_text: `tweet-${id}`, + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: id, + }, + core: { + user_results: { + result: { legacy: { screen_name: 'root', name: 'Root' } }, + }, + }, + }, + }, + }, + }, + }); + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [makeSearchEntry('1')], + }, + ], + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getAllSearchResults('query', { cursor: 'old-cursor' }); + + expect(result.success).toBe(true); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(1); + + const vars = JSON.parse(new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string); + expect(vars.cursor).toBe('old-cursor'); + }); }); describe('TwitterClient bookmarks', () => {