diff --git a/CHANGELOG.md b/CHANGELOG.md index ca1ebc7..d2df150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Query ID updater now tracks BookmarkFolderTimeline and keeps bookmark query IDs seeded. - `following`/`followers` JSON user fields are now camelCase (`followersCount`, `followingCount`, `isBlueVerified`, `profileImageUrl`, `createdAt`). - Cookie extraction timeout is now configurable (default 30s on macOS) via `--cookie-timeout` / `BIRD_COOKIE_TIMEOUT_MS` (thanks @tylerseymour). +- Search now paginates beyond 20 results when using `-n` (thanks @ryanh-ai). ## 0.4.1 — 2025-12-31 ### Added diff --git a/src/lib/twitter-client.ts b/src/lib/twitter-client.ts index 340ab95..6419c8c 100644 --- a/src/lib/twitter-client.ts +++ b/src/lib/twitter-client.ts @@ -924,6 +924,30 @@ export class TwitterClient { return tweets; } + private extractCursorFromInstructions( + instructions: + | Array<{ + entries?: Array<{ + content?: { + cursorType?: string; + value?: string; + }; + }>; + }> + | undefined, + cursorType = 'Bottom', + ): string | undefined { + for (const instruction of instructions ?? []) { + for (const entry of instruction.entries ?? []) { + const content = entry.content; + if (content?.cursorType === cursorType && typeof content.value === 'string' && content.value.length > 0) { + return content.value; + } + } + } + return undefined; + } + private buildArticleFeatures(): Record { return { rweb_video_screen_enabled: true, @@ -1650,25 +1674,30 @@ export class TwitterClient { * Search for tweets matching a query */ async search(query: string, count = 20): Promise { - const variables = { - rawQuery: query, - count, - querySource: 'typed_query', - product: 'Latest', - }; - const features = this.buildSearchFeatures(); + const pageSize = 20; + const seen = new Set(); + const tweets: TweetData[] = []; + let cursor: string | undefined; - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - }); - - const tryOnce = async () => { + const fetchPage = async (pageCount: number, pageCursor?: string) => { let lastError: string | undefined; let had404 = false; const queryIds = await this.getSearchTimelineQueryIds(); for (const queryId of queryIds) { + const variables = { + rawQuery: query, + count: pageCount, + querySource: 'typed_query', + product: 'Latest', + ...(pageCursor ? { cursor: pageCursor } : {}), + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + }); + const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params.toString()}`; try { @@ -1737,9 +1766,10 @@ export class TwitterClient { } const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; - const tweets = this.parseTweetsFromInstructions(instructions); + const pageTweets = this.parseTweetsFromInstructions(instructions); + const nextCursor = this.extractCursorFromInstructions(instructions); - return { success: true as const, tweets, had404 }; + return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); } @@ -1748,21 +1778,47 @@ export class TwitterClient { return { success: false as const, error: lastError ?? 'Unknown error fetching search results', had404 }; }; - const firstAttempt = await tryOnce(); - if (firstAttempt.success) { - return { success: true, tweets: firstAttempt.tweets }; - } - - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await tryOnce(); - if (secondAttempt.success) { - return { success: true, tweets: secondAttempt.tweets }; + const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { + const firstAttempt = await fetchPage(pageCount, pageCursor); + if (firstAttempt.success) { + return firstAttempt; } - return { success: false, error: secondAttempt.error }; + if (firstAttempt.had404) { + await this.refreshQueryIds(); + const secondAttempt = await fetchPage(pageCount, pageCursor); + if (secondAttempt.success) { + return secondAttempt; + } + return { success: false as const, error: secondAttempt.error }; + } + return { success: false as const, error: firstAttempt.error }; + }; + + while (tweets.length < count) { + const pageCount = Math.min(pageSize, count - tweets.length); + const page = await fetchWithRefresh(pageCount, cursor); + if (!page.success) { + return { success: false, error: page.error }; + } + + for (const tweet of page.tweets) { + if (seen.has(tweet.id)) { + continue; + } + seen.add(tweet.id); + tweets.push(tweet); + if (tweets.length >= count) { + break; + } + } + + if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) { + break; + } + cursor = page.cursor; } - return { success: false, error: firstAttempt.error }; + return { success: true, tweets }; } /** diff --git a/tests/twitter-client.test.ts b/tests/twitter-client.test.ts index 12c00f8..99a72ec 100644 --- a/tests/twitter-client.test.ts +++ b/tests/twitter-client.test.ts @@ -1167,6 +1167,81 @@ describe('TwitterClient', () => { expect(result.error).toContain('Unknown error fetching search results'); expect(mockFetch).not.toHaveBeenCalled(); }); + + it('paginates search results using the bottom cursor', async () => { + const makeSearchEntry = (id: string, text: string) => ({ + content: { + itemContent: { + tweet_results: { + result: { + rest_id: id, + legacy: { + full_text: text, + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: id, + }, + core: { + user_results: { + result: { legacy: { screen_name: 'root', name: 'Root' } }, + }, + }, + }, + }, + }, + }, + }); + + const makeResponse = (ids: string[], cursor?: string) => ({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [ + ...ids.map((id) => makeSearchEntry(id, `tweet-${id}`)), + ...(cursor ? [{ content: { cursorType: 'Bottom', value: cursor } }] : []), + ], + }, + ], + }, + }, + }, + }, + }); + + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeResponse(['1', '2'], 'cursor-1'), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => makeResponse(['2', '3']), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.search('needle', 3); + + expect(result.success).toBe(true); + expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1', '2', '3']); + expect(mockFetch).toHaveBeenCalledTimes(2); + + const firstVars = JSON.parse( + new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string, + ) as { cursor?: string }; + const secondVars = JSON.parse( + new URL(mockFetch.mock.calls[1][0] as string).searchParams.get('variables') as string, + ) as { cursor?: string }; + + expect(firstVars.cursor).toBeUndefined(); + expect(secondVars.cursor).toBe('cursor-1'); + }); }); describe('bookmarks', () => {