feat: add pagination support to search command

Adds --all, --max-pages, and --cursor options to search command, enabling users to fetch all search results through automatic pagination. Follows the same pattern as bookmarks pagination for consistency.
2026-01-12 06:29:27 +00:00
parent f351414046
commit e23c20a6a2
5 changed files with 205 additions and 13 deletions
@@ -156,7 +156,7 @@ const sportsNews = await client.getNews(10, {
 - `bird <tweet-id-or-url> [--json]` — shorthand for `read` when only a URL or ID is provided.
 - `bird replies <tweet-id-or-url> [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — list replies to a tweet.
 - `bird thread <tweet-id-or-url> [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — show the full conversation thread.
- `bird search "<query>" [-n count] [--json]` — search for tweets matching a query.
+- `bird search "<query>" [-n count] [--all] [--max-pages n] [--cursor string] [--json]` — search for tweets matching a query; `--max-pages` requires `--all` or `--cursor`.
 - `bird mentions [-n count] [--user @handle] [--json]` — find tweets mentioning a user (defaults to the authenticated user).
 - `bird user-tweets <@handle> [-n count] [--cursor string] [--max-pages n] [--delay ms] [--json]` — get tweets from a user's profile timeline.
 - `bird bookmarks [-n count] [--folder-id id] [--all] [--max-pages n] [--json]` — list your bookmarked tweets (or a specific bookmark folder); `--max-pages` requires `--all`.
@@ -17,6 +17,7 @@ Run:
 - `pnpm test:live`
 - `pnpm bird following --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird list-timeline <list-id> --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
+- `pnpm bird search "from:steipete" --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird home --count 5 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird home --count 5 --following --json --cookie-source chrome --chrome-profile Default`

@@ -9,13 +9,17 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void
    .description('Search for tweets')
    .argument('<query>', 'Search query (e.g., "@clawdbot" or "from:clawdbot")')
    .option('-n, --count <number>', 'Number of tweets to fetch', '10')
+    .option('--all', 'Fetch all search results (paged)')
+    .option('--max-pages <number>', 'Stop after N pages when using --all')
+    .option('--cursor <string>', 'Resume pagination from a cursor')
    .option('--json', 'Output as JSON')
    .option('--json-full', 'Output as JSON with full raw API response in _raw field')
-    .action(async (query: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => {
+    .action(async (query: string, cmdOpts: { count?: string; all?: boolean; maxPages?: string; cursor?: string; json?: boolean; jsonFull?: boolean }) => {
      const opts = program.opts();
      const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
      const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
      const count = Number.parseInt(cmdOpts.count || '10', 10);
+      const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;

      const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);

@@ -28,13 +32,33 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void
        process.exit(1);
      }

+      const usePagination = cmdOpts.all || cmdOpts.cursor;
+      if (maxPages !== undefined && !usePagination) {
+        console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
+        process.exit(1);
+      }
+      if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
+        console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
+        process.exit(1);
+      }
+      if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
+        console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
+        process.exit(1);
+      }
+
      const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
      const includeRaw = cmdOpts.jsonFull ?? false;
-      const result = await client.search(query, count, { includeRaw });
+      const searchOptions = { includeRaw };
+      const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor };
+      const result = usePagination
+        ? await client.getAllSearchResults(query, paginationOptions)
+        : await client.search(query, count, searchOptions);

      if (result.success) {
-        ctx.printTweets(result.tweets, {
-          json: cmdOpts.json || cmdOpts.jsonFull,
+        const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
+        ctx.printTweetsResult(result, {
+          json: isJson,
+          usePagination: Boolean(usePagination),
          emptyMessage: 'No tweets found.',
        });
      } else {
@@ -12,8 +12,16 @@ export interface SearchFetchOptions {
  includeRaw?: boolean;
 }

+/** Options for paged search methods */
+export interface SearchPaginationOptions extends SearchFetchOptions {
+  maxPages?: number;
+  /** Starting cursor for pagination (resume from previous fetch) */
+  cursor?: string;
+}
+
 export interface TwitterClientSearchMethods {
  search(query: string, count?: number, options?: SearchFetchOptions): Promise<SearchResult>;
+  getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise<SearchResult>;
 }

 function isQueryIdMismatch(payload: string): boolean {
@@ -50,12 +58,25 @@ export function withSearch<TBase extends AbstractConstructor<TwitterClientBase>>
     * Search for tweets matching a query
     */
    async search(query: string, count = 20, options: SearchFetchOptions = {}): Promise<SearchResult> {
-      const { includeRaw = false } = options;
+      return this.searchPaged(query, count, options);
+    }
+
+    /**
+     * Get all search results (paged)
+     */
+    async getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise<SearchResult> {
+      return this.searchPaged(query, Number.POSITIVE_INFINITY, options);
+    }
+
+    private async searchPaged(query: string, limit: number, options: SearchPaginationOptions = {}): Promise<SearchResult> {
      const features = buildSearchFeatures();
      const pageSize = 20;
      const seen = new Set<string>();
      const tweets: TweetData[] = [];
-      let cursor: string | undefined;
+      let cursor: string | undefined = options.cursor;
+      let nextCursor: string | undefined;
+      let pagesFetched = 0;
+      const { includeRaw = false, maxPages } = options;

      const fetchPage = async (pageCount: number, pageCursor?: string) => {
        let lastError: string | undefined;
@@ -184,31 +205,42 @@ export function withSearch<TBase extends AbstractConstructor<TwitterClientBase>>
        return { success: false as const, error: firstAttempt.error };
      };

-      while (tweets.length < count) {
-        const pageCount = Math.min(pageSize, count - tweets.length);
+      const unlimited = !Number.isFinite(limit);
+      while (unlimited || tweets.length < limit) {
+        const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
        const page = await fetchWithRefresh(pageCount, cursor);
        if (!page.success) {
          return { success: false, error: page.error };
        }
+        pagesFetched += 1;

+        let added = 0;
        for (const tweet of page.tweets) {
          if (seen.has(tweet.id)) {
            continue;
          }
          seen.add(tweet.id);
          tweets.push(tweet);
-          if (tweets.length >= count) {
+          added += 1;
+          if (!unlimited && tweets.length >= limit) {
            break;
          }
        }

-        if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
+        const pageCursor = page.cursor;
+        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
+          nextCursor = undefined;
          break;
        }
-        cursor = page.cursor;
+        if (maxPages && pagesFetched >= maxPages) {
+          nextCursor = pageCursor;
+          break;
+        }
+        cursor = pageCursor;
+        nextCursor = pageCursor;
      }

-      return { success: true, tweets };
+      return { success: true, tweets, nextCursor };
    }
  }

@@ -341,6 +341,141 @@ describe('TwitterClient search', () => {
    expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
    expect(mockFetch).toHaveBeenCalledTimes(2);
  });
+
+  it('respects maxPages when fetching all search results', async () => {
+    const makeSearchEntry = (id: string, text: string) => ({
+      content: {
+        itemContent: {
+          tweet_results: {
+            result: {
+              rest_id: id,
+              legacy: {
+                full_text: text,
+                created_at: '2024-01-01T00:00:00Z',
+                reply_count: 0,
+                retweet_count: 0,
+                favorite_count: 0,
+                conversation_id_str: id,
+              },
+              core: {
+                user_results: {
+                  result: { legacy: { screen_name: 'root', name: 'Root' } },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    mockFetch
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            search_by_raw_query: {
+              search_timeline: {
+                timeline: {
+                  instructions: [
+                    {
+                      entries: [
+                        makeSearchEntry('1', 'page 1'),
+                        { content: { cursorType: 'Bottom', value: 'cursor-1' } },
+                      ],
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            search_by_raw_query: {
+              search_timeline: {
+                timeline: {
+                  instructions: [
+                    {
+                      entries: [makeSearchEntry('2', 'page 2')],
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        }),
+      });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const result = await client.getAllSearchResults('query', { maxPages: 1 });
+
+    expect(result.success).toBe(true);
+    expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
+    expect(result.nextCursor).toBe('cursor-1');
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not return a stale cursor when search pagination ends', async () => {
+    const makeSearchEntry = (id: string) => ({
+      content: {
+        itemContent: {
+          tweet_results: {
+            result: {
+              rest_id: id,
+              legacy: {
+                full_text: `tweet-${id}`,
+                created_at: '2024-01-01T00:00:00Z',
+                reply_count: 0,
+                retweet_count: 0,
+                favorite_count: 0,
+                conversation_id_str: id,
+              },
+              core: {
+                user_results: {
+                  result: { legacy: { screen_name: 'root', name: 'Root' } },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      status: 200,
+      json: async () => ({
+        data: {
+          search_by_raw_query: {
+            search_timeline: {
+              timeline: {
+                instructions: [
+                  {
+                    entries: [makeSearchEntry('1')],
+                  },
+                ],
+              },
+            },
+          },
+        },
+      }),
+    });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const result = await client.getAllSearchResults('query', { cursor: 'old-cursor' });
+
+    expect(result.success).toBe(true);
+    expect(result.nextCursor).toBeUndefined();
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+
+    const vars = JSON.parse(new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string);
+    expect(vars.cursor).toBe('old-cursor');
+  });
 });

 describe('TwitterClient bookmarks', () => {