Merge pull request #42 from pjtf93/feat/search-pagination

feat: add pagination support to search command

CHANGELOG.md

@@ -10,6 +10,7 @@
 - `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala.
 - Long-form article tweets now render rich Draft.js content blocks/entities (#36) — thanks @crcatala.
 - `news`/`trending` command for Explore tabs with AI-curated headlines (#39) — thanks @aavetis.
+- `search` now supports pagination (`--all`, `--max-pages`, `--cursor`) (#42) — thanks @pjtf93.
 
 ### Changed
 - Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`).

README.md

@@ -156,7 +156,7 @@ const sportsNews = await client.getNews(10, {
 - `bird <tweet-id-or-url> [--json]` — shorthand for `read` when only a URL or ID is provided.
 - `bird replies <tweet-id-or-url> [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — list replies to a tweet.
 - `bird thread <tweet-id-or-url> [--all] [--max-pages n] [--cursor string] [--delay ms] [--json]` — show the full conversation thread.
-- `bird search "<query>" [-n count] [--json]` — search for tweets matching a query.
+- `bird search "<query>" [-n count] [--all] [--max-pages n] [--cursor string] [--json]` — search for tweets matching a query; `--max-pages` requires `--all` or `--cursor`.
 - `bird mentions [-n count] [--user @handle] [--json]` — find tweets mentioning a user (defaults to the authenticated user).
 - `bird user-tweets <@handle> [-n count] [--cursor string] [--max-pages n] [--delay ms] [--json]` — get tweets from a user's profile timeline.
 - `bird bookmarks [-n count] [--folder-id id] [--all] [--max-pages n] [--json]` — list your bookmarked tweets (or a specific bookmark folder); `--max-pages` requires `--all`.

docs/testing.md

@@ -17,6 +17,7 @@ Run:
 - `pnpm test:live`
 - `pnpm bird following --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird list-timeline <list-id> --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
+- `pnpm bird search "from:steipete" --all --max-pages 2 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird home --count 5 --json --cookie-source chrome --chrome-profile Default`
 - `pnpm bird home --count 5 --following --json --cookie-source chrome --chrome-profile Default`
 

src/commands/search.ts

@@ -9,39 +9,75 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void
     .description('Search for tweets')
     .argument('<query>', 'Search query (e.g., "@clawdbot" or "from:clawdbot")')
     .option('-n, --count <number>', 'Number of tweets to fetch', '10')
+    .option('--all', 'Fetch all search results (paged)')
+    .option('--max-pages <number>', 'Stop after N pages when using --all')
+    .option('--cursor <string>', 'Resume pagination from a cursor')
     .option('--json', 'Output as JSON')
     .option('--json-full', 'Output as JSON with full raw API response in _raw field')
-    .action(async (query: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => {
+    .action(
-      const opts = program.opts();
+      async (
-      const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
+        query: string,
-      const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
+        cmdOpts: {
-      const count = Number.parseInt(cmdOpts.count || '10', 10);
+          count?: string;
+          all?: boolean;
+          maxPages?: string;
+          cursor?: string;
+          json?: boolean;
+          jsonFull?: boolean;
+        },
+      ) => {
+        const opts = program.opts();
+        const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
+        const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
+        const count = Number.parseInt(cmdOpts.count || '10', 10);
+        const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
 
-      const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
+        const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
 
-      for (const warning of warnings) {
+        for (const warning of warnings) {
-        console.error(`${ctx.p('warn')}${warning}`);
+          console.error(`${ctx.p('warn')}${warning}`);
-      }
+        }
 
-      if (!cookies.authToken || !cookies.ct0) {
+        if (!cookies.authToken || !cookies.ct0) {
-        console.error(`${ctx.p('err')}Missing required credentials`);
+          console.error(`${ctx.p('err')}Missing required credentials`);
-        process.exit(1);
+          process.exit(1);
-      }
+        }
 
-      const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
+        const usePagination = cmdOpts.all || cmdOpts.cursor;
-      const includeRaw = cmdOpts.jsonFull ?? false;
+        if (maxPages !== undefined && !usePagination) {
-      const result = await client.search(query, count, { includeRaw });
+          console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
+          process.exit(1);
+        }
+        if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
+          console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
+          process.exit(1);
+        }
+        if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
+          console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
+          process.exit(1);
+        }
 
-      if (result.success) {
+        const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
-        ctx.printTweets(result.tweets, {
+        const includeRaw = cmdOpts.jsonFull ?? false;
-          json: cmdOpts.json || cmdOpts.jsonFull,
+        const searchOptions = { includeRaw };
-          emptyMessage: 'No tweets found.',
+        const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor };
-        });
+        const result = usePagination
-      } else {
+          ? await client.getAllSearchResults(query, paginationOptions)
-        console.error(`${ctx.p('err')}Search failed: ${result.error}`);
+          : await client.search(query, count, searchOptions);
-        process.exit(1);
+
-      }
+        if (result.success) {
-    });
+          const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
+          ctx.printTweetsResult(result, {
+            json: isJson,
+            usePagination: Boolean(usePagination),
+            emptyMessage: 'No tweets found.',
+          });
+        } else {
+          console.error(`${ctx.p('err')}Search failed: ${result.error}`);
+          process.exit(1);
+        }
+      },
+    );
 
   program
     .command('mentions')

src/lib/twitter-client-search.ts

@@ -12,8 +12,16 @@ export interface SearchFetchOptions {
   includeRaw?: boolean;
 }
 
+/** Options for paged search methods */
+export interface SearchPaginationOptions extends SearchFetchOptions {
+  maxPages?: number;
+  /** Starting cursor for pagination (resume from previous fetch) */
+  cursor?: string;
+}
+
 export interface TwitterClientSearchMethods {
   search(query: string, count?: number, options?: SearchFetchOptions): Promise<SearchResult>;
+  getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise<SearchResult>;
 }
 
 function isQueryIdMismatch(payload: string): boolean {
@@ -50,12 +58,29 @@ export function withSearch<TBase extends AbstractConstructor<TwitterClientBase>>
      * Search for tweets matching a query
      */
     async search(query: string, count = 20, options: SearchFetchOptions = {}): Promise<SearchResult> {
-      const { includeRaw = false } = options;
+      return this.searchPaged(query, count, options);
+    }
+
+    /**
+     * Get all search results (paged)
+     */
+    async getAllSearchResults(query: string, options?: SearchPaginationOptions): Promise<SearchResult> {
+      return this.searchPaged(query, Number.POSITIVE_INFINITY, options);
+    }
+
+    private async searchPaged(
+      query: string,
+      limit: number,
+      options: SearchPaginationOptions = {},
+    ): Promise<SearchResult> {
       const features = buildSearchFeatures();
       const pageSize = 20;
       const seen = new Set<string>();
       const tweets: TweetData[] = [];
-      let cursor: string | undefined;
+      let cursor: string | undefined = options.cursor;
+      let nextCursor: string | undefined;
+      let pagesFetched = 0;
+      const { includeRaw = false, maxPages } = options;
 
       const fetchPage = async (pageCount: number, pageCursor?: string) => {
         let lastError: string | undefined;
@@ -184,31 +209,42 @@ export function withSearch<TBase extends AbstractConstructor<TwitterClientBase>>
         return { success: false as const, error: firstAttempt.error };
       };
 
-      while (tweets.length < count) {
+      const unlimited = !Number.isFinite(limit);
-        const pageCount = Math.min(pageSize, count - tweets.length);
+      while (unlimited || tweets.length < limit) {
+        const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
         const page = await fetchWithRefresh(pageCount, cursor);
         if (!page.success) {
           return { success: false, error: page.error };
         }
+        pagesFetched += 1;
 
+        let added = 0;
         for (const tweet of page.tweets) {
           if (seen.has(tweet.id)) {
             continue;
           }
           seen.add(tweet.id);
           tweets.push(tweet);
-          if (tweets.length >= count) {
+          added += 1;
+          if (!unlimited && tweets.length >= limit) {
             break;
           }
         }
 
-        if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
+        const pageCursor = page.cursor;
+        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
+          nextCursor = undefined;
           break;
         }
-        cursor = page.cursor;
+        if (maxPages && pagesFetched >= maxPages) {
+          nextCursor = pageCursor;
+          break;
+        }
+        cursor = pageCursor;
+        nextCursor = pageCursor;
       }
 
-      return { success: true, tweets };
+      return { success: true, tweets, nextCursor };
     }
   }
 

tests/twitter-client.search-bookmarks.test.ts

@@ -341,6 +341,141 @@ describe('TwitterClient search', () => {
     expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
     expect(mockFetch).toHaveBeenCalledTimes(2);
   });
+
+  it('respects maxPages when fetching all search results', async () => {
+    const makeSearchEntry = (id: string, text: string) => ({
+      content: {
+        itemContent: {
+          tweet_results: {
+            result: {
+              rest_id: id,
+              legacy: {
+                full_text: text,
+                created_at: '2024-01-01T00:00:00Z',
+                reply_count: 0,
+                retweet_count: 0,
+                favorite_count: 0,
+                conversation_id_str: id,
+              },
+              core: {
+                user_results: {
+                  result: { legacy: { screen_name: 'root', name: 'Root' } },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    mockFetch
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            search_by_raw_query: {
+              search_timeline: {
+                timeline: {
+                  instructions: [
+                    {
+                      entries: [
+                        makeSearchEntry('1', 'page 1'),
+                        { content: { cursorType: 'Bottom', value: 'cursor-1' } },
+                      ],
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            search_by_raw_query: {
+              search_timeline: {
+                timeline: {
+                  instructions: [
+                    {
+                      entries: [makeSearchEntry('2', 'page 2')],
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        }),
+      });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const result = await client.getAllSearchResults('query', { maxPages: 1 });
+
+    expect(result.success).toBe(true);
+    expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
+    expect(result.nextCursor).toBe('cursor-1');
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not return a stale cursor when search pagination ends', async () => {
+    const makeSearchEntry = (id: string) => ({
+      content: {
+        itemContent: {
+          tweet_results: {
+            result: {
+              rest_id: id,
+              legacy: {
+                full_text: `tweet-${id}`,
+                created_at: '2024-01-01T00:00:00Z',
+                reply_count: 0,
+                retweet_count: 0,
+                favorite_count: 0,
+                conversation_id_str: id,
+              },
+              core: {
+                user_results: {
+                  result: { legacy: { screen_name: 'root', name: 'Root' } },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      status: 200,
+      json: async () => ({
+        data: {
+          search_by_raw_query: {
+            search_timeline: {
+              timeline: {
+                instructions: [
+                  {
+                    entries: [makeSearchEntry('1')],
+                  },
+                ],
+              },
+            },
+          },
+        },
+      }),
+    });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const result = await client.getAllSearchResults('query', { cursor: 'old-cursor' });
+
+    expect(result.success).toBe(true);
+    expect(result.nextCursor).toBeUndefined();
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+
+    const vars = JSON.parse(new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string);
+    expect(vars.cursor).toBe('old-cursor');
+  });
 });
 
 describe('TwitterClient bookmarks', () => {