Merge pull request #12 from titouv/paginate-fix

Paginate for bookmarks and likes

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixed
 - macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst.
+- Likes pagination now follows cursors and avoids stalling on duplicate pages (#12) — thanks @titouv.
 
 ## 0.6.0 — 2026-01-05
 

src/lib/twitter-client-timelines.ts

@@ -81,28 +81,33 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
         return { success: false, error: userResult.error ?? 'Could not determine current user' };
       }
 
-      const variables = {
+      const userId = userResult.user.id;
-        userId: userResult.user.id,
+      const pageSize = 20;
-        count,
+      const seen = new Set<string>();
-        includePromotedContent: false,
+      const tweets: TweetData[] = [];
-        withClientEventToken: false,
+      let cursor: string | undefined;
-        withBirdwatchNotes: false,
-        withVoice: true,
-      };
-
       const features = buildLikesFeatures();
 
-      const params = new URLSearchParams({
+      const fetchPage = async (pageCount: number, pageCursor?: string) => {
-        variables: JSON.stringify(variables),
-        features: JSON.stringify(features),
-      });
-
-      const tryOnce = async () => {
         let lastError: string | undefined;
         let had404 = false;
         const queryIds = await this.getLikesQueryIds();
 
         for (const queryId of queryIds) {
+          const variables = {
+            userId,
+            count: pageCount,
+            includePromotedContent: false,
+            withClientEventToken: false,
+            withBirdwatchNotes: false,
+            withVoice: true,
+            ...(pageCursor ? { cursor: pageCursor } : {}),
+          };
+
+          const params = new URLSearchParams({
+            variables: JSON.stringify(variables),
+            features: JSON.stringify(features),
+          });
           const url = `${TWITTER_API_BASE}/${queryId}/Likes?${params.toString()}`;
 
           try {
@@ -147,14 +152,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
               errors?: Array<{ message: string }>;
             };
 
-            if (data.errors && data.errors.length > 0) {
-              return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
-            }
-
             const instructions = data.data?.user?.result?.timeline?.timeline?.instructions;
-            const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
+            if (data.errors && data.errors.length > 0) {
+              const message = data.errors.map((e) => e.message).join(', ');
+              if (!instructions) {
+                if (message.includes('Query: Unspecified')) {
+                  lastError = message;
+                  continue;
+                }
+                return { success: false as const, error: message, had404 };
+              }
+            }
+            const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
+            const nextCursor = extractCursorFromInstructions(instructions);
 
-            return { success: true as const, tweets, had404 };
+            return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
           } catch (error) {
             lastError = error instanceof Error ? error.message : String(error);
           }
@@ -163,21 +175,52 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
         return { success: false as const, error: lastError ?? 'Unknown error fetching likes', had404 };
       };
 
-      const firstAttempt = await tryOnce();
+      const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
-      if (firstAttempt.success) {
+        const firstAttempt = await fetchPage(pageCount, pageCursor);
-        return { success: true, tweets: firstAttempt.tweets };
+        if (firstAttempt.success) {
-      }
+          return firstAttempt;
-
-      if (firstAttempt.had404) {
-        await this.refreshQueryIds();
-        const secondAttempt = await tryOnce();
-        if (secondAttempt.success) {
-          return { success: true, tweets: secondAttempt.tweets };
         }
-        return { success: false, error: secondAttempt.error };
+        const shouldRefresh =
+          firstAttempt.had404 ||
+          (typeof firstAttempt.error === 'string' && firstAttempt.error.includes('Query: Unspecified'));
+        if (shouldRefresh) {
+          await this.refreshQueryIds();
+          const secondAttempt = await fetchPage(pageCount, pageCursor);
+          if (secondAttempt.success) {
+            return secondAttempt;
+          }
+          return { success: false as const, error: secondAttempt.error };
+        }
+        return { success: false as const, error: firstAttempt.error };
+      };
+
+      while (tweets.length < count) {
+        const pageCount = Math.min(pageSize, count - tweets.length);
+        const page = await fetchWithRefresh(pageCount, cursor);
+        if (!page.success) {
+          return { success: false, error: page.error };
+        }
+
+        let added = 0;
+        for (const tweet of page.tweets) {
+          if (seen.has(tweet.id)) {
+            continue;
+          }
+          seen.add(tweet.id);
+          tweets.push(tweet);
+          added += 1;
+          if (tweets.length >= count) {
+            break;
+          }
+        }
+
+        if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) {
+          break;
+        }
+        cursor = page.cursor;
       }
 
-      return { success: false, error: firstAttempt.error };
+      return { success: true, tweets };
     }
 
     /**
@@ -329,19 +372,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
         }
         pagesFetched += 1;
 
+        let added = 0;
         for (const tweet of page.tweets) {
           if (seen.has(tweet.id)) {
             continue;
           }
           seen.add(tweet.id);
           tweets.push(tweet);
+          added += 1;
           if (!unlimited && tweets.length >= limit) {
             break;
           }
         }
 
         const pageCursor = page.cursor;
-        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
+        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
           nextCursor = undefined;
           break;
         }
@@ -510,19 +555,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
         }
         pagesFetched += 1;
 
+        let added = 0;
         for (const tweet of page.tweets) {
           if (seen.has(tweet.id)) {
             continue;
           }
           seen.add(tweet.id);
           tweets.push(tweet);
+          added += 1;
           if (!unlimited && tweets.length >= limit) {
             break;
           }
         }
 
         const pageCursor = page.cursor;
-        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
+        if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
           nextCursor = undefined;
           break;
         }

tests/twitter-client.following-likes.test.ts

@@ -365,6 +365,133 @@ describe('TwitterClient likes', () => {
     expect(parsedFeatures.graphql_timeline_v2_bookmark_timeline).toBeUndefined();
   });
 
+  it('paginates likes when count exceeds the first page', async () => {
+    mockFetch
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            user: {
+              result: {
+                timeline: {
+                  timeline: {
+                    instructions: [
+                      {
+                        entries: [
+                          {
+                            content: {
+                              itemContent: {
+                                tweet_results: {
+                                  result: {
+                                    rest_id: '1',
+                                    legacy: {
+                                      full_text: 'liked page 1',
+                                      created_at: '2024-01-01T00:00:00Z',
+                                      reply_count: 0,
+                                      retweet_count: 0,
+                                      favorite_count: 0,
+                                      conversation_id_str: '1',
+                                    },
+                                    core: {
+                                      user_results: {
+                                        result: {
+                                          rest_id: 'u1',
+                                          legacy: { screen_name: 'root', name: 'Root' },
+                                        },
+                                      },
+                                    },
+                                  },
+                                },
+                              },
+                            },
+                          },
+                          {
+                            content: {
+                              cursorType: 'Bottom',
+                              value: 'cursor-1',
+                            },
+                          },
+                        ],
+                      },
+                    ],
+                  },
+                },
+              },
+            },
+          },
+        }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            user: {
+              result: {
+                timeline: {
+                  timeline: {
+                    instructions: [
+                      {
+                        entries: [
+                          {
+                            content: {
+                              itemContent: {
+                                tweet_results: {
+                                  result: {
+                                    rest_id: '2',
+                                    legacy: {
+                                      full_text: 'liked page 2',
+                                      created_at: '2024-01-02T00:00:00Z',
+                                      reply_count: 0,
+                                      retweet_count: 0,
+                                      favorite_count: 0,
+                                      conversation_id_str: '2',
+                                    },
+                                    core: {
+                                      user_results: {
+                                        result: {
+                                          rest_id: 'u2',
+                                          legacy: { screen_name: 'root', name: 'Root' },
+                                        },
+                                      },
+                                    },
+                                  },
+                                },
+                              },
+                            },
+                          },
+                        ],
+                      },
+                    ],
+                  },
+                },
+              },
+            },
+          },
+        }),
+      });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const clientPrivate = client as unknown as TwitterClientPrivate;
+    clientPrivate.getCurrentUser = async () => ({
+      success: true,
+      user: { id: '42', username: 'tester', name: 'Tester' },
+    });
+    clientPrivate.getLikesQueryIds = async () => ['test'];
+
+    const result = await client.getLikes(3);
+
+    expect(result.success).toBe(true);
+    expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1', '2']);
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+
+    const secondVars = JSON.parse(
+      new URL(mockFetch.mock.calls[1][0] as string).searchParams.get('variables') as string,
+    );
+    expect(secondVars.cursor).toBe('cursor-1');
+  });
+
   it('returns an error when current user is unavailable', async () => {
     const client = new TwitterClient({ cookies: validCookies });
     const clientPrivate = client as unknown as TwitterClientPrivate;

tests/twitter-client.search-bookmarks.test.ts

@@ -691,6 +691,120 @@ describe('TwitterClient bookmarks', () => {
     expect(vars.cursor).toBe('cursor-1');
   });
 
+  it('stops paginating when a page only returns duplicates', async () => {
+    mockFetch
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            bookmark_timeline_v2: {
+              timeline: {
+                instructions: [
+                  {
+                    entries: [
+                      {
+                        content: {
+                          itemContent: {
+                            tweet_results: {
+                              result: {
+                                rest_id: '1',
+                                legacy: {
+                                  full_text: 'saved page 1',
+                                  created_at: '2024-01-01T00:00:00Z',
+                                  reply_count: 0,
+                                  retweet_count: 0,
+                                  favorite_count: 0,
+                                  conversation_id_str: '1',
+                                },
+                                core: {
+                                  user_results: {
+                                    result: {
+                                      rest_id: 'u1',
+                                      legacy: { screen_name: 'root', name: 'Root' },
+                                    },
+                                  },
+                                },
+                              },
+                            },
+                          },
+                        },
+                      },
+                      {
+                        content: {
+                          cursorType: 'Bottom',
+                          value: 'cursor-1',
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+            },
+          },
+        }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: async () => ({
+          data: {
+            bookmark_timeline_v2: {
+              timeline: {
+                instructions: [
+                  {
+                    entries: [
+                      {
+                        content: {
+                          itemContent: {
+                            tweet_results: {
+                              result: {
+                                rest_id: '1',
+                                legacy: {
+                                  full_text: 'saved page 2 duplicate',
+                                  created_at: '2024-01-02T00:00:00Z',
+                                  reply_count: 0,
+                                  retweet_count: 0,
+                                  favorite_count: 0,
+                                  conversation_id_str: '1',
+                                },
+                                core: {
+                                  user_results: {
+                                    result: {
+                                      rest_id: 'u1',
+                                      legacy: { screen_name: 'root', name: 'Root' },
+                                    },
+                                  },
+                                },
+                              },
+                            },
+                          },
+                        },
+                      },
+                      {
+                        content: {
+                          cursorType: 'Bottom',
+                          value: 'cursor-2',
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+            },
+          },
+        }),
+      });
+
+    const client = new TwitterClient({ cookies: validCookies });
+    const result = await client.getAllBookmarks();
+
+    expect(result.success).toBe(true);
+    expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
+    expect(result.nextCursor).toBeUndefined();
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+  });
+
   it('treats graphql errors as non-fatal when instructions are present', async () => {
     mockFetch.mockResolvedValueOnce({
       ok: true,