diff --git a/CHANGELOG.md b/CHANGELOG.md index 8faea90..671168d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed - macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst. +- Likes pagination now follows cursors and avoids stalling on duplicate pages (#12) — thanks @titouv. ## 0.6.0 — 2026-01-05 diff --git a/src/lib/twitter-client-timelines.ts b/src/lib/twitter-client-timelines.ts index 4786c6e..b7e7ba8 100644 --- a/src/lib/twitter-client-timelines.ts +++ b/src/lib/twitter-client-timelines.ts @@ -81,28 +81,33 @@ export function withTimelines(); + const tweets: TweetData[] = []; + let cursor: string | undefined; const features = buildLikesFeatures(); - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); - - const tryOnce = async () => { + const fetchPage = async (pageCount: number, pageCursor?: string) => { let lastError: string | undefined; let had404 = false; const queryIds = await this.getLikesQueryIds(); for (const queryId of queryIds) { + const variables = { + userId, + count: pageCount, + includePromotedContent: false, + withClientEventToken: false, + withBirdwatchNotes: false, + withVoice: true, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); const url = `${TWITTER_API_BASE}/${queryId}/Likes?${params.toString()}`; try { @@ -147,14 +152,21 @@ export function withTimelines; }; - if (data.errors && data.errors.length > 0) { - return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 }; - } - const instructions = data.data?.user?.result?.timeline?.timeline?.instructions; - const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + if (data.errors && data.errors.length > 0) { + const message = data.errors.map((e) => e.message).join(', '); + if (!instructions) { + if (message.includes('Query: Unspecified')) { + lastError = message; + continue; + } + return { success: false as const, error: message, had404 }; + } + } + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const nextCursor = extractCursorFromInstructions(instructions); - return { success: true as const, tweets, had404 }; + return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); } @@ -163,21 +175,52 @@ export function withTimelines { + const firstAttempt = await fetchPage(pageCount, pageCursor); + if (firstAttempt.success) { + return firstAttempt; } - return { success: false, error: secondAttempt.error }; + const shouldRefresh = + firstAttempt.had404 || + (typeof firstAttempt.error === 'string' && firstAttempt.error.includes('Query: Unspecified')); + if (shouldRefresh) { + await this.refreshQueryIds(); + const secondAttempt = await fetchPage(pageCount, pageCursor); + if (secondAttempt.success) { + return secondAttempt; + } + return { success: false as const, error: secondAttempt.error }; + } + return { success: false as const, error: firstAttempt.error }; + }; + + while (tweets.length < count) { + const pageCount = Math.min(pageSize, count - tweets.length); + const page = await fetchWithRefresh(pageCount, cursor); + if (!page.success) { + return { success: false, error: page.error }; + } + + let added = 0; + for (const tweet of page.tweets) { + if (seen.has(tweet.id)) { + continue; + } + seen.add(tweet.id); + tweets.push(tweet); + added += 1; + if (tweets.length >= count) { + break; + } + } + + if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) { + break; + } + cursor = page.cursor; } - return { success: false, error: firstAttempt.error }; + return { success: true, tweets }; } /** @@ -329,19 +372,21 @@ export function withTimelines= limit) { break; } } const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) { + if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { nextCursor = undefined; break; } @@ -510,19 +555,21 @@ export function withTimelines= limit) { break; } } const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) { + if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { nextCursor = undefined; break; } diff --git a/tests/twitter-client.following-likes.test.ts b/tests/twitter-client.following-likes.test.ts index 02719ca..872c649 100644 --- a/tests/twitter-client.following-likes.test.ts +++ b/tests/twitter-client.following-likes.test.ts @@ -365,6 +365,133 @@ describe('TwitterClient likes', () => { expect(parsedFeatures.graphql_timeline_v2_bookmark_timeline).toBeUndefined(); }); + it('paginates likes when count exceeds the first page', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'liked page 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'root', name: 'Root' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-1', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + user: { + result: { + timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '2', + legacy: { + full_text: 'liked page 2', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '2', + }, + core: { + user_results: { + result: { + rest_id: 'u2', + legacy: { screen_name: 'root', name: 'Root' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const clientPrivate = client as unknown as TwitterClientPrivate; + clientPrivate.getCurrentUser = async () => ({ + success: true, + user: { id: '42', username: 'tester', name: 'Tester' }, + }); + clientPrivate.getLikesQueryIds = async () => ['test']; + + const result = await client.getLikes(3); + + expect(result.success).toBe(true); + expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1', '2']); + expect(mockFetch).toHaveBeenCalledTimes(2); + + const secondVars = JSON.parse( + new URL(mockFetch.mock.calls[1][0] as string).searchParams.get('variables') as string, + ); + expect(secondVars.cursor).toBe('cursor-1'); + }); + it('returns an error when current user is unavailable', async () => { const client = new TwitterClient({ cookies: validCookies }); const clientPrivate = client as unknown as TwitterClientPrivate; diff --git a/tests/twitter-client.search-bookmarks.test.ts b/tests/twitter-client.search-bookmarks.test.ts index 197aa3a..07cbf3d 100644 --- a/tests/twitter-client.search-bookmarks.test.ts +++ b/tests/twitter-client.search-bookmarks.test.ts @@ -691,6 +691,120 @@ describe('TwitterClient bookmarks', () => { expect(vars.cursor).toBe('cursor-1'); }); + it('stops paginating when a page only returns duplicates', async () => { + mockFetch + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'saved page 1', + created_at: '2024-01-01T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'root', name: 'Root' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-1', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'saved page 2 duplicate', + created_at: '2024-01-02T00:00:00Z', + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + conversation_id_str: '1', + }, + core: { + user_results: { + result: { + rest_id: 'u1', + legacy: { screen_name: 'root', name: 'Root' }, + }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor-2', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getAllBookmarks(); + + expect(result.success).toBe(true); + expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']); + expect(result.nextCursor).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + it('treats graphql errors as non-fatal when instructions are present', async () => { mockFetch.mockResolvedValueOnce({ ok: true,