yuta/bird
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12 from titouv/paginate-fix

Browse Source 
Paginate for bookmarks and likes
This commit is contained in:
Peter Steinberger
2026-01-11 10:32:51 +00:00
committed by GitHub
parent f048c3d968 2e03bd8c07
commit 3c2107d5d1
4 changed files with 324 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -4,6 +4,7 @@
### Fixed
- macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst.
- Likes pagination now follows cursors and avoids stalling on duplicate pages (#12) — thanks @titouv.
## 0.6.0 — 2026-01-05
src/lib/twitter-client-timelines.ts
+78 -31
View File
@@ -81,28 +81,33 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
return { success: false, error: userResult.error ?? 'Could not determine current user' };
}
const variables = {
userId: userResult.user.id,
count,
includePromotedContent: false,
withClientEventToken: false,
withBirdwatchNotes: false,
withVoice: true,
};
const userId = userResult.user.id;
const pageSize = 20;
const seen = new Set<string>();
const tweets: TweetData[] = [];
let cursor: string | undefined;
const features = buildLikesFeatures();
const params = new URLSearchParams({
variables: JSON.stringify(variables),
features: JSON.stringify(features),
});
const tryOnce = async () => {
const fetchPage = async (pageCount: number, pageCursor?: string) => {
let lastError: string | undefined;
let had404 = false;
const queryIds = await this.getLikesQueryIds();
for (const queryId of queryIds) {
const variables = {
userId,
count: pageCount,
includePromotedContent: false,
withClientEventToken: false,
withBirdwatchNotes: false,
withVoice: true,
...(pageCursor ? { cursor: pageCursor } : {}),
};
const params = new URLSearchParams({
variables: JSON.stringify(variables),
features: JSON.stringify(features),
});
const url = `${TWITTER_API_BASE}/${queryId}/Likes?${params.toString()}`;
try {
@@ -147,14 +152,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
errors?: Array<{ message: string }>;
};
if (data.errors && data.errors.length > 0) {
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
}
const instructions = data.data?.user?.result?.timeline?.timeline?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
if (data.errors && data.errors.length > 0) {
const message = data.errors.map((e) => e.message).join(', ');
if (!instructions) {
if (message.includes('Query: Unspecified')) {
lastError = message;
continue;
}
return { success: false as const, error: message, had404 };
}
}
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const nextCursor = extractCursorFromInstructions(instructions);
return { success: true as const, tweets, had404 };
return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
}
@@ -163,21 +175,52 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
return { success: false as const, error: lastError ?? 'Unknown error fetching likes', had404 };
};
const firstAttempt = await tryOnce();
const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
const firstAttempt = await fetchPage(pageCount, pageCursor);
if (firstAttempt.success) {
return { success: true, tweets: firstAttempt.tweets };
return firstAttempt;
}
if (firstAttempt.had404) {
const shouldRefresh =
firstAttempt.had404 ||
(typeof firstAttempt.error === 'string' && firstAttempt.error.includes('Query: Unspecified'));
if (shouldRefresh) {
await this.refreshQueryIds();
const secondAttempt = await tryOnce();
const secondAttempt = await fetchPage(pageCount, pageCursor);
if (secondAttempt.success) {
return { success: true, tweets: secondAttempt.tweets };
return secondAttempt;
}
return { success: false, error: secondAttempt.error };
return { success: false as const, error: secondAttempt.error };
}
return { success: false as const, error: firstAttempt.error };
};
while (tweets.length < count) {
const pageCount = Math.min(pageSize, count - tweets.length);
const page = await fetchWithRefresh(pageCount, cursor);
if (!page.success) {
return { success: false, error: page.error };
}
return { success: false, error: firstAttempt.error };
let added = 0;
for (const tweet of page.tweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
tweets.push(tweet);
added += 1;
if (tweets.length >= count) {
break;
}
}
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) {
break;
}
cursor = page.cursor;
}
return { success: true, tweets };
}
/**
@@ -329,19 +372,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
}
pagesFetched += 1;
let added = 0;
for (const tweet of page.tweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
tweets.push(tweet);
added += 1;
if (!unlimited && tweets.length >= limit) {
break;
}
}
const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
@@ -510,19 +555,21 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
}
pagesFetched += 1;
let added = 0;
for (const tweet of page.tweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
tweets.push(tweet);
added += 1;
if (!unlimited && tweets.length >= limit) {
break;
}
}
const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
tests/twitter-client.following-likes.test.ts
+127
View File
@@ -365,6 +365,133 @@ describe('TwitterClient likes', () => {
expect(parsedFeatures.graphql_timeline_v2_bookmark_timeline).toBeUndefined();
});
it('paginates likes when count exceeds the first page', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: {
user: {
result: {
timeline: {
timeline: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'liked page 1',
created_at: '2024-01-01T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: {
user_results: {
result: {
rest_id: 'u1',
legacy: { screen_name: 'root', name: 'Root' },
},
},
},
},
},
},
},
},
{
content: {
cursorType: 'Bottom',
value: 'cursor-1',
},
},
],
},
],
},
},
},
},
},
}),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: {
user: {
result: {
timeline: {
timeline: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '2',
legacy: {
full_text: 'liked page 2',
created_at: '2024-01-02T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '2',
},
core: {
user_results: {
result: {
rest_id: 'u2',
legacy: { screen_name: 'root', name: 'Root' },
},
},
},
},
},
},
},
},
],
},
],
},
},
},
},
},
}),
});
const client = new TwitterClient({ cookies: validCookies });
const clientPrivate = client as unknown as TwitterClientPrivate;
clientPrivate.getCurrentUser = async () => ({
success: true,
user: { id: '42', username: 'tester', name: 'Tester' },
});
clientPrivate.getLikesQueryIds = async () => ['test'];
const result = await client.getLikes(3);
expect(result.success).toBe(true);
expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1', '2']);
expect(mockFetch).toHaveBeenCalledTimes(2);
const secondVars = JSON.parse(
new URL(mockFetch.mock.calls[1][0] as string).searchParams.get('variables') as string,
);
expect(secondVars.cursor).toBe('cursor-1');
});
it('returns an error when current user is unavailable', async () => {
const client = new TwitterClient({ cookies: validCookies });
const clientPrivate = client as unknown as TwitterClientPrivate;
tests/twitter-client.search-bookmarks.test.ts
+114
View File
@@ -691,6 +691,120 @@ describe('TwitterClient bookmarks', () => {
expect(vars.cursor).toBe('cursor-1');
});
it('stops paginating when a page only returns duplicates', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: {
bookmark_timeline_v2: {
timeline: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'saved page 1',
created_at: '2024-01-01T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: {
user_results: {
result: {
rest_id: 'u1',
legacy: { screen_name: 'root', name: 'Root' },
},
},
},
},
},
},
},
},
{
content: {
cursorType: 'Bottom',
value: 'cursor-1',
},
},
],
},
],
},
},
},
}),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: {
bookmark_timeline_v2: {
timeline: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'saved page 2 duplicate',
created_at: '2024-01-02T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: {
user_results: {
result: {
rest_id: 'u1',
legacy: { screen_name: 'root', name: 'Root' },
},
},
},
},
},
},
},
},
{
content: {
cursorType: 'Bottom',
value: 'cursor-2',
},
},
],
},
],
},
},
},
}),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getAllBookmarks();
expect(result.success).toBe(true);
expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
expect(result.nextCursor).toBeUndefined();
expect(mockFetch).toHaveBeenCalledTimes(2);
});
it('treats graphql errors as non-fatal when instructions are present', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,