fix: stabilize bookmark pagination cursors

This commit is contained in:
Peter Steinberger
2026-01-04 23:42:34 +00:00
parent 3442ba84c6
commit f765955116
5 changed files with 82 additions and 16 deletions
+1
View File
@@ -6,6 +6,7 @@
- Bookmark exports now support pagination (`--all`, `--max-pages`) with retries (#15) — thanks @Nano1337. - Bookmark exports now support pagination (`--all`, `--max-pages`) with retries (#15) — thanks @Nano1337.
- `lists` + `list-timeline` commands for Twitter Lists (#21) — thanks @harperreed - `lists` + `list-timeline` commands for Twitter Lists (#21) — thanks @harperreed
- Tweet JSON output now includes media items (photos, videos, GIFs) (#14) — thanks @Hormold - Tweet JSON output now includes media items (photos, videos, GIFs) (#14) — thanks @Hormold
- Bookmarks can resume pagination from a cursor (#26) — thanks @leonho
## 0.5.1 — 2026-01-01 ## 0.5.1 — 2026-01-01
### Changed ### Changed
+5 -6
View File
@@ -40,11 +40,11 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
process.exit(1); process.exit(1);
} }
if (maxPages !== undefined && !cmdOpts.all) { const usePagination = cmdOpts.all || cmdOpts.cursor;
console.error(`${ctx.p('err')}--max-pages requires --all.`); if (maxPages !== undefined && !usePagination) {
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
process.exit(1); process.exit(1);
} }
const usePagination = cmdOpts.all || cmdOpts.cursor;
if (!usePagination && (!Number.isFinite(count) || count <= 0)) { if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
process.exit(1); process.exit(1);
@@ -74,9 +74,8 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
if (result.success && result.tweets) { if (result.success && result.tweets) {
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.'; const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
const isJson = cmdOpts.json || cmdOpts.jsonFull; const isJson = cmdOpts.json || cmdOpts.jsonFull;
if (isJson && result.nextCursor) { if (isJson && usePagination) {
// Output with nextCursor for pagination console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor }, null, 2));
} else { } else {
ctx.printTweets(result.tweets, { json: isJson, emptyMessage }); ctx.printTweets(result.tweets, { json: isJson, emptyMessage });
} }
+16 -8
View File
@@ -201,6 +201,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
const seen = new Set<string>(); const seen = new Set<string>();
const tweets: TweetData[] = []; const tweets: TweetData[] = [];
let cursor: string | undefined = options.cursor; let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0; let pagesFetched = 0;
const { includeRaw = false, maxPages } = options; const { includeRaw = false, maxPages } = options;
@@ -339,17 +340,20 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
} }
} }
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) { const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
nextCursor = undefined;
break; break;
} }
if (maxPages && pagesFetched >= maxPages) { if (maxPages && pagesFetched >= maxPages) {
cursor = page.cursor; nextCursor = pageCursor;
break; break;
} }
cursor = page.cursor; cursor = pageCursor;
nextCursor = pageCursor;
} }
return { success: true, tweets, nextCursor: cursor }; return { success: true, tweets, nextCursor };
} }
private async getBookmarkFolderTimelinePaged( private async getBookmarkFolderTimelinePaged(
@@ -362,6 +366,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
const seen = new Set<string>(); const seen = new Set<string>();
const tweets: TweetData[] = []; const tweets: TweetData[] = [];
let cursor: string | undefined = options.cursor; let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0; let pagesFetched = 0;
const { includeRaw = false, maxPages } = options; const { includeRaw = false, maxPages } = options;
@@ -516,17 +521,20 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
} }
} }
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) { const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
nextCursor = undefined;
break; break;
} }
if (maxPages && pagesFetched >= maxPages) { if (maxPages && pagesFetched >= maxPages) {
cursor = page.cursor; nextCursor = pageCursor;
break; break;
} }
cursor = page.cursor; cursor = pageCursor;
nextCursor = pageCursor;
} }
return { success: true, tweets, nextCursor: cursor }; return { success: true, tweets, nextCursor };
} }
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> { private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
+2 -2
View File
@@ -4,7 +4,7 @@ import type { CliContext } from '../src/cli/shared.js';
import { registerBookmarksCommand } from '../src/commands/bookmarks.js'; import { registerBookmarksCommand } from '../src/commands/bookmarks.js';
describe('bookmarks command', () => { describe('bookmarks command', () => {
it('requires --all when --max-pages is provided', async () => { it('requires --all or --cursor when --max-pages is provided', async () => {
const program = new Command(); const program = new Command();
const ctx = { const ctx = {
resolveTimeoutFromOptions: () => undefined, resolveTimeoutFromOptions: () => undefined,
@@ -24,7 +24,7 @@ describe('bookmarks command', () => {
try { try {
await expect(program.parseAsync(['node', 'bird', 'bookmarks', '--max-pages', '2'])).rejects.toThrow('exit 1'); await expect(program.parseAsync(['node', 'bird', 'bookmarks', '--max-pages', '2'])).rejects.toThrow('exit 1');
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all')); expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor'));
} finally { } finally {
exitSpy.mockRestore(); exitSpy.mockRestore();
errorSpy.mockRestore(); errorSpy.mockRestore();
@@ -630,9 +630,67 @@ describe('TwitterClient bookmarks', () => {
expect(result.success).toBe(true); expect(result.success).toBe(true);
expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']); expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
expect(result.nextCursor).toBe('cursor-1');
expect(mockFetch).toHaveBeenCalledTimes(1); expect(mockFetch).toHaveBeenCalledTimes(1);
}); });
it('does not return a stale cursor when pagination ends', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: {
bookmark_timeline_v2: {
timeline: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'saved page 1',
created_at: '2024-01-01T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: {
user_results: {
result: {
rest_id: 'u1',
legacy: { screen_name: 'root', name: 'Root' },
},
},
},
},
},
},
},
},
],
},
],
},
},
},
}),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getAllBookmarks({ cursor: 'cursor-1' });
expect(result.success).toBe(true);
expect(result.nextCursor).toBeUndefined();
expect(mockFetch).toHaveBeenCalledTimes(1);
const vars = JSON.parse(new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string);
expect(vars.cursor).toBe('cursor-1');
});
it('treats graphql errors as non-fatal when instructions are present', async () => { it('treats graphql errors as non-fatal when instructions are present', async () => {
mockFetch.mockResolvedValueOnce({ mockFetch.mockResolvedValueOnce({
ok: true, ok: true,