fix: stabilize bookmark pagination cursors
This commit is contained in:
@@ -6,6 +6,7 @@
|
|||||||
- Bookmark exports now support pagination (`--all`, `--max-pages`) with retries (#15) — thanks @Nano1337.
|
- Bookmark exports now support pagination (`--all`, `--max-pages`) with retries (#15) — thanks @Nano1337.
|
||||||
- `lists` + `list-timeline` commands for Twitter Lists (#21) — thanks @harperreed
|
- `lists` + `list-timeline` commands for Twitter Lists (#21) — thanks @harperreed
|
||||||
- Tweet JSON output now includes media items (photos, videos, GIFs) (#14) — thanks @Hormold
|
- Tweet JSON output now includes media items (photos, videos, GIFs) (#14) — thanks @Hormold
|
||||||
|
- Bookmarks can resume pagination from a cursor (#26) — thanks @leonho
|
||||||
## 0.5.1 — 2026-01-01
|
## 0.5.1 — 2026-01-01
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|||||||
@@ -40,11 +40,11 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (maxPages !== undefined && !cmdOpts.all) {
|
const usePagination = cmdOpts.all || cmdOpts.cursor;
|
||||||
console.error(`${ctx.p('err')}--max-pages requires --all.`);
|
if (maxPages !== undefined && !usePagination) {
|
||||||
|
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
const usePagination = cmdOpts.all || cmdOpts.cursor;
|
|
||||||
if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
|
if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
|
||||||
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
|
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
@@ -74,9 +74,8 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
if (result.success && result.tweets) {
|
if (result.success && result.tweets) {
|
||||||
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
|
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
|
||||||
const isJson = cmdOpts.json || cmdOpts.jsonFull;
|
const isJson = cmdOpts.json || cmdOpts.jsonFull;
|
||||||
if (isJson && result.nextCursor) {
|
if (isJson && usePagination) {
|
||||||
// Output with nextCursor for pagination
|
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
|
||||||
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor }, null, 2));
|
|
||||||
} else {
|
} else {
|
||||||
ctx.printTweets(result.tweets, { json: isJson, emptyMessage });
|
ctx.printTweets(result.tweets, { json: isJson, emptyMessage });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -201,6 +201,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const tweets: TweetData[] = [];
|
const tweets: TweetData[] = [];
|
||||||
let cursor: string | undefined = options.cursor;
|
let cursor: string | undefined = options.cursor;
|
||||||
|
let nextCursor: string | undefined;
|
||||||
let pagesFetched = 0;
|
let pagesFetched = 0;
|
||||||
const { includeRaw = false, maxPages } = options;
|
const { includeRaw = false, maxPages } = options;
|
||||||
|
|
||||||
@@ -339,17 +340,20 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
|
const pageCursor = page.cursor;
|
||||||
|
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
|
||||||
|
nextCursor = undefined;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (maxPages && pagesFetched >= maxPages) {
|
if (maxPages && pagesFetched >= maxPages) {
|
||||||
cursor = page.cursor;
|
nextCursor = pageCursor;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
cursor = page.cursor;
|
cursor = pageCursor;
|
||||||
|
nextCursor = pageCursor;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: true, tweets, nextCursor: cursor };
|
return { success: true, tweets, nextCursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async getBookmarkFolderTimelinePaged(
|
private async getBookmarkFolderTimelinePaged(
|
||||||
@@ -362,6 +366,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const tweets: TweetData[] = [];
|
const tweets: TweetData[] = [];
|
||||||
let cursor: string | undefined = options.cursor;
|
let cursor: string | undefined = options.cursor;
|
||||||
|
let nextCursor: string | undefined;
|
||||||
let pagesFetched = 0;
|
let pagesFetched = 0;
|
||||||
const { includeRaw = false, maxPages } = options;
|
const { includeRaw = false, maxPages } = options;
|
||||||
|
|
||||||
@@ -516,17 +521,20 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
|
const pageCursor = page.cursor;
|
||||||
|
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
|
||||||
|
nextCursor = undefined;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (maxPages && pagesFetched >= maxPages) {
|
if (maxPages && pagesFetched >= maxPages) {
|
||||||
cursor = page.cursor;
|
nextCursor = pageCursor;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
cursor = page.cursor;
|
cursor = pageCursor;
|
||||||
|
nextCursor = pageCursor;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: true, tweets, nextCursor: cursor };
|
return { success: true, tweets, nextCursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import type { CliContext } from '../src/cli/shared.js';
|
|||||||
import { registerBookmarksCommand } from '../src/commands/bookmarks.js';
|
import { registerBookmarksCommand } from '../src/commands/bookmarks.js';
|
||||||
|
|
||||||
describe('bookmarks command', () => {
|
describe('bookmarks command', () => {
|
||||||
it('requires --all when --max-pages is provided', async () => {
|
it('requires --all or --cursor when --max-pages is provided', async () => {
|
||||||
const program = new Command();
|
const program = new Command();
|
||||||
const ctx = {
|
const ctx = {
|
||||||
resolveTimeoutFromOptions: () => undefined,
|
resolveTimeoutFromOptions: () => undefined,
|
||||||
@@ -24,7 +24,7 @@ describe('bookmarks command', () => {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await expect(program.parseAsync(['node', 'bird', 'bookmarks', '--max-pages', '2'])).rejects.toThrow('exit 1');
|
await expect(program.parseAsync(['node', 'bird', 'bookmarks', '--max-pages', '2'])).rejects.toThrow('exit 1');
|
||||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all'));
|
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor'));
|
||||||
} finally {
|
} finally {
|
||||||
exitSpy.mockRestore();
|
exitSpy.mockRestore();
|
||||||
errorSpy.mockRestore();
|
errorSpy.mockRestore();
|
||||||
|
|||||||
@@ -630,9 +630,67 @@ describe('TwitterClient bookmarks', () => {
|
|||||||
|
|
||||||
expect(result.success).toBe(true);
|
expect(result.success).toBe(true);
|
||||||
expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
|
expect(result.tweets?.map((tweet) => tweet.id)).toEqual(['1']);
|
||||||
|
expect(result.nextCursor).toBe('cursor-1');
|
||||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not return a stale cursor when pagination ends', async () => {
|
||||||
|
mockFetch.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: async () => ({
|
||||||
|
data: {
|
||||||
|
bookmark_timeline_v2: {
|
||||||
|
timeline: {
|
||||||
|
instructions: [
|
||||||
|
{
|
||||||
|
entries: [
|
||||||
|
{
|
||||||
|
content: {
|
||||||
|
itemContent: {
|
||||||
|
tweet_results: {
|
||||||
|
result: {
|
||||||
|
rest_id: '1',
|
||||||
|
legacy: {
|
||||||
|
full_text: 'saved page 1',
|
||||||
|
created_at: '2024-01-01T00:00:00Z',
|
||||||
|
reply_count: 0,
|
||||||
|
retweet_count: 0,
|
||||||
|
favorite_count: 0,
|
||||||
|
conversation_id_str: '1',
|
||||||
|
},
|
||||||
|
core: {
|
||||||
|
user_results: {
|
||||||
|
result: {
|
||||||
|
rest_id: 'u1',
|
||||||
|
legacy: { screen_name: 'root', name: 'Root' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const client = new TwitterClient({ cookies: validCookies });
|
||||||
|
const result = await client.getAllBookmarks({ cursor: 'cursor-1' });
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.nextCursor).toBeUndefined();
|
||||||
|
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||||
|
const vars = JSON.parse(new URL(mockFetch.mock.calls[0][0] as string).searchParams.get('variables') as string);
|
||||||
|
expect(vars.cursor).toBe('cursor-1');
|
||||||
|
});
|
||||||
|
|
||||||
it('treats graphql errors as non-fatal when instructions are present', async () => {
|
it('treats graphql errors as non-fatal when instructions are present', async () => {
|
||||||
mockFetch.mockResolvedValueOnce({
|
mockFetch.mockResolvedValueOnce({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
|||||||
Reference in New Issue
Block a user