feat: add cursor-based pagination to bookmarks command
- Add --cursor option to resume pagination from a specific point - Change --max-pages to enable pagination independently - Return nextCursor in JSON output for subsequent requests feat: add cursor param and nextCursor to pagination methods not modifying max-pages
This commit is contained in:
committed by
Peter Steinberger
parent
c4fa04e0b2
commit
3442ba84c6
@@ -11,6 +11,7 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
.option('--folder-id <id>', 'Bookmark folder (collection) id')
|
.option('--folder-id <id>', 'Bookmark folder (collection) id')
|
||||||
.option('--all', 'Fetch all bookmarks (paged)')
|
.option('--all', 'Fetch all bookmarks (paged)')
|
||||||
.option('--max-pages <number>', 'Stop after N pages when using --all')
|
.option('--max-pages <number>', 'Stop after N pages when using --all')
|
||||||
|
.option('--cursor <string>', 'Resume pagination from a cursor')
|
||||||
.option('--json', 'Output as JSON')
|
.option('--json', 'Output as JSON')
|
||||||
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
|
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
|
||||||
.action(
|
.action(
|
||||||
@@ -21,6 +22,7 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
folderId?: string;
|
folderId?: string;
|
||||||
all?: boolean;
|
all?: boolean;
|
||||||
maxPages?: string;
|
maxPages?: string;
|
||||||
|
cursor?: string;
|
||||||
}) => {
|
}) => {
|
||||||
const opts = program.opts();
|
const opts = program.opts();
|
||||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||||
@@ -42,7 +44,8 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
console.error(`${ctx.p('err')}--max-pages requires --all.`);
|
console.error(`${ctx.p('err')}--max-pages requires --all.`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
if (!cmdOpts.all && (!Number.isFinite(count) || count <= 0)) {
|
const usePagination = cmdOpts.all || cmdOpts.cursor;
|
||||||
|
if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
|
||||||
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
|
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
@@ -59,18 +62,24 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
|
|||||||
}
|
}
|
||||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||||
const timelineOptions = { includeRaw };
|
const timelineOptions = { includeRaw };
|
||||||
const paginationOptions = { includeRaw, maxPages };
|
const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor };
|
||||||
const result = folderId
|
const result = folderId
|
||||||
? cmdOpts.all
|
? usePagination
|
||||||
? await client.getAllBookmarkFolderTimeline(folderId, paginationOptions)
|
? await client.getAllBookmarkFolderTimeline(folderId, paginationOptions)
|
||||||
: await client.getBookmarkFolderTimeline(folderId, count, timelineOptions)
|
: await client.getBookmarkFolderTimeline(folderId, count, timelineOptions)
|
||||||
: cmdOpts.all
|
: usePagination
|
||||||
? await client.getAllBookmarks(paginationOptions)
|
? await client.getAllBookmarks(paginationOptions)
|
||||||
: await client.getBookmarks(count, timelineOptions);
|
: await client.getBookmarks(count, timelineOptions);
|
||||||
|
|
||||||
if (result.success && result.tweets) {
|
if (result.success && result.tweets) {
|
||||||
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
|
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
|
||||||
ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage });
|
const isJson = cmdOpts.json || cmdOpts.jsonFull;
|
||||||
|
if (isJson && result.nextCursor) {
|
||||||
|
// Output with nextCursor for pagination
|
||||||
|
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor }, null, 2));
|
||||||
|
} else {
|
||||||
|
ctx.printTweets(result.tweets, { json: isJson, emptyMessage });
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
console.error(`${ctx.p('err')}Failed to fetch bookmarks: ${result.error}`);
|
console.error(`${ctx.p('err')}Failed to fetch bookmarks: ${result.error}`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ export interface TimelineFetchOptions {
|
|||||||
/** Options for paged timeline fetch methods */
|
/** Options for paged timeline fetch methods */
|
||||||
export interface TimelinePaginationOptions extends TimelineFetchOptions {
|
export interface TimelinePaginationOptions extends TimelineFetchOptions {
|
||||||
maxPages?: number;
|
maxPages?: number;
|
||||||
|
/** Starting cursor for pagination (resume from previous fetch) */
|
||||||
|
cursor?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TwitterClientTimelineMethods {
|
export interface TwitterClientTimelineMethods {
|
||||||
@@ -198,7 +200,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
const pageSize = 20;
|
const pageSize = 20;
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const tweets: TweetData[] = [];
|
const tweets: TweetData[] = [];
|
||||||
let cursor: string | undefined;
|
let cursor: string | undefined = options.cursor;
|
||||||
let pagesFetched = 0;
|
let pagesFetched = 0;
|
||||||
const { includeRaw = false, maxPages } = options;
|
const { includeRaw = false, maxPages } = options;
|
||||||
|
|
||||||
@@ -341,12 +343,13 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (maxPages && pagesFetched >= maxPages) {
|
if (maxPages && pagesFetched >= maxPages) {
|
||||||
|
cursor = page.cursor;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
cursor = page.cursor;
|
cursor = page.cursor;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: true, tweets };
|
return { success: true, tweets, nextCursor: cursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async getBookmarkFolderTimelinePaged(
|
private async getBookmarkFolderTimelinePaged(
|
||||||
@@ -358,7 +361,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
const pageSize = 20;
|
const pageSize = 20;
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const tweets: TweetData[] = [];
|
const tweets: TweetData[] = [];
|
||||||
let cursor: string | undefined;
|
let cursor: string | undefined = options.cursor;
|
||||||
let pagesFetched = 0;
|
let pagesFetched = 0;
|
||||||
const { includeRaw = false, maxPages } = options;
|
const { includeRaw = false, maxPages } = options;
|
||||||
|
|
||||||
@@ -517,12 +520,13 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (maxPages && pagesFetched >= maxPages) {
|
if (maxPages && pagesFetched >= maxPages) {
|
||||||
|
cursor = page.cursor;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
cursor = page.cursor;
|
cursor = page.cursor;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: true, tweets };
|
return { success: true, tweets, nextCursor: cursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
||||||
|
|||||||
@@ -241,6 +241,8 @@ export interface SearchResult {
|
|||||||
success: boolean;
|
success: boolean;
|
||||||
tweets?: TweetData[];
|
tweets?: TweetData[];
|
||||||
error?: string;
|
error?: string;
|
||||||
|
/** Cursor for fetching the next page of results */
|
||||||
|
nextCursor?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CurrentUserResult {
|
export interface CurrentUserResult {
|
||||||
|
|||||||
@@ -297,21 +297,78 @@ export function findTweetInInstructions(
|
|||||||
result?: GraphqlTweetResult;
|
result?: GraphqlTweetResult;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
item?: {
|
||||||
|
itemContent?: {
|
||||||
|
tweet_results?: {
|
||||||
|
result?: GraphqlTweetResult;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
items?: Array<{
|
||||||
|
item?: {
|
||||||
|
itemContent?: {
|
||||||
|
tweet_results?: {
|
||||||
|
result?: GraphqlTweetResult;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
itemContent?: {
|
||||||
|
tweet_results?: {
|
||||||
|
result?: GraphqlTweetResult;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
content?: {
|
||||||
|
itemContent?: {
|
||||||
|
tweet_results?: {
|
||||||
|
result?: GraphqlTweetResult;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}>;
|
||||||
};
|
};
|
||||||
}>;
|
}>;
|
||||||
}>
|
}>
|
||||||
| undefined,
|
| undefined,
|
||||||
tweetId: string,
|
id: string,
|
||||||
) {
|
options: MapTweetResultOptions,
|
||||||
if (!instructions) {
|
): TweetData | undefined {
|
||||||
return undefined;
|
for (const instruction of instructions ?? []) {
|
||||||
|
for (const entry of instruction.entries ?? []) {
|
||||||
|
const content = entry.content;
|
||||||
|
if (!content) {
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const instruction of instructions) {
|
const itemTweet = content.itemContent?.tweet_results?.result;
|
||||||
for (const entry of instruction.entries || []) {
|
const itemResult = itemTweet ? unwrapTweetResult(itemTweet) : undefined;
|
||||||
const result = entry.content?.itemContent?.tweet_results?.result;
|
if (itemResult?.rest_id === id) {
|
||||||
if (result?.rest_id === tweetId) {
|
return mapTweetResult(itemResult, options);
|
||||||
return result;
|
}
|
||||||
|
|
||||||
|
const innerItemTweet = content.item?.itemContent?.tweet_results?.result;
|
||||||
|
const innerItemResult = innerItemTweet ? unwrapTweetResult(innerItemTweet) : undefined;
|
||||||
|
if (innerItemResult?.rest_id === id) {
|
||||||
|
return mapTweetResult(innerItemResult, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const item of content.items ?? []) {
|
||||||
|
const itemTweetResult = item.item?.itemContent?.tweet_results?.result;
|
||||||
|
const itemResult = itemTweetResult ? unwrapTweetResult(itemTweetResult) : undefined;
|
||||||
|
if (itemResult?.rest_id === id) {
|
||||||
|
return mapTweetResult(itemResult, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
const itemContentResult = item.itemContent?.tweet_results?.result;
|
||||||
|
const itemContentTweet = itemContentResult ? unwrapTweetResult(itemContentResult) : undefined;
|
||||||
|
if (itemContentTweet?.rest_id === id) {
|
||||||
|
return mapTweetResult(itemContentTweet, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
const itemEntryResult = item.content?.itemContent?.tweet_results?.result;
|
||||||
|
const itemEntryTweet = itemEntryResult ? unwrapTweetResult(itemEntryResult) : undefined;
|
||||||
|
if (itemEntryTweet?.rest_id === id) {
|
||||||
|
return mapTweetResult(itemEntryTweet, options);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user