feat: add cursor-based pagination to bookmarks command
- Add --cursor option to resume pagination from a specific point - Change --max-pages to enable pagination independently - Return nextCursor in JSON output for subsequent requests feat: add cursor param and nextCursor to pagination methods not modifying max-pages
This commit is contained in:
committed by
Peter Steinberger
parent
c4fa04e0b2
commit
3442ba84c6
@@ -13,6 +13,8 @@ export interface TimelineFetchOptions {
|
||||
/** Options for paged timeline fetch methods */
|
||||
export interface TimelinePaginationOptions extends TimelineFetchOptions {
|
||||
maxPages?: number;
|
||||
/** Starting cursor for pagination (resume from previous fetch) */
|
||||
cursor?: string;
|
||||
}
|
||||
|
||||
export interface TwitterClientTimelineMethods {
|
||||
@@ -198,7 +200,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
const pageSize = 20;
|
||||
const seen = new Set<string>();
|
||||
const tweets: TweetData[] = [];
|
||||
let cursor: string | undefined;
|
||||
let cursor: string | undefined = options.cursor;
|
||||
let pagesFetched = 0;
|
||||
const { includeRaw = false, maxPages } = options;
|
||||
|
||||
@@ -341,12 +343,13 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
break;
|
||||
}
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
cursor = page.cursor;
|
||||
break;
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
|
||||
return { success: true, tweets };
|
||||
return { success: true, tweets, nextCursor: cursor };
|
||||
}
|
||||
|
||||
private async getBookmarkFolderTimelinePaged(
|
||||
@@ -358,7 +361,7 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
const pageSize = 20;
|
||||
const seen = new Set<string>();
|
||||
const tweets: TweetData[] = [];
|
||||
let cursor: string | undefined;
|
||||
let cursor: string | undefined = options.cursor;
|
||||
let pagesFetched = 0;
|
||||
const { includeRaw = false, maxPages } = options;
|
||||
|
||||
@@ -517,12 +520,13 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
break;
|
||||
}
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
cursor = page.cursor;
|
||||
break;
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
|
||||
return { success: true, tweets };
|
||||
return { success: true, tweets, nextCursor: cursor };
|
||||
}
|
||||
|
||||
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
||||
|
||||
@@ -241,6 +241,8 @@ export interface SearchResult {
|
||||
success: boolean;
|
||||
tweets?: TweetData[];
|
||||
error?: string;
|
||||
/** Cursor for fetching the next page of results */
|
||||
nextCursor?: string;
|
||||
}
|
||||
|
||||
export interface CurrentUserResult {
|
||||
|
||||
@@ -297,21 +297,78 @@ export function findTweetInInstructions(
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
items?: Array<{
|
||||
item?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
}>
|
||||
| undefined,
|
||||
tweetId: string,
|
||||
) {
|
||||
if (!instructions) {
|
||||
return undefined;
|
||||
}
|
||||
id: string,
|
||||
options: MapTweetResultOptions,
|
||||
): TweetData | undefined {
|
||||
for (const instruction of instructions ?? []) {
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const content = entry.content;
|
||||
if (!content) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const instruction of instructions) {
|
||||
for (const entry of instruction.entries || []) {
|
||||
const result = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (result?.rest_id === tweetId) {
|
||||
return result;
|
||||
const itemTweet = content.itemContent?.tweet_results?.result;
|
||||
const itemResult = itemTweet ? unwrapTweetResult(itemTweet) : undefined;
|
||||
if (itemResult?.rest_id === id) {
|
||||
return mapTweetResult(itemResult, options);
|
||||
}
|
||||
|
||||
const innerItemTweet = content.item?.itemContent?.tweet_results?.result;
|
||||
const innerItemResult = innerItemTweet ? unwrapTweetResult(innerItemTweet) : undefined;
|
||||
if (innerItemResult?.rest_id === id) {
|
||||
return mapTweetResult(innerItemResult, options);
|
||||
}
|
||||
|
||||
for (const item of content.items ?? []) {
|
||||
const itemTweetResult = item.item?.itemContent?.tweet_results?.result;
|
||||
const itemResult = itemTweetResult ? unwrapTweetResult(itemTweetResult) : undefined;
|
||||
if (itemResult?.rest_id === id) {
|
||||
return mapTweetResult(itemResult, options);
|
||||
}
|
||||
|
||||
const itemContentResult = item.itemContent?.tweet_results?.result;
|
||||
const itemContentTweet = itemContentResult ? unwrapTweetResult(itemContentResult) : undefined;
|
||||
if (itemContentTweet?.rest_id === id) {
|
||||
return mapTweetResult(itemContentTweet, options);
|
||||
}
|
||||
|
||||
const itemEntryResult = item.content?.itemContent?.tweet_results?.result;
|
||||
const itemEntryTweet = itemEntryResult ? unwrapTweetResult(itemEntryResult) : undefined;
|
||||
if (itemEntryTweet?.rest_id === id) {
|
||||
return mapTweetResult(itemEntryTweet, options);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user