feat: add pagination support for thread and replies commands
Add --all, --max-pages, --delay, and --cursor options to both thread and replies commands, enabling fetching of all conversation content with rate-limit-friendly pagination. Changes: - Modified fetchTweetDetail() to accept optional cursor parameter - Added getRepliesPaged() and getThreadPaged() methods with pagination loop - Updated CLI commands with new options (--all, --max-pages, --delay, --cursor) - Added TweetDetailPaginationOptions interface - JSON output includes nextCursor for scripting/resumption Options: - --all: Fetch all pages (no hard limit) - --max-pages <n>: Limit number of pages when using --all - --delay <ms>: Delay between page fetches (default: 1000ms) - --cursor <string>: Resume from a previous cursor Follows patterns established by bookmarks command and PR #34 (user-tweets). Co-authored-by: Christian Catalan <[email protected]>
This commit is contained in:
cc-vps
committed by
Peter Steinberger
Peter Steinberger
co-authored by
Christian Catalan
Christian Catalan
parent
c7e2ec0495
commit
90037ce880
+151
-49
@@ -49,72 +49,174 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
|
||||
.command('replies')
|
||||
.description('List replies to a tweet (by ID or URL)')
|
||||
.argument('<tweet-id-or-url>', 'Tweet ID or URL')
|
||||
.option('--all', 'Fetch all replies (paged)')
|
||||
.option('--max-pages <number>', 'Stop after N pages when using --all')
|
||||
.option('--delay <ms>', 'Delay in ms between page fetches (default: 1000)')
|
||||
.option('--cursor <string>', 'Resume pagination from a cursor')
|
||||
.option('--json', 'Output as JSON')
|
||||
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
|
||||
.action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => {
|
||||
const opts = program.opts();
|
||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
|
||||
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
|
||||
.action(
|
||||
async (
|
||||
tweetIdOrUrl: string,
|
||||
cmdOpts: {
|
||||
all?: boolean;
|
||||
maxPages?: string;
|
||||
delay?: string;
|
||||
cursor?: string;
|
||||
json?: boolean;
|
||||
jsonFull?: boolean;
|
||||
},
|
||||
) => {
|
||||
const opts = program.opts();
|
||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
|
||||
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
|
||||
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
|
||||
const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000;
|
||||
|
||||
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
|
||||
const usePagination = cmdOpts.all || cmdOpts.cursor;
|
||||
if (maxPages !== undefined && !usePagination) {
|
||||
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
|
||||
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
|
||||
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const warning of warnings) {
|
||||
console.error(`${ctx.p('warn')}${warning}`);
|
||||
}
|
||||
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
|
||||
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
console.error(`${ctx.p('err')}Missing required credentials`);
|
||||
process.exit(1);
|
||||
}
|
||||
for (const warning of warnings) {
|
||||
console.error(`${ctx.p('warn')}${warning}`);
|
||||
}
|
||||
|
||||
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
|
||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||
const result = await client.getReplies(tweetId, { includeRaw });
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
console.error(`${ctx.p('err')}Missing required credentials`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (result.success && result.tweets) {
|
||||
ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No replies found.' });
|
||||
} else {
|
||||
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
|
||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||
|
||||
const result = usePagination
|
||||
? await client.getRepliesPaged(tweetId, {
|
||||
includeRaw,
|
||||
maxPages,
|
||||
cursor: cmdOpts.cursor,
|
||||
pageDelayMs,
|
||||
})
|
||||
: await client.getReplies(tweetId, { includeRaw });
|
||||
|
||||
if (result.success && result.tweets) {
|
||||
const isJson = cmdOpts.json || cmdOpts.jsonFull;
|
||||
if (isJson && usePagination) {
|
||||
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
|
||||
} else {
|
||||
ctx.printTweets(result.tweets, { json: isJson, emptyMessage: 'No replies found.' });
|
||||
}
|
||||
|
||||
// Show pagination hint if there's more
|
||||
if (result.nextCursor && !isJson) {
|
||||
console.error(`${ctx.p('info')}More replies available. Use --cursor "${result.nextCursor}" to continue.`);
|
||||
}
|
||||
} else {
|
||||
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
program
|
||||
.command('thread')
|
||||
.description('Show the full conversation thread containing the tweet')
|
||||
.argument('<tweet-id-or-url>', 'Tweet ID or URL')
|
||||
.option('--all', 'Fetch all thread tweets (paged)')
|
||||
.option('--max-pages <number>', 'Stop after N pages when using --all')
|
||||
.option('--delay <ms>', 'Delay in ms between page fetches (default: 1000)')
|
||||
.option('--cursor <string>', 'Resume pagination from a cursor')
|
||||
.option('--json', 'Output as JSON')
|
||||
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
|
||||
.action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => {
|
||||
const opts = program.opts();
|
||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
|
||||
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
|
||||
.action(
|
||||
async (
|
||||
tweetIdOrUrl: string,
|
||||
cmdOpts: {
|
||||
all?: boolean;
|
||||
maxPages?: string;
|
||||
delay?: string;
|
||||
cursor?: string;
|
||||
json?: boolean;
|
||||
jsonFull?: boolean;
|
||||
},
|
||||
) => {
|
||||
const opts = program.opts();
|
||||
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
|
||||
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
|
||||
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
|
||||
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
|
||||
const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000;
|
||||
|
||||
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
|
||||
const usePagination = cmdOpts.all || cmdOpts.cursor;
|
||||
if (maxPages !== undefined && !usePagination) {
|
||||
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
|
||||
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
|
||||
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const warning of warnings) {
|
||||
console.error(`${ctx.p('warn')}${warning}`);
|
||||
}
|
||||
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
|
||||
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
console.error(`${ctx.p('err')}Missing required credentials`);
|
||||
process.exit(1);
|
||||
}
|
||||
for (const warning of warnings) {
|
||||
console.error(`${ctx.p('warn')}${warning}`);
|
||||
}
|
||||
|
||||
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
|
||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||
const result = await client.getThread(tweetId, { includeRaw });
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
console.error(`${ctx.p('err')}Missing required credentials`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (result.success && result.tweets) {
|
||||
ctx.printTweets(result.tweets, {
|
||||
json: cmdOpts.json || cmdOpts.jsonFull,
|
||||
emptyMessage: 'No thread tweets found.',
|
||||
});
|
||||
} else {
|
||||
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
|
||||
const includeRaw = cmdOpts.jsonFull ?? false;
|
||||
|
||||
const result = usePagination
|
||||
? await client.getThreadPaged(tweetId, {
|
||||
includeRaw,
|
||||
maxPages,
|
||||
cursor: cmdOpts.cursor,
|
||||
pageDelayMs,
|
||||
})
|
||||
: await client.getThread(tweetId, { includeRaw });
|
||||
|
||||
if (result.success && result.tweets) {
|
||||
const isJson = cmdOpts.json || cmdOpts.jsonFull;
|
||||
if (isJson && usePagination) {
|
||||
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
|
||||
} else {
|
||||
ctx.printTweets(result.tweets, {
|
||||
json: isJson,
|
||||
emptyMessage: 'No thread tweets found.',
|
||||
});
|
||||
}
|
||||
|
||||
// Show pagination hint if there's more
|
||||
if (result.nextCursor && !isJson) {
|
||||
console.error(
|
||||
`${ctx.p('info')}More thread tweets available. Use --cursor "${result.nextCursor}" to continue.`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
|
||||
import { TWITTER_API_BASE } from './twitter-client-constants.js';
|
||||
import { buildArticleFeatures, buildArticleFieldToggles, buildTweetDetailFeatures } from './twitter-client-features.js';
|
||||
import type { GetTweetResult, GraphqlTweetResult, SearchResult } from './twitter-client-types.js';
|
||||
import type { GetTweetResult, GraphqlTweetResult, SearchResult, TweetData } from './twitter-client-types.js';
|
||||
import {
|
||||
extractArticleText,
|
||||
extractCursorFromInstructions,
|
||||
findTweetInInstructions,
|
||||
firstText,
|
||||
mapTweetResult,
|
||||
@@ -16,10 +17,22 @@ export interface TweetFetchOptions {
|
||||
includeRaw?: boolean;
|
||||
}
|
||||
|
||||
/** Options for paginated tweet detail fetch */
|
||||
export interface TweetDetailPaginationOptions extends TweetFetchOptions {
|
||||
/** Maximum number of pages to fetch (default: unlimited when using pagination) */
|
||||
maxPages?: number;
|
||||
/** Starting cursor for pagination (resume from previous fetch) */
|
||||
cursor?: string;
|
||||
/** Delay in milliseconds between page fetches (default: 1000) */
|
||||
pageDelayMs?: number;
|
||||
}
|
||||
|
||||
export interface TwitterClientTweetDetailMethods {
|
||||
getTweet(tweetId: string, options?: TweetFetchOptions): Promise<GetTweetResult>;
|
||||
getReplies(tweetId: string, options?: TweetFetchOptions): Promise<SearchResult>;
|
||||
getThread(tweetId: string, options?: TweetFetchOptions): Promise<SearchResult>;
|
||||
getRepliesPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise<SearchResult>;
|
||||
getThreadPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise<SearchResult>;
|
||||
}
|
||||
|
||||
export function withTweetDetails<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
@@ -110,7 +123,14 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
|
||||
return {};
|
||||
}
|
||||
|
||||
private async fetchTweetDetail(tweetId: string): Promise<
|
||||
private async sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
private async fetchTweetDetail(
|
||||
tweetId: string,
|
||||
cursor?: string,
|
||||
): Promise<
|
||||
| {
|
||||
success: true;
|
||||
data: {
|
||||
@@ -141,6 +161,7 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
|
||||
withQuickPromoteEligibilityTweetFields: true,
|
||||
withBirdwatchNotes: true,
|
||||
withVoice: true,
|
||||
...(cursor ? { cursor } : {}),
|
||||
};
|
||||
|
||||
const features = {
|
||||
@@ -340,6 +361,135 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
|
||||
|
||||
return { success: true, tweets: thread };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get replies to a tweet with pagination support
|
||||
*/
|
||||
async getRepliesPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
|
||||
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
|
||||
const seen = new Set<string>();
|
||||
const allReplies: TweetData[] = [];
|
||||
let cursor: string | undefined = options.cursor;
|
||||
let nextCursor: string | undefined;
|
||||
let pagesFetched = 0;
|
||||
|
||||
while (true) {
|
||||
// Add delay between pages (but not before the first page)
|
||||
if (pagesFetched > 0 && pageDelayMs > 0) {
|
||||
await this.sleep(pageDelayMs);
|
||||
}
|
||||
|
||||
const response = await this.fetchTweetDetail(tweetId, cursor);
|
||||
if (!response.success) {
|
||||
// If we have some replies already, return them with the error
|
||||
if (allReplies.length > 0) {
|
||||
return { success: true, tweets: allReplies, nextCursor: cursor, error: response.error };
|
||||
}
|
||||
return response;
|
||||
}
|
||||
pagesFetched += 1;
|
||||
|
||||
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
|
||||
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId);
|
||||
|
||||
for (const reply of replies) {
|
||||
if (seen.has(reply.id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(reply.id);
|
||||
allReplies.push(reply);
|
||||
}
|
||||
|
||||
const pageCursor = extractCursorFromInstructions(instructions);
|
||||
if (!pageCursor || pageCursor === cursor || replies.length === 0) {
|
||||
nextCursor = undefined;
|
||||
break;
|
||||
}
|
||||
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
nextCursor = pageCursor;
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = pageCursor;
|
||||
nextCursor = pageCursor;
|
||||
}
|
||||
|
||||
return { success: true, tweets: allReplies, nextCursor };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get full conversation thread with pagination support
|
||||
*/
|
||||
async getThreadPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
|
||||
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
|
||||
const seen = new Set<string>();
|
||||
const allTweets: TweetData[] = [];
|
||||
let cursor: string | undefined = options.cursor;
|
||||
let nextCursor: string | undefined;
|
||||
let pagesFetched = 0;
|
||||
let rootId: string | undefined;
|
||||
|
||||
while (true) {
|
||||
// Add delay between pages (but not before the first page)
|
||||
if (pagesFetched > 0 && pageDelayMs > 0) {
|
||||
await this.sleep(pageDelayMs);
|
||||
}
|
||||
|
||||
const response = await this.fetchTweetDetail(tweetId, cursor);
|
||||
if (!response.success) {
|
||||
// If we have some tweets already, return them with the error
|
||||
if (allTweets.length > 0) {
|
||||
return { success: true, tweets: allTweets, nextCursor: cursor, error: response.error };
|
||||
}
|
||||
return response;
|
||||
}
|
||||
pagesFetched += 1;
|
||||
|
||||
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
|
||||
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
|
||||
// Determine root conversation ID from first page
|
||||
if (!rootId) {
|
||||
const target = tweets.find((t) => t.id === tweetId);
|
||||
rootId = target?.conversationId || tweetId;
|
||||
}
|
||||
|
||||
const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId);
|
||||
|
||||
for (const tweet of threadTweets) {
|
||||
if (seen.has(tweet.id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(tweet.id);
|
||||
allTweets.push(tweet);
|
||||
}
|
||||
|
||||
const pageCursor = extractCursorFromInstructions(instructions);
|
||||
if (!pageCursor || pageCursor === cursor || threadTweets.length === 0) {
|
||||
nextCursor = undefined;
|
||||
break;
|
||||
}
|
||||
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
nextCursor = pageCursor;
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = pageCursor;
|
||||
nextCursor = pageCursor;
|
||||
}
|
||||
|
||||
// Sort by creation time
|
||||
allTweets.sort((a, b) => {
|
||||
const aTime = a.createdAt ? Date.parse(a.createdAt) : 0;
|
||||
const bTime = b.createdAt ? Date.parse(b.createdAt) : 0;
|
||||
return aTime - bTime;
|
||||
});
|
||||
|
||||
return { success: true, tweets: allTweets, nextCursor };
|
||||
}
|
||||
}
|
||||
|
||||
return TwitterClientTweetDetails;
|
||||
|
||||
Reference in New Issue
Block a user