yuta/bird
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #49 from steipete/refactor/pagination-cleanup

Browse Source 
refactor: unify pagination helpers
This commit is contained in:
Peter Steinberger
2026-01-12 02:59:07 +00:00
committed by GitHub
parent e4d2773d46 d5a527d5fb
commit b51258c4e8
15 changed files with 307 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -9,6 +9,9 @@
- `user-tweets` command to fetch a user's profile timeline (#34) — thanks @crcatala.
- `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala.
### Changed
- Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`).
### Fixed
- macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst.
- Terminal hyperlinks now sanitize control characters before emitting OSC 8 sequences (#29) — thanks @mafulafunk.
src/cli/pagination.ts
+79
View File
@@ -0,0 +1,79 @@
export type PaginationCmdOpts = {
all?: boolean;
maxPages?: string;
cursor?: string;
delay?: string;
};
export function parsePositiveIntFlag(
raw: string | undefined,
flagName: string,
): { ok: true; value: number | undefined } | { ok: false; error: string } {
if (raw === undefined) {
return { ok: true, value: undefined };
}
const value = Number.parseInt(raw, 10);
if (!Number.isFinite(value) || value <= 0) {
return { ok: false, error: `Invalid ${flagName}. Expected a positive integer.` };
}
return { ok: true, value };
}
export function parseNonNegativeIntFlag(
raw: string | undefined,
flagName: string,
defaultValue: number,
): { ok: true; value: number } | { ok: false; error: string } {
const value = Number.parseInt(raw ?? String(defaultValue), 10);
if (!Number.isFinite(value) || value < 0) {
return { ok: false, error: `Invalid ${flagName}. Expected a non-negative integer.` };
}
return { ok: true, value };
}
export function parsePaginationFlags(
cmdOpts: PaginationCmdOpts,
opts?: {
maxPagesImpliesPagination?: boolean;
defaultDelayMs?: number;
includeDelay?: boolean;
},
):
| {
ok: true;
usePagination: boolean;
maxPages?: number;
cursor?: string;
pageDelayMs?: number;
}
| { ok: false; error: string } {
const maxPagesImpliesPagination = opts?.maxPagesImpliesPagination ?? false;
const includeDelay = opts?.includeDelay ?? false;
const defaultDelayMs = opts?.defaultDelayMs ?? 1000;
const maxPages = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages');
if (!maxPages.ok) {
return maxPages;
}
const usePagination = Boolean(
cmdOpts.all || cmdOpts.cursor || (maxPagesImpliesPagination && maxPages.value !== undefined),
);
let pageDelayMs: number | undefined;
if (includeDelay) {
const delay = parseNonNegativeIntFlag(cmdOpts.delay, '--delay', defaultDelayMs);
if (!delay.ok) {
return delay;
}
pageDelayMs = delay.value;
}
return {
ok: true,
usePagination,
maxPages: maxPages.value,
cursor: cmdOpts.cursor,
pageDelayMs,
};
}
src/commands/bookmarks.ts
+8 -6
View File
@@ -1,4 +1,5 @@
import type { Command } from 'commander';
import { parsePositiveIntFlag } from '../cli/pagination.js';
import type { CliContext } from '../cli/shared.js';
import { extractBookmarkFolderId } from '../lib/extract-bookmark-folder-id.js';
import { TwitterClient } from '../lib/twitter-client.js';
@@ -27,7 +28,12 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
const opts = program.opts();
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const count = Number.parseInt(cmdOpts.count || '20', 10);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const maxPagesParsed = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages');
if (!maxPagesParsed.ok) {
console.error(`${ctx.p('err')}${maxPagesParsed.error}`);
process.exit(1);
}
const maxPages = maxPagesParsed.value;
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
@@ -49,10 +55,6 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
process.exit(1);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
const client = new TwitterClient({ cookies, timeoutMs });
const folderId = cmdOpts.folderId ? extractBookmarkFolderId(cmdOpts.folderId) : null;
@@ -71,7 +73,7 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi
? await client.getAllBookmarks(paginationOptions)
: await client.getBookmarks(count, timelineOptions);
if (result.success && result.tweets) {
if (result.success) {
const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.';
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage });
src/commands/home.ts
+2 -2
View File
@@ -38,10 +38,10 @@ export function registerHomeCommand(program: Command, ctx: CliContext): void {
? await client.getHomeLatestTimeline(count, { includeRaw })
: await client.getHomeTimeline(count, { includeRaw });
if (result.success && result.tweets) {
if (result.success) {
const feedType = cmdOpts.following ? 'Following' : 'For You';
const emptyMessage = `No tweets found in ${feedType} timeline.`;
const isJson = cmdOpts.json || cmdOpts.jsonFull;
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweets(result.tweets, { json: isJson, emptyMessage });
} else {
console.error(`${ctx.p('err')}Failed to fetch home timeline: ${result.error}`);
src/commands/lists.ts
+10 -8
View File
@@ -2,6 +2,7 @@
// ABOUTME: Supports listing owned lists, memberships, and list timelines.
import type { Command } from 'commander';
import { parsePaginationFlags } from '../cli/pagination.js';
import type { CliContext } from '../cli/shared.js';
import { extractListId } from '../lib/extract-list-id.js';
import { hyperlink } from '../lib/output.js';
@@ -99,7 +100,12 @@ export function registerListsCommand(program: Command, ctx: CliContext): void {
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const count = Number.parseInt(cmdOpts.count || '20', 10);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true });
if (!pagination.ok) {
console.error(`${ctx.p('err')}${pagination.error}`);
process.exit(1);
}
const listId = extractListId(listIdOrUrl);
if (!listId) {
@@ -107,15 +113,11 @@ export function registerListsCommand(program: Command, ctx: CliContext): void {
process.exit(2);
}
const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined);
const usePagination = pagination.usePagination;
if (!usePagination && (!Number.isFinite(count) || count <= 0)) {
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
process.exit(1);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
@@ -131,13 +133,13 @@ export function registerListsCommand(program: Command, ctx: CliContext): void {
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const timelineOptions = { includeRaw };
const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor };
const paginationOptions = { includeRaw, maxPages: pagination.maxPages, cursor: pagination.cursor };
const result = usePagination
? await client.getAllListTimeline(listId, paginationOptions)
: await client.getListTimeline(listId, count, timelineOptions);
if (result.success && result.tweets) {
if (result.success) {
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweetsResult(result, {
json: isJson,
src/commands/read.ts
+31 -40
View File
@@ -1,4 +1,5 @@
import type { Command } from 'commander';
import { parsePaginationFlags } from '../cli/pagination.js';
import type { CliContext } from '../cli/shared.js';
import { formatStatsLine } from '../lib/output.js';
import { TwitterClient } from '../lib/twitter-client.js';
@@ -71,16 +72,10 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10);
const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined);
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true, includeDelay: true });
if (!pagination.ok) {
console.error(`${ctx.p('err')}${pagination.error}`);
process.exit(1);
}
@@ -98,29 +93,30 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = usePagination
const result = pagination.usePagination
? await client.getRepliesPaged(tweetId, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
maxPages: pagination.maxPages,
cursor: pagination.cursor,
pageDelayMs: pagination.pageDelayMs,
})
: await client.getReplies(tweetId, { includeRaw });
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
if (result.tweets) {
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage: 'No replies found.' });
ctx.printTweetsResult(result, {
json: isJson,
usePagination: pagination.usePagination,
emptyMessage: 'No replies found.',
});
// Show pagination hint if there's more
if (result.nextCursor && !isJson) {
console.error(`${ctx.p('info')}More replies available. Use --cursor "${result.nextCursor}" to continue.`);
}
}
if (!result.success) {
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
process.exit(1);
}
} else {
if (!result.success) {
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
process.exit(1);
}
@@ -153,16 +149,10 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10);
const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined);
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true, includeDelay: true });
if (!pagination.ok) {
console.error(`${ctx.p('err')}${pagination.error}`);
process.exit(1);
}
@@ -180,18 +170,22 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = usePagination
const result = pagination.usePagination
? await client.getThreadPaged(tweetId, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
maxPages: pagination.maxPages,
cursor: pagination.cursor,
pageDelayMs: pagination.pageDelayMs,
})
: await client.getThread(tweetId, { includeRaw });
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
if (result.tweets) {
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage: 'No thread tweets found.' });
ctx.printTweetsResult(result, {
json: isJson,
usePagination: pagination.usePagination,
emptyMessage: 'No thread tweets found.',
});
// Show pagination hint if there's more
if (result.nextCursor && !isJson) {
@@ -199,12 +193,9 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
`${ctx.p('info')}More thread tweets available. Use --cursor "${result.nextCursor}" to continue.`,
);
}
}
if (!result.success) {
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
process.exit(1);
}
} else {
if (!result.success) {
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
process.exit(1);
}
src/commands/search.ts
+10 -4
View File
@@ -32,8 +32,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void
const includeRaw = cmdOpts.jsonFull ?? false;
const result = await client.search(query, count, { includeRaw });
if (result.success && result.tweets) {
ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No tweets found.' });
if (result.success) {
ctx.printTweets(result.tweets, {
json: cmdOpts.json || cmdOpts.jsonFull,
emptyMessage: 'No tweets found.',
});
} else {
console.error(`${ctx.p('err')}Search failed: ${result.error}`);
process.exit(1);
@@ -90,8 +93,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void
const includeRaw = cmdOpts.jsonFull ?? false;
const result = await client.search(query, count, { includeRaw });
if (result.success && result.tweets) {
ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No mentions found.' });
if (result.success) {
ctx.printTweets(result.tweets, {
json: cmdOpts.json || cmdOpts.jsonFull,
emptyMessage: 'No mentions found.',
});
} else {
console.error(`${ctx.p('err')}Failed to fetch mentions: ${result.error}`);
process.exit(1);
src/commands/user-tweets.ts
+17 -8
View File
@@ -1,4 +1,5 @@
import type { Command } from 'commander';
import { parseNonNegativeIntFlag, parsePositiveIntFlag } from '../cli/pagination.js';
import type { CliContext } from '../cli/shared.js';
import { normalizeHandle } from '../lib/normalize-handle.js';
import { TwitterClient } from '../lib/twitter-client.js';
@@ -45,8 +46,20 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const count = Number.parseInt(cmdOpts.count || '20', 10);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10);
const maxPagesParsed = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages');
if (!maxPagesParsed.ok) {
console.error(`${ctx.p('err')}${maxPagesParsed.error}`);
process.exit(2);
}
const maxPages = maxPagesParsed.value;
const delayParsed = parseNonNegativeIntFlag(cmdOpts.delay, '--delay', 1000);
if (!delayParsed.ok) {
console.error(`${ctx.p('err')}${delayParsed.error}`);
process.exit(2);
}
const pageDelayMs = delayParsed.value;
// Validate inputs
if (!Number.isFinite(count) || count <= 0) {
@@ -62,14 +75,10 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
);
process.exit(2);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0 || maxPages > hardMaxPages)) {
if (maxPages !== undefined && maxPages > hardMaxPages) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer (max: ${hardMaxPages}).`);
process.exit(2);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
process.exit(2);
}
// Normalize handle (strip @ if present)
const username = normalizeHandle(handle);
@@ -113,7 +122,7 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
pageDelayMs,
});
if (result.success && result.tweets) {
if (result.success) {
const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull);
ctx.printTweetsResult(result, {
json: isJson,
src/commands/users.ts
+1 -1
View File
@@ -351,7 +351,7 @@ export function registerUserCommands(program: Command, ctx: CliContext): void {
const includeRaw = cmdOpts.jsonFull ?? false;
const result = await client.getLikes(count, { includeRaw });
if (result.success && result.tweets) {
if (result.success) {
ctx.printTweets(result.tweets, {
json: cmdOpts.json || cmdOpts.jsonFull,
emptyMessage: 'No liked tweets found.',
src/lib/paginate-cursor.ts
+73
View File
@@ -0,0 +1,73 @@
export type CursorPage<T> =
| {
success: true;
items: T[];
cursor?: string;
}
| {
success: false;
error: string;
};
export type CursorPaginationResult<T> =
| {
success: true;
items: T[];
nextCursor?: string;
}
| {
success: false;
error: string;
items?: T[];
nextCursor?: string;
};
export async function paginateCursor<T>(opts: {
cursor?: string;
maxPages?: number;
pageDelayMs?: number;
sleep: (ms: number) => Promise<void>;
getKey: (item: T) => string;
fetchPage: (cursor?: string) => Promise<CursorPage<T>>;
}): Promise<CursorPaginationResult<T>> {
const { maxPages, pageDelayMs = 1000 } = opts;
const seen = new Set<string>();
const items: T[] = [];
let cursor: string | undefined = opts.cursor;
let pagesFetched = 0;
while (true) {
if (pagesFetched > 0 && pageDelayMs > 0) {
await opts.sleep(pageDelayMs);
}
const page = await opts.fetchPage(cursor);
if (!page.success) {
if (items.length > 0) {
return { success: false, error: page.error, items, nextCursor: cursor };
}
return page;
}
pagesFetched += 1;
for (const item of page.items) {
const key = opts.getKey(item);
if (seen.has(key)) {
continue;
}
seen.add(key);
items.push(item);
}
const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor) {
return { success: true, items, nextCursor: undefined };
}
if (maxPages !== undefined && pagesFetched >= maxPages) {
return { success: true, items, nextCursor: pageCursor };
}
cursor = pageCursor;
}
}
src/lib/twitter-client-base.ts
+4
View File
@@ -41,6 +41,10 @@ export abstract class TwitterClientBase {
protected abstract getCurrentUser(): Promise<CurrentUserResult>;
protected async sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
protected async getQueryId(operationName: OperationName): Promise<string> {
const cached = await runtimeQueryIds.getQueryId(operationName);
return cached ?? QUERY_IDS[operationName];
src/lib/twitter-client-media.ts
-4
View File
@@ -28,10 +28,6 @@ export function withMedia<TBase extends AbstractConstructor<TwitterClientBase>>(
return null;
}
private async sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async uploadMedia(input: { data: Uint8Array; mimeType: string; alt?: string }): Promise<UploadMediaResult> {
const category = this.mediaCategoryForMime(input.mimeType);
if (!category) {
src/lib/twitter-client-tweet-detail.ts
+55 -98
View File
@@ -1,3 +1,4 @@
import { paginateCursor } from './paginate-cursor.js';
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
import { TWITTER_API_BASE } from './twitter-client-constants.js';
import { buildArticleFeatures, buildArticleFieldToggles, buildTweetDetailFeatures } from './twitter-client-features.js';
@@ -123,10 +124,6 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
return {};
}
private async sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
private async fetchTweetDetail(
tweetId: string,
cursor?: string,
@@ -367,56 +364,37 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
*/
async getRepliesPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
const seen = new Set<string>();
const allReplies: TweetData[] = [];
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
while (true) {
// Add delay between pages (but not before the first page)
if (pagesFetched > 0 && pageDelayMs > 0) {
await this.sleep(pageDelayMs);
}
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
// If we have some replies already, return them with the error
if (allReplies.length > 0) {
return { success: false, tweets: allReplies, nextCursor: cursor, error: response.error };
const result = await paginateCursor<TweetData>({
cursor: options.cursor,
maxPages,
pageDelayMs,
sleep: async (ms) => this.sleep(ms),
getKey: (tweet) => tweet.id,
fetchPage: async (cursor) => {
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
return response;
}
return response;
}
pagesFetched += 1;
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId);
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId);
const pageCursor = extractCursorFromInstructions(instructions);
for (const reply of replies) {
if (seen.has(reply.id)) {
continue;
}
seen.add(reply.id);
allReplies.push(reply);
}
return { success: true as const, items: replies, cursor: pageCursor };
},
});
const pageCursor = extractCursorFromInstructions(instructions);
if (!pageCursor || pageCursor === cursor) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
if (result.success) {
return { success: true, tweets: result.items, nextCursor: result.nextCursor };
}
return { success: true, tweets: allReplies, nextCursor };
if (result.items) {
return { success: false, tweets: result.items, nextCursor: result.nextCursor, error: result.error };
}
return { success: false, error: result.error };
}
/**
@@ -424,71 +402,50 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
*/
async getThreadPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
const seen = new Set<string>();
const allTweets: TweetData[] = [];
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
let rootId: string | undefined;
while (true) {
// Add delay between pages (but not before the first page)
if (pagesFetched > 0 && pageDelayMs > 0) {
await this.sleep(pageDelayMs);
}
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
// If we have some tweets already, return them with the error
if (allTweets.length > 0) {
return { success: false, tweets: allTweets, nextCursor: cursor, error: response.error };
const result = await paginateCursor<TweetData>({
cursor: options.cursor,
maxPages,
pageDelayMs,
sleep: async (ms) => this.sleep(ms),
getKey: (tweet) => tweet.id,
fetchPage: async (cursor) => {
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
return response;
}
return response;
}
pagesFetched += 1;
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
// Determine root conversation ID from first page
if (!rootId) {
const target = tweets.find((t) => t.id === tweetId);
rootId = target?.conversationId || tweetId;
}
const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId);
for (const tweet of threadTweets) {
if (seen.has(tweet.id)) {
continue;
if (!rootId) {
const target = tweets.find((t) => t.id === tweetId);
rootId = target?.conversationId || tweetId;
}
seen.add(tweet.id);
allTweets.push(tweet);
}
const pageCursor = extractCursorFromInstructions(instructions);
if (!pageCursor || pageCursor === cursor) {
nextCursor = undefined;
break;
}
const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId);
const pageCursor = extractCursorFromInstructions(instructions);
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
return { success: true as const, items: threadTweets, cursor: pageCursor };
},
});
cursor = pageCursor;
nextCursor = pageCursor;
}
// Sort by creation time
allTweets.sort((a, b) => {
const sortedTweets = (result.items ?? []).slice().sort((a, b) => {
const aTime = a.createdAt ? Date.parse(a.createdAt) : 0;
const bTime = b.createdAt ? Date.parse(b.createdAt) : 0;
return aTime - bTime;
});
return { success: true, tweets: allTweets, nextCursor };
if (result.success) {
return { success: true, tweets: sortedTweets, nextCursor: result.nextCursor };
}
if (result.items) {
return { success: false, tweets: sortedTweets, nextCursor: result.nextCursor, error: result.error };
}
return { success: false, error: result.error };
}
}
src/lib/twitter-client-types.ts
+14 -7
View File
@@ -253,13 +253,20 @@ export interface GetTweetResult {
error?: string;
}
export interface SearchResult {
success: boolean;
tweets?: TweetData[];
error?: string;
/** Cursor for fetching the next page of results */
nextCursor?: string;
}
export type SearchResult =
| {
success: true;
tweets: TweetData[];
/** Cursor for fetching the next page of results */
nextCursor?: string;
}
| {
success: false;
error: string;
tweets?: TweetData[];
/** Cursor for fetching the next page of results */
nextCursor?: string;
};
export interface CurrentUserResult {
success: boolean;
src/lib/twitter-client-user-tweets.ts
-4
View File
@@ -40,10 +40,6 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
return Array.from(new Set([primary, 'Wms1GvIiHXAPBaCr9KblaA']));
}
private async sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Get tweets from a user's profile timeline (single page).
*/