From d5a527d5fbf71901672edfeee1af1d0d4949733a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 12 Jan 2026 02:58:00 +0000 Subject: [PATCH] refactor: unify pagination helpers --- CHANGELOG.md | 3 + src/cli/pagination.ts | 79 +++++++++++++ src/commands/bookmarks.ts | 14 ++- src/commands/home.ts | 4 +- src/commands/lists.ts | 18 +-- src/commands/read.ts | 71 +++++------- src/commands/search.ts | 14 ++- src/commands/user-tweets.ts | 25 ++-- src/commands/users.ts | 2 +- src/lib/paginate-cursor.ts | 73 ++++++++++++ src/lib/twitter-client-base.ts | 4 + src/lib/twitter-client-media.ts | 4 - src/lib/twitter-client-tweet-detail.ts | 153 +++++++++---------------- src/lib/twitter-client-types.ts | 21 ++-- src/lib/twitter-client-user-tweets.ts | 4 - 15 files changed, 307 insertions(+), 182 deletions(-) create mode 100644 src/cli/pagination.ts create mode 100644 src/lib/paginate-cursor.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e69762b..4cf54f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ - `user-tweets` command to fetch a user's profile timeline (#34) — thanks @crcatala. - `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala. +### Changed +- Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`). + ### Fixed - macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst. - Terminal hyperlinks now sanitize control characters before emitting OSC 8 sequences (#29) — thanks @mafulafunk. diff --git a/src/cli/pagination.ts b/src/cli/pagination.ts new file mode 100644 index 0000000..b259bc7 --- /dev/null +++ b/src/cli/pagination.ts @@ -0,0 +1,79 @@ +export type PaginationCmdOpts = { + all?: boolean; + maxPages?: string; + cursor?: string; + delay?: string; +}; + +export function parsePositiveIntFlag( + raw: string | undefined, + flagName: string, +): { ok: true; value: number | undefined } | { ok: false; error: string } { + if (raw === undefined) { + return { ok: true, value: undefined }; + } + const value = Number.parseInt(raw, 10); + if (!Number.isFinite(value) || value <= 0) { + return { ok: false, error: `Invalid ${flagName}. Expected a positive integer.` }; + } + return { ok: true, value }; +} + +export function parseNonNegativeIntFlag( + raw: string | undefined, + flagName: string, + defaultValue: number, +): { ok: true; value: number } | { ok: false; error: string } { + const value = Number.parseInt(raw ?? String(defaultValue), 10); + if (!Number.isFinite(value) || value < 0) { + return { ok: false, error: `Invalid ${flagName}. Expected a non-negative integer.` }; + } + return { ok: true, value }; +} + +export function parsePaginationFlags( + cmdOpts: PaginationCmdOpts, + opts?: { + maxPagesImpliesPagination?: boolean; + defaultDelayMs?: number; + includeDelay?: boolean; + }, +): + | { + ok: true; + usePagination: boolean; + maxPages?: number; + cursor?: string; + pageDelayMs?: number; + } + | { ok: false; error: string } { + const maxPagesImpliesPagination = opts?.maxPagesImpliesPagination ?? false; + const includeDelay = opts?.includeDelay ?? false; + const defaultDelayMs = opts?.defaultDelayMs ?? 1000; + + const maxPages = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages'); + if (!maxPages.ok) { + return maxPages; + } + + const usePagination = Boolean( + cmdOpts.all || cmdOpts.cursor || (maxPagesImpliesPagination && maxPages.value !== undefined), + ); + + let pageDelayMs: number | undefined; + if (includeDelay) { + const delay = parseNonNegativeIntFlag(cmdOpts.delay, '--delay', defaultDelayMs); + if (!delay.ok) { + return delay; + } + pageDelayMs = delay.value; + } + + return { + ok: true, + usePagination, + maxPages: maxPages.value, + cursor: cmdOpts.cursor, + pageDelayMs, + }; +} diff --git a/src/commands/bookmarks.ts b/src/commands/bookmarks.ts index a4a5fde..abda8cb 100644 --- a/src/commands/bookmarks.ts +++ b/src/commands/bookmarks.ts @@ -1,4 +1,5 @@ import type { Command } from 'commander'; +import { parsePositiveIntFlag } from '../cli/pagination.js'; import type { CliContext } from '../cli/shared.js'; import { extractBookmarkFolderId } from '../lib/extract-bookmark-folder-id.js'; import { TwitterClient } from '../lib/twitter-client.js'; @@ -27,7 +28,12 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '20', 10); - const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; + const maxPagesParsed = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages'); + if (!maxPagesParsed.ok) { + console.error(`${ctx.p('err')}${maxPagesParsed.error}`); + process.exit(1); + } + const maxPages = maxPagesParsed.value; const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); @@ -49,10 +55,6 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); process.exit(1); } - if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { - console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); - process.exit(1); - } const client = new TwitterClient({ cookies, timeoutMs }); const folderId = cmdOpts.folderId ? extractBookmarkFolderId(cmdOpts.folderId) : null; @@ -71,7 +73,7 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi ? await client.getAllBookmarks(paginationOptions) : await client.getBookmarks(count, timelineOptions); - if (result.success && result.tweets) { + if (result.success) { const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.'; const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage }); diff --git a/src/commands/home.ts b/src/commands/home.ts index 5355b20..876cf35 100644 --- a/src/commands/home.ts +++ b/src/commands/home.ts @@ -38,10 +38,10 @@ export function registerHomeCommand(program: Command, ctx: CliContext): void { ? await client.getHomeLatestTimeline(count, { includeRaw }) : await client.getHomeTimeline(count, { includeRaw }); - if (result.success && result.tweets) { + if (result.success) { const feedType = cmdOpts.following ? 'Following' : 'For You'; const emptyMessage = `No tweets found in ${feedType} timeline.`; - const isJson = cmdOpts.json || cmdOpts.jsonFull; + const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); ctx.printTweets(result.tweets, { json: isJson, emptyMessage }); } else { console.error(`${ctx.p('err')}Failed to fetch home timeline: ${result.error}`); diff --git a/src/commands/lists.ts b/src/commands/lists.ts index e969499..d7297b7 100644 --- a/src/commands/lists.ts +++ b/src/commands/lists.ts @@ -2,6 +2,7 @@ // ABOUTME: Supports listing owned lists, memberships, and list timelines. import type { Command } from 'commander'; +import { parsePaginationFlags } from '../cli/pagination.js'; import type { CliContext } from '../cli/shared.js'; import { extractListId } from '../lib/extract-list-id.js'; import { hyperlink } from '../lib/output.js'; @@ -99,7 +100,12 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '20', 10); - const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; + + const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true }); + if (!pagination.ok) { + console.error(`${ctx.p('err')}${pagination.error}`); + process.exit(1); + } const listId = extractListId(listIdOrUrl); if (!listId) { @@ -107,15 +113,11 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { process.exit(2); } - const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined); + const usePagination = pagination.usePagination; if (!usePagination && (!Number.isFinite(count) || count <= 0)) { console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); process.exit(1); } - if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { - console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); - process.exit(1); - } const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); @@ -131,13 +133,13 @@ export function registerListsCommand(program: Command, ctx: CliContext): void { const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); const includeRaw = cmdOpts.jsonFull ?? false; const timelineOptions = { includeRaw }; - const paginationOptions = { includeRaw, maxPages, cursor: cmdOpts.cursor }; + const paginationOptions = { includeRaw, maxPages: pagination.maxPages, cursor: pagination.cursor }; const result = usePagination ? await client.getAllListTimeline(listId, paginationOptions) : await client.getListTimeline(listId, count, timelineOptions); - if (result.success && result.tweets) { + if (result.success) { const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); ctx.printTweetsResult(result, { json: isJson, diff --git a/src/commands/read.ts b/src/commands/read.ts index 7e371ae..5a353d4 100644 --- a/src/commands/read.ts +++ b/src/commands/read.ts @@ -1,4 +1,5 @@ import type { Command } from 'commander'; +import { parsePaginationFlags } from '../cli/pagination.js'; import type { CliContext } from '../cli/shared.js'; import { formatStatsLine } from '../lib/output.js'; import { TwitterClient } from '../lib/twitter-client.js'; @@ -71,16 +72,10 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const tweetId = ctx.extractTweetId(tweetIdOrUrl); - const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; - const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10); - const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined); - if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { - console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); - process.exit(1); - } - if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) { - console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`); + const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true, includeDelay: true }); + if (!pagination.ok) { + console.error(`${ctx.p('err')}${pagination.error}`); process.exit(1); } @@ -98,29 +93,30 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); const includeRaw = cmdOpts.jsonFull ?? false; - const result = usePagination + const result = pagination.usePagination ? await client.getRepliesPaged(tweetId, { includeRaw, - maxPages, - cursor: cmdOpts.cursor, - pageDelayMs, + maxPages: pagination.maxPages, + cursor: pagination.cursor, + pageDelayMs: pagination.pageDelayMs, }) : await client.getReplies(tweetId, { includeRaw }); + const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); if (result.tweets) { - const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); - ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage: 'No replies found.' }); + ctx.printTweetsResult(result, { + json: isJson, + usePagination: pagination.usePagination, + emptyMessage: 'No replies found.', + }); // Show pagination hint if there's more if (result.nextCursor && !isJson) { console.error(`${ctx.p('info')}More replies available. Use --cursor "${result.nextCursor}" to continue.`); } + } - if (!result.success) { - console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`); - process.exit(1); - } - } else { + if (!result.success) { console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`); process.exit(1); } @@ -153,16 +149,10 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const tweetId = ctx.extractTweetId(tweetIdOrUrl); - const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; - const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10); - const usePagination = Boolean(cmdOpts.all || cmdOpts.cursor || maxPages !== undefined); - if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) { - console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`); - process.exit(1); - } - if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) { - console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`); + const pagination = parsePaginationFlags(cmdOpts, { maxPagesImpliesPagination: true, includeDelay: true }); + if (!pagination.ok) { + console.error(`${ctx.p('err')}${pagination.error}`); process.exit(1); } @@ -180,18 +170,22 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); const includeRaw = cmdOpts.jsonFull ?? false; - const result = usePagination + const result = pagination.usePagination ? await client.getThreadPaged(tweetId, { includeRaw, - maxPages, - cursor: cmdOpts.cursor, - pageDelayMs, + maxPages: pagination.maxPages, + cursor: pagination.cursor, + pageDelayMs: pagination.pageDelayMs, }) : await client.getThread(tweetId, { includeRaw }); + const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); if (result.tweets) { - const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); - ctx.printTweetsResult(result, { json: isJson, usePagination, emptyMessage: 'No thread tweets found.' }); + ctx.printTweetsResult(result, { + json: isJson, + usePagination: pagination.usePagination, + emptyMessage: 'No thread tweets found.', + }); // Show pagination hint if there's more if (result.nextCursor && !isJson) { @@ -199,12 +193,9 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { `${ctx.p('info')}More thread tweets available. Use --cursor "${result.nextCursor}" to continue.`, ); } + } - if (!result.success) { - console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`); - process.exit(1); - } - } else { + if (!result.success) { console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`); process.exit(1); } diff --git a/src/commands/search.ts b/src/commands/search.ts index c1af99e..7311a07 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -32,8 +32,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void const includeRaw = cmdOpts.jsonFull ?? false; const result = await client.search(query, count, { includeRaw }); - if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No tweets found.' }); + if (result.success) { + ctx.printTweets(result.tweets, { + json: cmdOpts.json || cmdOpts.jsonFull, + emptyMessage: 'No tweets found.', + }); } else { console.error(`${ctx.p('err')}Search failed: ${result.error}`); process.exit(1); @@ -90,8 +93,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void const includeRaw = cmdOpts.jsonFull ?? false; const result = await client.search(query, count, { includeRaw }); - if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No mentions found.' }); + if (result.success) { + ctx.printTweets(result.tweets, { + json: cmdOpts.json || cmdOpts.jsonFull, + emptyMessage: 'No mentions found.', + }); } else { console.error(`${ctx.p('err')}Failed to fetch mentions: ${result.error}`); process.exit(1); diff --git a/src/commands/user-tweets.ts b/src/commands/user-tweets.ts index e09a67a..001bc18 100644 --- a/src/commands/user-tweets.ts +++ b/src/commands/user-tweets.ts @@ -1,4 +1,5 @@ import type { Command } from 'commander'; +import { parseNonNegativeIntFlag, parsePositiveIntFlag } from '../cli/pagination.js'; import type { CliContext } from '../cli/shared.js'; import { normalizeHandle } from '../lib/normalize-handle.js'; import { TwitterClient } from '../lib/twitter-client.js'; @@ -45,8 +46,20 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '20', 10); - const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined; - const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10); + + const maxPagesParsed = parsePositiveIntFlag(cmdOpts.maxPages, '--max-pages'); + if (!maxPagesParsed.ok) { + console.error(`${ctx.p('err')}${maxPagesParsed.error}`); + process.exit(2); + } + const maxPages = maxPagesParsed.value; + + const delayParsed = parseNonNegativeIntFlag(cmdOpts.delay, '--delay', 1000); + if (!delayParsed.ok) { + console.error(`${ctx.p('err')}${delayParsed.error}`); + process.exit(2); + } + const pageDelayMs = delayParsed.value; // Validate inputs if (!Number.isFinite(count) || count <= 0) { @@ -62,14 +75,10 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo ); process.exit(2); } - if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0 || maxPages > hardMaxPages)) { + if (maxPages !== undefined && maxPages > hardMaxPages) { console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer (max: ${hardMaxPages}).`); process.exit(2); } - if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) { - console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`); - process.exit(2); - } // Normalize handle (strip @ if present) const username = normalizeHandle(handle); @@ -113,7 +122,7 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo pageDelayMs, }); - if (result.success && result.tweets) { + if (result.success) { const isJson = Boolean(cmdOpts.json || cmdOpts.jsonFull); ctx.printTweetsResult(result, { json: isJson, diff --git a/src/commands/users.ts b/src/commands/users.ts index 9c834cf..77cc26e 100644 --- a/src/commands/users.ts +++ b/src/commands/users.ts @@ -351,7 +351,7 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { const includeRaw = cmdOpts.jsonFull ?? false; const result = await client.getLikes(count, { includeRaw }); - if (result.success && result.tweets) { + if (result.success) { ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No liked tweets found.', diff --git a/src/lib/paginate-cursor.ts b/src/lib/paginate-cursor.ts new file mode 100644 index 0000000..96e9e55 --- /dev/null +++ b/src/lib/paginate-cursor.ts @@ -0,0 +1,73 @@ +export type CursorPage = + | { + success: true; + items: T[]; + cursor?: string; + } + | { + success: false; + error: string; + }; + +export type CursorPaginationResult = + | { + success: true; + items: T[]; + nextCursor?: string; + } + | { + success: false; + error: string; + items?: T[]; + nextCursor?: string; + }; + +export async function paginateCursor(opts: { + cursor?: string; + maxPages?: number; + pageDelayMs?: number; + sleep: (ms: number) => Promise; + getKey: (item: T) => string; + fetchPage: (cursor?: string) => Promise>; +}): Promise> { + const { maxPages, pageDelayMs = 1000 } = opts; + const seen = new Set(); + const items: T[] = []; + let cursor: string | undefined = opts.cursor; + let pagesFetched = 0; + + while (true) { + if (pagesFetched > 0 && pageDelayMs > 0) { + await opts.sleep(pageDelayMs); + } + + const page = await opts.fetchPage(cursor); + if (!page.success) { + if (items.length > 0) { + return { success: false, error: page.error, items, nextCursor: cursor }; + } + return page; + } + pagesFetched += 1; + + for (const item of page.items) { + const key = opts.getKey(item); + if (seen.has(key)) { + continue; + } + seen.add(key); + items.push(item); + } + + const pageCursor = page.cursor; + if (!pageCursor || pageCursor === cursor) { + return { success: true, items, nextCursor: undefined }; + } + + if (maxPages !== undefined && pagesFetched >= maxPages) { + return { success: true, items, nextCursor: pageCursor }; + } + + cursor = pageCursor; + } +} diff --git a/src/lib/twitter-client-base.ts b/src/lib/twitter-client-base.ts index 9b9f7c8..2da5a3b 100644 --- a/src/lib/twitter-client-base.ts +++ b/src/lib/twitter-client-base.ts @@ -41,6 +41,10 @@ export abstract class TwitterClientBase { protected abstract getCurrentUser(): Promise; + protected async sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); + } + protected async getQueryId(operationName: OperationName): Promise { const cached = await runtimeQueryIds.getQueryId(operationName); return cached ?? QUERY_IDS[operationName]; diff --git a/src/lib/twitter-client-media.ts b/src/lib/twitter-client-media.ts index bb2afcd..141ed85 100644 --- a/src/lib/twitter-client-media.ts +++ b/src/lib/twitter-client-media.ts @@ -28,10 +28,6 @@ export function withMedia>( return null; } - private async sleep(ms: number): Promise { - await new Promise((resolve) => setTimeout(resolve, ms)); - } - async uploadMedia(input: { data: Uint8Array; mimeType: string; alt?: string }): Promise { const category = this.mediaCategoryForMime(input.mimeType); if (!category) { diff --git a/src/lib/twitter-client-tweet-detail.ts b/src/lib/twitter-client-tweet-detail.ts index 27f8553..a17881e 100644 --- a/src/lib/twitter-client-tweet-detail.ts +++ b/src/lib/twitter-client-tweet-detail.ts @@ -1,3 +1,4 @@ +import { paginateCursor } from './paginate-cursor.js'; import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js'; import { TWITTER_API_BASE } from './twitter-client-constants.js'; import { buildArticleFeatures, buildArticleFieldToggles, buildTweetDetailFeatures } from './twitter-client-features.js'; @@ -123,10 +124,6 @@ export function withTweetDetails { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - private async fetchTweetDetail( tweetId: string, cursor?: string, @@ -367,56 +364,37 @@ export function withTweetDetails { const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options; - const seen = new Set(); - const allReplies: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - while (true) { - // Add delay between pages (but not before the first page) - if (pagesFetched > 0 && pageDelayMs > 0) { - await this.sleep(pageDelayMs); - } - - const response = await this.fetchTweetDetail(tweetId, cursor); - if (!response.success) { - // If we have some replies already, return them with the error - if (allReplies.length > 0) { - return { success: false, tweets: allReplies, nextCursor: cursor, error: response.error }; + const result = await paginateCursor({ + cursor: options.cursor, + maxPages, + pageDelayMs, + sleep: async (ms) => this.sleep(ms), + getKey: (tweet) => tweet.id, + fetchPage: async (cursor) => { + const response = await this.fetchTweetDetail(tweetId, cursor); + if (!response.success) { + return response; } - return response; - } - pagesFetched += 1; - const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; - const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId); + const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId); + const pageCursor = extractCursorFromInstructions(instructions); - for (const reply of replies) { - if (seen.has(reply.id)) { - continue; - } - seen.add(reply.id); - allReplies.push(reply); - } + return { success: true as const, items: replies, cursor: pageCursor }; + }, + }); - const pageCursor = extractCursorFromInstructions(instructions); - if (!pageCursor || pageCursor === cursor) { - nextCursor = undefined; - break; - } - - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - - cursor = pageCursor; - nextCursor = pageCursor; + if (result.success) { + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; } - return { success: true, tweets: allReplies, nextCursor }; + if (result.items) { + return { success: false, tweets: result.items, nextCursor: result.nextCursor, error: result.error }; + } + + return { success: false, error: result.error }; } /** @@ -424,71 +402,50 @@ export function withTweetDetails { const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options; - const seen = new Set(); - const allTweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; let rootId: string | undefined; - while (true) { - // Add delay between pages (but not before the first page) - if (pagesFetched > 0 && pageDelayMs > 0) { - await this.sleep(pageDelayMs); - } - - const response = await this.fetchTweetDetail(tweetId, cursor); - if (!response.success) { - // If we have some tweets already, return them with the error - if (allTweets.length > 0) { - return { success: false, tweets: allTweets, nextCursor: cursor, error: response.error }; + const result = await paginateCursor({ + cursor: options.cursor, + maxPages, + pageDelayMs, + sleep: async (ms) => this.sleep(ms), + getKey: (tweet) => tweet.id, + fetchPage: async (cursor) => { + const response = await this.fetchTweetDetail(tweetId, cursor); + if (!response.success) { + return response; } - return response; - } - pagesFetched += 1; - const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; - const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - // Determine root conversation ID from first page - if (!rootId) { - const target = tweets.find((t) => t.id === tweetId); - rootId = target?.conversationId || tweetId; - } - - const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId); - - for (const tweet of threadTweets) { - if (seen.has(tweet.id)) { - continue; + if (!rootId) { + const target = tweets.find((t) => t.id === tweetId); + rootId = target?.conversationId || tweetId; } - seen.add(tweet.id); - allTweets.push(tweet); - } - const pageCursor = extractCursorFromInstructions(instructions); - if (!pageCursor || pageCursor === cursor) { - nextCursor = undefined; - break; - } + const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId); + const pageCursor = extractCursorFromInstructions(instructions); - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } + return { success: true as const, items: threadTweets, cursor: pageCursor }; + }, + }); - cursor = pageCursor; - nextCursor = pageCursor; - } - - // Sort by creation time - allTweets.sort((a, b) => { + const sortedTweets = (result.items ?? []).slice().sort((a, b) => { const aTime = a.createdAt ? Date.parse(a.createdAt) : 0; const bTime = b.createdAt ? Date.parse(b.createdAt) : 0; return aTime - bTime; }); - return { success: true, tweets: allTweets, nextCursor }; + if (result.success) { + return { success: true, tweets: sortedTweets, nextCursor: result.nextCursor }; + } + + if (result.items) { + return { success: false, tweets: sortedTweets, nextCursor: result.nextCursor, error: result.error }; + } + + return { success: false, error: result.error }; } } diff --git a/src/lib/twitter-client-types.ts b/src/lib/twitter-client-types.ts index c27c80c..eff281a 100644 --- a/src/lib/twitter-client-types.ts +++ b/src/lib/twitter-client-types.ts @@ -253,13 +253,20 @@ export interface GetTweetResult { error?: string; } -export interface SearchResult { - success: boolean; - tweets?: TweetData[]; - error?: string; - /** Cursor for fetching the next page of results */ - nextCursor?: string; -} +export type SearchResult = + | { + success: true; + tweets: TweetData[]; + /** Cursor for fetching the next page of results */ + nextCursor?: string; + } + | { + success: false; + error: string; + tweets?: TweetData[]; + /** Cursor for fetching the next page of results */ + nextCursor?: string; + }; export interface CurrentUserResult { success: boolean; diff --git a/src/lib/twitter-client-user-tweets.ts b/src/lib/twitter-client-user-tweets.ts index 42b2bcc..2660273 100644 --- a/src/lib/twitter-client-user-tweets.ts +++ b/src/lib/twitter-client-user-tweets.ts @@ -40,10 +40,6 @@ export function withUserTweets { - await new Promise((resolve) => setTimeout(resolve, ms)); - } - /** * Get tweets from a user's profile timeline (single page). */