feat: add pagination support for thread and replies commands

Add --all, --max-pages, --delay, and --cursor options to both thread
and replies commands, enabling fetching of all conversation content
with rate-limit-friendly pagination.

Changes:
- Modified fetchTweetDetail() to accept optional cursor parameter
- Added getRepliesPaged() and getThreadPaged() methods with pagination loop
- Updated CLI commands with new options (--all, --max-pages, --delay, --cursor)
- Added TweetDetailPaginationOptions interface
- JSON output includes nextCursor for scripting/resumption

Options:
- --all: Fetch all pages (no hard limit)
- --max-pages <n>: Limit number of pages when using --all
- --delay <ms>: Delay between page fetches (default: 1000ms)
- --cursor <string>: Resume from a previous cursor

Follows patterns established by bookmarks command and PR #34 (user-tweets).

Co-authored-by: Christian Catalan <[email protected]>
This commit is contained in:
cc-vps
2026-01-12 02:35:36 +00:00
committed by Peter Steinberger
co-authored by Christian Catalan
parent c7e2ec0495
commit 90037ce880
4 changed files with 852 additions and 51 deletions
+151 -49
View File
@@ -49,72 +49,174 @@ export function registerReadCommands(program: Command, ctx: CliContext): void {
.command('replies')
.description('List replies to a tweet (by ID or URL)')
.argument('<tweet-id-or-url>', 'Tweet ID or URL')
.option('--all', 'Fetch all replies (paged)')
.option('--max-pages <number>', 'Stop after N pages when using --all')
.option('--delay <ms>', 'Delay in ms between page fetches (default: 1000)')
.option('--cursor <string>', 'Resume pagination from a cursor')
.option('--json', 'Output as JSON')
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
.action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => {
const opts = program.opts();
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
.action(
async (
tweetIdOrUrl: string,
cmdOpts: {
all?: boolean;
maxPages?: string;
delay?: string;
cursor?: string;
json?: boolean;
jsonFull?: boolean;
},
) => {
const opts = program.opts();
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000;
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
const usePagination = cmdOpts.all || cmdOpts.cursor;
if (maxPages !== undefined && !usePagination) {
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
process.exit(1);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
process.exit(1);
}
for (const warning of warnings) {
console.error(`${ctx.p('warn')}${warning}`);
}
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
if (!cookies.authToken || !cookies.ct0) {
console.error(`${ctx.p('err')}Missing required credentials`);
process.exit(1);
}
for (const warning of warnings) {
console.error(`${ctx.p('warn')}${warning}`);
}
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = await client.getReplies(tweetId, { includeRaw });
if (!cookies.authToken || !cookies.ct0) {
console.error(`${ctx.p('err')}Missing required credentials`);
process.exit(1);
}
if (result.success && result.tweets) {
ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No replies found.' });
} else {
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
process.exit(1);
}
});
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = usePagination
? await client.getRepliesPaged(tweetId, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
})
: await client.getReplies(tweetId, { includeRaw });
if (result.success && result.tweets) {
const isJson = cmdOpts.json || cmdOpts.jsonFull;
if (isJson && usePagination) {
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
} else {
ctx.printTweets(result.tweets, { json: isJson, emptyMessage: 'No replies found.' });
}
// Show pagination hint if there's more
if (result.nextCursor && !isJson) {
console.error(`${ctx.p('info')}More replies available. Use --cursor "${result.nextCursor}" to continue.`);
}
} else {
console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`);
process.exit(1);
}
},
);
program
.command('thread')
.description('Show the full conversation thread containing the tweet')
.argument('<tweet-id-or-url>', 'Tweet ID or URL')
.option('--all', 'Fetch all thread tweets (paged)')
.option('--max-pages <number>', 'Stop after N pages when using --all')
.option('--delay <ms>', 'Delay in ms between page fetches (default: 1000)')
.option('--cursor <string>', 'Resume pagination from a cursor')
.option('--json', 'Output as JSON')
.option('--json-full', 'Output as JSON with full raw API response in _raw field')
.action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => {
const opts = program.opts();
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
.action(
async (
tweetIdOrUrl: string,
cmdOpts: {
all?: boolean;
maxPages?: string;
delay?: string;
cursor?: string;
json?: boolean;
jsonFull?: boolean;
},
) => {
const opts = program.opts();
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const tweetId = ctx.extractTweetId(tweetIdOrUrl);
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = cmdOpts.delay ? Number.parseInt(cmdOpts.delay, 10) : 1000;
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
const usePagination = cmdOpts.all || cmdOpts.cursor;
if (maxPages !== undefined && !usePagination) {
console.error(`${ctx.p('err')}--max-pages requires --all or --cursor.`);
process.exit(1);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer.`);
process.exit(1);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
console.error(`${ctx.p('err')}Invalid --delay. Expected a non-negative integer.`);
process.exit(1);
}
for (const warning of warnings) {
console.error(`${ctx.p('warn')}${warning}`);
}
const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts);
if (!cookies.authToken || !cookies.ct0) {
console.error(`${ctx.p('err')}Missing required credentials`);
process.exit(1);
}
for (const warning of warnings) {
console.error(`${ctx.p('warn')}${warning}`);
}
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = await client.getThread(tweetId, { includeRaw });
if (!cookies.authToken || !cookies.ct0) {
console.error(`${ctx.p('err')}Missing required credentials`);
process.exit(1);
}
if (result.success && result.tweets) {
ctx.printTweets(result.tweets, {
json: cmdOpts.json || cmdOpts.jsonFull,
emptyMessage: 'No thread tweets found.',
});
} else {
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
process.exit(1);
}
});
const client = new TwitterClient({ cookies, timeoutMs, quoteDepth });
const includeRaw = cmdOpts.jsonFull ?? false;
const result = usePagination
? await client.getThreadPaged(tweetId, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
})
: await client.getThread(tweetId, { includeRaw });
if (result.success && result.tweets) {
const isJson = cmdOpts.json || cmdOpts.jsonFull;
if (isJson && usePagination) {
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
} else {
ctx.printTweets(result.tweets, {
json: isJson,
emptyMessage: 'No thread tweets found.',
});
}
// Show pagination hint if there's more
if (result.nextCursor && !isJson) {
console.error(
`${ctx.p('info')}More thread tweets available. Use --cursor "${result.nextCursor}" to continue.`,
);
}
} else {
console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`);
process.exit(1);
}
},
);
}
+152 -2
View File
@@ -1,9 +1,10 @@
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
import { TWITTER_API_BASE } from './twitter-client-constants.js';
import { buildArticleFeatures, buildArticleFieldToggles, buildTweetDetailFeatures } from './twitter-client-features.js';
import type { GetTweetResult, GraphqlTweetResult, SearchResult } from './twitter-client-types.js';
import type { GetTweetResult, GraphqlTweetResult, SearchResult, TweetData } from './twitter-client-types.js';
import {
extractArticleText,
extractCursorFromInstructions,
findTweetInInstructions,
firstText,
mapTweetResult,
@@ -16,10 +17,22 @@ export interface TweetFetchOptions {
includeRaw?: boolean;
}
/** Options for paginated tweet detail fetch */
export interface TweetDetailPaginationOptions extends TweetFetchOptions {
/** Maximum number of pages to fetch (default: unlimited when using pagination) */
maxPages?: number;
/** Starting cursor for pagination (resume from previous fetch) */
cursor?: string;
/** Delay in milliseconds between page fetches (default: 1000) */
pageDelayMs?: number;
}
export interface TwitterClientTweetDetailMethods {
getTweet(tweetId: string, options?: TweetFetchOptions): Promise<GetTweetResult>;
getReplies(tweetId: string, options?: TweetFetchOptions): Promise<SearchResult>;
getThread(tweetId: string, options?: TweetFetchOptions): Promise<SearchResult>;
getRepliesPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise<SearchResult>;
getThreadPaged(tweetId: string, options?: TweetDetailPaginationOptions): Promise<SearchResult>;
}
export function withTweetDetails<TBase extends AbstractConstructor<TwitterClientBase>>(
@@ -110,7 +123,14 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
return {};
}
private async fetchTweetDetail(tweetId: string): Promise<
private async sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
private async fetchTweetDetail(
tweetId: string,
cursor?: string,
): Promise<
| {
success: true;
data: {
@@ -141,6 +161,7 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
withQuickPromoteEligibilityTweetFields: true,
withBirdwatchNotes: true,
withVoice: true,
...(cursor ? { cursor } : {}),
};
const features = {
@@ -340,6 +361,135 @@ export function withTweetDetails<TBase extends AbstractConstructor<TwitterClient
return { success: true, tweets: thread };
}
/**
* Get replies to a tweet with pagination support
*/
async getRepliesPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
const seen = new Set<string>();
const allReplies: TweetData[] = [];
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
while (true) {
// Add delay between pages (but not before the first page)
if (pagesFetched > 0 && pageDelayMs > 0) {
await this.sleep(pageDelayMs);
}
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
// If we have some replies already, return them with the error
if (allReplies.length > 0) {
return { success: true, tweets: allReplies, nextCursor: cursor, error: response.error };
}
return response;
}
pagesFetched += 1;
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId);
for (const reply of replies) {
if (seen.has(reply.id)) {
continue;
}
seen.add(reply.id);
allReplies.push(reply);
}
const pageCursor = extractCursorFromInstructions(instructions);
if (!pageCursor || pageCursor === cursor || replies.length === 0) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
}
return { success: true, tweets: allReplies, nextCursor };
}
/**
* Get full conversation thread with pagination support
*/
async getThreadPaged(tweetId: string, options: TweetDetailPaginationOptions = {}): Promise<SearchResult> {
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
const seen = new Set<string>();
const allTweets: TweetData[] = [];
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
let rootId: string | undefined;
while (true) {
// Add delay between pages (but not before the first page)
if (pagesFetched > 0 && pageDelayMs > 0) {
await this.sleep(pageDelayMs);
}
const response = await this.fetchTweetDetail(tweetId, cursor);
if (!response.success) {
// If we have some tweets already, return them with the error
if (allTweets.length > 0) {
return { success: true, tweets: allTweets, nextCursor: cursor, error: response.error };
}
return response;
}
pagesFetched += 1;
const instructions = response.data.threaded_conversation_with_injections_v2?.instructions;
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
// Determine root conversation ID from first page
if (!rootId) {
const target = tweets.find((t) => t.id === tweetId);
rootId = target?.conversationId || tweetId;
}
const threadTweets = tweets.filter((tweet) => tweet.conversationId === rootId);
for (const tweet of threadTweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
allTweets.push(tweet);
}
const pageCursor = extractCursorFromInstructions(instructions);
if (!pageCursor || pageCursor === cursor || threadTweets.length === 0) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
}
// Sort by creation time
allTweets.sort((a, b) => {
const aTime = a.createdAt ? Date.parse(a.createdAt) : 0;
const bTime = b.createdAt ? Date.parse(b.createdAt) : 0;
return aTime - bTime;
});
return { success: true, tweets: allTweets, nextCursor };
}
}
return TwitterClientTweetDetails;
+152
View File
@@ -0,0 +1,152 @@
import { Command } from 'commander';
import { describe, expect, it, vi } from 'vitest';
import type { CliContext } from '../src/cli/shared.js';
import { registerReadCommands } from '../src/commands/read.js';
describe('replies command', () => {
const createMockContext = () =>
({
resolveTimeoutFromOptions: () => undefined,
resolveQuoteDepthFromOptions: () => 1,
extractTweetId: (input: string) => input,
resolveCredentialsFromOptions: async () => ({
cookies: { authToken: 'auth', ct0: 'ct0', cookieHeader: 'auth=auth; ct0=ct0' },
warnings: [],
}),
p: () => '',
printTweets: () => undefined,
}) as unknown as CliContext;
it('requires --all or --cursor when --max-pages is provided', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(program.parseAsync(['node', 'bird', 'replies', '123', '--max-pages', '2'])).rejects.toThrow(
'exit 1',
);
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
it('validates --max-pages is a positive integer', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(
program.parseAsync(['node', 'bird', 'replies', '123', '--all', '--max-pages', '-1']),
).rejects.toThrow('exit 1');
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --max-pages'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
it('validates --delay is a non-negative integer', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(program.parseAsync(['node', 'bird', 'replies', '123', '--all', '--delay', '-100'])).rejects.toThrow(
'exit 1',
);
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --delay'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
});
describe('thread command', () => {
const createMockContext = () =>
({
resolveTimeoutFromOptions: () => undefined,
resolveQuoteDepthFromOptions: () => 1,
extractTweetId: (input: string) => input,
resolveCredentialsFromOptions: async () => ({
cookies: { authToken: 'auth', ct0: 'ct0', cookieHeader: 'auth=auth; ct0=ct0' },
warnings: [],
}),
p: () => '',
printTweets: () => undefined,
}) as unknown as CliContext;
it('requires --all or --cursor when --max-pages is provided', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--max-pages', '2'])).rejects.toThrow('exit 1');
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--max-pages requires --all or --cursor'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
it('validates --max-pages is a positive integer', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--all', '--max-pages', '0'])).rejects.toThrow(
'exit 1',
);
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --max-pages'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
it('validates --delay is a non-negative integer', async () => {
const program = new Command();
registerReadCommands(program, createMockContext());
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit ${code}`);
}) as never);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
try {
await expect(program.parseAsync(['node', 'bird', 'thread', '123', '--all', '--delay', 'abc'])).rejects.toThrow(
'exit 1',
);
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid --delay'));
} finally {
exitSpy.mockRestore();
errorSpy.mockRestore();
}
});
});
+397
View File
@@ -0,0 +1,397 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { TwitterClient } from '../src/lib/twitter-client.js';
import { validCookies } from './twitter-client-fixtures.js';
const originalFetch = global.fetch;
afterEach(() => {
global.fetch = originalFetch;
vi.restoreAllMocks();
});
describe('TwitterClient pagination for thread and replies', () => {
let mockFetch: ReturnType<typeof vi.fn>;
beforeEach(() => {
mockFetch = vi.fn();
global.fetch = mockFetch as unknown as typeof fetch;
});
const makeConversationPayload = (tweetIds: string[], cursor?: string, inReplyTo?: string, conversationId = '1') => ({
data: {
threaded_conversation_with_injections_v2: {
instructions: [
{
type: 'TimelineAddEntries',
entries: [
...tweetIds.map((id, i) => ({
entryId: `tweet-${id}`,
content: {
itemContent: {
tweet_results: {
result: {
rest_id: id,
legacy: {
full_text: `tweet ${id}`,
created_at: `2024-01-0${i + 1}T00:00:00Z`,
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: conversationId,
...(inReplyTo ? { in_reply_to_status_id_str: inReplyTo } : {}),
},
core: {
user_results: { result: { legacy: { screen_name: `user${id}`, name: `User ${id}` } } },
},
},
},
},
},
})),
// Add cursor entry if provided
...(cursor
? [
{
entryId: `cursor-bottom-${Date.now()}`,
content: {
cursorType: 'Bottom',
value: cursor,
},
},
]
: []),
],
},
],
},
},
});
describe('getRepliesPaged', () => {
it('returns replies from single page when no cursor in response', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: async () => makeConversationPayload(['2', '3'], undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1');
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(2);
expect(result.nextCursor).toBeUndefined();
expect(mockFetch).toHaveBeenCalledTimes(1);
});
it('fetches multiple pages when cursor is present and --all is used', async () => {
// First page returns cursor, second page does not
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['4', '5'], undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1', { pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(4);
expect(result.tweets?.map((t) => t.id)).toEqual(['2', '3', '4', '5']);
expect(result.nextCursor).toBeUndefined();
expect(mockFetch).toHaveBeenCalledTimes(2);
});
it('respects maxPages option', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['4', '5'], 'cursor-page-3', '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1', { maxPages: 2, pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(4);
expect(result.nextCursor).toBe('cursor-page-3');
expect(mockFetch).toHaveBeenCalledTimes(2);
});
it('resumes from provided cursor', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: async () => makeConversationPayload(['4', '5'], undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1', { cursor: 'resume-cursor', pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(2);
// Verify cursor was passed in the request
const url = mockFetch.mock.calls[0][0] as string;
expect(url).toContain('resume-cursor');
});
it('deduplicates tweets across pages', async () => {
// Second page returns a duplicate tweet
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['3', '4'], undefined, '1'), // '3' is duplicate
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1', { pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(3); // 2, 3, 4 (no duplicate)
expect(result.tweets?.map((t) => t.id)).toEqual(['2', '3', '4']);
});
it('returns partial results on error after first page', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['2', '3'], 'cursor-page-2', '1'),
})
.mockResolvedValueOnce({
ok: false,
status: 500,
text: async () => 'Internal Server Error',
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getRepliesPaged('1', { pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(2);
expect(result.error).toContain('500');
expect(result.nextCursor).toBe('cursor-page-2');
});
});
describe('getThreadPaged', () => {
it('returns thread tweets from single page', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: async () => makeConversationPayload(['1', '2', '3'], undefined, undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getThreadPaged('1');
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(3);
expect(result.nextCursor).toBeUndefined();
});
it('fetches multiple pages for thread', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['1', '2'], 'cursor-page-2', undefined, '1'),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['3', '4'], undefined, undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getThreadPaged('1', { pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(4);
expect(mockFetch).toHaveBeenCalledTimes(2);
});
it('sorts thread tweets by creation time', async () => {
// Return tweets in non-chronological order
const payload = {
data: {
threaded_conversation_with_injections_v2: {
instructions: [
{
type: 'TimelineAddEntries',
entries: [
{
entryId: 'tweet-3',
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '3',
legacy: {
full_text: 'tweet 3',
created_at: '2024-01-03T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: { user_results: { result: { legacy: { screen_name: 'user3', name: 'User 3' } } } },
},
},
},
},
},
{
entryId: 'tweet-1',
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'tweet 1',
created_at: '2024-01-01T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: { user_results: { result: { legacy: { screen_name: 'user1', name: 'User 1' } } } },
},
},
},
},
},
],
},
],
},
},
};
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: async () => payload,
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getThreadPaged('1');
expect(result.success).toBe(true);
expect(result.tweets?.map((t) => t.id)).toEqual(['1', '3']); // Sorted by date
});
it('filters by conversation ID', async () => {
// Include a tweet from a different conversation
const payload = {
data: {
threaded_conversation_with_injections_v2: {
instructions: [
{
type: 'TimelineAddEntries',
entries: [
{
entryId: 'tweet-1',
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '1',
legacy: {
full_text: 'tweet 1',
created_at: '2024-01-01T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '1',
},
core: { user_results: { result: { legacy: { screen_name: 'user1', name: 'User 1' } } } },
},
},
},
},
},
{
entryId: 'tweet-99',
content: {
itemContent: {
tweet_results: {
result: {
rest_id: '99',
legacy: {
full_text: 'different conversation',
created_at: '2024-01-02T00:00:00Z',
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
conversation_id_str: '99', // Different conversation
},
core: { user_results: { result: { legacy: { screen_name: 'user99', name: 'User 99' } } } },
},
},
},
},
},
],
},
],
},
},
};
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: async () => payload,
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getThreadPaged('1');
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(1);
expect(result.tweets?.[0].id).toBe('1');
});
it('respects maxPages for thread', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['1', '2'], 'cursor-page-2', undefined, '1'),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => makeConversationPayload(['3', '4'], 'cursor-page-3', undefined, '1'),
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getThreadPaged('1', { maxPages: 2, pageDelayMs: 0 });
expect(result.success).toBe(true);
expect(result.tweets?.length).toBe(4);
expect(result.nextCursor).toBe('cursor-page-3');
expect(mockFetch).toHaveBeenCalledTimes(2);
});
});
});