yuta/bird
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: paginate user-tweets by --count

Browse Source
This commit is contained in:
Peter Steinberger
2026-01-12 01:35:00 +00:00
parent df98e827f6
commit 4eac0927c1
4 changed files with 69 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/commands/user-tweets.ts
+26 -28
View File
@@ -12,7 +12,7 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
.description("Get tweets from a user's profile timeline")
.argument('<handle>', 'Username to fetch tweets from (e.g., @steipete or steipete)')
.option('-n, --count <number>', 'Number of tweets to fetch', '20')
.option('--pages <number>', 'Number of pages to fetch (max: 10)', '1')
.option('--max-pages <number>', 'Stop after N pages (max: 10)')
.option('--delay <ms>', 'Delay in ms between page fetches', '1000')
.option('--cursor <string>', 'Resume pagination from a cursor')
.option('--json', 'Output as JSON')
@@ -23,7 +23,8 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
`\n${ctx.colors.section('Command Examples')}\n${[
formatExample('bird user-tweets @steipete', 'Get recent tweets from a user'),
formatExample('bird user-tweets steipete -n 10', 'Get 10 tweets (@ is optional)'),
formatExample('bird user-tweets @steipete --pages 3', 'Fetch 3 pages of tweets'),
formatExample('bird user-tweets @steipete -n 50', 'Fetch 50 tweets (paged)'),
formatExample('bird user-tweets @steipete --max-pages 3 -n 200', 'Safety cap (max 3 pages)'),
formatExample('bird user-tweets @steipete --json', 'Output as JSON'),
formatExample('bird user-tweets @steipete --cursor "DAABCg..."', 'Resume from cursor'),
].join('\n')}`,
@@ -33,7 +34,7 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
handle: string,
cmdOpts: {
count?: string;
pages?: string;
maxPages?: string;
delay?: string;
cursor?: string;
json?: boolean;
@@ -44,7 +45,7 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
const timeoutMs = ctx.resolveTimeoutFromOptions(opts);
const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts);
const count = Number.parseInt(cmdOpts.count || '20', 10);
const maxPages = Math.min(Number.parseInt(cmdOpts.pages || '1', 10), 10); // Hard cap at 10 pages
const maxPages = cmdOpts.maxPages ? Number.parseInt(cmdOpts.maxPages, 10) : undefined;
const pageDelayMs = Number.parseInt(cmdOpts.delay || '1000', 10);
// Validate inputs
@@ -52,8 +53,17 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`);
process.exit(2);
}
if (!Number.isFinite(maxPages) || maxPages <= 0) {
console.error(`${ctx.p('err')}Invalid --pages. Expected a positive integer (max: 10).`);
const pageSize = 20;
const hardMaxPages = 10;
const hardMaxTweets = pageSize * hardMaxPages;
if (count > hardMaxTweets) {
console.error(
`${ctx.p('err')}Invalid --count. Max ${hardMaxTweets} tweets per run (safety cap: ${hardMaxPages} pages). Use --cursor to continue.`,
);
process.exit(2);
}
if (maxPages !== undefined && (!Number.isFinite(maxPages) || maxPages <= 0 || maxPages > hardMaxPages)) {
console.error(`${ctx.p('err')}Invalid --max-pages. Expected a positive integer (max: ${hardMaxPages}).`);
process.exit(2);
}
if (!Number.isFinite(pageDelayMs) || pageDelayMs < 0) {
@@ -95,33 +105,21 @@ export function registerUserTweetsCommand(program: Command, ctx: CliContext): vo
console.error(`${ctx.p('info')}Fetching tweets from ${displayName}...`);
const includeRaw = cmdOpts.jsonFull ?? false;
const usePagination = maxPages > 1 || cmdOpts.cursor;
let result: Awaited<ReturnType<typeof client.getUserTweetsPaged>>;
if (usePagination) {
result = await client.getUserTweetsPaged(userLookup.userId, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
});
} else {
result = await client.getUserTweets(userLookup.userId, count, { includeRaw });
}
const wantsPaginationOutput = Boolean(cmdOpts.cursor) || maxPages !== undefined || count > pageSize;
const result = await client.getUserTweetsPaged(userLookup.userId, count, {
includeRaw,
maxPages,
cursor: cmdOpts.cursor,
pageDelayMs,
});
if (result.success && result.tweets) {
// Trim to count if single page
let tweets = result.tweets;
if (!usePagination && tweets.length > count) {
tweets = tweets.slice(0, count);
}
const isJson = cmdOpts.json || cmdOpts.jsonFull;
if (isJson && usePagination) {
if (isJson && wantsPaginationOutput) {
// Include nextCursor for pagination consumers
console.log(JSON.stringify({ tweets, nextCursor: result.nextCursor ?? null }, null, 2));
console.log(JSON.stringify({ tweets: result.tweets, nextCursor: result.nextCursor ?? null }, null, 2));
} else {
ctx.printTweets(tweets, {
ctx.printTweets(result.tweets, {
json: isJson,
emptyMessage: `No tweets found for @${username}.`,
});
src/lib/twitter-client-user-tweets.ts
+36 -27
View File
@@ -33,7 +33,7 @@ export interface UserLookupResult {
export interface TwitterClientUserTweetsMethods {
getUserIdByUsername(username: string): Promise<UserLookupResult>;
getUserTweets(userId: string, count?: number, options?: UserTweetsFetchOptions): Promise<SearchResult>;
getUserTweetsPaged(userId: string, options?: UserTweetsPaginationOptions): Promise<SearchResult>;
getUserTweetsPaged(userId: string, limit: number, options?: UserTweetsPaginationOptions): Promise<SearchResult>;
}
export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBase>>(
@@ -169,7 +169,7 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
/**
* Look up a user's ID by their username/handle.
* Uses Twitter's REST API endpoint.
* Uses GraphQL UserByScreenName first, then falls back to REST on transient failures.
*/
async getUserIdByUsername(username: string): Promise<UserLookupResult> {
// Normalize and validate handle
@@ -245,24 +245,22 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
* Get tweets from a user's profile timeline (single page).
*/
async getUserTweets(userId: string, count = 20, options: UserTweetsFetchOptions = {}): Promise<SearchResult> {
const result = await this.getUserTweetsPaged(userId, {
...options,
maxPages: 1,
});
// Trim to requested count
if (result.success && result.tweets && result.tweets.length > count) {
result.tweets = result.tweets.slice(0, count);
}
return result;
return this.getUserTweetsPaged(userId, count, options);
}
/**
* Get tweets from a user's profile timeline with pagination support.
*/
async getUserTweetsPaged(userId: string, options: UserTweetsPaginationOptions = {}): Promise<SearchResult> {
const { includeRaw = false, maxPages = 1, pageDelayMs = 1000 } = options;
async getUserTweetsPaged(
userId: string,
limit: number,
options: UserTweetsPaginationOptions = {},
): Promise<SearchResult> {
if (!Number.isFinite(limit) || limit <= 0) {
return { success: false, error: `Invalid limit: ${limit}` };
}
const { includeRaw = false, maxPages, pageDelayMs = 1000 } = options;
const features = buildUserTweetsFeatures();
const pageSize = 20;
const seen = new Set<string>();
@@ -270,15 +268,18 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
const hardMaxPages = 10;
const computedMaxPages = Math.max(1, Math.ceil(limit / pageSize));
const effectiveMaxPages = Math.min(hardMaxPages, maxPages ?? computedMaxPages);
const fetchPage = async (pageCursor?: string) => {
const fetchPage = async (pageCount: number, pageCursor?: string) => {
let lastError: string | undefined;
let had404 = false;
const queryIds = await this.getUserTweetsQueryIds();
const variables = {
userId,
count: pageSize,
count: pageCount,
includePromotedContent: false, // Filter out ads
withQuickPromoteEligibilityTweetFields: true,
withVoice: true,
@@ -396,14 +397,14 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
return { success: false as const, error: lastError ?? 'Unknown error fetching user tweets', had404 };
};
const fetchWithRefresh = async (pageCursor?: string) => {
const firstAttempt = await fetchPage(pageCursor);
const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
const firstAttempt = await fetchPage(pageCount, pageCursor);
if (firstAttempt.success) {
return firstAttempt;
}
if (firstAttempt.had404) {
await this.refreshQueryIds();
const secondAttempt = await fetchPage(pageCursor);
const secondAttempt = await fetchPage(pageCount, pageCursor);
if (secondAttempt.success) {
return secondAttempt;
}
@@ -412,36 +413,44 @@ export function withUserTweets<TBase extends AbstractConstructor<TwitterClientBa
return { success: false as const, error: firstAttempt.error };
};
while (pagesFetched < maxPages) {
while (tweets.length < limit) {
// Add delay between pages (but not before the first page)
if (pagesFetched > 0 && pageDelayMs > 0) {
await this.sleep(pageDelayMs);
}
const page = await fetchWithRefresh(cursor);
const remaining = limit - tweets.length;
const pageCount = Math.min(pageSize, remaining);
const page = await fetchWithRefresh(pageCount, cursor);
if (!page.success) {
// If we have some tweets already, return them with the error
if (tweets.length > 0) {
return { success: true, tweets, nextCursor: cursor, error: page.error };
}
return { success: false, error: page.error };
}
pagesFetched += 1;
let added = 0;
for (const tweet of page.tweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
tweets.push(tweet);
added += 1;
if (tweets.length >= limit) {
break;
}
}
const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
if (pagesFetched >= effectiveMaxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
}
src/lib/twitter-client.ts
+1 -1
View File
@@ -26,7 +26,7 @@ type TwitterClientInstance = TwitterClientBase &
const MixedTwitterClient = withUserTweets(
withUsers(
withLists(
withHome(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase)))))))),
withHome(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase))))))),
),
),
) as AbstractConstructor<TwitterClientInstance>;
tests/twitter-client.user-tweets.test.ts
+6 -6
View File
@@ -299,7 +299,7 @@ describe('TwitterClient getUserTweets', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', { maxPages: 1 });
const result = await client.getUserTweetsPaged('12345', 20, { maxPages: 1 });
expect(result.success).toBe(true);
expect(result.nextCursor).toBe('next-cursor-abc');
@@ -408,7 +408,7 @@ describe('TwitterClient getUserTweetsPaged', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', {
const result = await client.getUserTweetsPaged('12345', 40, {
maxPages: 2,
pageDelayMs: 0, // No delay for tests
});
@@ -435,7 +435,7 @@ describe('TwitterClient getUserTweetsPaged', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', {
const result = await client.getUserTweetsPaged('12345', 40, {
maxPages: 2,
pageDelayMs: 0,
});
@@ -459,7 +459,7 @@ describe('TwitterClient getUserTweetsPaged', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', {
const result = await client.getUserTweetsPaged('12345', 100, {
maxPages: 5,
pageDelayMs: 0,
});
@@ -488,7 +488,7 @@ describe('TwitterClient getUserTweetsPaged', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', {
const result = await client.getUserTweetsPaged('12345', 40, {
maxPages: 2,
pageDelayMs: 0,
});
@@ -506,7 +506,7 @@ describe('TwitterClient getUserTweetsPaged', () => {
});
const client = new TwitterClient({ cookies: validCookies });
const result = await client.getUserTweetsPaged('12345', {
const result = await client.getUserTweetsPaged('12345', 20, {
maxPages: 1,
cursor: 'resume-cursor-xyz',
pageDelayMs: 0,