feat: add pagination support to list-timeline command
Adds --all, --max-pages, and --cursor options to list-timeline command, enabling users to fetch all tweets from a list through automatic pagination. Follows the same pattern as bookmarks pagination for consistency. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
This commit is contained in:
committed by
Peter Steinberger
co-authored by
Claude Sonnet 4.5
parent
bda980fa4d
commit
803bd25b0c
@@ -4,14 +4,15 @@
|
||||
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
|
||||
import { TWITTER_API_BASE } from './twitter-client-constants.js';
|
||||
import { buildListsFeatures } from './twitter-client-features.js';
|
||||
import type { TimelineFetchOptions } from './twitter-client-timelines.js';
|
||||
import type { GraphqlTweetResult, ListsResult, SearchResult, TwitterList } from './twitter-client-types.js';
|
||||
import { parseTweetsFromInstructions } from './twitter-client-utils.js';
|
||||
import type { TimelineFetchOptions, TimelinePaginationOptions } from './twitter-client-timelines.js';
|
||||
import type { GraphqlTweetResult, ListsResult, SearchResult, TweetData, TwitterList } from './twitter-client-types.js';
|
||||
import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js';
|
||||
|
||||
export interface TwitterClientListMethods {
|
||||
getOwnedLists(count?: number): Promise<ListsResult>;
|
||||
getListMemberships(count?: number): Promise<ListsResult>;
|
||||
getListTimeline(listId: string, count?: number, options?: TimelineFetchOptions): Promise<SearchResult>;
|
||||
getAllListTimeline(listId: string, options?: TimelinePaginationOptions): Promise<SearchResult>;
|
||||
}
|
||||
|
||||
interface GraphqlListResult {
|
||||
@@ -329,25 +330,49 @@ export function withLists<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
* Get tweets from a list timeline
|
||||
*/
|
||||
async getListTimeline(listId: string, count = 20, options: TimelineFetchOptions = {}): Promise<SearchResult> {
|
||||
const { includeRaw = false } = options;
|
||||
return this.getListTimelinePaged(listId, count, options);
|
||||
}
|
||||
|
||||
const variables = {
|
||||
listId,
|
||||
count,
|
||||
};
|
||||
/**
|
||||
* Get all tweets from a list timeline (paginated)
|
||||
*/
|
||||
async getAllListTimeline(listId: string, options?: TimelinePaginationOptions): Promise<SearchResult> {
|
||||
return this.getListTimelinePaged(listId, Number.POSITIVE_INFINITY, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal paginated list timeline fetcher
|
||||
*/
|
||||
private async getListTimelinePaged(
|
||||
listId: string,
|
||||
limit: number,
|
||||
options: TimelinePaginationOptions = {},
|
||||
): Promise<SearchResult> {
|
||||
const features = buildListsFeatures();
|
||||
const pageSize = 20;
|
||||
const seen = new Set<string>();
|
||||
const tweets: TweetData[] = [];
|
||||
let cursor: string | undefined = options.cursor;
|
||||
let nextCursor: string | undefined;
|
||||
let pagesFetched = 0;
|
||||
const { includeRaw = false, maxPages } = options;
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
const tryOnce = async () => {
|
||||
const fetchPage = async (pageCount: number, pageCursor?: string) => {
|
||||
let lastError: string | undefined;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getListTimelineQueryIds();
|
||||
|
||||
const variables = {
|
||||
listId,
|
||||
count: pageCount,
|
||||
...(pageCursor ? { cursor: pageCursor } : {}),
|
||||
};
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`;
|
||||
|
||||
@@ -396,9 +421,10 @@ export function withLists<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
}
|
||||
|
||||
const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions;
|
||||
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
const nextCursor = extractCursorFromInstructions(instructions);
|
||||
|
||||
return { success: true as const, tweets, had404 };
|
||||
return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@@ -407,21 +433,56 @@ export function withLists<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 };
|
||||
};
|
||||
|
||||
const firstAttempt = await tryOnce();
|
||||
if (firstAttempt.success) {
|
||||
return { success: true, tweets: firstAttempt.tweets };
|
||||
}
|
||||
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
const secondAttempt = await tryOnce();
|
||||
if (secondAttempt.success) {
|
||||
return { success: true, tweets: secondAttempt.tweets };
|
||||
const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
|
||||
const firstAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (firstAttempt.success) {
|
||||
return firstAttempt;
|
||||
}
|
||||
return { success: false, error: secondAttempt.error };
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
const secondAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (secondAttempt.success) {
|
||||
return secondAttempt;
|
||||
}
|
||||
return { success: false as const, error: secondAttempt.error };
|
||||
}
|
||||
return { success: false as const, error: firstAttempt.error };
|
||||
};
|
||||
|
||||
const unlimited = !Number.isFinite(limit);
|
||||
while (unlimited || tweets.length < limit) {
|
||||
const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
|
||||
const page = await fetchWithRefresh(pageCount, cursor);
|
||||
if (!page.success) {
|
||||
return { success: false, error: page.error };
|
||||
}
|
||||
pagesFetched += 1;
|
||||
|
||||
for (const tweet of page.tweets) {
|
||||
if (seen.has(tweet.id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(tweet.id);
|
||||
tweets.push(tweet);
|
||||
if (!unlimited && tweets.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const pageCursor = page.cursor;
|
||||
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0) {
|
||||
nextCursor = undefined;
|
||||
break;
|
||||
}
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
nextCursor = pageCursor;
|
||||
break;
|
||||
}
|
||||
cursor = pageCursor;
|
||||
nextCursor = pageCursor;
|
||||
}
|
||||
|
||||
return { success: false, error: firstAttempt.error };
|
||||
return { success: true, tweets, nextCursor };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user