Add paged bookmark export with gating and retries
This commit is contained in:
Haoli
+321
-129
@@ -1,8 +1,8 @@
|
||||
import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js';
|
||||
import { TWITTER_API_BASE } from './twitter-client-constants.js';
|
||||
import { buildBookmarksFeatures, buildLikesFeatures } from './twitter-client-features.js';
|
||||
import type { GraphqlTweetResult, SearchResult } from './twitter-client-types.js';
|
||||
import { parseTweetsFromInstructions } from './twitter-client-utils.js';
|
||||
import type { GraphqlTweetResult, SearchResult, TweetData } from './twitter-client-types.js';
|
||||
import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js';
|
||||
|
||||
/** Options for timeline fetch methods */
|
||||
export interface TimelineFetchOptions {
|
||||
@@ -10,10 +10,17 @@ export interface TimelineFetchOptions {
|
||||
includeRaw?: boolean;
|
||||
}
|
||||
|
||||
/** Options for paged timeline fetch methods */
|
||||
export interface TimelinePaginationOptions extends TimelineFetchOptions {
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface TwitterClientTimelineMethods {
|
||||
getBookmarks(count?: number, options?: TimelineFetchOptions): Promise<SearchResult>;
|
||||
getAllBookmarks(options?: TimelinePaginationOptions): Promise<SearchResult>;
|
||||
getLikes(count?: number, options?: TimelineFetchOptions): Promise<SearchResult>;
|
||||
getBookmarkFolderTimeline(folderId: string, count?: number, options?: TimelineFetchOptions): Promise<SearchResult>;
|
||||
getAllBookmarkFolderTimeline(folderId: string, options?: TimelinePaginationOptions): Promise<SearchResult>;
|
||||
}
|
||||
|
||||
export function withTimelines<TBase extends AbstractConstructor<TwitterClientBase>>(
|
||||
@@ -25,6 +32,17 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
super(...args);
|
||||
}
|
||||
|
||||
private logBookmarksDebug(message: string, data?: Record<string, unknown>): void {
|
||||
if (process.env.BIRD_DEBUG_BOOKMARKS !== '1') {
|
||||
return;
|
||||
}
|
||||
if (data) {
|
||||
console.error(`[bird][debug][bookmarks] ${message}`, JSON.stringify(data));
|
||||
} else {
|
||||
console.error(`[bird][debug][bookmarks] ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async getBookmarksQueryIds(): Promise<string[]> {
|
||||
const primary = await this.getQueryId('Bookmarks');
|
||||
return Array.from(new Set([primary, 'RV1g3b8n_SGOHwkqKYSCFw', 'tmd4ifV8RHltzn8ymGg1aw']));
|
||||
@@ -44,99 +62,11 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
* Get the authenticated user's bookmarks
|
||||
*/
|
||||
async getBookmarks(count = 20, options: TimelineFetchOptions = {}): Promise<SearchResult> {
|
||||
const { includeRaw = false } = options;
|
||||
const variables = {
|
||||
count,
|
||||
includePromotedContent: false,
|
||||
withDownvotePerspective: false,
|
||||
withReactionsMetadata: false,
|
||||
withReactionsPerspective: false,
|
||||
};
|
||||
return this.getBookmarksPaged(count, options);
|
||||
}
|
||||
|
||||
const features = buildBookmarksFeatures();
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
const tryOnce = async () => {
|
||||
let lastError: string | undefined;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getBookmarksQueryIds();
|
||||
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/Bookmarks?${params.toString()}`;
|
||||
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
|
||||
if (response.status === 404) {
|
||||
had404 = true;
|
||||
lastError = `HTTP ${response.status}`;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
data?: {
|
||||
bookmark_timeline_v2?: {
|
||||
timeline?: {
|
||||
instructions?: Array<{
|
||||
entries?: Array<{
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
};
|
||||
errors?: Array<{ message: string }>;
|
||||
};
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
|
||||
}
|
||||
|
||||
const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions;
|
||||
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
|
||||
return { success: true as const, tweets, had404 };
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false as const, error: lastError ?? 'Unknown error fetching bookmarks', had404 };
|
||||
};
|
||||
|
||||
const firstAttempt = await tryOnce();
|
||||
if (firstAttempt.success) {
|
||||
return { success: true, tweets: firstAttempt.tweets };
|
||||
}
|
||||
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
const secondAttempt = await tryOnce();
|
||||
if (secondAttempt.success) {
|
||||
return { success: true, tweets: secondAttempt.tweets };
|
||||
}
|
||||
return { success: false, error: secondAttempt.error };
|
||||
}
|
||||
|
||||
return { success: false, error: firstAttempt.error };
|
||||
async getAllBookmarks(options?: TimelinePaginationOptions): Promise<SearchResult> {
|
||||
return this.getBookmarksPaged(Number.POSITIVE_INFINITY, options);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -256,24 +186,35 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
count = 20,
|
||||
options: TimelineFetchOptions = {},
|
||||
): Promise<SearchResult> {
|
||||
const { includeRaw = false } = options;
|
||||
const variablesWithCount = {
|
||||
bookmark_collection_id: folderId,
|
||||
includePromotedContent: true,
|
||||
count,
|
||||
};
|
||||
return this.getBookmarkFolderTimelinePaged(folderId, count, options);
|
||||
}
|
||||
|
||||
const variablesWithoutCount = {
|
||||
bookmark_collection_id: folderId,
|
||||
includePromotedContent: true,
|
||||
};
|
||||
async getAllBookmarkFolderTimeline(folderId: string, options?: TimelinePaginationOptions): Promise<SearchResult> {
|
||||
return this.getBookmarkFolderTimelinePaged(folderId, Number.POSITIVE_INFINITY, options);
|
||||
}
|
||||
|
||||
private async getBookmarksPaged(limit: number, options: TimelinePaginationOptions = {}): Promise<SearchResult> {
|
||||
const features = buildBookmarksFeatures();
|
||||
const pageSize = 20;
|
||||
const seen = new Set<string>();
|
||||
const tweets: TweetData[] = [];
|
||||
let cursor: string | undefined;
|
||||
let pagesFetched = 0;
|
||||
const { includeRaw = false, maxPages } = options;
|
||||
|
||||
const tryOnce = async (variables: Record<string, unknown>) => {
|
||||
const fetchPage = async (pageCount: number, pageCursor?: string) => {
|
||||
let lastError: string | undefined;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getBookmarkFolderQueryIds();
|
||||
const queryIds = await this.getBookmarksQueryIds();
|
||||
|
||||
const variables = {
|
||||
count: pageCount,
|
||||
includePromotedContent: false,
|
||||
withDownvotePerspective: false,
|
||||
withReactionsMetadata: false,
|
||||
withReactionsPerspective: false,
|
||||
...(pageCursor ? { cursor: pageCursor } : {}),
|
||||
};
|
||||
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
@@ -281,10 +222,15 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
});
|
||||
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/BookmarkFolderTimeline?${params.toString()}`;
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/Bookmarks?${params.toString()}`;
|
||||
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
this.logBookmarksDebug('request bookmarks page', {
|
||||
queryId,
|
||||
pageCount,
|
||||
hasCursor: Boolean(pageCursor),
|
||||
});
|
||||
const response = await this.fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
@@ -292,17 +238,23 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
if (response.status === 404) {
|
||||
had404 = true;
|
||||
lastError = `HTTP ${response.status}`;
|
||||
this.logBookmarksDebug('bookmarks 404', { queryId });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
this.logBookmarksDebug('bookmarks non-200', {
|
||||
queryId,
|
||||
status: response.status,
|
||||
body: text.slice(0, 200),
|
||||
});
|
||||
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
data?: {
|
||||
bookmark_collection_timeline?: {
|
||||
bookmark_timeline_v2?: {
|
||||
timeline?: {
|
||||
instructions?: Array<{
|
||||
entries?: Array<{
|
||||
@@ -321,43 +273,283 @@ export function withTimelines<TBase extends AbstractConstructor<TwitterClientBas
|
||||
errors?: Array<{ message: string }>;
|
||||
};
|
||||
|
||||
const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions;
|
||||
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
const nextCursor = extractCursorFromInstructions(instructions);
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
|
||||
this.logBookmarksDebug('bookmarks graphql errors (non-fatal)', { queryId, errors: data.errors });
|
||||
if (!instructions) {
|
||||
lastError = data.errors.map((e) => e.message).join(', ');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const instructions = data.data?.bookmark_collection_timeline?.timeline?.instructions;
|
||||
const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
this.logBookmarksDebug('bookmarks page parsed', {
|
||||
queryId,
|
||||
tweets: pageTweets.length,
|
||||
hasNextCursor: Boolean(nextCursor),
|
||||
});
|
||||
|
||||
return { success: true as const, tweets, had404 };
|
||||
return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
this.logBookmarksDebug('bookmarks request error', { queryId, error: lastError });
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false as const, error: lastError ?? 'Unknown error fetching bookmark folder', had404 };
|
||||
return { success: false as const, error: lastError ?? 'Unknown error fetching bookmarks', had404 };
|
||||
};
|
||||
|
||||
let firstAttempt = await tryOnce(variablesWithCount);
|
||||
if (!firstAttempt.success && firstAttempt.error?.includes('Variable "$count"')) {
|
||||
firstAttempt = await tryOnce(variablesWithoutCount);
|
||||
}
|
||||
if (firstAttempt.success) {
|
||||
return { success: true, tweets: firstAttempt.tweets };
|
||||
const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
|
||||
const firstAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (firstAttempt.success) {
|
||||
return firstAttempt;
|
||||
}
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
const secondAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (secondAttempt.success) {
|
||||
return secondAttempt;
|
||||
}
|
||||
return { success: false as const, error: secondAttempt.error };
|
||||
}
|
||||
return { success: false as const, error: firstAttempt.error };
|
||||
};
|
||||
|
||||
const unlimited = !Number.isFinite(limit);
|
||||
while (unlimited || tweets.length < limit) {
|
||||
const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
|
||||
const page = await fetchWithRefresh(pageCount, cursor);
|
||||
if (!page.success) {
|
||||
return { success: false, error: page.error };
|
||||
}
|
||||
pagesFetched += 1;
|
||||
|
||||
for (const tweet of page.tweets) {
|
||||
if (seen.has(tweet.id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(tweet.id);
|
||||
tweets.push(tweet);
|
||||
if (!unlimited && tweets.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
|
||||
break;
|
||||
}
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
break;
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
let secondAttempt = await tryOnce(variablesWithCount);
|
||||
if (!secondAttempt.success && secondAttempt.error?.includes('Variable "$count"')) {
|
||||
secondAttempt = await tryOnce(variablesWithoutCount);
|
||||
return { success: true, tweets };
|
||||
}
|
||||
|
||||
private async getBookmarkFolderTimelinePaged(
|
||||
folderId: string,
|
||||
limit: number,
|
||||
options: TimelinePaginationOptions = {},
|
||||
): Promise<SearchResult> {
|
||||
const features = buildBookmarksFeatures();
|
||||
const pageSize = 20;
|
||||
const seen = new Set<string>();
|
||||
const tweets: TweetData[] = [];
|
||||
let cursor: string | undefined;
|
||||
let pagesFetched = 0;
|
||||
const { includeRaw = false, maxPages } = options;
|
||||
|
||||
const buildVariables = (pageCount: number, pageCursor: string | undefined, includeCount: boolean) => ({
|
||||
bookmark_collection_id: folderId,
|
||||
includePromotedContent: true,
|
||||
...(includeCount ? { count: pageCount } : {}),
|
||||
...(pageCursor ? { cursor: pageCursor } : {}),
|
||||
});
|
||||
|
||||
const fetchPage = async (pageCount: number, pageCursor?: string) => {
|
||||
let lastError: string | undefined;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getBookmarkFolderQueryIds();
|
||||
|
||||
const tryOnce = async (variables: Record<string, unknown>) => {
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/BookmarkFolderTimeline?${params.toString()}`;
|
||||
|
||||
try {
|
||||
this.logBookmarksDebug('request bookmark folder page', {
|
||||
queryId,
|
||||
pageCount,
|
||||
hasCursor: Boolean(pageCursor),
|
||||
includeCount: Object.prototype.hasOwnProperty.call(variables, 'count'),
|
||||
});
|
||||
const response = await this.fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
|
||||
if (response.status === 404) {
|
||||
had404 = true;
|
||||
lastError = `HTTP ${response.status}`;
|
||||
this.logBookmarksDebug('bookmark folder 404', { queryId });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
this.logBookmarksDebug('bookmark folder non-200', {
|
||||
queryId,
|
||||
status: response.status,
|
||||
body: text.slice(0, 200),
|
||||
});
|
||||
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
data?: {
|
||||
bookmark_collection_timeline?: {
|
||||
timeline?: {
|
||||
instructions?: Array<{
|
||||
entries?: Array<{
|
||||
content?: {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: GraphqlTweetResult;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
};
|
||||
errors?: Array<{ message: string }>;
|
||||
};
|
||||
|
||||
const instructions = data.data?.bookmark_collection_timeline?.timeline?.instructions;
|
||||
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
|
||||
const nextCursor = extractCursorFromInstructions(instructions);
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
this.logBookmarksDebug('bookmark folder graphql errors (non-fatal)', { queryId, errors: data.errors });
|
||||
if (!instructions) {
|
||||
lastError = data.errors.map((e) => e.message).join(', ');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.logBookmarksDebug('bookmark folder page parsed', {
|
||||
queryId,
|
||||
tweets: pageTweets.length,
|
||||
hasNextCursor: Boolean(nextCursor),
|
||||
});
|
||||
|
||||
return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
this.logBookmarksDebug('bookmark folder request error', { queryId, error: lastError });
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false as const, error: lastError ?? 'Unknown error fetching bookmark folder', had404 };
|
||||
};
|
||||
|
||||
let attempt = await tryOnce(buildVariables(pageCount, pageCursor, true));
|
||||
if (!attempt.success && attempt.error?.includes('Variable "$count"')) {
|
||||
attempt = await tryOnce(buildVariables(pageCount, pageCursor, false));
|
||||
}
|
||||
if (secondAttempt.success) {
|
||||
return { success: true, tweets: secondAttempt.tweets };
|
||||
|
||||
if (!attempt.success && attempt.error?.includes('Variable "$cursor"') && pageCursor) {
|
||||
return {
|
||||
success: false as const,
|
||||
error: 'Bookmark folder pagination rejected the cursor parameter',
|
||||
had404: attempt.had404,
|
||||
};
|
||||
}
|
||||
return { success: false, error: secondAttempt.error };
|
||||
|
||||
return attempt;
|
||||
};
|
||||
|
||||
const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
|
||||
const firstAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (firstAttempt.success) {
|
||||
return firstAttempt;
|
||||
}
|
||||
if (firstAttempt.had404) {
|
||||
await this.refreshQueryIds();
|
||||
const secondAttempt = await fetchPage(pageCount, pageCursor);
|
||||
if (secondAttempt.success) {
|
||||
return secondAttempt;
|
||||
}
|
||||
return { success: false as const, error: secondAttempt.error };
|
||||
}
|
||||
return { success: false as const, error: firstAttempt.error };
|
||||
};
|
||||
|
||||
const unlimited = !Number.isFinite(limit);
|
||||
while (unlimited || tweets.length < limit) {
|
||||
const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
|
||||
const page = await fetchWithRefresh(pageCount, cursor);
|
||||
if (!page.success) {
|
||||
return { success: false, error: page.error };
|
||||
}
|
||||
pagesFetched += 1;
|
||||
|
||||
for (const tweet of page.tweets) {
|
||||
if (seen.has(tweet.id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(tweet.id);
|
||||
tweets.push(tweet);
|
||||
if (!unlimited && tweets.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!page.cursor || page.cursor === cursor || page.tweets.length === 0) {
|
||||
break;
|
||||
}
|
||||
if (maxPages && pagesFetched >= maxPages) {
|
||||
break;
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
|
||||
return { success: false, error: firstAttempt.error };
|
||||
return { success: true, tweets };
|
||||
}
|
||||
|
||||
private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
|
||||
const maxRetries = 2;
|
||||
const baseDelayMs = 500;
|
||||
const retryable = new Set([429, 500, 502, 503, 504]);
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
|
||||
const response = await this.fetchWithTimeout(url, init);
|
||||
if (!retryable.has(response.status) || attempt === maxRetries) {
|
||||
return response;
|
||||
}
|
||||
this.logBookmarksDebug('retrying bookmarks request', {
|
||||
status: response.status,
|
||||
attempt,
|
||||
});
|
||||
|
||||
// Retry-After supports delta-seconds only; HTTP-date falls back to backoff.
|
||||
const retryAfter = response.headers.get('retry-after');
|
||||
const retryAfterMs = retryAfter ? Number.parseInt(retryAfter, 10) * 1000 : Number.NaN;
|
||||
const backoffMs = Number.isFinite(retryAfterMs)
|
||||
? retryAfterMs
|
||||
: baseDelayMs * 2 ** attempt + Math.floor(Math.random() * baseDelayMs);
|
||||
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
||||
}
|
||||
|
||||
return this.fetchWithTimeout(url, init);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user