feat(graphql): runtime query-id refresh
This commit is contained in:
@@ -5,6 +5,7 @@
|
|||||||
### Added
|
### Added
|
||||||
- Output controls: `--plain`, `--no-emoji`, `--no-color` (respects `NO_COLOR`).
|
- Output controls: `--plain`, `--no-emoji`, `--no-color` (respects `NO_COLOR`).
|
||||||
- `help` command: `bird help <command>`.
|
- `help` command: `bird help <command>`.
|
||||||
|
- Runtime GraphQL query ID refresh: `bird query-ids --fresh` (cached on disk; auto-retry on 404).
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- CLI `--version`: read version from `package.json`/`VERSION` (no hardcoded string) + append git sha when available.
|
- CLI `--version`: read version from `package.json`/`VERSION` (no hardcoded string) + append git sha when available.
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ bird replies 1234567890123456789
|
|||||||
bird search "from:steipete" -n 5
|
bird search "from:steipete" -n 5
|
||||||
bird mentions -n 5
|
bird mentions -n 5
|
||||||
bird mentions --user @steipete -n 5
|
bird mentions --user @steipete -n 5
|
||||||
|
|
||||||
|
# Refresh GraphQL query IDs cache (no rebuild)
|
||||||
|
bird query-ids --fresh
|
||||||
```
|
```
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
@@ -38,6 +41,7 @@ bird mentions --user @steipete -n 5
|
|||||||
- `bird tweet "<text>"` — post a new tweet.
|
- `bird tweet "<text>"` — post a new tweet.
|
||||||
- `bird reply <tweet-id-or-url> "<text>"` — reply to a tweet using its ID or URL.
|
- `bird reply <tweet-id-or-url> "<text>"` — reply to a tweet using its ID or URL.
|
||||||
- `bird help [command]` — show help (or help for a subcommand).
|
- `bird help [command]` — show help (or help for a subcommand).
|
||||||
|
- `bird query-ids [--fresh] [--json]` — inspect or refresh cached GraphQL query IDs.
|
||||||
- `bird read <tweet-id-or-url> [--json]` — fetch tweet content as text or JSON.
|
- `bird read <tweet-id-or-url> [--json]` — fetch tweet content as text or JSON.
|
||||||
- `bird <tweet-id-or-url> [--json]` — shorthand for `read` when only a URL or ID is provided.
|
- `bird <tweet-id-or-url> [--json]` — shorthand for `read` when only a URL or ID is provided.
|
||||||
- `bird replies <tweet-id-or-url> [--json]` — list replies to a tweet.
|
- `bird replies <tweet-id-or-url> [--json]` — list replies to a tweet.
|
||||||
@@ -101,6 +105,16 @@ Environment shortcuts:
|
|||||||
- `read` returns full text for Notes and Articles when present.
|
- `read` returns full text for Notes and Articles when present.
|
||||||
- Use `--plain` for stable, script-friendly output (no emoji, no color).
|
- Use `--plain` for stable, script-friendly output (no emoji, no color).
|
||||||
|
|
||||||
|
## Query IDs (GraphQL)
|
||||||
|
|
||||||
|
X rotates GraphQL query IDs frequently. `bird` caches refreshed IDs on disk and retries automatically on 404s.
|
||||||
|
|
||||||
|
Refresh on demand:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bird query-ids --fresh
|
||||||
|
```
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
- `0`: success
|
- `0`: success
|
||||||
- `1`: runtime error (network/auth/etc)
|
- `1`: runtime error (network/auth/etc)
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ import {
|
|||||||
resolveOutputConfigFromCommander,
|
resolveOutputConfigFromCommander,
|
||||||
statusPrefix,
|
statusPrefix,
|
||||||
} from './lib/output.js';
|
} from './lib/output.js';
|
||||||
|
import { runtimeQueryIds } from './lib/runtime-query-ids.js';
|
||||||
import { SweetisticsClient } from './lib/sweetistics-client.js';
|
import { SweetisticsClient } from './lib/sweetistics-client.js';
|
||||||
import { type TweetData, TwitterClient } from './lib/twitter-client.js';
|
import { type TweetData, TwitterClient } from './lib/twitter-client.js';
|
||||||
import { getCliVersion } from './lib/version.js';
|
import { getCliVersion } from './lib/version.js';
|
||||||
@@ -110,6 +111,7 @@ const config = loadConfig();
|
|||||||
const KNOWN_COMMANDS = new Set([
|
const KNOWN_COMMANDS = new Set([
|
||||||
'tweet',
|
'tweet',
|
||||||
'reply',
|
'reply',
|
||||||
|
'query-ids',
|
||||||
'read',
|
'read',
|
||||||
'replies',
|
'replies',
|
||||||
'thread',
|
'thread',
|
||||||
@@ -267,6 +269,63 @@ program
|
|||||||
cmd.outputHelp();
|
cmd.outputHelp();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
program
|
||||||
|
.command('query-ids')
|
||||||
|
.description('Show or refresh cached Twitter GraphQL query IDs')
|
||||||
|
.option('--json', 'Output as JSON')
|
||||||
|
.option('--fresh', 'Force refresh (downloads X client bundles)', false)
|
||||||
|
.action(async (cmdOpts: { json?: boolean; fresh?: boolean }) => {
|
||||||
|
const operations = [
|
||||||
|
'CreateTweet',
|
||||||
|
'CreateRetweet',
|
||||||
|
'FavoriteTweet',
|
||||||
|
'TweetDetail',
|
||||||
|
'SearchTimeline',
|
||||||
|
'UserArticlesTweets',
|
||||||
|
];
|
||||||
|
|
||||||
|
if (cmdOpts.fresh) {
|
||||||
|
console.error(`${p('info')}Refreshing GraphQL query IDs…`);
|
||||||
|
await runtimeQueryIds.refresh(operations, { force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const info = await runtimeQueryIds.getSnapshotInfo();
|
||||||
|
if (!info) {
|
||||||
|
if (cmdOpts.json) {
|
||||||
|
console.log(JSON.stringify({ cached: false, cachePath: runtimeQueryIds.cachePath }, null, 2));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log(`${p('warn')}No cached query IDs yet.`);
|
||||||
|
console.log(`${p('info')}Run: bird query-ids --fresh`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cmdOpts.json) {
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
cached: true,
|
||||||
|
cachePath: info.cachePath,
|
||||||
|
fetchedAt: info.snapshot.fetchedAt,
|
||||||
|
isFresh: info.isFresh,
|
||||||
|
ageMs: info.ageMs,
|
||||||
|
ids: info.snapshot.ids,
|
||||||
|
discovery: info.snapshot.discovery,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`${p('ok')}GraphQL query IDs cached`);
|
||||||
|
console.log(`path: ${info.cachePath}`);
|
||||||
|
console.log(`fetched_at: ${info.snapshot.fetchedAt}`);
|
||||||
|
console.log(`fresh: ${info.isFresh ? 'yes' : 'no'}`);
|
||||||
|
console.log(`ops: ${Object.keys(info.snapshot.ids).length}`);
|
||||||
|
});
|
||||||
|
|
||||||
// Tweet command
|
// Tweet command
|
||||||
program
|
program
|
||||||
.command('tweet')
|
.command('tweet')
|
||||||
|
|||||||
@@ -0,0 +1,303 @@
|
|||||||
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||||
|
import { homedir } from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
const DEFAULT_CACHE_FILENAME = 'query-ids-cache.json';
|
||||||
|
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
const DISCOVERY_PAGES = [
|
||||||
|
'https://x.com/?lang=en',
|
||||||
|
'https://x.com/explore',
|
||||||
|
'https://x.com/notifications',
|
||||||
|
'https://x.com/settings/profile',
|
||||||
|
];
|
||||||
|
|
||||||
|
const BUNDLE_URL_REGEX = /https:\/\/abs\.twimg\.com\/responsive-web\/client-web(?:-legacy)?\/[A-Za-z0-9.-]+\.js/g;
|
||||||
|
|
||||||
|
const OPERATION_PATTERNS = [
|
||||||
|
{
|
||||||
|
regex: /e\.exports=\{queryId\s*:\s*["']([^"']+)["']\s*,\s*operationName\s*:\s*["']([^"']+)["']/gs,
|
||||||
|
operationGroup: 2,
|
||||||
|
queryIdGroup: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
regex: /e\.exports=\{operationName\s*:\s*["']([^"']+)["']\s*,\s*queryId\s*:\s*["']([^"']+)["']/gs,
|
||||||
|
operationGroup: 1,
|
||||||
|
queryIdGroup: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
regex: /operationName\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)queryId\s*[:=]\s*["']([^"']+)["']/gs,
|
||||||
|
operationGroup: 1,
|
||||||
|
queryIdGroup: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
regex: /queryId\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)operationName\s*[:=]\s*["']([^"']+)["']/gs,
|
||||||
|
operationGroup: 3,
|
||||||
|
queryIdGroup: 1,
|
||||||
|
},
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const HEADERS = {
|
||||||
|
'User-Agent':
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
|
||||||
|
Accept: 'text/html,application/json;q=0.9,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RuntimeQueryIdSnapshot = {
|
||||||
|
fetchedAt: string;
|
||||||
|
ttlMs: number;
|
||||||
|
ids: Record<string, string>;
|
||||||
|
discovery: {
|
||||||
|
pages: string[];
|
||||||
|
bundles: string[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RuntimeQueryIdSnapshotInfo = {
|
||||||
|
snapshot: RuntimeQueryIdSnapshot;
|
||||||
|
cachePath: string;
|
||||||
|
ageMs: number;
|
||||||
|
isFresh: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RuntimeQueryIdsOptions = {
|
||||||
|
cachePath?: string;
|
||||||
|
ttlMs?: number;
|
||||||
|
fetchImpl?: typeof fetch;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function fetchText(fetchImpl: typeof fetch, url: string): Promise<string> {
|
||||||
|
const response = await fetchImpl(url, { headers: HEADERS });
|
||||||
|
if (!response.ok) {
|
||||||
|
const body = await response.text().catch(() => '');
|
||||||
|
throw new Error(`HTTP ${response.status} for ${url}: ${body.slice(0, 120)}`);
|
||||||
|
}
|
||||||
|
return response.text();
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDefaultCachePath(): string {
|
||||||
|
const override = process.env.BIRD_QUERY_IDS_CACHE;
|
||||||
|
if (override && override.trim().length > 0) {
|
||||||
|
return path.resolve(override.trim());
|
||||||
|
}
|
||||||
|
return path.join(homedir(), '.config', 'bird', DEFAULT_CACHE_FILENAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSnapshot(raw: unknown): RuntimeQueryIdSnapshot | null {
|
||||||
|
if (!raw || typeof raw !== 'object') return null;
|
||||||
|
const record = raw as Record<string, unknown>;
|
||||||
|
const fetchedAt = typeof record.fetchedAt === 'string' ? record.fetchedAt : null;
|
||||||
|
const ttlMs = typeof record.ttlMs === 'number' && Number.isFinite(record.ttlMs) ? record.ttlMs : null;
|
||||||
|
const ids = record.ids && typeof record.ids === 'object' ? (record.ids as Record<string, unknown>) : null;
|
||||||
|
const discovery =
|
||||||
|
record.discovery && typeof record.discovery === 'object' ? (record.discovery as Record<string, unknown>) : null;
|
||||||
|
|
||||||
|
if (!fetchedAt || !ttlMs || !ids || !discovery) return null;
|
||||||
|
const pages = Array.isArray(discovery.pages) ? discovery.pages : null;
|
||||||
|
const bundles = Array.isArray(discovery.bundles) ? discovery.bundles : null;
|
||||||
|
if (!pages || !bundles) return null;
|
||||||
|
|
||||||
|
const normalizedIds: Record<string, string> = {};
|
||||||
|
for (const [key, value] of Object.entries(ids)) {
|
||||||
|
if (typeof value === 'string' && value.trim().length > 0) {
|
||||||
|
normalizedIds[key] = value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
fetchedAt,
|
||||||
|
ttlMs,
|
||||||
|
ids: normalizedIds,
|
||||||
|
discovery: {
|
||||||
|
pages: pages.filter((p) => typeof p === 'string') as string[],
|
||||||
|
bundles: bundles.filter((b) => typeof b === 'string') as string[],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readSnapshotFromDisk(cachePath: string): Promise<RuntimeQueryIdSnapshot | null> {
|
||||||
|
try {
|
||||||
|
const raw = await readFile(cachePath, 'utf8');
|
||||||
|
return parseSnapshot(JSON.parse(raw));
|
||||||
|
} catch (error) {
|
||||||
|
const code = error && typeof error === 'object' ? String((error as { code?: unknown }).code ?? '') : '';
|
||||||
|
if (code !== 'ENOENT') return null;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeSnapshotToDisk(cachePath: string, snapshot: RuntimeQueryIdSnapshot): Promise<void> {
|
||||||
|
await mkdir(path.dirname(cachePath), { recursive: true });
|
||||||
|
await writeFile(cachePath, `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverBundles(fetchImpl: typeof fetch): Promise<string[]> {
|
||||||
|
const bundles = new Set<string>();
|
||||||
|
for (const page of DISCOVERY_PAGES) {
|
||||||
|
try {
|
||||||
|
const html = await fetchText(fetchImpl, page);
|
||||||
|
for (const match of html.matchAll(BUNDLE_URL_REGEX)) {
|
||||||
|
bundles.add(match[0]);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore discovery page failures; other pages often work
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const discovered = [...bundles];
|
||||||
|
if (discovered.length === 0) {
|
||||||
|
throw new Error('No client bundles discovered; x.com layout may have changed.');
|
||||||
|
}
|
||||||
|
return discovered;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractOperations(
|
||||||
|
bundleContents: string,
|
||||||
|
bundleLabel: string,
|
||||||
|
targets: Set<string>,
|
||||||
|
discovered: Map<string, { queryId: string; bundle: string }>,
|
||||||
|
): void {
|
||||||
|
for (const pattern of OPERATION_PATTERNS) {
|
||||||
|
pattern.regex.lastIndex = 0;
|
||||||
|
while (true) {
|
||||||
|
const match = pattern.regex.exec(bundleContents);
|
||||||
|
if (match === null) break;
|
||||||
|
const operationName = match[pattern.operationGroup];
|
||||||
|
const queryId = match[pattern.queryIdGroup];
|
||||||
|
if (!operationName || !queryId) continue;
|
||||||
|
if (!targets.has(operationName)) continue;
|
||||||
|
if (!/^[a-zA-Z0-9_-]+$/.test(queryId)) continue;
|
||||||
|
if (discovered.has(operationName)) continue;
|
||||||
|
discovered.set(operationName, { queryId, bundle: bundleLabel });
|
||||||
|
if (discovered.size === targets.size) return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchAndExtract(
|
||||||
|
fetchImpl: typeof fetch,
|
||||||
|
bundleUrls: string[],
|
||||||
|
targets: Set<string>,
|
||||||
|
): Promise<Map<string, { queryId: string; bundle: string }>> {
|
||||||
|
const discovered = new Map<string, { queryId: string; bundle: string }>();
|
||||||
|
const CONCURRENCY = 6;
|
||||||
|
|
||||||
|
for (let i = 0; i < bundleUrls.length; i += CONCURRENCY) {
|
||||||
|
const chunk = bundleUrls.slice(i, i + CONCURRENCY);
|
||||||
|
await Promise.all(
|
||||||
|
chunk.map(async (url) => {
|
||||||
|
if (discovered.size === targets.size) return;
|
||||||
|
const label = url.split('/').at(-1) ?? url;
|
||||||
|
try {
|
||||||
|
const js = await fetchText(fetchImpl, url);
|
||||||
|
extractOperations(js, label, targets, discovered);
|
||||||
|
} catch {
|
||||||
|
// ignore failed bundles
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if (discovered.size === targets.size) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return discovered;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createRuntimeQueryIdStore(options: RuntimeQueryIdsOptions = {}) {
|
||||||
|
const fetchImpl = options.fetchImpl ?? fetch;
|
||||||
|
const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
||||||
|
const cachePath = options.cachePath ? path.resolve(options.cachePath) : resolveDefaultCachePath();
|
||||||
|
|
||||||
|
let memorySnapshot: RuntimeQueryIdSnapshot | null = null;
|
||||||
|
let loadOnce: Promise<RuntimeQueryIdSnapshot | null> | null = null;
|
||||||
|
let refreshInFlight: Promise<RuntimeQueryIdSnapshotInfo | null> | null = null;
|
||||||
|
|
||||||
|
const loadSnapshot = async (): Promise<RuntimeQueryIdSnapshot | null> => {
|
||||||
|
if (memorySnapshot) return memorySnapshot;
|
||||||
|
if (!loadOnce) {
|
||||||
|
loadOnce = (async () => {
|
||||||
|
const fromDisk = await readSnapshotFromDisk(cachePath);
|
||||||
|
memorySnapshot = fromDisk;
|
||||||
|
return fromDisk;
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
return loadOnce;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getSnapshotInfo = async (): Promise<RuntimeQueryIdSnapshotInfo | null> => {
|
||||||
|
const snapshot = await loadSnapshot();
|
||||||
|
if (!snapshot) return null;
|
||||||
|
const fetchedAtMs = new Date(snapshot.fetchedAt).getTime();
|
||||||
|
const ageMs = Number.isFinite(fetchedAtMs) ? Math.max(0, Date.now() - fetchedAtMs) : Number.POSITIVE_INFINITY;
|
||||||
|
const effectiveTtl = Number.isFinite(snapshot.ttlMs) ? snapshot.ttlMs : ttlMs;
|
||||||
|
const isFresh = ageMs <= effectiveTtl;
|
||||||
|
return { snapshot, cachePath, ageMs, isFresh };
|
||||||
|
};
|
||||||
|
|
||||||
|
const getQueryId = async (operationName: string): Promise<string | null> => {
|
||||||
|
const info = await getSnapshotInfo();
|
||||||
|
if (!info) return null;
|
||||||
|
return info.snapshot.ids[operationName] ?? null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const refresh = async (
|
||||||
|
operationNames: string[],
|
||||||
|
opts: { force?: boolean } = {},
|
||||||
|
): Promise<RuntimeQueryIdSnapshotInfo | null> => {
|
||||||
|
if (refreshInFlight) return refreshInFlight;
|
||||||
|
|
||||||
|
refreshInFlight = (async () => {
|
||||||
|
const current = await getSnapshotInfo();
|
||||||
|
if (!opts.force && current?.isFresh) {
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
const targets = new Set(operationNames);
|
||||||
|
const bundleUrls = await discoverBundles(fetchImpl);
|
||||||
|
const discovered = await fetchAndExtract(fetchImpl, bundleUrls, targets);
|
||||||
|
if (discovered.size === 0) {
|
||||||
|
return current ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ids: Record<string, string> = {};
|
||||||
|
for (const name of operationNames) {
|
||||||
|
const entry = discovered.get(name);
|
||||||
|
if (entry?.queryId) {
|
||||||
|
ids[name] = entry.queryId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const snapshot: RuntimeQueryIdSnapshot = {
|
||||||
|
fetchedAt: new Date().toISOString(),
|
||||||
|
ttlMs,
|
||||||
|
ids,
|
||||||
|
discovery: {
|
||||||
|
pages: [...DISCOVERY_PAGES],
|
||||||
|
bundles: bundleUrls.map((url) => url.split('/').at(-1) ?? url),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await writeSnapshotToDisk(cachePath, snapshot);
|
||||||
|
memorySnapshot = snapshot;
|
||||||
|
|
||||||
|
return getSnapshotInfo();
|
||||||
|
})().finally(() => {
|
||||||
|
refreshInFlight = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
return refreshInFlight;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
cachePath,
|
||||||
|
ttlMs,
|
||||||
|
getSnapshotInfo,
|
||||||
|
getQueryId,
|
||||||
|
refresh,
|
||||||
|
clearMemory() {
|
||||||
|
memorySnapshot = null;
|
||||||
|
loadOnce = null;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const runtimeQueryIds = createRuntimeQueryIdStore();
|
||||||
+94
-21
@@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
import type { TwitterCookies } from './cookies.js';
|
import type { TwitterCookies } from './cookies.js';
|
||||||
import queryIds from './query-ids.json' with { type: 'json' };
|
import queryIds from './query-ids.json' with { type: 'json' };
|
||||||
|
import { runtimeQueryIds } from './runtime-query-ids.js';
|
||||||
|
|
||||||
const TWITTER_API_BASE = 'https://x.com/i/api/graphql';
|
const TWITTER_API_BASE = 'https://x.com/i/api/graphql';
|
||||||
const TWITTER_GRAPHQL_POST_URL = 'https://x.com/i/api/graphql';
|
const TWITTER_GRAPHQL_POST_URL = 'https://x.com/i/api/graphql';
|
||||||
@@ -27,13 +28,7 @@ const QUERY_IDS: Record<OperationName, string> = {
|
|||||||
...(queryIds as Partial<Record<OperationName, string>>),
|
...(queryIds as Partial<Record<OperationName, string>>),
|
||||||
};
|
};
|
||||||
|
|
||||||
const TWEET_DETAIL_QUERY_IDS = Array.from(
|
const TARGET_QUERY_ID_OPERATIONS = Object.keys(FALLBACK_QUERY_IDS) as Array<OperationName>;
|
||||||
new Set([QUERY_IDS.TweetDetail, '97JF30KziU00483E_8elBA', 'aFvUsJm2c-oDkJV75blV6g']),
|
|
||||||
);
|
|
||||||
|
|
||||||
const SEARCH_TIMELINE_QUERY_IDS = Array.from(
|
|
||||||
new Set([QUERY_IDS.SearchTimeline, 'M1jEez78PEfVfbQLvlWMvQ', '5h0kNbk3ii97rmfY6CdgAA', 'Tp1sewRU1AsZpBWhqCZicQ']),
|
|
||||||
);
|
|
||||||
|
|
||||||
type GraphqlTweetResult = {
|
type GraphqlTweetResult = {
|
||||||
rest_id?: string;
|
rest_id?: string;
|
||||||
@@ -266,6 +261,32 @@ export class TwitterClient {
|
|||||||
this.timeoutMs = options.timeoutMs;
|
this.timeoutMs = options.timeoutMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async getQueryId(operationName: OperationName): Promise<string> {
|
||||||
|
const cached = await runtimeQueryIds.getQueryId(operationName);
|
||||||
|
return cached ?? QUERY_IDS[operationName];
|
||||||
|
}
|
||||||
|
|
||||||
|
private async refreshQueryIds(): Promise<void> {
|
||||||
|
if (process.env.NODE_ENV === 'test') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await runtimeQueryIds.refresh(TARGET_QUERY_ID_OPERATIONS, { force: true });
|
||||||
|
} catch {
|
||||||
|
// ignore refresh failures; callers will fall back to baked-in IDs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getTweetDetailQueryIds(): Promise<string[]> {
|
||||||
|
const primary = await this.getQueryId('TweetDetail');
|
||||||
|
return Array.from(new Set([primary, '97JF30KziU00483E_8elBA', 'aFvUsJm2c-oDkJV75blV6g']));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getSearchTimelineQueryIds(): Promise<string[]> {
|
||||||
|
const primary = await this.getQueryId('SearchTimeline');
|
||||||
|
return Array.from(new Set([primary, 'M1jEez78PEfVfbQLvlWMvQ', '5h0kNbk3ii97rmfY6CdgAA', 'Tp1sewRU1AsZpBWhqCZicQ']));
|
||||||
|
}
|
||||||
|
|
||||||
private async fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
private async fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
||||||
if (!this.timeoutMs || this.timeoutMs <= 0) {
|
if (!this.timeoutMs || this.timeoutMs <= 0) {
|
||||||
return fetch(url, init);
|
return fetch(url, init);
|
||||||
@@ -738,7 +759,8 @@ export class TwitterClient {
|
|||||||
fieldToggles: JSON.stringify(this.buildArticleFieldToggles()),
|
fieldToggles: JSON.stringify(this.buildArticleFieldToggles()),
|
||||||
});
|
});
|
||||||
|
|
||||||
const url = `${TWITTER_API_BASE}/${QUERY_IDS.UserArticlesTweets}/UserArticlesTweets?${params}`;
|
const queryId = await this.getQueryId('UserArticlesTweets');
|
||||||
|
const url = `${TWITTER_API_BASE}/${queryId}/UserArticlesTweets?${params}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders() });
|
const response = await this.fetchWithTimeout(url, { method: 'GET', headers: this.getHeaders() });
|
||||||
@@ -875,8 +897,12 @@ export class TwitterClient {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let lastError: string | undefined;
|
let lastError: string | undefined;
|
||||||
|
let had404 = false;
|
||||||
|
|
||||||
for (const queryId of TWEET_DETAIL_QUERY_IDS) {
|
const tryOnce = async () => {
|
||||||
|
const queryIds = await this.getTweetDetailQueryIds();
|
||||||
|
|
||||||
|
for (const queryId of queryIds) {
|
||||||
const url = `${TWITTER_API_BASE}/${queryId}/TweetDetail?${params}`;
|
const url = `${TWITTER_API_BASE}/${queryId}/TweetDetail?${params}`;
|
||||||
const response = await this.fetchWithTimeout(url, {
|
const response = await this.fetchWithTimeout(url, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
@@ -887,6 +913,8 @@ export class TwitterClient {
|
|||||||
return await parseResponse(response);
|
return await parseResponse(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
had404 = true;
|
||||||
|
|
||||||
const postResponse = await this.fetchWithTimeout(`${TWITTER_API_BASE}/${queryId}/TweetDetail`, {
|
const postResponse = await this.fetchWithTimeout(`${TWITTER_API_BASE}/${queryId}/TweetDetail`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: this.getHeaders(),
|
headers: this.getHeaders(),
|
||||||
@@ -900,7 +928,20 @@ export class TwitterClient {
|
|||||||
lastError = 'HTTP 404';
|
lastError = 'HTTP 404';
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: false, error: lastError ?? 'Unknown error fetching tweet detail' };
|
return { success: false as const, error: lastError ?? 'Unknown error fetching tweet detail' };
|
||||||
|
};
|
||||||
|
|
||||||
|
const firstAttempt = await tryOnce();
|
||||||
|
if (firstAttempt.success) {
|
||||||
|
return firstAttempt;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (had404) {
|
||||||
|
await this.refreshQueryIds();
|
||||||
|
return await tryOnce();
|
||||||
|
}
|
||||||
|
|
||||||
|
return firstAttempt;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||||
}
|
}
|
||||||
@@ -1070,13 +1111,14 @@ export class TwitterClient {
|
|||||||
variables: Record<string, unknown>,
|
variables: Record<string, unknown>,
|
||||||
features: Record<string, boolean>,
|
features: Record<string, boolean>,
|
||||||
): Promise<TweetResult> {
|
): Promise<TweetResult> {
|
||||||
const queryId = QUERY_IDS.CreateTweet;
|
let queryId = await this.getQueryId('CreateTweet');
|
||||||
const urlWithOperation = `${TWITTER_API_BASE}/${queryId}/CreateTweet`;
|
let urlWithOperation = `${TWITTER_API_BASE}/${queryId}/CreateTweet`;
|
||||||
|
|
||||||
const body = JSON.stringify({ variables, features, queryId });
|
const buildBody = () => JSON.stringify({ variables, features, queryId });
|
||||||
|
let body = buildBody();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await this.fetchWithTimeout(urlWithOperation, {
|
let response = await this.fetchWithTimeout(urlWithOperation, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: this.getHeaders(),
|
headers: this.getHeaders(),
|
||||||
body,
|
body,
|
||||||
@@ -1084,6 +1126,18 @@ export class TwitterClient {
|
|||||||
|
|
||||||
// Twitter increasingly prefers POST to /i/api/graphql with queryId in the payload.
|
// Twitter increasingly prefers POST to /i/api/graphql with queryId in the payload.
|
||||||
// If the operation URL 404s, retry the generic endpoint.
|
// If the operation URL 404s, retry the generic endpoint.
|
||||||
|
if (response.status === 404) {
|
||||||
|
await this.refreshQueryIds();
|
||||||
|
queryId = await this.getQueryId('CreateTweet');
|
||||||
|
urlWithOperation = `${TWITTER_API_BASE}/${queryId}/CreateTweet`;
|
||||||
|
body = buildBody();
|
||||||
|
|
||||||
|
response = await this.fetchWithTimeout(urlWithOperation, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: this.getHeaders(),
|
||||||
|
body,
|
||||||
|
});
|
||||||
|
|
||||||
if (response.status === 404) {
|
if (response.status === 404) {
|
||||||
const retry = await this.fetchWithTimeout(TWITTER_GRAPHQL_POST_URL, {
|
const retry = await this.fetchWithTimeout(TWITTER_GRAPHQL_POST_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@@ -1107,6 +1161,7 @@ export class TwitterClient {
|
|||||||
|
|
||||||
return { success: false, error: 'Tweet created but no ID returned' };
|
return { success: false, error: 'Tweet created but no ID returned' };
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
@@ -1158,12 +1213,16 @@ export class TwitterClient {
|
|||||||
|
|
||||||
const features = this.buildSearchFeatures();
|
const features = this.buildSearchFeatures();
|
||||||
|
|
||||||
let lastError: string | undefined;
|
|
||||||
|
|
||||||
for (const queryId of SEARCH_TIMELINE_QUERY_IDS) {
|
|
||||||
const params = new URLSearchParams({
|
const params = new URLSearchParams({
|
||||||
variables: JSON.stringify(variables),
|
variables: JSON.stringify(variables),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const tryOnce = async () => {
|
||||||
|
let lastError: string | undefined;
|
||||||
|
let had404 = false;
|
||||||
|
const queryIds = await this.getSearchTimelineQueryIds();
|
||||||
|
|
||||||
|
for (const queryId of queryIds) {
|
||||||
const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params}`;
|
const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -1174,13 +1233,14 @@ export class TwitterClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (response.status === 404) {
|
if (response.status === 404) {
|
||||||
|
had404 = true;
|
||||||
lastError = `HTTP ${response.status}`;
|
lastError = `HTTP ${response.status}`;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}` };
|
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = (await response.json()) as {
|
const data = (await response.json()) as {
|
||||||
@@ -1227,19 +1287,32 @@ export class TwitterClient {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (data.errors && data.errors.length > 0) {
|
if (data.errors && data.errors.length > 0) {
|
||||||
return { success: false, error: data.errors.map((e) => e.message).join(', ') };
|
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
|
||||||
}
|
}
|
||||||
|
|
||||||
const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions;
|
const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions;
|
||||||
const tweets = this.parseTweetsFromInstructions(instructions);
|
const tweets = this.parseTweetsFromInstructions(instructions);
|
||||||
|
|
||||||
return { success: true, tweets };
|
return { success: true as const, tweets, had404 };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
lastError = error instanceof Error ? error.message : String(error);
|
lastError = error instanceof Error ? error.message : String(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: false, error: lastError ?? 'Unknown error fetching search results' };
|
return { success: false as const, error: lastError ?? 'Unknown error fetching search results', had404 };
|
||||||
|
};
|
||||||
|
|
||||||
|
const firstAttempt = await tryOnce();
|
||||||
|
if (firstAttempt.success) return { success: true, tweets: firstAttempt.tweets };
|
||||||
|
|
||||||
|
if (firstAttempt.had404) {
|
||||||
|
await this.refreshQueryIds();
|
||||||
|
const secondAttempt = await tryOnce();
|
||||||
|
if (secondAttempt.success) return { success: true, tweets: secondAttempt.tweets };
|
||||||
|
return { success: false, error: secondAttempt.error };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: false, error: firstAttempt.error };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ describe('cli-args', () => {
|
|||||||
const known = new Set([
|
const known = new Set([
|
||||||
'tweet',
|
'tweet',
|
||||||
'reply',
|
'reply',
|
||||||
|
'query-ids',
|
||||||
'read',
|
'read',
|
||||||
'replies',
|
'replies',
|
||||||
'thread',
|
'thread',
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import { mkdir, readFile } from 'node:fs/promises';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { createRuntimeQueryIdStore } from '../src/lib/runtime-query-ids.js';
|
||||||
|
|
||||||
|
describe('runtime-query-ids', () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refreshes IDs by scanning discovered bundles and persists cache', async () => {
|
||||||
|
const cacheDir = path.join(os.tmpdir(), `bird-test-${randomUUID()}`);
|
||||||
|
await mkdir(cacheDir, { recursive: true });
|
||||||
|
const cachePath = path.join(cacheDir, 'query-ids.json');
|
||||||
|
|
||||||
|
const html = `
|
||||||
|
<html>
|
||||||
|
<script src="https://abs.twimg.com/responsive-web/client-web/main.test.js"></script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
const js = [
|
||||||
|
`e.exports={queryId:"AAA",operationName:"CreateTweet"}`,
|
||||||
|
`e.exports={queryId:"BBB",operationName:"CreateRetweet"}`,
|
||||||
|
`e.exports={queryId:"CCC",operationName:"FavoriteTweet"}`,
|
||||||
|
`e.exports={queryId:"DDD",operationName:"TweetDetail"}`,
|
||||||
|
`e.exports={queryId:"EEE",operationName:"SearchTimeline"}`,
|
||||||
|
`e.exports={queryId:"FFF",operationName:"UserArticlesTweets"}`,
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const fetchMock = vi.fn(async (url: string | URL) => {
|
||||||
|
const asString = String(url);
|
||||||
|
if (asString.startsWith('https://x.com/')) {
|
||||||
|
return new Response(html, { status: 200 });
|
||||||
|
}
|
||||||
|
if (asString.startsWith('https://abs.twimg.com/')) {
|
||||||
|
return new Response(js, { status: 200 });
|
||||||
|
}
|
||||||
|
return new Response('not found', { status: 404 });
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
|
||||||
|
const store = createRuntimeQueryIdStore({
|
||||||
|
cachePath,
|
||||||
|
ttlMs: 1000,
|
||||||
|
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||||
|
});
|
||||||
|
const info = await store.refresh(
|
||||||
|
['CreateTweet', 'CreateRetweet', 'FavoriteTweet', 'TweetDetail', 'SearchTimeline', 'UserArticlesTweets'],
|
||||||
|
{ force: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(info?.snapshot.ids.CreateTweet).toBe('AAA');
|
||||||
|
expect(info?.snapshot.ids.TweetDetail).toBe('DDD');
|
||||||
|
expect(info?.cachePath).toBe(cachePath);
|
||||||
|
|
||||||
|
const raw = await readFile(cachePath, 'utf8');
|
||||||
|
const parsed = JSON.parse(raw) as { ids?: Record<string, string> };
|
||||||
|
expect(parsed.ids?.SearchTimeline).toBe('EEE');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -85,6 +85,11 @@ describe('TwitterClient', () => {
|
|||||||
|
|
||||||
it('retries CreateTweet via /i/api/graphql when operation URL 404s', async () => {
|
it('retries CreateTweet via /i/api/graphql when operation URL 404s', async () => {
|
||||||
mockFetch
|
mockFetch
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
text: async () => '',
|
||||||
|
})
|
||||||
.mockResolvedValueOnce({
|
.mockResolvedValueOnce({
|
||||||
ok: false,
|
ok: false,
|
||||||
status: 404,
|
status: 404,
|
||||||
@@ -111,12 +116,12 @@ describe('TwitterClient', () => {
|
|||||||
|
|
||||||
expect(result.success).toBe(true);
|
expect(result.success).toBe(true);
|
||||||
expect(result.tweetId).toBe('1234567890');
|
expect(result.tweetId).toBe('1234567890');
|
||||||
expect(mockFetch).toHaveBeenCalledTimes(2);
|
expect(mockFetch).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
const [firstUrl] = mockFetch.mock.calls[0];
|
const [firstUrl] = mockFetch.mock.calls[0];
|
||||||
const [secondUrl] = mockFetch.mock.calls[1];
|
const [thirdUrl] = mockFetch.mock.calls[2];
|
||||||
expect(String(firstUrl)).toContain('/CreateTweet');
|
expect(String(firstUrl)).toContain('/CreateTweet');
|
||||||
expect(String(secondUrl)).toBe('https://x.com/i/api/graphql');
|
expect(String(thirdUrl)).toBe('https://x.com/i/api/graphql');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should handle API errors', async () => {
|
it('should handle API errors', async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user