Add engine switch and Sweetistics endpoints for tweet reading
This commit is contained in:
Peter Steinberger
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Fetches current Twitter/X GraphQL query IDs from public client bundles and
|
||||
* updates src/lib/query-ids.json. Inspired by Sweetistics' extractor.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const TARGET_OPERATIONS = [
|
||||
'CreateTweet',
|
||||
'CreateRetweet',
|
||||
'FavoriteTweet',
|
||||
'TweetDetail',
|
||||
'SearchTimeline',
|
||||
] as const;
|
||||
|
||||
type OperationName = (typeof TARGET_OPERATIONS)[number];
|
||||
|
||||
const DISCOVERY_PAGES = [
|
||||
'https://x.com/?lang=en',
|
||||
'https://x.com/explore',
|
||||
'https://x.com/notifications',
|
||||
'https://x.com/settings/profile',
|
||||
];
|
||||
|
||||
const BUNDLE_URL_REGEX =
|
||||
/https:\/\/abs\.twimg\.com\/responsive-web\/client-web(?:-legacy)?\/[A-Za-z0-9.-]+\.js/g;
|
||||
|
||||
const OPERATION_PATTERNS = [
|
||||
{
|
||||
regex: /operationName\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)queryId\s*[:=]\s*["']([^"']+)["']/gs,
|
||||
operationGroup: 1,
|
||||
queryIdGroup: 3,
|
||||
},
|
||||
{
|
||||
regex: /queryId\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)operationName\s*[:=]\s*["']([^"']+)["']/gs,
|
||||
operationGroup: 3,
|
||||
queryIdGroup: 1,
|
||||
},
|
||||
] as const;
|
||||
|
||||
const QUERY_IDS_PATH = path.resolve(process.cwd(), 'src/lib/query-ids.json');
|
||||
const HEADERS = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/json;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
};
|
||||
|
||||
interface DiscoveredOperation {
|
||||
queryId: string;
|
||||
bundle: string;
|
||||
}
|
||||
|
||||
async function readExistingIds(): Promise<Record<OperationName, string>> {
|
||||
try {
|
||||
const contents = await fs.readFile(QUERY_IDS_PATH, 'utf8');
|
||||
const parsed = JSON.parse(contents) as Record<string, string>;
|
||||
const result: Partial<Record<OperationName, string>> = {};
|
||||
for (const op of TARGET_OPERATIONS) {
|
||||
if (typeof parsed[op] === 'string' && parsed[op].trim().length > 0) {
|
||||
result[op] = parsed[op].trim();
|
||||
}
|
||||
}
|
||||
return result as Record<OperationName, string>;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
console.warn('[warn] Failed to read existing query IDs:', error);
|
||||
}
|
||||
return {} as Record<OperationName, string>;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
const response = await fetch(url, { headers: HEADERS });
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`HTTP ${response.status} for ${url}: ${body.slice(0, 120)}`);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function discoverBundles(): Promise<string[]> {
|
||||
const bundles = new Set<string>();
|
||||
for (const page of DISCOVERY_PAGES) {
|
||||
try {
|
||||
const html = await fetchText(page);
|
||||
for (const match of html.matchAll(BUNDLE_URL_REGEX)) {
|
||||
bundles.add(match[0]);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`[warn] Could not fetch ${page}:`, error instanceof Error ? error.message : error);
|
||||
}
|
||||
}
|
||||
|
||||
const discovered = Array.from(bundles);
|
||||
if (discovered.length === 0) {
|
||||
throw new Error('No client bundles discovered; x.com layout may have changed.');
|
||||
}
|
||||
return discovered;
|
||||
}
|
||||
|
||||
function extractOperations(
|
||||
bundleContents: string,
|
||||
bundleLabel: string,
|
||||
targets: Set<OperationName>,
|
||||
discovered: Map<OperationName, DiscoveredOperation>,
|
||||
): void {
|
||||
for (const pattern of OPERATION_PATTERNS) {
|
||||
pattern.regex.lastIndex = 0; // reset stateful regex
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = pattern.regex.exec(bundleContents)) !== null) {
|
||||
const operationName = match[pattern.operationGroup];
|
||||
const queryId = match[pattern.queryIdGroup];
|
||||
if (!operationName || !queryId) continue;
|
||||
|
||||
if (!targets.has(operationName as OperationName)) continue;
|
||||
if (!/^[a-zA-Z0-9_-]+$/.test(queryId)) continue;
|
||||
const op = operationName as OperationName;
|
||||
if (discovered.has(op)) continue;
|
||||
discovered.set(op, { queryId, bundle: bundleLabel });
|
||||
if (discovered.size === targets.size) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAndExtract(
|
||||
bundleUrls: string[],
|
||||
targets: Set<OperationName>,
|
||||
): Promise<Map<OperationName, DiscoveredOperation>> {
|
||||
const discovered = new Map<OperationName, DiscoveredOperation>();
|
||||
const CONCURRENCY = 6;
|
||||
|
||||
for (let i = 0; i < bundleUrls.length; i += CONCURRENCY) {
|
||||
const chunk = bundleUrls.slice(i, i + CONCURRENCY);
|
||||
await Promise.all(
|
||||
chunk.map(async (url) => {
|
||||
if (discovered.size === targets.size) {
|
||||
return;
|
||||
}
|
||||
const label = url.split('/').at(-1) ?? url;
|
||||
try {
|
||||
const js = await fetchText(url);
|
||||
extractOperations(js, label, targets, discovered);
|
||||
} catch (error) {
|
||||
console.warn(`[warn] Failed to scan ${label}:`, error instanceof Error ? error.message : error);
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (discovered.size === targets.size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return discovered;
|
||||
}
|
||||
|
||||
async function writeIds(ids: Record<OperationName, string>): Promise<void> {
|
||||
const ordered: Record<OperationName, string> = {} as Record<OperationName, string>;
|
||||
for (const op of TARGET_OPERATIONS) {
|
||||
if (ids[op]) {
|
||||
ordered[op] = ids[op];
|
||||
}
|
||||
}
|
||||
const json = `${JSON.stringify(ordered, null, 2)}\n`;
|
||||
await fs.mkdir(path.dirname(QUERY_IDS_PATH), { recursive: true });
|
||||
await fs.writeFile(QUERY_IDS_PATH, json, 'utf8');
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log('[info] Discovering Twitter/X client bundles…');
|
||||
const bundleUrls = await discoverBundles();
|
||||
console.log(`[info] Found ${bundleUrls.length} bundles`);
|
||||
|
||||
const targets = new Set<OperationName>(TARGET_OPERATIONS);
|
||||
const existing = await readExistingIds();
|
||||
|
||||
const discovered = await fetchAndExtract(bundleUrls, targets);
|
||||
if (discovered.size === 0) {
|
||||
throw new Error('No query IDs discovered; extraction patterns may need an update.');
|
||||
}
|
||||
|
||||
const nextIds: Record<OperationName, string> = { ...existing };
|
||||
for (const op of TARGET_OPERATIONS) {
|
||||
const found = discovered.get(op);
|
||||
if (found?.queryId) {
|
||||
nextIds[op] = found.queryId;
|
||||
}
|
||||
}
|
||||
|
||||
await writeIds(nextIds);
|
||||
|
||||
for (const op of TARGET_OPERATIONS) {
|
||||
const previous = existing[op];
|
||||
const current = nextIds[op];
|
||||
const source = discovered.get(op)?.bundle ?? 'existing file';
|
||||
if (previous && current && previous !== current) {
|
||||
console.log(`✅ ${op}: ${previous} → ${current} (${source})`);
|
||||
} else if (current) {
|
||||
console.log(`✅ ${op}: ${current} (${source})`);
|
||||
} else {
|
||||
console.warn(`⚠️ ${op}: not found (kept previous value if present)`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[info] Updated ${QUERY_IDS_PATH}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[error]', error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user