From 3815ebfa03944c942743c304308d1e7c41353799 Mon Sep 17 00:00:00 2001 From: cc-vps Date: Fri, 2 Jan 2026 07:59:03 -0800 Subject: [PATCH] feat: add --json-full flag to include raw API response in _raw field Add a new --json-full flag to tweet-fetching commands that outputs JSON with an additional _raw field containing the full GraphQL response. This provides power users access to additional data not exposed in the curated TweetData interface (media, entities, hashtags, cards, etc.) while maintaining backward compatibility with --json. Supported commands: - read, replies, thread (TweetDetail API) - search, mentions (SearchTimeline API) - bookmarks (Bookmarks API) - likes (Likes API) Not supported: following, followers (uses mixed GraphQL/REST fallback with different response structures) Implementation: - Add _raw?: GraphqlTweetResult optional field to TweetData type - Add includeRaw option to mapTweetResult() and parseTweetsFromInstructions() - Add TweetFetchOptions, SearchFetchOptions, TimelineFetchOptions interfaces - Update client methods to accept options parameter - Add --json-full flag to CLI commands - Update help text to document the new flag - Add comprehensive test coverage (16 tests) Co-authored-by: Christian Catalan --- src/cli/program.ts | 2 + src/commands/bookmarks.ts | 10 +- src/commands/read.ts | 27 +- src/commands/search.ts | 16 +- src/commands/users.ts | 11 +- src/lib/index.ts | 3 + src/lib/twitter-client-search.ts | 13 +- src/lib/twitter-client-timelines.ts | 31 +- src/lib/twitter-client-tweet-detail.ts | 27 +- src/lib/twitter-client-types.ts | 3 + src/lib/twitter-client-utils.ts | 37 +- tests/help-output.test.ts | 49 +++ tests/twitter-client.json-full.test.ts | 477 +++++++++++++++++++++++++ 13 files changed, 658 insertions(+), 48 deletions(-) create mode 100644 tests/twitter-client.json-full.test.ts diff --git a/src/cli/program.ts b/src/cli/program.ts index e0101d1..0ca048c 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -82,6 +82,8 @@ export function createProgram(ctx: CliContext): Command { formatExample('bird [--json]', 'Shorthand for `bird read `'), ].join('\n\n')}\n\n${ctx.colors.section('JSON Output')}\n${ctx.colors.muted( ` Add ${ctx.colors.option('--json')} to: read, replies, thread, search, mentions, bookmarks, likes, following, followers, query-ids`, + )}\n${ctx.colors.muted( + ` Add ${ctx.colors.option('--json-full')} to include raw API response in ${ctx.colors.argument('_raw')} field (tweet commands only)`, )}\n${ctx.colors.muted(` (Run ${ctx.colors.command('bird --help')} to see per-command flags.)`)}`, ); diff --git a/src/commands/bookmarks.ts b/src/commands/bookmarks.ts index 2a583bf..06ef2cb 100644 --- a/src/commands/bookmarks.ts +++ b/src/commands/bookmarks.ts @@ -10,7 +10,8 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi .option('-n, --count ', 'Number of bookmarks to fetch', '20') .option('--folder-id ', 'Bookmark folder (collection) id') .option('--json', 'Output as JSON') - .action(async (cmdOpts: { count?: string; json?: boolean; folderId?: string }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean; folderId?: string }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const count = Number.parseInt(cmdOpts.count || '20', 10); @@ -32,13 +33,14 @@ export function registerBookmarksCommand(program: Command, ctx: CliContext): voi console.error(`${ctx.p('err')}Invalid --folder-id. Expected numeric ID or https://x.com/i/bookmarks/.`); process.exit(1); } + const includeRaw = cmdOpts.jsonFull ?? false; const result = folderId - ? await client.getBookmarkFolderTimeline(folderId, count) - : await client.getBookmarks(count); + ? await client.getBookmarkFolderTimeline(folderId, count, { includeRaw }) + : await client.getBookmarks(count, { includeRaw }); if (result.success && result.tweets) { const emptyMessage = folderId ? 'No bookmarks found in folder.' : 'No bookmarks found.'; - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage }); + ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage }); } else { console.error(`${ctx.p('err')}Failed to fetch bookmarks: ${result.error}`); process.exit(1); diff --git a/src/commands/read.ts b/src/commands/read.ts index 4965484..3a1056d 100644 --- a/src/commands/read.ts +++ b/src/commands/read.ts @@ -9,7 +9,8 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { .description('Read/fetch a tweet by ID or URL') .argument('', 'Tweet ID or URL to read') .option('--json', 'Output as JSON') - .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -28,10 +29,11 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.getTweet(tweetId); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.getTweet(tweetId, { includeRaw }); if (result.success && result.tweet) { - if (cmdOpts.json) { + if (cmdOpts.json || cmdOpts.jsonFull) { console.log(JSON.stringify(result.tweet, null, 2)); } else { console.log(`@${result.tweet.author.username} (${result.tweet.author.name}):`); @@ -52,7 +54,8 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { .description('List replies to a tweet (by ID or URL)') .argument('', 'Tweet ID or URL') .option('--json', 'Output as JSON') - .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -70,10 +73,11 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.getReplies(tweetId); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.getReplies(tweetId, { includeRaw }); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage: 'No replies found.' }); + ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No replies found.' }); } else { console.error(`${ctx.p('err')}Failed to fetch replies: ${result.error}`); process.exit(1); @@ -85,7 +89,8 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { .description('Show the full conversation thread containing the tweet') .argument('', 'Tweet ID or URL') .option('--json', 'Output as JSON') - .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (tweetIdOrUrl: string, cmdOpts: { json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -103,10 +108,14 @@ export function registerReadCommands(program: Command, ctx: CliContext): void { } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.getThread(tweetId); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.getThread(tweetId, { includeRaw }); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage: 'No thread tweets found.' }); + ctx.printTweets(result.tweets, { + json: cmdOpts.json || cmdOpts.jsonFull, + emptyMessage: 'No thread tweets found.', + }); } else { console.error(`${ctx.p('err')}Failed to fetch thread: ${result.error}`); process.exit(1); diff --git a/src/commands/search.ts b/src/commands/search.ts index 298260a..c1af99e 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -10,7 +10,8 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void .argument('', 'Search query (e.g., "@clawdbot" or "from:clawdbot")') .option('-n, --count ', 'Number of tweets to fetch', '10') .option('--json', 'Output as JSON') - .action(async (query: string, cmdOpts: { count?: string; json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (query: string, cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -28,10 +29,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.search(query, count); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.search(query, count, { includeRaw }); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage: 'No tweets found.' }); + ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No tweets found.' }); } else { console.error(`${ctx.p('err')}Search failed: ${result.error}`); process.exit(1); @@ -44,7 +46,8 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void .option('-u, --user ', 'User handle (e.g. @steipete)') .option('-n, --count ', 'Number of tweets to fetch', '10') .option('--json', 'Output as JSON') - .action(async (cmdOpts: { user?: string; count?: string; json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (cmdOpts: { user?: string; count?: string; json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -84,10 +87,11 @@ export function registerSearchCommands(program: Command, ctx: CliContext): void } } - const result = await client.search(query, count); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.search(query, count, { includeRaw }); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage: 'No mentions found.' }); + ctx.printTweets(result.tweets, { json: cmdOpts.json || cmdOpts.jsonFull, emptyMessage: 'No mentions found.' }); } else { console.error(`${ctx.p('err')}Failed to fetch mentions: ${result.error}`); process.exit(1); diff --git a/src/commands/users.ts b/src/commands/users.ts index 0e989ed..62ce548 100644 --- a/src/commands/users.ts +++ b/src/commands/users.ts @@ -130,7 +130,8 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { .description('Get your liked tweets') .option('-n, --count ', 'Number of likes to fetch', '20') .option('--json', 'Output as JSON') - .action(async (cmdOpts: { count?: string; json?: boolean }) => { + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (cmdOpts: { count?: string; json?: boolean; jsonFull?: boolean }) => { const opts = program.opts(); const timeoutMs = ctx.resolveTimeoutFromOptions(opts); const quoteDepth = ctx.resolveQuoteDepthFromOptions(opts); @@ -148,10 +149,14 @@ export function registerUserCommands(program: Command, ctx: CliContext): void { } const client = new TwitterClient({ cookies, timeoutMs, quoteDepth }); - const result = await client.getLikes(count); + const includeRaw = cmdOpts.jsonFull ?? false; + const result = await client.getLikes(count, { includeRaw }); if (result.success && result.tweets) { - ctx.printTweets(result.tweets, { json: cmdOpts.json, emptyMessage: 'No liked tweets found.' }); + ctx.printTweets(result.tweets, { + json: cmdOpts.json || cmdOpts.jsonFull, + emptyMessage: 'No liked tweets found.', + }); } else { console.error(`${ctx.p('err')}Failed to fetch likes: ${result.error}`); process.exit(1); diff --git a/src/lib/index.ts b/src/lib/index.ts index ea39a50..1906406 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -18,4 +18,7 @@ export { type TwitterClientOptions, type TwitterUser, } from './twitter-client.js'; +export type { SearchFetchOptions } from './twitter-client-search.js'; +export type { TimelineFetchOptions } from './twitter-client-timelines.js'; +export type { TweetFetchOptions } from './twitter-client-tweet-detail.js'; export type { TweetResult, UploadMediaResult } from './twitter-client-types.js'; diff --git a/src/lib/twitter-client-search.ts b/src/lib/twitter-client-search.ts index d7ae437..769315a 100644 --- a/src/lib/twitter-client-search.ts +++ b/src/lib/twitter-client-search.ts @@ -4,8 +4,14 @@ import { buildSearchFeatures } from './twitter-client-features.js'; import type { SearchResult, TweetData } from './twitter-client-types.js'; import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js'; +/** Options for search methods */ +export interface SearchFetchOptions { + /** Include raw GraphQL response in `_raw` field */ + includeRaw?: boolean; +} + export interface TwitterClientSearchMethods { - search(query: string, count?: number): Promise; + search(query: string, count?: number, options?: SearchFetchOptions): Promise; } export function withSearch>( @@ -20,7 +26,8 @@ export function withSearch> /** * Search for tweets matching a query */ - async search(query: string, count = 20): Promise { + async search(query: string, count = 20, options: SearchFetchOptions = {}): Promise { + const { includeRaw = false } = options; const features = buildSearchFeatures(); const pageSize = 20; const seen = new Set(); @@ -113,7 +120,7 @@ export function withSearch> } const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; - const pageTweets = parseTweetsFromInstructions(instructions, this.quoteDepth); + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); const nextCursor = extractCursorFromInstructions(instructions); return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; diff --git a/src/lib/twitter-client-timelines.ts b/src/lib/twitter-client-timelines.ts index 1cc7bb2..119836c 100644 --- a/src/lib/twitter-client-timelines.ts +++ b/src/lib/twitter-client-timelines.ts @@ -4,10 +4,16 @@ import { buildBookmarksFeatures, buildLikesFeatures } from './twitter-client-fea import type { GraphqlTweetResult, SearchResult } from './twitter-client-types.js'; import { parseTweetsFromInstructions } from './twitter-client-utils.js'; +/** Options for timeline fetch methods */ +export interface TimelineFetchOptions { + /** Include raw GraphQL response in `_raw` field */ + includeRaw?: boolean; +} + export interface TwitterClientTimelineMethods { - getBookmarks(count?: number): Promise; - getLikes(count?: number): Promise; - getBookmarkFolderTimeline(folderId: string, count?: number): Promise; + getBookmarks(count?: number, options?: TimelineFetchOptions): Promise; + getLikes(count?: number, options?: TimelineFetchOptions): Promise; + getBookmarkFolderTimeline(folderId: string, count?: number, options?: TimelineFetchOptions): Promise; } export function withTimelines>( @@ -37,7 +43,8 @@ export function withTimelines { + async getBookmarks(count = 20, options: TimelineFetchOptions = {}): Promise { + const { includeRaw = false } = options; const variables = { count, includePromotedContent: false, @@ -104,7 +111,7 @@ export function withTimelines { + async getLikes(count = 20, options: TimelineFetchOptions = {}): Promise { + const { includeRaw = false } = options; const userResult = await this.getCurrentUser(); if (!userResult.success || !userResult.user) { return { success: false, error: userResult.error ?? 'Could not determine current user' }; @@ -212,7 +220,7 @@ export function withTimelines { + async getBookmarkFolderTimeline( + folderId: string, + count = 20, + options: TimelineFetchOptions = {}, + ): Promise { + const { includeRaw = false } = options; const variablesWithCount = { bookmark_collection_id: folderId, includePromotedContent: true, @@ -313,7 +326,7 @@ export function withTimelines; - getReplies(tweetId: string): Promise; - getThread(tweetId: string): Promise; + getTweet(tweetId: string, options?: TweetFetchOptions): Promise; + getReplies(tweetId: string, options?: TweetFetchOptions): Promise; + getThread(tweetId: string, options?: TweetFetchOptions): Promise; } export function withTweetDetails>( @@ -246,7 +252,8 @@ export function withTweetDetails { + async getTweet(tweetId: string, options: TweetFetchOptions = {}): Promise { + const { includeRaw = false } = options; const response = await this.fetchTweetDetail(tweetId); if (!response.success) { return response; @@ -271,7 +278,7 @@ export function withTweetDetails { + async getReplies(tweetId: string, options: TweetFetchOptions = {}): Promise { + const { includeRaw = false } = options; const response = await this.fetchTweetDetail(tweetId); if (!response.success) { return response; } const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; - const tweets = parseTweetsFromInstructions(instructions, this.quoteDepth); + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); const replies = tweets.filter((tweet) => tweet.inReplyToStatusId === tweetId); return { success: true, tweets: replies }; @@ -310,14 +318,15 @@ export function withTweetDetails { + async getThread(tweetId: string, options: TweetFetchOptions = {}): Promise { + const { includeRaw = false } = options; const response = await this.fetchTweetDetail(tweetId); if (!response.success) { return response; } const instructions = response.data.threaded_conversation_with_injections_v2?.instructions; - const tweets = parseTweetsFromInstructions(instructions, this.quoteDepth); + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); const target = tweets.find((t) => t.id === tweetId); const rootId = target?.conversationId || tweetId; diff --git a/src/lib/twitter-client-types.ts b/src/lib/twitter-client-types.ts index 52fe7ed..8923475 100644 --- a/src/lib/twitter-client-types.ts +++ b/src/lib/twitter-client-types.ts @@ -184,6 +184,9 @@ export interface TweetData { inReplyToStatusId?: string; // Optional quoted tweet; depth controlled by quoteDepth (default: 1). quotedTweet?: TweetData; + // Raw GraphQL response (included when --json-full is used). + // Structure may change as Twitter's API evolves. + _raw?: GraphqlTweetResult; } export interface GetTweetResult { diff --git a/src/lib/twitter-client-utils.ts b/src/lib/twitter-client-utils.ts index 4654538..26a5da9 100644 --- a/src/lib/twitter-client-utils.ts +++ b/src/lib/twitter-client-utils.ts @@ -163,7 +163,19 @@ export function unwrapTweetResult(result: GraphqlTweetResult | undefined): Graph return result; } -export function mapTweetResult(result: GraphqlTweetResult | undefined, quoteDepth: number): TweetData | undefined { +export interface MapTweetResultOptions { + quoteDepth: number; + includeRaw?: boolean; +} + +export function mapTweetResult( + result: GraphqlTweetResult | undefined, + quoteDepthOrOptions: number | MapTweetResultOptions, +): TweetData | undefined { + const options: MapTweetResultOptions = + typeof quoteDepthOrOptions === 'number' ? { quoteDepth: quoteDepthOrOptions } : quoteDepthOrOptions; + const { quoteDepth, includeRaw = false } = options; + const userResult = result?.core?.user_results?.result; const userLegacy = userResult?.legacy; const userCore = userResult?.core; @@ -183,11 +195,11 @@ export function mapTweetResult(result: GraphqlTweetResult | undefined, quoteDept if (quoteDepth > 0) { const quotedResult = unwrapTweetResult(result.quoted_status_result?.result); if (quotedResult) { - quotedTweet = mapTweetResult(quotedResult, quoteDepth - 1); + quotedTweet = mapTweetResult(quotedResult, { quoteDepth: quoteDepth - 1, includeRaw }); } } - return { + const tweetData: TweetData = { id: result.rest_id, text, createdAt: result.legacy?.created_at, @@ -203,6 +215,12 @@ export function mapTweetResult(result: GraphqlTweetResult | undefined, quoteDept authorId: userId, quotedTweet, }; + + if (includeRaw) { + tweetData._raw = result; + } + + return tweetData; } export function findTweetInInstructions( @@ -294,6 +312,11 @@ export function collectTweetResultsFromEntry(entry: { return results; } +export interface ParseTweetsOptions { + quoteDepth: number; + includeRaw?: boolean; +} + export function parseTweetsFromInstructions( instructions: | Array<{ @@ -336,8 +359,12 @@ export function parseTweetsFromInstructions( }>; }> | undefined, - quoteDepth: number, + quoteDepthOrOptions: number | ParseTweetsOptions, ): TweetData[] { + const options: ParseTweetsOptions = + typeof quoteDepthOrOptions === 'number' ? { quoteDepth: quoteDepthOrOptions } : quoteDepthOrOptions; + const { quoteDepth, includeRaw = false } = options; + const tweets: TweetData[] = []; const seen = new Set(); @@ -345,7 +372,7 @@ export function parseTweetsFromInstructions( for (const entry of instruction.entries ?? []) { const results = collectTweetResultsFromEntry(entry); for (const result of results) { - const mapped = mapTweetResult(result, quoteDepth); + const mapped = mapTweetResult(result, { quoteDepth, includeRaw }); if (!mapped || seen.has(mapped.id)) { continue; } diff --git a/tests/help-output.test.ts b/tests/help-output.test.ts index dc17792..6abbd30 100644 --- a/tests/help-output.test.ts +++ b/tests/help-output.test.ts @@ -21,6 +21,8 @@ describe('root help output', () => { expect(help).toContain('bird '); expect(help).toContain('--json'); + expect(help).toContain('--json-full'); + expect(help).toContain('_raw'); }); it('shows global options in subcommand help', () => { @@ -48,4 +50,51 @@ describe('root help output', () => { expect(help).toContain('--timeout'); expect(help).toContain('--quote-depth'); }); + + it('shows --json-full option in read command help', () => { + const ctx = createCliContext([]); + const program = createProgram(ctx); + const cmd = program.commands.find((c) => c.name() === 'read'); + if (!cmd) { + throw new Error('Expected "read" command to be registered'); + } + + let help = ''; + const output = { + writeOut: (s) => { + help += s; + }, + writeErr: () => {}, + }; + program.configureOutput(output); + cmd.configureOutput(output); + + cmd.outputHelp(); + + expect(help).toContain('--json-full'); + expect(help).toContain('_raw'); + }); + + it('shows --json-full option in search command help', () => { + const ctx = createCliContext([]); + const program = createProgram(ctx); + const cmd = program.commands.find((c) => c.name() === 'search'); + if (!cmd) { + throw new Error('Expected "search" command to be registered'); + } + + let help = ''; + const output = { + writeOut: (s) => { + help += s; + }, + writeErr: () => {}, + }; + program.configureOutput(output); + cmd.configureOutput(output); + + cmd.outputHelp(); + + expect(help).toContain('--json-full'); + }); }); diff --git a/tests/twitter-client.json-full.test.ts b/tests/twitter-client.json-full.test.ts new file mode 100644 index 0000000..8cbb69a --- /dev/null +++ b/tests/twitter-client.json-full.test.ts @@ -0,0 +1,477 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { TwitterClient } from '../src/lib/twitter-client.js'; +import { mapTweetResult, parseTweetsFromInstructions } from '../src/lib/twitter-client-utils.js'; +import { validCookies } from './twitter-client-fixtures.js'; + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); +}); + +describe('mapTweetResult with includeRaw', () => { + const makeTweetResult = (id: string, text: string, username = `user${id}`) => ({ + rest_id: id, + legacy: { + full_text: text, + created_at: '2024-01-01T00:00:00Z', + reply_count: 1, + retweet_count: 2, + favorite_count: 3, + conversation_id_str: id, + }, + core: { + user_results: { + result: { + rest_id: `u${id}`, + legacy: { screen_name: username, name: `User ${id}` }, + }, + }, + }, + // Extra fields that should only be in _raw + entities: { + urls: [{ url: 'https://t.co/abc', expanded_url: 'https://example.com' }], + hashtags: [{ text: 'test' }], + }, + extended_entities: { + media: [{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/test.jpg' }], + }, + }); + + it('does not include _raw by default', () => { + const result = makeTweetResult('1', 'hello'); + const mapped = mapTweetResult(result, { quoteDepth: 1 }); + + expect(mapped).toBeDefined(); + expect(mapped?.id).toBe('1'); + expect(mapped?.text).toBe('hello'); + expect(mapped?._raw).toBeUndefined(); + }); + + it('does not include _raw when includeRaw is false', () => { + const result = makeTweetResult('1', 'hello'); + const mapped = mapTweetResult(result, { quoteDepth: 1, includeRaw: false }); + + expect(mapped).toBeDefined(); + expect(mapped?._raw).toBeUndefined(); + }); + + it('includes _raw when includeRaw is true', () => { + const result = makeTweetResult('1', 'hello'); + const mapped = mapTweetResult(result, { quoteDepth: 1, includeRaw: true }); + + expect(mapped).toBeDefined(); + expect(mapped?.id).toBe('1'); + expect(mapped?.text).toBe('hello'); + expect(mapped?._raw).toBeDefined(); + expect(mapped?._raw).toBe(result); + // Verify the raw data contains fields not in the curated output + expect(mapped?._raw?.entities).toBeDefined(); + expect(mapped?._raw?.extended_entities).toBeDefined(); + }); + + it('includes _raw in quoted tweets when includeRaw is true', () => { + const quoted = makeTweetResult('2', 'quoted tweet'); + const root = makeTweetResult('1', 'root tweet'); + (root as Record).quoted_status_result = { result: quoted }; + + const mapped = mapTweetResult(root, { quoteDepth: 1, includeRaw: true }); + + expect(mapped?._raw).toBeDefined(); + expect(mapped?.quotedTweet?._raw).toBeDefined(); + expect(mapped?.quotedTweet?._raw).toBe(quoted); + }); + + it('does not include _raw in quoted tweets when includeRaw is false', () => { + const quoted = makeTweetResult('2', 'quoted tweet'); + const root = makeTweetResult('1', 'root tweet'); + (root as Record).quoted_status_result = { result: quoted }; + + const mapped = mapTweetResult(root, { quoteDepth: 1, includeRaw: false }); + + expect(mapped?._raw).toBeUndefined(); + expect(mapped?.quotedTweet?._raw).toBeUndefined(); + }); + + it('works with legacy number-only quoteDepth parameter', () => { + const result = makeTweetResult('1', 'hello'); + // This tests backward compatibility + const mapped = mapTweetResult(result, 1); + + expect(mapped).toBeDefined(); + expect(mapped?.id).toBe('1'); + expect(mapped?._raw).toBeUndefined(); + }); +}); + +describe('parseTweetsFromInstructions with includeRaw', () => { + const makeInstruction = (tweets: Array<{ id: string; text: string }>) => [ + { + entries: tweets.map((t) => ({ + content: { + itemContent: { + tweet_results: { + result: { + rest_id: t.id, + legacy: { + full_text: t.text, + created_at: '2024-01-01T00:00:00Z', + }, + core: { + user_results: { + result: { + legacy: { screen_name: `user${t.id}`, name: `User ${t.id}` }, + }, + }, + }, + }, + }, + }, + }, + })), + }, + ]; + + it('does not include _raw by default', () => { + const instructions = makeInstruction([{ id: '1', text: 'tweet 1' }]); + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: 1 }); + + expect(tweets).toHaveLength(1); + expect(tweets[0]._raw).toBeUndefined(); + }); + + it('includes _raw when includeRaw is true', () => { + const instructions = makeInstruction([ + { id: '1', text: 'tweet 1' }, + { id: '2', text: 'tweet 2' }, + ]); + const tweets = parseTweetsFromInstructions(instructions, { quoteDepth: 1, includeRaw: true }); + + expect(tweets).toHaveLength(2); + expect(tweets[0]._raw).toBeDefined(); + expect(tweets[0]._raw?.rest_id).toBe('1'); + expect(tweets[1]._raw).toBeDefined(); + expect(tweets[1]._raw?.rest_id).toBe('2'); + }); + + it('works with legacy number-only quoteDepth parameter', () => { + const instructions = makeInstruction([{ id: '1', text: 'tweet 1' }]); + // This tests backward compatibility + const tweets = parseTweetsFromInstructions(instructions, 1); + + expect(tweets).toHaveLength(1); + expect(tweets[0]._raw).toBeUndefined(); + }); +}); + +describe('TwitterClient getTweet with includeRaw option', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch; + }); + + const mockTweetResponse = { + data: { + tweetResult: { + result: { + rest_id: '12345', + legacy: { + full_text: 'Test tweet', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + reply_count: 1, + retweet_count: 2, + favorite_count: 3, + }, + core: { + user_results: { + result: { + rest_id: 'u123', + legacy: { screen_name: 'testuser', name: 'Test User' }, + }, + }, + }, + entities: { + urls: [{ url: 'https://t.co/abc', expanded_url: 'https://example.com' }], + }, + }, + }, + }, + }; + + it('does not include _raw by default', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockTweetResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getTweet('12345'); + + expect(result.success).toBe(true); + expect(result.tweet?._raw).toBeUndefined(); + }); + + it('includes _raw when includeRaw option is true', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockTweetResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getTweet('12345', { includeRaw: true }); + + expect(result.success).toBe(true); + expect(result.tweet?._raw).toBeDefined(); + expect(result.tweet?._raw?.rest_id).toBe('12345'); + expect(result.tweet?._raw?.entities).toBeDefined(); + }); +}); + +describe('TwitterClient getReplies with includeRaw option', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch; + }); + + const mockRepliesResponse = { + data: { + threaded_conversation_with_injections_v2: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '99999', + legacy: { + full_text: 'A reply', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + in_reply_to_status_id_str: '12345', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'replier', name: 'Replier' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }; + + it('includes _raw in replies when includeRaw is true', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockRepliesResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getReplies('12345', { includeRaw: true }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0]._raw).toBeDefined(); + expect(result.tweets?.[0]._raw?.rest_id).toBe('99999'); + }); +}); + +describe('TwitterClient getThread with includeRaw option', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch; + }); + + const mockThreadResponse = { + data: { + threaded_conversation_with_injections_v2: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '12345', + legacy: { + full_text: 'Thread start', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + conversation_id_str: '12345', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'author', name: 'Author' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }; + + it('includes _raw in thread tweets when includeRaw is true', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockThreadResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getThread('12345', { includeRaw: true }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0]._raw).toBeDefined(); + }); +}); + +describe('TwitterClient search with includeRaw option', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch; + }); + + const mockSearchResponse = { + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '11111', + legacy: { + full_text: 'Search result', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'found', name: 'Found User' }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }; + + it('includes _raw in search results when includeRaw is true', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockSearchResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.search('test query', 10, { includeRaw: true }); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0]._raw).toBeDefined(); + expect(result.tweets?.[0]._raw?.rest_id).toBe('11111'); + }); + + it('does not include _raw in search results by default', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockSearchResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.search('test query', 10); + + expect(result.success).toBe(true); + expect(result.tweets?.[0]._raw).toBeUndefined(); + }); +}); + +describe('--json-full flag behavior (without --json)', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch; + }); + + const mockTweetResponse = { + data: { + tweetResult: { + result: { + rest_id: '12345', + legacy: { + full_text: 'Test tweet', + created_at: 'Mon Jan 01 00:00:00 +0000 2024', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'testuser', name: 'Test User' }, + }, + }, + }, + }, + }, + }, + }; + + it('includeRaw works independently without requiring other json options', async () => { + // This tests that --json-full alone (without --json) will include _raw + // The CLI handles output format separately, but the client should work with just includeRaw + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockTweetResponse, + }); + + const client = new TwitterClient({ cookies: validCookies }); + // Only passing includeRaw: true, simulating --json-full without --json + const result = await client.getTweet('12345', { includeRaw: true }); + + expect(result.success).toBe(true); + expect(result.tweet).toBeDefined(); + expect(result.tweet?._raw).toBeDefined(); + // Verify the result is JSON-serializable (would be used for JSON output) + const serialized = JSON.stringify(result.tweet); + expect(serialized).toContain('"_raw"'); + expect(serialized).toContain('"id":"12345"'); + }); +});