diff --git a/CHANGELOG.md b/CHANGELOG.md index 8509607..df5755f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - `list-timeline` now supports pagination (`--all`, `--max-pages`, `--cursor`) (#30) — thanks @zheli. +- `home` command for the "For You" and "Following" home timelines (#31) — thanks @odysseus0. ### Fixed - macOS cookie extraction now supports Brave keychain storage (#40) — thanks @gakonst. diff --git a/src/cli/program.ts b/src/cli/program.ts index 584032d..d901702 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import { registerBookmarksCommand } from '../commands/bookmarks.js'; import { registerCheckCommand } from '../commands/check.js'; import { registerHelpCommand } from '../commands/help.js'; +import { registerHomeCommand } from '../commands/home.js'; import { registerListsCommand } from '../commands/lists.js'; import { registerPostCommands } from '../commands/post.js'; import { registerQueryIdsCommand } from '../commands/query-ids.js'; @@ -28,6 +29,7 @@ export const KNOWN_COMMANDS = new Set([ 'likes', 'lists', 'list-timeline', + 'home', 'help', 'whoami', 'check', @@ -131,6 +133,7 @@ export function createProgram(ctx: CliContext): Command { registerBookmarksCommand(program, ctx); registerUnbookmarkCommand(program, ctx); registerListsCommand(program, ctx); + registerHomeCommand(program, ctx); registerUserCommands(program, ctx); registerCheckCommand(program, ctx); diff --git a/src/commands/home.ts b/src/commands/home.ts new file mode 100644 index 0000000..5355b20 --- /dev/null +++ b/src/commands/home.ts @@ -0,0 +1,51 @@ +import type { Command } from 'commander'; +import type { CliContext } from '../cli/shared.js'; +import { TwitterClient } from '../lib/twitter-client.js'; + +export function registerHomeCommand(program: Command, ctx: CliContext): void { + program + .command('home') + .description('Get your home timeline ("For You" feed)') + .option('-n, --count ', 'Number of tweets to fetch', '20') + .option('--following', 'Get "Following" feed (chronological) instead of "For You"') + .option('--json', 'Output as JSON') + .option('--json-full', 'Output as JSON with full raw API response in _raw field') + .action(async (cmdOpts: { count?: string; following?: boolean; json?: boolean; jsonFull?: boolean }) => { + const opts = program.opts(); + const timeoutMs = ctx.resolveTimeoutFromOptions(opts); + const count = Number.parseInt(cmdOpts.count || '20', 10); + + const { cookies, warnings } = await ctx.resolveCredentialsFromOptions(opts); + + for (const warning of warnings) { + console.error(`${ctx.p('warn')}${warning}`); + } + + if (!cookies.authToken || !cookies.ct0) { + console.error(`${ctx.p('err')}Missing required credentials`); + process.exit(1); + } + + if (!Number.isFinite(count) || count <= 0) { + console.error(`${ctx.p('err')}Invalid --count. Expected a positive integer.`); + process.exit(1); + } + + const client = new TwitterClient({ cookies, timeoutMs }); + const includeRaw = cmdOpts.jsonFull ?? false; + + const result = cmdOpts.following + ? await client.getHomeLatestTimeline(count, { includeRaw }) + : await client.getHomeTimeline(count, { includeRaw }); + + if (result.success && result.tweets) { + const feedType = cmdOpts.following ? 'Following' : 'For You'; + const emptyMessage = `No tweets found in ${feedType} timeline.`; + const isJson = cmdOpts.json || cmdOpts.jsonFull; + ctx.printTweets(result.tweets, { json: isJson, emptyMessage }); + } else { + console.error(`${ctx.p('err')}Failed to fetch home timeline: ${result.error}`); + process.exit(1); + } + }); +} diff --git a/src/lib/index.ts b/src/lib/index.ts index 1906406..fd9018f 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -18,6 +18,7 @@ export { type TwitterClientOptions, type TwitterUser, } from './twitter-client.js'; +export type { HomeTimelineFetchOptions } from './twitter-client-home.js'; export type { SearchFetchOptions } from './twitter-client-search.js'; export type { TimelineFetchOptions } from './twitter-client-timelines.js'; export type { TweetFetchOptions } from './twitter-client-tweet-detail.js'; diff --git a/src/lib/twitter-client-constants.ts b/src/lib/twitter-client-constants.ts index 23c4dd2..09febbb 100644 --- a/src/lib/twitter-client-constants.ts +++ b/src/lib/twitter-client-constants.ts @@ -30,6 +30,8 @@ export const FALLBACK_QUERY_IDS = { ListMemberships: 'BlEXXdARdSeL_0KyKHHvvg', ListLatestTweetsTimeline: '2TemLyqrMpTeAmysdbnVqw', ListByRestId: 'wXzyA5vM_aVkBL9G8Vp3kw', + HomeTimeline: 'edseUwk9sP5Phz__9TIRnA', + HomeLatestTimeline: 'iOEZpOdfekFsxSlPQCQtPg', } as const; export type OperationName = keyof typeof FALLBACK_QUERY_IDS; diff --git a/src/lib/twitter-client-features.ts b/src/lib/twitter-client-features.ts index 67c9bd2..7454964 100644 --- a/src/lib/twitter-client-features.ts +++ b/src/lib/twitter-client-features.ts @@ -226,6 +226,12 @@ export function buildListsFeatures(): Record { }); } +export function buildHomeTimelineFeatures(): Record { + return applyFeatureOverrides('homeTimeline', { + ...buildTimelineFeatures(), + }); +} + export function buildFollowingFeatures(): Record { return applyFeatureOverrides('following', { rweb_video_screen_enabled: true, diff --git a/src/lib/twitter-client-home.ts b/src/lib/twitter-client-home.ts new file mode 100644 index 0000000..a4b5ada --- /dev/null +++ b/src/lib/twitter-client-home.ts @@ -0,0 +1,190 @@ +import type { AbstractConstructor, Mixin, TwitterClientBase } from './twitter-client-base.js'; +import { TWITTER_API_BASE } from './twitter-client-constants.js'; +import { buildHomeTimelineFeatures } from './twitter-client-features.js'; +import type { GraphqlTweetResult, SearchResult, TweetData } from './twitter-client-types.js'; +import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js'; + +/** Options for home timeline fetch methods */ +export interface HomeTimelineFetchOptions { + /** Include raw GraphQL response in `_raw` field */ + includeRaw?: boolean; +} + +export interface TwitterClientHomeMethods { + getHomeTimeline(count?: number, options?: HomeTimelineFetchOptions): Promise; + getHomeLatestTimeline(count?: number, options?: HomeTimelineFetchOptions): Promise; +} + +export function withHome>( + Base: TBase, +): Mixin { + abstract class TwitterClientHome extends Base { + // biome-ignore lint/complexity/noUselessConstructor lint/suspicious/noExplicitAny: TS mixin constructor requirement. + constructor(...args: any[]) { + super(...args); + } + + private async getHomeTimelineQueryIds(): Promise { + const primary = await this.getQueryId('HomeTimeline'); + return Array.from(new Set([primary, 'edseUwk9sP5Phz__9TIRnA'])); + } + + private async getHomeLatestTimelineQueryIds(): Promise { + const primary = await this.getQueryId('HomeLatestTimeline'); + return Array.from(new Set([primary, 'iOEZpOdfekFsxSlPQCQtPg'])); + } + + /** + * Get the authenticated user's "For You" home timeline + */ + async getHomeTimeline(count = 20, options: HomeTimelineFetchOptions = {}): Promise { + return this.fetchHomeTimeline('HomeTimeline', count, options); + } + + /** + * Get the authenticated user's "Following" (latest/chronological) home timeline + */ + async getHomeLatestTimeline(count = 20, options: HomeTimelineFetchOptions = {}): Promise { + return this.fetchHomeTimeline('HomeLatestTimeline', count, options); + } + + private async fetchHomeTimeline( + operation: 'HomeTimeline' | 'HomeLatestTimeline', + count: number, + options: HomeTimelineFetchOptions, + ): Promise { + const { includeRaw = false } = options; + const features = buildHomeTimelineFeatures(); + const pageSize = 20; + const seen = new Set(); + const tweets: TweetData[] = []; + let cursor: string | undefined; + + const fetchPage = async (pageCount: number, pageCursor?: string) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = + operation === 'HomeTimeline' + ? await this.getHomeTimelineQueryIds() + : await this.getHomeLatestTimelineQueryIds(); + + for (const queryId of queryIds) { + const variables = { + count: pageCount, + includePromotedContent: true, + latestControlAvailable: true, + requestContext: 'launch', + withCommunity: true, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; + + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); + + const url = `${TWITTER_API_BASE}/${queryId}/${operation}?${params.toString()}`; + + try { + const response = await this.fetchWithTimeout(url, { + method: 'GET', + headers: this.getHeaders(), + }); + + if (response.status === 404) { + had404 = true; + lastError = `HTTP ${response.status}`; + continue; + } + + if (!response.ok) { + const text = await response.text(); + return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; + } + + const data = (await response.json()) as { + data?: { + home?: { + home_timeline_urt?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + }>; + }>; + }; + }; + }; + errors?: Array<{ message: string }>; + }; + + if (data.errors && data.errors.length > 0) { + return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 }; + } + + const instructions = data.data?.home?.home_timeline_urt?.instructions; + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const nextCursor = extractCursorFromInstructions(instructions); + + return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + } + + return { success: false as const, error: lastError ?? 'Unknown error fetching home timeline', had404 }; + }; + + const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { + const firstAttempt = await fetchPage(pageCount, pageCursor); + if (firstAttempt.success) { + return firstAttempt; + } + if (firstAttempt.had404) { + await this.refreshQueryIds(); + const secondAttempt = await fetchPage(pageCount, pageCursor); + if (secondAttempt.success) { + return secondAttempt; + } + return { success: false as const, error: secondAttempt.error }; + } + return { success: false as const, error: firstAttempt.error }; + }; + + while (tweets.length < count) { + const pageCount = Math.min(pageSize, count - tweets.length); + const page = await fetchWithRefresh(pageCount, cursor); + if (!page.success) { + return { success: false, error: page.error }; + } + + let added = 0; + for (const tweet of page.tweets) { + if (seen.has(tweet.id)) { + continue; + } + seen.add(tweet.id); + tweets.push(tweet); + added += 1; + if (tweets.length >= count) { + break; + } + } + + if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) { + break; + } + cursor = page.cursor; + } + + return { success: true, tweets }; + } + } + + return TwitterClientHome; +} diff --git a/src/lib/twitter-client.ts b/src/lib/twitter-client.ts index 9b1bc20..e32001d 100644 --- a/src/lib/twitter-client.ts +++ b/src/lib/twitter-client.ts @@ -1,6 +1,7 @@ import type { AbstractConstructor } from './twitter-client-base.js'; import { TwitterClientBase } from './twitter-client-base.js'; import { type TwitterClientBookmarkMethods, withBookmarks } from './twitter-client-bookmarks.js'; +import { type TwitterClientHomeMethods, withHome } from './twitter-client-home.js'; import { type TwitterClientListMethods, withLists } from './twitter-client-lists.js'; import { type TwitterClientMediaMethods, withMedia } from './twitter-client-media.js'; import { type TwitterClientPostingMethods, withPosting } from './twitter-client-posting.js'; @@ -11,6 +12,7 @@ import { type TwitterClientUserMethods, withUsers } from './twitter-client-users type TwitterClientInstance = TwitterClientBase & TwitterClientBookmarkMethods & + TwitterClientHomeMethods & TwitterClientListMethods & TwitterClientMediaMethods & TwitterClientPostingMethods & @@ -20,7 +22,9 @@ type TwitterClientInstance = TwitterClientBase & TwitterClientUserMethods; const MixedTwitterClient = withUsers( - withLists(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase))))))), + withLists( + withHome(withTimelines(withSearch(withTweetDetails(withPosting(withBookmarks(withMedia(TwitterClientBase))))))), + ), ) as AbstractConstructor; export class TwitterClient extends MixedTwitterClient {} diff --git a/tests/twitter-client.home-timeline.test.ts b/tests/twitter-client.home-timeline.test.ts new file mode 100644 index 0000000..607e82a --- /dev/null +++ b/tests/twitter-client.home-timeline.test.ts @@ -0,0 +1,344 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { TwitterClient } from '../src/lib/twitter-client.js'; +import { validCookies } from './twitter-client-fixtures.js'; + +const originalFetch = global.fetch; + +afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); +}); + +// Realistic fixture based on actual Twitter API response structure +const makeHomeTimelineResponse = (tweetId: string, text: string, username: string) => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + __typename: 'Tweet', + rest_id: tweetId, + legacy: { + full_text: text, + created_at: 'Mon Jan 06 00:00:00 +0000 2025', + reply_count: 5, + retweet_count: 10, + favorite_count: 25, + conversation_id_str: tweetId, + }, + core: { + user_results: { + result: { + __typename: 'User', + rest_id: 'u123', + legacy: { + screen_name: username, + name: `${username} Name`, + }, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, +}); + +describe('TwitterClient home timeline', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + global.fetch = mockFetch as unknown as typeof fetch; + }); + + describe('getHomeTimeline', () => { + it('should return tweets from For You feed', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => makeHomeTimelineResponse('123456', 'Hello from For You feed!', 'testuser'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeTimeline(1); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0].id).toBe('123456'); + expect(result.tweets?.[0].text).toBe('Hello from For You feed!'); + expect(result.tweets?.[0].author.username).toBe('testuser'); + }); + + it('should return error on non-ok response', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 401, + text: async () => 'Unauthorized', + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeTimeline(1); + + expect(result.success).toBe(false); + expect(result.error).toContain('401'); + }); + + it('should return error when API returns errors', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + errors: [{ message: 'Rate limit exceeded' }], + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeTimeline(1); + + expect(result.success).toBe(false); + expect(result.error).toContain('Rate limit exceeded'); + }); + }); + + describe('getHomeLatestTimeline', () => { + it('should return tweets from Following feed', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => makeHomeTimelineResponse('789012', 'Hello from Following feed!', 'chronouser'), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeLatestTimeline(1); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(result.tweets?.[0].id).toBe('789012'); + expect(result.tweets?.[0].text).toBe('Hello from Following feed!'); + expect(result.tweets?.[0].author.username).toBe('chronouser'); + }); + + it('should return error on non-ok response', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 403, + text: async () => 'Forbidden', + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeLatestTimeline(1); + + expect(result.success).toBe(false); + expect(result.error).toContain('403'); + }); + }); + + describe('pagination', () => { + it('should deduplicate tweets across pages', async () => { + // First page + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor123', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + // Second page with duplicate + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', // duplicate + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet2', + legacy: { full_text: 'Second tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u2', legacy: { screen_name: 'user2', name: 'User 2' } }, + }, + }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeTimeline(2); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(2); + expect(result.tweets?.[0].id).toBe('tweet1'); + expect(result.tweets?.[1].id).toBe('tweet2'); + }); + + it('stops when a page only returns duplicates', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor123', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: { + home: { + home_timeline_urt: { + instructions: [ + { + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: 'tweet1', + legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' }, + core: { + user_results: { + result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } }, + }, + }, + }, + }, + }, + }, + }, + { + content: { + cursorType: 'Bottom', + value: 'cursor456', + }, + }, + ], + }, + ], + }, + }, + }, + }), + }); + + const client = new TwitterClient({ cookies: validCookies }); + const result = await client.getHomeTimeline(5); + + expect(result.success).toBe(true); + expect(result.tweets).toHaveLength(1); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + }); +});