feat: add news command to fetch AI-curated headlines
Add comprehensive news/trending feature that fetches AI-generated news headlines from Twitter's "For You" page. This feature specifically targets the AI-curated news items that Twitter displays under "Today's News" section, not generic trending hashtags. - Uses Twitter's ExplorePage GraphQL API endpoint to access the Explore page timeline data where AI headlines are located - AI headlines are identified by the explicit `is_ai_trend: true` flag in the API response - Fallback heuristic detection for full-sentence headlines (5+ words) with "News" or time indicators in social context - Headlines are found in "stories-*" modules within the initialTimeline structure under a "Today's News" header - Follows the existing mixin pattern (withNews) for composability - Supports both AI-only filtering and mixed trending/AI results - `bird news` - Fetch news and trending topics (mixed results) - `bird news --ai-only` - Fetch ONLY AI-curated headlines - `bird news -n <count>` - Limit number of results - `bird news --json` - Output as JSON - `bird news --json-full` - Include raw API response - `bird news --with-tweets` - Enrich with related tweets - `bird trending` - Alias for news command - AI headlines are clearly marked with "AI · " category prefix - Automatic deduplication of duplicate headlines - Rich formatting with category, time, post count, and URLs - src/commands/news.ts - CLI command implementation - src/lib/twitter-client-news.ts - Core news fetching functionality - tests/commands.news.test.ts - Command validation tests (7 tests) - tests/twitter-client-coverage.news.test.ts - API coverage tests (7 tests) - README.md - Added documentation for news command - src/cli/program.ts - Registered news command - src/lib/index.ts - Exported news-related types - src/lib/twitter-client.ts - Integrated withNews mixin - src/lib/query-ids.json - Added ExplorePage query ID - src/lib/twitter-client-constants.ts - Added ExplorePage constant - src/lib/twitter-client-features.ts - Added buildExploreFeatures() - scripts/update-query-ids.ts - Added ExplorePage to update script - All 268 existing tests continue to pass - Added 14 new tests covering command validation and API functionality - Tested with real Twitter data confirming AI headline detection - All TypeScript compilation and linting checks pass Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
This commit is contained in:
aavetis
committed by
Peter Steinberger
Peter Steinberger
co-authored by
Claude Sonnet 4.5
Claude Sonnet 4.5
parent
abc6db411d
commit
57d1498f1d
@@ -0,0 +1,154 @@
|
||||
import { Command } from 'commander';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { CliContext } from '../src/cli/shared.js';
|
||||
import { registerNewsCommand } from '../src/commands/news.js';
|
||||
|
||||
describe('news command', () => {
|
||||
let program: Command;
|
||||
let mockContext: Partial<CliContext>;
|
||||
|
||||
beforeEach(() => {
|
||||
program = new Command();
|
||||
mockContext = {
|
||||
resolveTimeoutFromOptions: () => 30000,
|
||||
resolveQuoteDepthFromOptions: () => undefined,
|
||||
resolveCredentialsFromOptions: async () => ({
|
||||
cookies: {
|
||||
authToken: 'auth',
|
||||
ct0: 'ct0',
|
||||
cookieHeader: 'auth=auth; ct0=ct0',
|
||||
},
|
||||
warnings: [],
|
||||
}),
|
||||
p: (type: string) => `[${type}] `,
|
||||
colors: {
|
||||
accent: (text: string) => text,
|
||||
command: (text: string) => text,
|
||||
muted: (text: string) => text,
|
||||
section: (text: string) => text,
|
||||
},
|
||||
l: (key: string) => key,
|
||||
};
|
||||
});
|
||||
|
||||
it('requires positive count value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--count', '0'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects negative count value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--count', '-5'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects non-numeric count value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--count', 'abc'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--count must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('requires positive tweets-per-item value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', '0'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects negative tweets-per-item value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', '-3'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects non-numeric tweets-per-item value', async () => {
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news', '--tweets-per-item', 'xyz'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('--tweets-per-item must be a positive number'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('requires both authToken and ct0 credentials', async () => {
|
||||
mockContext.resolveCredentialsFromOptions = async () => ({
|
||||
cookies: {
|
||||
authToken: '',
|
||||
ct0: '',
|
||||
cookieHeader: '',
|
||||
},
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
registerNewsCommand(program, mockContext as CliContext);
|
||||
const exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
}) as never);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
await expect(program.parseAsync(['node', 'bird', 'news'])).rejects.toThrow('exit 1');
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Missing required credentials'));
|
||||
} finally {
|
||||
exitSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,360 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { TwitterClient } from '../src/lib/twitter-client.js';
|
||||
|
||||
const validCookies = {
|
||||
authToken: 'test_auth_token',
|
||||
ct0: 'test_ct0_token',
|
||||
cookieHeader: 'auth_token=test_auth_token; ct0=test_ct0_token',
|
||||
source: 'test',
|
||||
};
|
||||
|
||||
type ResponseLike = {
|
||||
ok: boolean;
|
||||
status: number;
|
||||
json: () => Promise<unknown>;
|
||||
text: () => Promise<string>;
|
||||
};
|
||||
|
||||
const makeResponse = (overrides: Partial<ResponseLike> = {}): ResponseLike => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async (): Promise<unknown> => ({}),
|
||||
text: async (): Promise<string> => '',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('TwitterClient news API coverage', () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('getNews', () => {
|
||||
it('returns news items from ExplorePage', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'AI Breakthrough in Machine Learning',
|
||||
social_context: {
|
||||
text: 'AI · 2h ago · 15.5K posts',
|
||||
},
|
||||
trend_url: {
|
||||
url: 'https://x.com/hashtag/AI',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(1);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items).toBeDefined();
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('AI Breakthrough in Machine Learning');
|
||||
expect(result.items?.[0].category).toBe('AI · AI');
|
||||
expect(result.items?.[0].timeAgo).toBe('2h ago');
|
||||
expect(result.items?.[0].postCount).toBe(15500);
|
||||
expect(result.items?.[0].url).toBe('https://x.com/hashtag/AI');
|
||||
});
|
||||
|
||||
it('filters to AI-only items when aiOnly is true', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'AI News',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: false,
|
||||
name: 'Regular News',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10, { aiOnly: true });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('AI News');
|
||||
});
|
||||
|
||||
it('returns error for non-ok responses', async () => {
|
||||
const mockFetch = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(makeResponse({ ok: false, status: 500, text: async () => 'Server error' }));
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('HTTP 500');
|
||||
});
|
||||
|
||||
it('returns error when API returns errors', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
errors: [{ message: 'Rate limited' }, { message: 'Too many requests' }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('Rate limited');
|
||||
expect(result.error).toContain('Too many requests');
|
||||
});
|
||||
|
||||
it('returns error when no news items found', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('No news items found');
|
||||
});
|
||||
|
||||
it('deduplicates headlines', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'Duplicate Headline',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'Duplicate Headline',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('Duplicate Headline');
|
||||
});
|
||||
|
||||
it('respects count parameter', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 1',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 2',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-3',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 3',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(2);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user