feat: add news command to fetch AI-curated headlines
Add comprehensive news/trending feature that fetches AI-generated news headlines from Twitter's "For You" page. This feature specifically targets the AI-curated news items that Twitter displays under "Today's News" section, not generic trending hashtags. - Uses Twitter's ExplorePage GraphQL API endpoint to access the Explore page timeline data where AI headlines are located - AI headlines are identified by the explicit `is_ai_trend: true` flag in the API response - Fallback heuristic detection for full-sentence headlines (5+ words) with "News" or time indicators in social context - Headlines are found in "stories-*" modules within the initialTimeline structure under a "Today's News" header - Follows the existing mixin pattern (withNews) for composability - Supports both AI-only filtering and mixed trending/AI results - `bird news` - Fetch news and trending topics (mixed results) - `bird news --ai-only` - Fetch ONLY AI-curated headlines - `bird news -n <count>` - Limit number of results - `bird news --json` - Output as JSON - `bird news --json-full` - Include raw API response - `bird news --with-tweets` - Enrich with related tweets - `bird trending` - Alias for news command - AI headlines are clearly marked with "AI · " category prefix - Automatic deduplication of duplicate headlines - Rich formatting with category, time, post count, and URLs - src/commands/news.ts - CLI command implementation - src/lib/twitter-client-news.ts - Core news fetching functionality - tests/commands.news.test.ts - Command validation tests (7 tests) - tests/twitter-client-coverage.news.test.ts - API coverage tests (7 tests) - README.md - Added documentation for news command - src/cli/program.ts - Registered news command - src/lib/index.ts - Exported news-related types - src/lib/twitter-client.ts - Integrated withNews mixin - src/lib/query-ids.json - Added ExplorePage query ID - src/lib/twitter-client-constants.ts - Added ExplorePage constant - src/lib/twitter-client-features.ts - Added buildExploreFeatures() - scripts/update-query-ids.ts - Added ExplorePage to update script - All 268 existing tests continue to pass - Added 14 new tests covering command validation and API functionality - Tested with real Twitter data confirming AI headline detection - All TypeScript compilation and linting checks pass Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
This commit is contained in:
aavetis
committed by
Peter Steinberger
Peter Steinberger
co-authored by
Claude Sonnet 4.5
Claude Sonnet 4.5
parent
abc6db411d
commit
57d1498f1d
@@ -0,0 +1,360 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { TwitterClient } from '../src/lib/twitter-client.js';
|
||||
|
||||
const validCookies = {
|
||||
authToken: 'test_auth_token',
|
||||
ct0: 'test_ct0_token',
|
||||
cookieHeader: 'auth_token=test_auth_token; ct0=test_ct0_token',
|
||||
source: 'test',
|
||||
};
|
||||
|
||||
type ResponseLike = {
|
||||
ok: boolean;
|
||||
status: number;
|
||||
json: () => Promise<unknown>;
|
||||
text: () => Promise<string>;
|
||||
};
|
||||
|
||||
const makeResponse = (overrides: Partial<ResponseLike> = {}): ResponseLike => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async (): Promise<unknown> => ({}),
|
||||
text: async (): Promise<string> => '',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('TwitterClient news API coverage', () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('getNews', () => {
|
||||
it('returns news items from ExplorePage', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'AI Breakthrough in Machine Learning',
|
||||
social_context: {
|
||||
text: 'AI · 2h ago · 15.5K posts',
|
||||
},
|
||||
trend_url: {
|
||||
url: 'https://x.com/hashtag/AI',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(1);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items).toBeDefined();
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('AI Breakthrough in Machine Learning');
|
||||
expect(result.items?.[0].category).toBe('AI · AI');
|
||||
expect(result.items?.[0].timeAgo).toBe('2h ago');
|
||||
expect(result.items?.[0].postCount).toBe(15500);
|
||||
expect(result.items?.[0].url).toBe('https://x.com/hashtag/AI');
|
||||
});
|
||||
|
||||
it('filters to AI-only items when aiOnly is true', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'AI News',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: false,
|
||||
name: 'Regular News',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10, { aiOnly: true });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('AI News');
|
||||
});
|
||||
|
||||
it('returns error for non-ok responses', async () => {
|
||||
const mockFetch = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(makeResponse({ ok: false, status: 500, text: async () => 'Server error' }));
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('HTTP 500');
|
||||
});
|
||||
|
||||
it('returns error when API returns errors', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
errors: [{ message: 'Rate limited' }, { message: 'Too many requests' }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('Rate limited');
|
||||
expect(result.error).toContain('Too many requests');
|
||||
});
|
||||
|
||||
it('returns error when no news items found', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('No news items found');
|
||||
});
|
||||
|
||||
it('deduplicates headlines', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'Duplicate Headline',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'Duplicate Headline',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(10);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(1);
|
||||
expect(result.items?.[0].headline).toBe('Duplicate Headline');
|
||||
});
|
||||
|
||||
it('respects count parameter', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValueOnce(
|
||||
makeResponse({
|
||||
json: async () => ({
|
||||
data: {
|
||||
explore_page: {
|
||||
body: {
|
||||
initialTimeline: {
|
||||
timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: 'test-entry-1',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 1',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-2',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 2',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: 'test-entry-3',
|
||||
content: {
|
||||
items: [
|
||||
{
|
||||
itemContent: {
|
||||
is_ai_trend: true,
|
||||
name: 'News 3',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
global.fetch = mockFetch as unknown as typeof fetch;
|
||||
|
||||
const client = new TwitterClient({ cookies: validCookies });
|
||||
const result = await client.getNews(2);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.items?.length).toBe(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user