fix: harden article rich content rendering (#36) (thanks @crcatala)

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Rich text output now shows article previews, quoted tweets, and media links (#32) — thanks @odysseus0.
 - `user-tweets` command to fetch a user's profile timeline (#34) — thanks @crcatala.
 - `replies` and `thread` now support pagination (`--all`, `--max-pages`, `--cursor`, `--delay`) (#35) — thanks @crcatala.
+- Long-form article tweets now render rich Draft.js content blocks/entities (#36) — thanks @crcatala.
 
 ### Changed
 - Library typing: `SearchResult` is now a discriminated union (so `error` only exists when `success: false`).

src/lib/twitter-client-search.ts

@@ -4,6 +4,8 @@ import { buildSearchFeatures } from './twitter-client-features.js';
 import type { SearchResult, TweetData } from './twitter-client-types.js';
 import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js';
 
+const RAW_QUERY_MISSING_REGEX = /must be defined/i;
+
 /** Options for search methods */
 export interface SearchFetchOptions {
   /** Include raw GraphQL response in `_raw` field */
@@ -24,7 +26,7 @@ function isQueryIdMismatch(payload: string): boolean {
         if (error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED') {
           return true;
         }
-        if (error?.path?.includes('rawQuery') && /must be defined/i.test(error.message ?? '')) {
+        if (error?.path?.includes('rawQuery') && RAW_QUERY_MISSING_REGEX.test(error.message ?? '')) {
           return true;
         }
         return false;
@@ -143,7 +145,9 @@ export function withSearch<TBase extends AbstractConstructor<TwitterClientBase>>
             };
 
             if (data.errors && data.errors.length > 0) {
-              const shouldRefreshQueryIds = data.errors.some((error) => error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED');
+              const shouldRefreshQueryIds = data.errors.some(
+                (error) => error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED',
+              );
               return {
                 success: false as const,
                 error: data.errors.map((e) => e.message).join(', '),

src/lib/twitter-client-types.ts

@@ -137,17 +137,26 @@ export type GraphqlTweetResult = {
             type: string;
             text: string;
             data?: Record<string, unknown>;
-            entityRanges: Array<{ key: number; offset: number; length: number }>;
+            entityRanges?: Array<{ key: number; offset: number; length: number }>;
-            inlineStyleRanges: Array<{ offset: number; length: number; style: string }>;
+            inlineStyleRanges?: Array<{ offset: number; length: number; style: string }>;
           }>;
-          entityMap: Array<{
+          entityMap?:
+            | Array<{
                 key: string;
                 value: {
                   type: string;
                   mutability: string;
                   data: Record<string, unknown>;
                 };
-          }>;
+              }>
+            | Record<
+                string,
+                {
+                  type: string;
+                  mutability: string;
+                  data: Record<string, unknown>;
+                }
+              >;
         };
       };
     };

src/lib/twitter-client-utils.ts

@@ -92,8 +92,8 @@ interface ContentBlock {
   data?: {
     mentions?: Array<{ fromIndex: number; toIndex: number; text: string }>;
   };
-  entityRanges: EntityRange[];
+  entityRanges?: EntityRange[];
-  inlineStyleRanges: InlineStyleRange[];
+  inlineStyleRanges?: InlineStyleRange[];
 }
 
 /** Entity data for different entity types */
@@ -116,7 +116,7 @@ interface EntityMapEntry {
 /** Draft.js content state structure */
 interface ContentState {
   blocks: ContentBlock[];
-  entityMap: EntityMapEntry[];
+  entityMap?: Array<EntityMapEntry> | Record<string, EntityValue>;
 }
 
 /**
@@ -128,14 +128,24 @@ export function renderContentState(contentState: ContentState | undefined): stri
     return undefined;
   }
 
-  // Build entity lookup map from array format
+  // Build entity lookup map from array/object formats
   const entityMap = new Map<number, EntityValue>();
-  for (const entry of contentState.entityMap ?? []) {
+  const rawEntityMap = contentState.entityMap ?? [];
+  if (Array.isArray(rawEntityMap)) {
+    for (const entry of rawEntityMap) {
       const key = Number.parseInt(entry.key, 10);
       if (!Number.isNaN(key)) {
         entityMap.set(key, entry.value);
       }
     }
+  } else {
+    for (const [key, value] of Object.entries(rawEntityMap)) {
+      const keyNumber = Number.parseInt(key, 10);
+      if (!Number.isNaN(keyNumber)) {
+        entityMap.set(keyNumber, value);
+      }
+    }
+  }
 
   const outputLines: string[] = [];
   let orderedListCounter = 0;
@@ -239,7 +249,7 @@ function renderBlockText(block: ContentBlock, entityMap: Map<number, EntityValue
 
   // Handle LINK entities by appending URL in markdown format
   // Process in reverse order to not mess up offsets
-  const linkRanges = block.entityRanges
+  const linkRanges = (block.entityRanges ?? [])
     .filter((range) => {
       const entity = entityMap.get(range.key);
       return entity?.type === 'LINK' && entity.data.url;
@@ -262,11 +272,12 @@ function renderBlockText(block: ContentBlock, entityMap: Map<number, EntityValue
  * Renders an atomic block by looking up its entity and returning appropriate content.
  */
 function renderAtomicBlock(block: ContentBlock, entityMap: Map<number, EntityValue>): string | undefined {
-  if (block.entityRanges.length === 0) {
+  const entityRanges = block.entityRanges ?? [];
+  if (entityRanges.length === 0) {
     return undefined;
   }
 
-  const entityKey = block.entityRanges[0].key;
+  const entityKey = entityRanges[0].key;
   const entity = entityMap.get(entityKey);
 
   if (!entity) {
@@ -332,9 +343,18 @@ export function extractArticleText(result: GraphqlTweetResult | undefined): stri
   const richBody = renderContentState(contentState);
   if (richBody) {
     // Rich content found - prepend title if not already included
-    if (title && !richBody.startsWith(title)) {
+    if (title) {
+      const normalizedTitle = title.trim();
+      const trimmedBody = richBody.trimStart();
+      const headingMatches = [`# ${normalizedTitle}`, `## ${normalizedTitle}`, `### ${normalizedTitle}`];
+      const hasTitle =
+        trimmedBody === normalizedTitle ||
+        trimmedBody.startsWith(`${normalizedTitle}\n`) ||
+        headingMatches.some((heading) => trimmedBody.startsWith(heading));
+      if (!hasTitle) {
         return `${title}\n\n${richBody}`;
       }
+    }
     return richBody;
   }
 

tests/extract-article-text.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from 'vitest';
+import type { GraphqlTweetResult } from '../src/lib/twitter-client-types.js';
+import { extractArticleText } from '../src/lib/twitter-client-utils.js';
+
+describe('extractArticleText', () => {
+  it('does not duplicate title when rich content starts with a heading', () => {
+    const result = {
+      rest_id: '1',
+      article: {
+        title: 'Hello World',
+        article_results: {
+          result: {
+            title: 'Hello World',
+            content_state: {
+              blocks: [
+                {
+                  key: '1',
+                  type: 'header-one',
+                  text: 'Hello World',
+                  entityRanges: [],
+                  inlineStyleRanges: [],
+                },
+              ],
+              entityMap: [],
+            },
+          },
+        },
+      },
+    } as GraphqlTweetResult;
+
+    expect(extractArticleText(result)).toBe('# Hello World');
+  });
+});

tests/render-content-state.test.ts

@@ -114,6 +114,27 @@ describe('renderContentState', () => {
     expect(result).toBe('```bash\necho "hello"\n```');
   });
 
+  it('handles entityMap object form', () => {
+    const result = renderContentState({
+      blocks: [
+        {
+          key: '1',
+          type: 'atomic',
+          text: ' ',
+          entityRanges: [{ key: 0, offset: 0, length: 1 }],
+        },
+      ],
+      entityMap: {
+        0: {
+          type: 'MARKDOWN',
+          mutability: 'Mutable',
+          data: { markdown: '```js\nconsole.log("ok")\n```' },
+        },
+      },
+    });
+    expect(result).toBe('```js\nconsole.log("ok")\n```');
+  });
+
   it('renders DIVIDER entity as horizontal rule', () => {
     const result = renderContentState({
       blocks: [
@@ -290,6 +311,14 @@ describe('renderContentState', () => {
     expect(result).toBe('Content\n\nMore content');
   });
 
+  it('handles missing entityRanges on text blocks', () => {
+    const result = renderContentState({
+      blocks: [{ key: '1', type: 'unstyled', text: 'Content' }],
+      entityMap: [],
+    });
+    expect(result).toBe('Content');
+  });
+
   it('handles atomic block with missing entity gracefully', () => {
     const result = renderContentState({
       blocks: [