From 6d6385ef2628b890b612088114b33fe4a276ab5a Mon Sep 17 00:00:00 2001 From: Martin Menestret Date: Tue, 7 Apr 2026 10:15:53 +0200 Subject: [PATCH 1/3] feat: add sync-likes, sync-timeline, and sync-feed commands Add three new sync commands that extend ft beyond bookmarks: - `ft sync-likes `: sync liked tweets via GraphQL (Likes endpoint) - `ft sync-timeline `: sync user's own tweets (UserTweets endpoint) - `ft sync-feed`: sync Following/chronological feed (HomeLatestTimeline endpoint) All synced data is stored in separate JSONL files (likes.jsonl, timeline.jsonl, feed.jsonl) and merged into the unified SQLite FTS5 index via `ft index`. A new `source` column (schema v5) enables filtering across all query commands: ft search "AI" --source likes ft list --source feed ft stats --source likes Implementation details: - New `src/graphql-user-sync.ts` module handles all three endpoints, reusing shared helpers (convertTweetToRecord, mergeRecords, buildHeaders, parseSnowflake, snowflakeToIso) from graphql-bookmarks.ts - Feed sync is session-scoped (no userId needed), likes/timeline resolve userId via UserByScreenName GraphQL query - CLI commands factored via registerUserSyncCommand to avoid boilerplate - --source validated against allowed values (bookmarks, likes, timeline, feed) - 19 new tests covering response parsing for both user timeline and feed response shapes, cursor extraction, conversation modules, ingestedVia assignment, and likedAt conversion --- CLAUDE.md | 15 +- README.md | 31 +- src/bookmarks-db.ts | 86 ++++-- src/cli.ts | 144 +++++++++- src/graphql-bookmarks.ts | 22 +- src/graphql-user-sync.ts | 491 ++++++++++++++++++++++++++++++++ src/paths.ts | 36 +++ src/types.ts | 3 +- tests/graphql-user-sync.test.ts | 391 +++++++++++++++++++++++++ 9 files changed, 1178 insertions(+), 41 deletions(-) create mode 100644 src/graphql-user-sync.ts create mode 100644 tests/graphql-user-sync.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 91daf6e..a4ed60e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,16 +29,21 @@ Single CLI application built with Commander.js. All data stored in `~/.ft-bookma | `src/bookmarks-viz.ts` | ANSI terminal dashboard | | `src/chrome-cookies.ts` | Chrome cookie extraction (macOS Keychain) | | `src/xauth.ts` | OAuth 2.0 flow | +| `src/graphql-user-sync.ts` | GraphQL sync for likes, timeline, and feed | | `src/db.ts` | WASM SQLite layer (sql.js-fts5) | ### Data flow ``` -Chrome cookies → GraphQL API → JSONL cache → SQLite FTS5 index - ↓ - Regex classification - ↓ - Search / List / Viz +Chrome cookies → GraphQL API → JSONL caches → SQLite FTS5 index + │ (bookmarks.jsonl, + │ likes.jsonl, + │ timeline.jsonl, + │ feed.jsonl) + ↓ + Regex classification + ↓ + Search / List / Viz ``` ### Dependencies diff --git a/README.md b/README.md index f979fbb..d58faef 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,12 @@ Requires Node.js 20+. Chrome recommended for session sync; OAuth available for a # 1. Sync your bookmarks (needs Chrome logged into X) ft sync -# 2. Search them -ft search "distributed systems" +# 2. Sync likes, your timeline, or your feed +ft sync-likes yourhandle +ft sync-feed -# 3. Explore -ft viz -ft categories -ft stats +# 3. Search them +ft search "distributed systems" ``` On first run, `ft sync` extracts your X session from Chrome and downloads your bookmarks into `~/.ft-bookmarks/`. @@ -43,6 +42,9 @@ On first run, `ft sync` extracts your X session from Chrome and downloads your b | `ft sync --folder ` | Sync a single folder by name (exact or unambiguous prefix) | | `ft sync --classify` | Sync then classify new bookmarks with LLM | | `ft sync --api` | Sync via OAuth API (cross-platform) | +| `ft sync-likes ` | Sync liked tweets (no API required) | +| `ft sync-timeline ` | Sync your own tweets (no API required) | +| `ft sync-feed` | Sync your Following feed (no API required) | | `ft auth` | Set up OAuth for API-based sync (optional) | ### Search and browse @@ -50,8 +52,10 @@ On first run, `ft sync` extracts your X session from Chrome and downloads your b | Command | Description | |---------|-------------| | `ft search ` | Full-text search with BM25 ranking | -| `ft list` | Filter by author, date, category, domain, or folder | +| `ft search --source likes` | Search within a specific source | +| `ft list` | Filter by author, date, category, domain, folder, or source | | `ft list --folder ` | Show bookmarks in an X bookmark folder | +| `ft list --source ` | Filter by source (bookmarks, likes, timeline, feed) | | `ft show ` | Show one bookmark in detail | | `ft sample ` | Random sample from a category | | `ft stats` | Top authors, languages, date range | @@ -112,6 +116,8 @@ Then ask your agent: > "I bookmarked a number of new open source AI memory tools. Pick the best one and figure out how to incorporate it in this repo." +> "What topics have I liked the most this month?" +> > "Every day please sync any new X bookmarks using the Field Theory CLI." Works with Claude Code, Codex, or any agent with shell access. @@ -119,9 +125,15 @@ Works with Claude Code, Codex, or any agent with shell access. ## Scheduling ```bash -# Sync every morning at 7am +# Sync bookmarks every morning at 7am 0 7 * * * ft sync +# Sync likes daily +0 7 * * * ft sync-likes yourhandle + +# Sync feed every 6 hours +0 */6 * * * ft sync-feed + # Sync and classify every morning 0 7 * * * ft sync --classify ``` @@ -133,6 +145,9 @@ All data is stored locally at `~/.ft-bookmarks/`: ``` ~/.ft-bookmarks/ bookmarks.jsonl # raw bookmark cache (one per line) + likes.jsonl # liked tweets cache + timeline.jsonl # your own tweets cache + feed.jsonl # Following feed cache bookmarks.db # SQLite FTS5 search index bookmarks-meta.json # sync metadata oauth-token.json # OAuth token (if using API mode, chmod 600) diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index f9d02fa..86fb7de 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -2,12 +2,18 @@ import type { Database } from 'sql.js'; import { openDb, saveDb } from './db.js'; import { parseTimestampMs, toIsoDate } from './date-utils.js'; import { readJsonLines } from './fs.js'; -import { twitterBookmarksCachePath, twitterBookmarksIndexPath } from './paths.js'; +import { + twitterBookmarksCachePath, + twitterBookmarksIndexPath, + twitterLikesCachePath, + twitterTimelineCachePath, + twitterFeedCachePath, +} from './paths.js'; import type { BookmarkRecord, QuotedTweetSnapshot } from './types.js'; import { classifyCorpus, formatClassificationSummary } from './bookmark-classify.js'; import type { ClassificationSummary } from './bookmark-classify.js'; -const SCHEMA_VERSION = 6; +const SCHEMA_VERSION = 7; export interface SearchResult { id: string; @@ -26,6 +32,8 @@ export interface SearchOptions { before?: string; after?: string; folder?: string; + /** Filter by source: bookmarks, likes, timeline, feed */ + source?: string; } export interface BookmarkTimelineItem { @@ -64,6 +72,8 @@ export interface BookmarkTimelineFilters { category?: string; domain?: string; folder?: string; + /** Filter by source: bookmarks, likes, timeline, feed */ + source?: string; sort?: 'asc' | 'desc'; limit?: number; offset?: number; @@ -179,6 +189,10 @@ function buildBookmarkWhereClause(filters: BookmarkTimelineFilters): { ); params.push(filters.folder); } + if (filters.source) { + conditions.push(`b.source = ?`); + params.push(filters.source); + } return { where: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '', @@ -239,10 +253,12 @@ function initSchema(db: Database): void { article_site TEXT, enriched_at TEXT, folder_ids TEXT, - folder_names TEXT + folder_names TEXT, + source TEXT DEFAULT 'bookmarks' )`); db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_author ON bookmarks(author_handle)`); + db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_source ON bookmarks(source)`); db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_posted ON bookmarks(posted_at)`); db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_language ON bookmarks(language)`); db.run(`CREATE INDEX IF NOT EXISTS idx_bookmarks_category ON bookmarks(primary_category)`); @@ -314,6 +330,9 @@ function ensureMigrations(db: Database): void { ensureColumn(db, 'bookmarks', 'folder_ids', 'TEXT'); ensureColumn(db, 'bookmarks', 'folder_names', 'TEXT'); + ensureColumn(db, 'bookmarks', 'source', "TEXT DEFAULT 'bookmarks'"); + db.run('CREATE INDEX IF NOT EXISTS idx_bookmarks_source ON bookmarks(source)'); + // FTS rebuild: only if the FTS table is missing the article_text column. // Check via a zero-row SELECT so we don't rebuild unnecessarily. if (!ftsHasColumn(db, 'article_text')) { @@ -350,7 +369,7 @@ function serializeJsonArray(values: string[] | undefined | null): string | null return JSON.stringify(values); } -function insertRecord(db: Database, r: BookmarkRecord, preserved?: PreservedBookmarkFields): void { +function insertRecord(db: Database, r: BookmarkRecord, source: string = 'bookmarks', preserved?: PreservedBookmarkFields): void { // Extract GitHub URLs (kept inline — no LLM needed for URL parsing) const text = r.text ?? ''; const githubMatches = text.match(/github\.com\/[\w.-]+\/[\w.-]+/gi) ?? []; @@ -358,7 +377,7 @@ function insertRecord(db: Database, r: BookmarkRecord, preserved?: PreservedBook const githubUrls = [...new Set([...githubMatches.map((m) => `https://${m}`), ...githubFromLinks])]; db.run( - `INSERT OR REPLACE INTO bookmarks VALUES (${Array(37).fill('?').join(',')})`, + `INSERT OR REPLACE INTO bookmarks VALUES (${Array(38).fill('?').join(',')})`, [ r.id, r.tweetId, @@ -397,14 +416,31 @@ function insertRecord(db: Database, r: BookmarkRecord, preserved?: PreservedBook preserved?.enrichedAt ?? null, serializeJsonArray(r.folderIds) ?? preserved?.folderIds ?? null, serializeJsonArray(r.folderNames) ?? preserved?.folderNames ?? null, + source, ] ); } export async function buildIndex(options?: { force?: boolean }): Promise<{ dbPath: string; recordCount: number; newRecords: number }> { - const cachePath = twitterBookmarksCachePath(); const dbPath = twitterBookmarksIndexPath(); - const records = await readJsonLines(cachePath); + + // Collect records from all sources + const sources: Array<{ path: string; source: string }> = [ + { path: twitterBookmarksCachePath(), source: 'bookmarks' }, + { path: twitterLikesCachePath(), source: 'likes' }, + { path: twitterTimelineCachePath(), source: 'timeline' }, + { path: twitterFeedCachePath(), source: 'feed' }, + ]; + + const taggedRecords: Array<{ record: BookmarkRecord; source: string }> = []; + for (const { path, source } of sources) { + try { + const records = await readJsonLines(path); + for (const record of records) { + taggedRecords.push({ record, source }); + } + } catch { /* file may not exist */ } + } const db = await openDb(dbPath); try { @@ -448,13 +484,13 @@ export async function buildIndex(options?: { force?: boolean }): Promise<{ dbPat } } catch { /* table may be empty */ } - const newRecords: BookmarkRecord[] = records.filter(r => !existingRows.has(r.id)); + const newEntries = taggedRecords.filter(({ record }) => !existingRows.has(record.id)); - if (records.length > 0) { + if (taggedRecords.length > 0) { db.run('BEGIN TRANSACTION'); try { - for (const record of records) { - insertRecord(db, record, existingRows.get(record.id)); + for (const { record, source } of taggedRecords) { + insertRecord(db, record, source, existingRows.get(record.id)); } db.run('COMMIT'); } catch (err) { @@ -468,7 +504,7 @@ export async function buildIndex(options?: { force?: boolean }): Promise<{ dbPat saveDb(db, dbPath); const totalRows = db.exec('SELECT COUNT(*) FROM bookmarks')[0]?.values[0]?.[0] as number; - return { dbPath, recordCount: totalRows, newRecords: newRecords.length }; + return { dbPath, recordCount: totalRows, newRecords: newEntries.length }; } finally { db.close(); } @@ -530,6 +566,10 @@ export async function searchBookmarks(options: SearchOptions): Promise 0 ? `WHERE ${conditions.join(' AND ')}` : ''; @@ -779,7 +819,7 @@ export async function getBookmarkById(id: string): Promise row[0]) ); const topAuthorsRows = db.exec( `SELECT author_handle, COUNT(*) as c FROM bookmarks - WHERE author_handle IS NOT NULL - GROUP BY author_handle ORDER BY c DESC LIMIT 15` + WHERE author_handle IS NOT NULL ${sourceAnd} + GROUP BY author_handle ORDER BY c DESC LIMIT 15`, + src ? [src] : [] ); const topAuthors = (topAuthorsRows[0]?.values ?? []).map((r) => ({ handle: r[0] as string, @@ -809,8 +854,9 @@ export async function getStats(): Promise<{ const langRows = db.exec( `SELECT language, COUNT(*) as c FROM bookmarks - WHERE language IS NOT NULL - GROUP BY language ORDER BY c DESC LIMIT 10` + WHERE language IS NOT NULL ${sourceAnd} + GROUP BY language ORDER BY c DESC LIMIT 10`, + src ? [src] : [] ); const languageBreakdown = (langRows[0]?.values ?? []).map((r) => ({ language: r[0] as string, diff --git a/src/cli.ts b/src/cli.ts index 0c98402..16a42cd 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -7,6 +7,18 @@ import { syncBookmarksGraphQL, syncGaps, syncBookmarkFolders } from './graphql-b import type { SyncProgress, GapFillProgress, FolderSyncProgress } from './graphql-bookmarks.js'; import type { BookmarkFolder } from './types.js'; import { fetchBookmarkMediaBatch } from './bookmark-media.js'; +import { syncUserTimeline, type UserSyncOptions } from './graphql-user-sync.js'; +import { + twitterLikesCachePath, + twitterLikesMetaPath, + twitterLikesBackfillStatePath, + twitterTimelineCachePath, + twitterTimelineMetaPath, + twitterTimelineBackfillStatePath, + twitterFeedCachePath, + twitterFeedMetaPath, + twitterFeedBackfillStatePath, +} from './paths.js'; import { buildIndex, searchBookmarks, @@ -89,9 +101,18 @@ const FRIENDLY_STOP_REASONS: Record = { 'caught up to newest stored bookmark': 'All caught up \u2014 no new bookmarks since last sync.', 'no new bookmarks (stale)': 'Sync complete \u2014 reached the end of new bookmarks.', 'end of bookmarks': 'Sync complete \u2014 all bookmarks fetched.', + 'caught up to newest stored like': 'All caught up \u2014 no new likes since last sync.', + 'no new likes (stale)': 'Sync complete \u2014 reached the end of new likes.', + 'end of likes': 'Sync complete \u2014 all likes fetched.', + 'caught up to newest stored tweet': 'All caught up \u2014 no new tweets since last sync.', + 'no new tweets (stale)': 'Sync complete \u2014 reached the end of new tweets.', + 'end of tweets': 'Sync complete \u2014 all tweets fetched.', + 'caught up to newest stored feed item': 'All caught up \u2014 no new feed items since last sync.', + 'no new feed items (stale)': 'Sync complete \u2014 reached the end of new feed items.', + 'end of feed items': 'Sync complete \u2014 all feed items fetched.', 'max runtime reached': 'Paused after 30 minutes. Run again to continue.', 'max pages reached': 'Paused after reaching page limit. Run again to continue.', - 'target additions reached': 'Reached target bookmark count.', + 'target additions reached': 'Reached target count.', }; function friendlyStopReason(raw?: string): string { @@ -788,6 +809,118 @@ export function buildCli() { } }); + // ── sync-likes / sync-timeline / sync-feed ───────────────────────────── + // + // Shared helper: extracts cookies from --cookies flag, builds sync options, + // runs syncUserTimeline with a spinner, and prints the result. + // + + const VALID_SOURCES = ['bookmarks', 'likes', 'timeline', 'feed'] as const; + + function parseSource(value: string): string { + if (!(VALID_SOURCES as readonly string[]).includes(value)) { + throw new Error(`Invalid source: ${value}. Valid values: ${VALID_SOURCES.join(', ')}`); + } + return value; + } + + interface UserSyncCmdConfig { + type: import('./graphql-user-sync.js').UserSyncType; + label: string; + paths: { cache: string; meta: string; state: string }; + } + + function parseCookieOptions(options: any): { csrfToken?: string; cookieHeader?: string } { + if (!options.cookies || !Array.isArray(options.cookies) || options.cookies.length === 0) { + return {}; + } + const csrfToken = String(options.cookies[0]); + const authToken = options.cookies.length > 1 ? String(options.cookies[1]) : undefined; + const parts = [`ct0=${csrfToken}`]; + if (authToken) parts.push(`auth_token=${authToken}`); + return { csrfToken, cookieHeader: parts.join('; ') }; + } + + function addBrowserOptions(cmd: import('commander').Command): import('commander').Command { + return cmd + .option('--max-pages ', 'Max pages to fetch', (v: string) => Number(v), 500) + .option('--target-adds ', 'Stop after N new items', (v: string) => Number(v)) + .option('--delay-ms ', 'Delay between requests in ms', (v: string) => Number(v), 600) + .option('--max-minutes ', 'Max runtime in minutes', (v: string) => Number(v), 30) + .option('--browser ', 'Browser to read session from (chrome, chromium, brave, firefox, ...)') + .option('--cookies ', 'Pass ct0 and auth_token directly (skips browser extraction)') + .option('--chrome-user-data-dir ', 'Chrome-family user-data directory') + .option('--chrome-profile-directory ', 'Chrome-family profile name') + .option('--firefox-profile-dir ', 'Firefox profile directory'); + } + + function registerUserSyncCommand(cfg: UserSyncCmdConfig, description: string, hasScreenNameArg: boolean): void { + let cmd = program.command(cfg.type === 'timeline' ? 'sync-timeline' : cfg.type === 'likes' ? 'sync-likes' : 'sync-feed') + .description(description); + if (hasScreenNameArg) { + cmd = cmd.argument('', 'Your X screen name (without @)'); + } + cmd = addBrowserOptions(cmd); + cmd.action(safe(async (...args: any[]) => { + const screenName = hasScreenNameArg ? String(args[0]) : undefined; + const options = hasScreenNameArg ? args[1] : args[0]; + ensureDataDir(); + const startTime = Date.now(); + let lastSync: SyncProgress = { page: 0, totalFetched: 0, newAdded: 0, running: true, done: false }; + const spinner = createSpinner(() => { + const elapsed = Math.round((Date.now() - startTime) / 1000); + return `Syncing ${cfg.label}... ${lastSync.newAdded} new \u2502 page ${lastSync.page} \u2502 ${elapsed}s`; + }); + + const { csrfToken, cookieHeader } = parseCookieOptions(options); + + const result = await runWithSpinner(spinner, () => syncUserTimeline( + cfg.type, + cfg.paths, + { + screenName, + incremental: true, + maxPages: Number(options.maxPages) || 500, + targetAdds: typeof options.targetAdds === 'number' && !Number.isNaN(options.targetAdds) ? options.targetAdds : undefined, + delayMs: Number(options.delayMs) || 600, + maxMinutes: Number(options.maxMinutes) || 30, + browser: options.browser ? String(options.browser) : undefined, + csrfToken, + cookieHeader, + chromeUserDataDir: options.chromeUserDataDir ? String(options.chromeUserDataDir) : undefined, + chromeProfileDirectory: options.chromeProfileDirectory ? String(options.chromeProfileDirectory) : undefined, + firefoxProfileDir: options.firefoxProfileDir ? String(options.firefoxProfileDir) : undefined, + onProgress: (status: SyncProgress) => { + lastSync = status; + spinner.update(); + }, + } + )); + + console.log(`\n \u2713 ${result.added} new ${cfg.label} synced (${result.totalBookmarks} total)`); + console.log(` ${friendlyStopReason(result.stopReason)}`); + console.log(` \u2713 Data: ${dataDir()}\n`); + })); + } + + registerUserSyncCommand( + { type: 'likes', label: 'likes', paths: { cache: twitterLikesCachePath(), meta: twitterLikesMetaPath(), state: twitterLikesBackfillStatePath() } }, + 'Sync liked tweets from X into your local database', + true, + ); + + registerUserSyncCommand( + { type: 'timeline', label: 'tweets', paths: { cache: twitterTimelineCachePath(), meta: twitterTimelineMetaPath(), state: twitterTimelineBackfillStatePath() } }, + 'Sync your own tweets from X into your local database', + true, + ); + + registerUserSyncCommand( + { type: 'feed', label: 'feed items', paths: { cache: twitterFeedCachePath(), meta: twitterFeedMetaPath(), state: twitterFeedBackfillStatePath() } }, + 'Sync your Following feed from X into your local database', + false, + ); + // ── search ────────────────────────────────────────────────────────────── program @@ -797,6 +930,7 @@ export function buildCli() { .option('--author ', 'Filter by author handle') .option('--after ', 'Bookmarks posted after this date (YYYY-MM-DD)') .option('--before ', 'Bookmarks posted before this date (YYYY-MM-DD)') + .option('--source ', 'Filter by source (bookmarks, likes, timeline, feed)', parseSource) .option('--limit ', 'Max results', (v: string) => Number(v), 20) .action(safe(async (query: string, options) => { if (!requireIndex()) return; @@ -805,6 +939,7 @@ export function buildCli() { author: options.author ? String(options.author) : undefined, after: options.after ? String(options.after) : undefined, before: options.before ? String(options.before) : undefined, + source: options.source ? String(options.source) : undefined, limit: Number(options.limit) || 20, }); console.log(formatSearchResults(results)); @@ -822,6 +957,7 @@ export function buildCli() { .option('--category ', 'Filter by category') .option('--domain ', 'Filter by domain') .option('--folder ', 'Filter by X bookmark folder name (exact or unambiguous prefix)') + .option('--source ', 'Filter by source (bookmarks, likes, timeline, feed)', parseSource) .option('--limit ', 'Max results', (v: string) => Number(v), 30) .option('--offset ', 'Offset into results', (v: string) => Number(v), 0) .option('--json', 'JSON output') @@ -851,6 +987,7 @@ export function buildCli() { category: options.category ? String(options.category) : undefined, domain: options.domain ? String(options.domain) : undefined, folder: resolvedFolder, + source: options.source ? String(options.source) : undefined, limit: Number(options.limit) || 30, offset: Number(options.offset) || 0, }); @@ -900,9 +1037,10 @@ export function buildCli() { program .command('stats') .description('Aggregate statistics from your bookmarks') - .action(safe(async () => { + .option('--source ', 'Filter by source (bookmarks, likes, timeline, feed)', parseSource) + .action(safe(async (options: any) => { if (!requireIndex()) return; - const stats = await getStats(); + const stats = await getStats({ source: options.source ? String(options.source) : undefined }); console.log(`Bookmarks: ${stats.totalBookmarks}`); console.log(`Unique authors: ${stats.uniqueAuthors}`); console.log(`Date range: ${stats.dateRange.earliest?.slice(0, 10) ?? '?'} to ${stats.dateRange.latest?.slice(0, 10) ?? '?'}`); diff --git a/src/graphql-bookmarks.ts b/src/graphql-bookmarks.ts index 624ff7a..37fb4bf 100644 --- a/src/graphql-bookmarks.ts +++ b/src/graphql-bookmarks.ts @@ -9,9 +9,9 @@ import { exportBookmarksForSyncSeed, updateQuotedTweets, updateBookmarkText, upd import type { ArticleUpdate } from './bookmarks-db.js'; import { fetchArticle, resolveTcoLink } from './bookmark-enrich.js'; -const CHROME_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'; +export const CHROME_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'; -const X_PUBLIC_BEARER = +export const X_PUBLIC_BEARER = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; const BOOKMARKS_QUERY_ID = 'Z9GWmP0kP2dajyckAaDUBw'; @@ -106,7 +106,7 @@ export interface SyncResult { statePath: string; } -function parseSnowflake(value?: string | null): bigint | null { +export function parseSnowflake(value?: string | null): bigint | null { if (!value || !/^\d+$/.test(value)) return null; try { return BigInt(value); @@ -205,7 +205,7 @@ function buildUrl(cursor?: string, count = 20): string { return `https://x.com/i/api/graphql/${BOOKMARKS_QUERY_ID}/${BOOKMARKS_OPERATION}?${params}`; } -function buildHeaders(csrfToken: string, cookieHeader?: string): Record { +export function buildHeaders(csrfToken: string, cookieHeader?: string): Record { return { authorization: `Bearer ${X_PUBLIC_BEARER}`, 'x-csrf-token': csrfToken, @@ -351,6 +351,20 @@ export function convertTweetToRecord(tweetResult: any, now: string): BookmarkRec }; } +const TWITTER_SNOWFLAKE_EPOCH = 1288834974657n; + +export function snowflakeToIso(snowflake: string): string | null { + try { + const id = BigInt(snowflake); + const ms = Number(id >> 22n) + Number(TWITTER_SNOWFLAKE_EPOCH); + const date = new Date(ms); + return Number.isFinite(date.getTime()) ? date.toISOString() : null; + } catch { + return null; + } +} + + export function parseBookmarksResponse(json: any, now?: string): PageResult { const ts = now ?? new Date().toISOString(); const instructions = json?.data?.bookmark_timeline_v2?.timeline?.instructions ?? []; diff --git a/src/graphql-user-sync.ts b/src/graphql-user-sync.ts new file mode 100644 index 0000000..65b78b4 --- /dev/null +++ b/src/graphql-user-sync.ts @@ -0,0 +1,491 @@ +/** + * Generic GraphQL sync engine for user-scoped timelines (likes, user tweets) + * and the authenticated user's home feed (Following/chronological). + * + * - likes / timeline: require a userId resolved from screenName. + * - feed: uses the authenticated session directly (no userId needed). + */ +import { readJsonLines, writeJsonLines, readJson, writeJson, pathExists } from './fs.js'; +import { ensureDataDir } from './paths.js'; +import { loadChromeSessionConfig } from './config.js'; +import { extractChromeXCookies } from './chrome-cookies.js'; +import { extractFirefoxXCookies } from './firefox-cookies.js'; +import { + convertTweetToRecord, + mergeRecords, + snowflakeToIso, + parseSnowflake, + CHROME_UA, + X_PUBLIC_BEARER, + buildHeaders, + type SyncOptions, + type SyncProgress, + type SyncResult, +} from './graphql-bookmarks.js'; +import type { BookmarkBackfillState, BookmarkCacheMeta, BookmarkRecord } from './types.js'; + +// ── Query IDs (extracted from X's JS bundles, April 2026) ────────── + +const LIKES_QUERY_ID = 'KPuet6dGbC8LB2sOLx7tZQ'; +const LIKES_OPERATION = 'Likes'; + +const USER_TWEETS_QUERY_ID = 'x3B_xLqC0yZawOB7WQhaVQ'; +const USER_TWEETS_OPERATION = 'UserTweets'; + +const FEED_QUERY_ID = '2ee46L1AFXmnTa0EvUog-Q'; +const FEED_OPERATION = 'HomeLatestTimeline'; + +const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; +const USER_BY_SCREEN_NAME_OPERATION = 'UserByScreenName'; + +// Feature flags shared by user timeline queries +const USER_TIMELINE_FEATURES = { + rweb_video_screen_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: false, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: false, + responsive_web_grok_share_attachment_enabled: false, + responsive_web_grok_annotations_enabled: false, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + freedom_of_speech_not_reach_fetch_enabled: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_enhance_cards_enabled: false, +}; + +const USER_BY_SCREEN_NAME_FEATURES = { + hidden_profile_subscriptions_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + rweb_tipjar_consumption_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + subscriptions_verification_info_is_identity_verified_enabled: true, + subscriptions_verification_info_verified_since_enabled: true, + highlights_tweets_tab_ui_enabled: true, + responsive_web_twitter_article_notes_tab_enabled: true, + subscriptions_feature_can_gift_premium: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, +}; + +export type UserSyncType = 'likes' | 'timeline' | 'feed'; + +interface UserSyncConfig { + queryId: string; + operationName: string; + cachePath: string; + metaPath: string; + statePath: string; + ingestedVia: BookmarkRecord['ingestedVia']; + label: string; + /** true when the endpoint does not require a userId (e.g. home feed). */ + sessionScoped: boolean; +} + +function getConfig(type: UserSyncType, paths: { cache: string; meta: string; state: string }): UserSyncConfig { + if (type === 'likes') { + return { + queryId: LIKES_QUERY_ID, + operationName: LIKES_OPERATION, + cachePath: paths.cache, + metaPath: paths.meta, + statePath: paths.state, + ingestedVia: 'graphql-likes', + label: 'likes', + sessionScoped: false, + }; + } + if (type === 'feed') { + return { + queryId: FEED_QUERY_ID, + operationName: FEED_OPERATION, + cachePath: paths.cache, + metaPath: paths.meta, + statePath: paths.state, + ingestedVia: 'graphql-feed', + label: 'feed items', + sessionScoped: true, + }; + } + return { + queryId: USER_TWEETS_QUERY_ID, + operationName: USER_TWEETS_OPERATION, + cachePath: paths.cache, + metaPath: paths.meta, + statePath: paths.state, + ingestedVia: 'graphql-timeline', + label: 'tweets', + sessionScoped: false, + }; +} + +// ── Resolve userId from screen name ──────────────────────────────── + +function buildUserByScreenNameUrl(screenName: string): string { + const variables = { screen_name: screenName, withSafetyModeUserFields: true }; + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(USER_BY_SCREEN_NAME_FEATURES), + fieldToggles: JSON.stringify({ withAuxiliaryUserLabels: false }), + }); + return `https://x.com/i/api/graphql/${USER_BY_SCREEN_NAME_QUERY_ID}/${USER_BY_SCREEN_NAME_OPERATION}?${params}`; +} + +async function resolveUserId( + screenName: string, + csrfToken: string, + cookieHeader?: string +): Promise { + const url = buildUserByScreenNameUrl(screenName); + const response = await fetch(url, { headers: buildHeaders(csrfToken, cookieHeader) }); + if (!response.ok) { + const text = await response.text(); + throw new Error( + `Failed to resolve userId for @${screenName}: HTTP ${response.status}\n${text.slice(0, 300)}` + ); + } + const json = (await response.json()) as any; + const userId = json?.data?.user?.result?.rest_id; + if (!userId) { + throw new Error(`Could not find userId for @${screenName}. Make sure the account exists and is not suspended.`); + } + return userId; +} + +// ── GraphQL request helpers ──────────────────────────────────────── + +function buildUserTimelineUrl(config: UserSyncConfig, userId: string, cursor?: string): string { + const variables: Record = { + userId, + count: 20, + includePromotedContent: false, + withQuickPromoteEligibilityTweetFields: false, + withVoice: true, + withV2Timeline: true, + }; + if (cursor) variables.cursor = cursor; + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(USER_TIMELINE_FEATURES), + fieldToggles: JSON.stringify({ withArticlePlainText: false }), + }); + return `https://x.com/i/api/graphql/${config.queryId}/${config.operationName}?${params}`; +} + +function buildFeedUrl(config: UserSyncConfig, cursor?: string): string { + const variables: Record = { + count: 20, + includePromotedContent: false, + latestControlAvailable: true, + requestContext: 'launch', + }; + if (cursor) variables.cursor = cursor; + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(USER_TIMELINE_FEATURES), + fieldToggles: JSON.stringify({ withArticlePlainText: false }), + }); + return `https://x.com/i/api/graphql/${config.queryId}/${config.operationName}?${params}`; +} + +function buildRequestUrl(config: UserSyncConfig, userId: string | undefined, cursor?: string): string { + if (config.sessionScoped) { + return buildFeedUrl(config, cursor); + } + if (!userId) { + throw new Error(`userId is required for ${config.operationName}`); + } + return buildUserTimelineUrl(config, userId, cursor); +} + +// ── Response parsing ─────────────────────────────────────────────── + +interface PageResult { + records: BookmarkRecord[]; + nextCursor?: string; +} + +export function parseUserTimelineResponse(json: any, ingestedVia: BookmarkRecord['ingestedVia'], now?: string): PageResult { + const ts = now ?? new Date().toISOString(); + + // User timeline responses nest under data.user.result.timeline_v2.timeline + // Home feed responses nest under data.home.home_timeline_urt + const timeline = json?.data?.user?.result?.timeline_v2?.timeline + ?? json?.data?.user?.result?.timeline?.timeline; + const instructions = timeline?.instructions + ?? json?.data?.home?.home_timeline_urt?.instructions + ?? []; + + const entries: any[] = []; + for (const inst of instructions) { + if (inst.type === 'TimelineAddEntries' && Array.isArray(inst.entries)) { + entries.push(...inst.entries); + } + } + + const records: BookmarkRecord[] = []; + let nextCursor: string | undefined; + + for (const entry of entries) { + if (entry.entryId?.startsWith('cursor-bottom')) { + nextCursor = entry.content?.value; + continue; + } + + // User timeline entries can be nested under itemContent or items (for conversation modules) + const tweetResult = + entry?.content?.itemContent?.tweet_results?.result + ?? entry?.content?.items?.[0]?.item?.itemContent?.tweet_results?.result; + if (!tweetResult) continue; + + const record = convertTweetToRecord(tweetResult, ts); + if (record) { + record.ingestedVia = ingestedVia; + if (ingestedVia === 'graphql-likes' && entry.sortIndex) { + // sortIndex on likes entries is a snowflake for when the like happened + record.likedAt = snowflakeToIso(entry.sortIndex) ?? entry.sortIndex; + } + records.push(record); + } + } + + return { records, nextCursor }; +} + +// ── Fetch with retry ─────────────────────────────────────────────── + +async function fetchPageWithRetry( + config: UserSyncConfig, + userId: string | undefined, + csrfToken: string, + cursor?: string, + cookieHeader?: string +): Promise { + let lastError: Error | undefined; + + for (let attempt = 0; attempt < 4; attempt++) { + const response = await fetch(buildRequestUrl(config, userId, cursor), { + headers: buildHeaders(csrfToken, cookieHeader), + }); + + if (response.status === 429) { + const waitSec = Math.min(15 * Math.pow(2, attempt), 120); + lastError = new Error(`Rate limited (429) on attempt ${attempt + 1}`); + await new Promise((r) => setTimeout(r, waitSec * 1000)); + continue; + } + + if (response.status >= 500) { + lastError = new Error(`Server error (${response.status}) on attempt ${attempt + 1}`); + await new Promise((r) => setTimeout(r, 5000 * (attempt + 1))); + continue; + } + + if (!response.ok) { + const text = await response.text(); + throw new Error( + `GraphQL ${config.operationName} API returned ${response.status}.\n` + + `Response: ${text.slice(0, 300)}\n\n` + + (response.status === 401 || response.status === 403 + ? 'Fix: Your X session may have expired. Open your browser, go to https://x.com, and make sure you are logged in. Then retry.' + : 'This may be a temporary issue. Try again in a few minutes.') + ); + } + + const json = await response.json(); + return parseUserTimelineResponse(json, config.ingestedVia); + } + + throw lastError ?? new Error(`GraphQL ${config.operationName} API: all retry attempts failed.`); +} + +// ── Extract cookies (shared logic) ───────────────────────────────── + +function extractCookies(options: SyncOptions): { csrfToken: string; cookieHeader?: string } { + if (options.csrfToken) { + return { csrfToken: options.csrfToken, cookieHeader: options.cookieHeader }; + } + + const config = loadChromeSessionConfig({ browserId: options.browser }); + + if (config.browser.cookieBackend === 'firefox') { + const cookies = extractFirefoxXCookies(options.firefoxProfileDir); + return { csrfToken: cookies.csrfToken, cookieHeader: cookies.cookieHeader }; + } + + const chromeDir = options.chromeUserDataDir ?? config.chromeUserDataDir; + const chromeProfile = options.chromeProfileDirectory ?? config.chromeProfileDirectory; + const cookies = extractChromeXCookies(chromeDir, chromeProfile, config.browser); + return { csrfToken: cookies.csrfToken, cookieHeader: cookies.cookieHeader }; +} + +// ── Main sync function ───────────────────────────────────────────── + +export interface UserSyncOptions extends SyncOptions { + /** X screen name (without @) — required for likes/timeline, unused for feed. */ + screenName?: string; +} + +export async function syncUserTimeline( + type: UserSyncType, + paths: { cache: string; meta: string; state: string }, + options: UserSyncOptions +): Promise { + const config = getConfig(type, paths); + const incremental = options.incremental ?? true; + const maxPages = options.maxPages ?? 500; + const delayMs = options.delayMs ?? 600; + const maxMinutes = options.maxMinutes ?? 30; + const stalePageLimit = options.stalePageLimit ?? 3; + const checkpointEvery = options.checkpointEvery ?? 25; + + const { csrfToken, cookieHeader } = extractCookies(options); + + ensureDataDir(); + + // Resolve userId from screen name (only for likes/timeline) + let userId: string | undefined; + if (!config.sessionScoped) { + if (!options.screenName) { + throw new Error(`screenName is required for ${type} sync`); + } + userId = await resolveUserId(options.screenName, csrfToken, cookieHeader); + } + + // Load existing records + let existing: BookmarkRecord[] = []; + if (await pathExists(config.cachePath)) { + existing = await readJsonLines(config.cachePath); + } + + const newestKnownId = incremental + ? existing.slice().sort((a, b) => { + const aId = parseSnowflake(a.tweetId) ?? 0n; + const bId = parseSnowflake(b.tweetId) ?? 0n; + return aId > bId ? -1 : aId < bId ? 1 : 0; + })[0]?.id + : undefined; + + const previousMeta = (await pathExists(config.metaPath)) + ? await readJson(config.metaPath) + : undefined; + const prevState: BookmarkBackfillState = (await pathExists(config.statePath)) + ? await readJson(config.statePath) + : { provider: 'twitter', totalRuns: 0, totalAdded: 0, lastAdded: 0, lastSeenIds: [] }; + + const started = Date.now(); + let page = 0; + let totalAdded = 0; + let stalePages = 0; + let cursor: string | undefined; + const allSeenIds: string[] = []; + let stopReason = 'unknown'; + + while (page < maxPages) { + if (Date.now() - started > maxMinutes * 60_000) { + stopReason = 'max runtime reached'; + break; + } + + const result = await fetchPageWithRetry(config, userId, csrfToken, cursor, cookieHeader); + page += 1; + + if (result.records.length === 0 && !result.nextCursor) { + stopReason = `end of ${config.label}`; + break; + } + + const { merged, added } = mergeRecords(existing, result.records); + existing = merged; + totalAdded += added; + result.records.forEach((r) => allSeenIds.push(r.id)); + const reachedLatestStored = Boolean(newestKnownId) && result.records.some((record) => record.id === newestKnownId); + + stalePages = added === 0 ? stalePages + 1 : 0; + + options.onProgress?.({ + page, + totalFetched: allSeenIds.length, + newAdded: totalAdded, + running: true, + done: false, + }); + + if (options.targetAdds && totalAdded >= options.targetAdds) { + stopReason = 'target additions reached'; + break; + } + if (reachedLatestStored) { + stopReason = `caught up to newest stored ${config.label === 'likes' ? 'like' : config.label === 'tweets' ? 'tweet' : 'feed item'}`; + break; + } + if (stalePages >= stalePageLimit) { + stopReason = `no new ${config.label} (stale)`; + break; + } + if (!result.nextCursor) { + stopReason = `end of ${config.label}`; + break; + } + + if (page % checkpointEvery === 0) await writeJsonLines(config.cachePath, existing); + + cursor = result.nextCursor; + if (page < maxPages) await new Promise((r) => setTimeout(r, delayMs)); + } + + if (stopReason === 'unknown') stopReason = page >= maxPages ? 'max pages reached' : 'unknown'; + + const syncedAt = new Date().toISOString(); + await writeJsonLines(config.cachePath, existing); + await writeJson(config.metaPath, { + provider: 'twitter', + schemaVersion: 1, + lastFullSyncAt: incremental ? previousMeta?.lastFullSyncAt : syncedAt, + lastIncrementalSyncAt: incremental ? syncedAt : previousMeta?.lastIncrementalSyncAt, + totalBookmarks: existing.length, + } satisfies BookmarkCacheMeta); + await writeJson(config.statePath, { + provider: 'twitter', + lastRunAt: syncedAt, + totalRuns: prevState.totalRuns + 1, + totalAdded: prevState.totalAdded + totalAdded, + lastAdded: totalAdded, + lastSeenIds: allSeenIds.slice(-20), + stopReason, + } satisfies BookmarkBackfillState); + + options.onProgress?.({ + page, + totalFetched: allSeenIds.length, + newAdded: totalAdded, + running: false, + done: true, + stopReason, + }); + + return { + added: totalAdded, + totalBookmarks: existing.length, + pages: page, + stopReason, + cachePath: config.cachePath, + statePath: config.statePath, + }; +} \ No newline at end of file diff --git a/src/paths.ts b/src/paths.ts index 852ad30..1ed5f0c 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -48,6 +48,42 @@ export function twitterBookmarksIndexPath(): string { return path.join(dataDir(), 'bookmarks.db'); } +export function twitterLikesCachePath(): string { + return path.join(dataDir(), 'likes.jsonl'); +} + +export function twitterLikesMetaPath(): string { + return path.join(dataDir(), 'likes-meta.json'); +} + +export function twitterLikesBackfillStatePath(): string { + return path.join(dataDir(), 'likes-backfill-state.json'); +} + +export function twitterTimelineCachePath(): string { + return path.join(dataDir(), 'timeline.jsonl'); +} + +export function twitterTimelineMetaPath(): string { + return path.join(dataDir(), 'timeline-meta.json'); +} + +export function twitterTimelineBackfillStatePath(): string { + return path.join(dataDir(), 'timeline-backfill-state.json'); +} + +export function twitterFeedCachePath(): string { + return path.join(dataDir(), 'feed.jsonl'); +} + +export function twitterFeedMetaPath(): string { + return path.join(dataDir(), 'feed-meta.json'); +} + +export function twitterFeedBackfillStatePath(): string { + return path.join(dataDir(), 'feed-backfill-state.json'); +} + export function preferencesPath(): string { return path.join(dataDir(), '.preferences'); } diff --git a/src/types.ts b/src/types.ts index bad472e..33394ee 100644 --- a/src/types.ts +++ b/src/types.ts @@ -79,7 +79,8 @@ export interface BookmarkRecord { mediaObjects?: BookmarkMediaObject[]; links?: string[]; tags?: string[]; - ingestedVia?: 'api' | 'browser' | 'graphql'; + likedAt?: string | null; + ingestedVia?: 'api' | 'browser' | 'graphql' | 'graphql-likes' | 'graphql-timeline' | 'graphql-feed'; /** Parallel arrays of folder IDs and display names this bookmark is in on X. */ folderIds?: string[]; folderNames?: string[]; diff --git a/tests/graphql-user-sync.test.ts b/tests/graphql-user-sync.test.ts new file mode 100644 index 0000000..738d04d --- /dev/null +++ b/tests/graphql-user-sync.test.ts @@ -0,0 +1,391 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseUserTimelineResponse } from '../src/graphql-user-sync.js'; +import type { BookmarkRecord } from '../src/types.js'; + +const NOW = '2026-03-28T00:00:00.000Z'; + +function makeTweetResult(overrides: Record = {}) { + return { + rest_id: '1234567890', + legacy: { + id_str: '1234567890', + full_text: 'Hello world, this is a test tweet!', + created_at: 'Tue Mar 10 12:00:00 +0000 2026', + favorite_count: 42, + retweet_count: 5, + reply_count: 3, + quote_count: 1, + bookmark_count: 7, + conversation_id_str: '1234567890', + lang: 'en', + entities: { + urls: [ + { expanded_url: 'https://example.com/article', url: 'https://t.co/abc' }, + { expanded_url: 'https://t.co/internal', url: 'https://t.co/def' }, + ], + }, + extended_entities: { + media: [ + { + type: 'photo', + media_url_https: 'https://pbs.twimg.com/media/example.jpg', + expanded_url: 'https://x.com/user/status/1234567890/photo/1', + original_info: { width: 1200, height: 800 }, + ext_alt_text: 'A test image', + }, + ], + }, + ...overrides.legacy, + }, + core: { + user_results: { + result: { + rest_id: '9876', + core: { screen_name: 'testuser', name: 'Test User' }, + avatar: { image_url: 'https://pbs.twimg.com/profile_images/9876/photo.jpg' }, + legacy: { + description: 'I test things', + followers_count: 1000, + friends_count: 200, + location: 'San Francisco', + verified: false, + }, + is_blue_verified: true, + ...overrides.userResult, + }, + }, + }, + views: { count: '15000' }, + ...overrides.tweet, + }; +} + +function makeUserTimelineResponse(tweetResults: any[], bottomCursor?: string) { + const entries = tweetResults.map((tr, i) => ({ + entryId: `tweet-${i}`, + content: { + itemContent: { + tweet_results: { result: tr }, + }, + }, + })); + + if (bottomCursor !== undefined) { + entries.push({ + entryId: 'cursor-bottom-123', + content: { value: bottomCursor } as any, + }); + } + + return { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [ + { type: 'TimelineAddEntries', entries }, + ], + }, + }, + }, + }, + }, + }; +} + +function makeFeedResponse(tweetResults: any[], bottomCursor?: string) { + const entries = tweetResults.map((tr, i) => ({ + entryId: `tweet-${i}`, + content: { + itemContent: { + tweet_results: { result: tr }, + }, + }, + })); + + if (bottomCursor !== undefined) { + entries.push({ + entryId: 'cursor-bottom-456', + content: { value: bottomCursor } as any, + }); + } + + return { + data: { + home: { + home_timeline_urt: { + instructions: [ + { type: 'TimelineAddEntries', entries }, + ], + }, + }, + }, + }; +} + +// ── User timeline response parsing ─────────────────────────────── + +test('parseUserTimelineResponse: parses user timeline entries', () => { + const tr = makeTweetResult(); + const resp = makeUserTimelineResponse([tr]); + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 1); + assert.equal(records[0].id, '1234567890'); + assert.equal(records[0].text, 'Hello world, this is a test tweet!'); +}); + +test('parseUserTimelineResponse: parses multiple user timeline entries', () => { + const tr1 = makeTweetResult(); + const tr2 = makeTweetResult({ legacy: { id_str: '2222222', full_text: 'Second tweet' } }); + const resp = makeUserTimelineResponse([tr1, tr2]); + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 2); +}); + +// ── Feed response parsing ──────────────────────────────────────── + +test('parseUserTimelineResponse: parses feed response entries', () => { + const tr = makeTweetResult(); + const resp = makeFeedResponse([tr]); + const { records } = parseUserTimelineResponse(resp, 'graphql-feed', NOW); + + assert.equal(records.length, 1); + assert.equal(records[0].id, '1234567890'); +}); + +test('parseUserTimelineResponse: parses multiple feed entries', () => { + const tr1 = makeTweetResult(); + const tr2 = makeTweetResult({ legacy: { id_str: '3333333', full_text: 'Feed tweet' } }); + const resp = makeFeedResponse([tr1, tr2]); + const { records } = parseUserTimelineResponse(resp, 'graphql-feed', NOW); + + assert.equal(records.length, 2); +}); + +// ── Cursor extraction ──────────────────────────────────────────── + +test('parseUserTimelineResponse: extracts bottom cursor from user timeline', () => { + const tr = makeTweetResult(); + const resp = makeUserTimelineResponse([tr], 'cursor-abc-123'); + const { nextCursor } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(nextCursor, 'cursor-abc-123'); +}); + +test('parseUserTimelineResponse: extracts bottom cursor from feed', () => { + const tr = makeTweetResult(); + const resp = makeFeedResponse([tr], 'cursor-feed-xyz'); + const { nextCursor } = parseUserTimelineResponse(resp, 'graphql-feed', NOW); + + assert.equal(nextCursor, 'cursor-feed-xyz'); +}); + +test('parseUserTimelineResponse: no cursor when not present', () => { + const resp = makeUserTimelineResponse([makeTweetResult()]); + const { nextCursor } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(nextCursor, undefined); +}); + +// ── Conversation modules ───────────────────────────────────────── + +test('parseUserTimelineResponse: extracts tweet from conversation module items', () => { + const tr = makeTweetResult(); + const resp = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [{ + type: 'TimelineAddEntries', + entries: [{ + entryId: 'conversationthread-111', + content: { + items: [{ + item: { + itemContent: { + tweet_results: { result: tr }, + }, + }, + }], + }, + }], + }], + }, + }, + }, + }, + }, + }; + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 1); + assert.equal(records[0].id, '1234567890'); +}); + +// ── Empty / missing data ───────────────────────────────────────── + +test('parseUserTimelineResponse: returns empty when json is empty object', () => { + const { records, nextCursor } = parseUserTimelineResponse({}, 'graphql-timeline', NOW); + + assert.equal(records.length, 0); + assert.equal(nextCursor, undefined); +}); + +test('parseUserTimelineResponse: returns empty when json is null', () => { + const { records, nextCursor } = parseUserTimelineResponse(null, 'graphql-timeline', NOW); + + assert.equal(records.length, 0); + assert.equal(nextCursor, undefined); +}); + +test('parseUserTimelineResponse: returns empty when json is undefined', () => { + const { records, nextCursor } = parseUserTimelineResponse(undefined, 'graphql-timeline', NOW); + + assert.equal(records.length, 0); + assert.equal(nextCursor, undefined); +}); + +test('parseUserTimelineResponse: returns empty when instructions array is empty', () => { + const resp = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [], + }, + }, + }, + }, + }, + }; + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 0); +}); + +test('parseUserTimelineResponse: skips entries with no tweet_results', () => { + const resp = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [{ + type: 'TimelineAddEntries', + entries: [ + { entryId: 'tweet-1', content: {} }, + { entryId: 'tweet-2', content: { itemContent: { tweet_results: { result: makeTweetResult() } } } }, + ], + }], + }, + }, + }, + }, + }, + }; + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 1); +}); + +// ── ingestedVia assignment ─────────────────────────────────────── + +test('parseUserTimelineResponse: sets ingestedVia to graphql-timeline', () => { + const resp = makeUserTimelineResponse([makeTweetResult()]); + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records[0].ingestedVia, 'graphql-timeline'); +}); + +test('parseUserTimelineResponse: sets ingestedVia to graphql-likes', () => { + const resp = makeUserTimelineResponse([makeTweetResult()]); + const { records } = parseUserTimelineResponse(resp, 'graphql-likes', NOW); + + assert.equal(records[0].ingestedVia, 'graphql-likes'); +}); + +test('parseUserTimelineResponse: sets ingestedVia to graphql-feed', () => { + const resp = makeFeedResponse([makeTweetResult()]); + const { records } = parseUserTimelineResponse(resp, 'graphql-feed', NOW); + + assert.equal(records[0].ingestedVia, 'graphql-feed'); +}); + +// ── likedAt extraction ─────────────────────────────────────────── + +test('parseUserTimelineResponse: sets likedAt from sortIndex when ingestedVia is graphql-likes', () => { + const tr = makeTweetResult(); + const resp = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [{ + type: 'TimelineAddEntries', + entries: [{ + entryId: 'tweet-0', + sortIndex: '2031116076166176768', + content: { + itemContent: { tweet_results: { result: tr } }, + }, + }], + }], + }, + }, + }, + }, + }, + }; + const { records } = parseUserTimelineResponse(resp, 'graphql-likes', NOW); + + assert.equal(records.length, 1); + // likedAt should be converted from snowflake to ISO date + assert.ok(records[0].likedAt, 'likedAt should be defined'); + assert.ok(!Number.isNaN(Date.parse(records[0].likedAt!)), `likedAt should be a valid ISO date, got: ${records[0].likedAt}`); +}); + +test('parseUserTimelineResponse: does not set likedAt when ingestedVia is not graphql-likes', () => { + const tr = makeTweetResult(); + const resp = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [{ + type: 'TimelineAddEntries', + entries: [{ + entryId: 'tweet-0', + sortIndex: '2031116076166176768', + content: { + itemContent: { tweet_results: { result: tr } }, + }, + }], + }], + }, + }, + }, + }, + }, + }; + const { records } = parseUserTimelineResponse(resp, 'graphql-timeline', NOW); + + assert.equal(records.length, 1); + assert.equal(records[0].likedAt, undefined); +}); + +test('parseUserTimelineResponse: does not set likedAt when sortIndex is absent', () => { + const resp = makeUserTimelineResponse([makeTweetResult()]); + const { records } = parseUserTimelineResponse(resp, 'graphql-likes', NOW); + + assert.equal(records.length, 1); + assert.equal(records[0].likedAt, undefined); +}); From 8da0b6729c8a6019c5e9ec2641353ad81b3cb657 Mon Sep 17 00:00:00 2001 From: Martin Menestret Date: Tue, 7 Apr 2026 10:39:48 +0200 Subject: [PATCH 2/3] fix: add ensureMigrations to getStats and rebuildIndex after user sync commands getStats now calls ensureMigrations(db) to avoid crashing on pre-v5 databases when --source is used. sync-likes, sync-timeline, and sync-feed now rebuild the search index after syncing, matching the existing sync command behavior. --- src/bookmarks-db.ts | 1 + src/cli.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index 86fb7de..2cdecec 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -828,6 +828,7 @@ export async function getStats(options?: { source?: string }): Promise<{ }> { const dbPath = twitterBookmarksIndexPath(); const db = await openDb(dbPath); + ensureMigrations(db); const src = options?.source; const sourceFilter = src ? 'WHERE source = ?' : ''; diff --git a/src/cli.ts b/src/cli.ts index 16a42cd..ace1bc3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -900,6 +900,7 @@ export function buildCli() { console.log(`\n \u2713 ${result.added} new ${cfg.label} synced (${result.totalBookmarks} total)`); console.log(` ${friendlyStopReason(result.stopReason)}`); console.log(` \u2713 Data: ${dataDir()}\n`); + await rebuildIndex(result.added); })); } From 39a2d2af9d928e7976a17b9557b6b481875eaeb4 Mon Sep 17 00:00:00 2001 From: Martin Menestret Date: Mon, 13 Apr 2026 16:56:33 +0200 Subject: [PATCH 3/3] fix: align sync commands with updated SyncResult and rebuildIndex signatures --- src/cli.ts | 2 +- src/graphql-user-sync.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index ace1bc3..0fa5060 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -900,7 +900,7 @@ export function buildCli() { console.log(`\n \u2713 ${result.added} new ${cfg.label} synced (${result.totalBookmarks} total)`); console.log(` ${friendlyStopReason(result.stopReason)}`); console.log(` \u2713 Data: ${dataDir()}\n`); - await rebuildIndex(result.added); + await rebuildIndex(); })); } diff --git a/src/graphql-user-sync.ts b/src/graphql-user-sync.ts index 65b78b4..562d37a 100644 --- a/src/graphql-user-sync.ts +++ b/src/graphql-user-sync.ts @@ -482,7 +482,9 @@ export async function syncUserTimeline( return { added: totalAdded, + bookmarkedAtRepaired: 0, totalBookmarks: existing.length, + bookmarkedAtMissing: 0, pages: page, stopReason, cachePath: config.cachePath,