Skip to content

fix: use transactionsHub endpoint so re-categorizations appear in CSV… #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CarterDunn wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: use transactionsHub endpoint so re-categorizations appear in CSV… #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 131 additions & 35 deletions content.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,48 +140,35 @@ function generateTraceId() {
}

/**
* Main API entry point: Orchestrates fetching recent + historical data
* Main API entry point.
*
* IMPORTANT: The fast GetTransactionsList endpoint returns STALE categories —
* it ignores any re-categorizations the user made in the Credit Karma UI
* (verified 2026-06: a transaction recategorized "Transfer" -> "Travel &
* vacation" weeks earlier still came back as "Transfer" from the list
* endpoint, while the transactionsHub endpoint — the one Credit Karma's own
* transactions page uses — returned the corrected category).
*
* So the paginated transactionsHub endpoint is the primary source for the
* entire date range. GetTransactionsList is only a fallback if hub fails.
*/
async function fetchTransactionsViaAPI(startDate, endDate, onProgress, signal) {
// 1. Fetch recent data using the fast GetTransactionsList endpoint
console.log('[API] Phase 1: Fetching recent transactions...');
let transactions = await fetchRecentTransactions(startDate, endDate, onProgress, signal);
console.log('[API] Fetching via transactionsHub (respects user re-categorizations)...');

if (!transactions) {
transactions = [];
}
const endDateTime = new Date(endDate);
endDateTime.setHours(23, 59, 59, 999);

// Sort to find the oldest date we have
transactions.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
const transactions = await fetchHistoricalTransactions(startDate, endDateTime, onProgress, signal);

let oldestDate = new Date();
if (transactions.length > 0) {
oldestDate = new Date(transactions[transactions.length - 1].date);
return transactions;
}

const reqStartDate = new Date(startDate);

// Check if we need to fetch older history
// Allow 7 day buffer
const daysDiff = (oldestDate - reqStartDate) / (1000 * 60 * 60 * 24);

if (daysDiff > 7) {
console.log(`[API] Gap detected: ${daysDiff.toFixed(1)} days. Phase 2: Fetching historical data...`);
if (onProgress) onProgress(`Fetching older history (${daysDiff.toFixed(0)} days gap)...`);

// Start from the day before our oldest known transaction
const historyEndDate = new Date(oldestDate);
historyEndDate.setDate(historyEndDate.getDate() - 1);

const historicalTransactions = await fetchHistoricalTransactions(startDate, historyEndDate, onProgress, signal);

if (historicalTransactions.length > 0) {
console.log(`[API] Merging ${historicalTransactions.length} historical transactions`);
transactions = [...transactions, ...historicalTransactions];
}
}

return transactions;
// Fallback: fast list endpoint. WARNING: its categories do not reflect
// user re-categorizations, so only use it if the hub returned nothing.
console.warn('[API] Hub endpoint returned no data — falling back to GetTransactionsList (categories may be stale)');
if (onProgress) onProgress('Hub endpoint failed, using fallback...');
return (await fetchRecentTransactions(startDate, endDate, onProgress, signal)) || [];
}

/**
Expand Down Expand Up @@ -433,7 +420,7 @@ async function fetchHistoricalTransactions(startDate, endDate, onProgress, signa

// Rate limiting delay
// Adaptive: If we are scanning (skipping), go faster. If collecting, go slower.
const delay = newItemsInPage > 0 ? 800 : 300;
const delay = newItemsInPage > 0 ? 400 : 250;
await new Promise(resolve => setTimeout(resolve, delay));

retryCount = 0; // Reset retries on success
Expand Down Expand Up @@ -654,6 +641,94 @@ async function enrichTransactionsWithAccountNames(transactions, onProgress) {
return enrichedTransactions;
}

// ============================================
// Debug: Raw API response dump
// ============================================

/**
* Fetch raw, unprocessed responses from BOTH transaction endpoints and save
* them as a JSON file. Used to diagnose schema issues (e.g. which field holds
* user re-categorizations) without any of the extension's field mapping.
* @param {number} maxHubPages - how many pages of the hub endpoint to fetch
*/
async function debugDumpRawResponses(maxHubPages = 5, onProgress) {
const headers = await getAuthHeaders();
const dump = {
generatedAt: new Date().toISOString(),
note: 'Raw GraphQL responses. transactionsList = GetTransactionsList (fast bulk endpoint). hubPages = GetTransactions (paginated endpoint used by the Credit Karma transactions page UI).',
transactionsList: null,
hubPages: []
};

// 1. Raw GetTransactionsList response
if (onProgress) onProgress('Debug: fetching GetTransactionsList...');
try {
const listResponse = await fetch(CONFIG.API_ENDPOINT, {
method: 'POST',
headers: headers,
credentials: 'include',
body: JSON.stringify({
extensions: { persistedQuery: { sha256Hash: CONFIG.TRANSACTIONS_LIST_HASH, version: 1 } },
operationName: "GetTransactionsList",
variables: { input: { accountInput: {}, categoryInput: { categoryId: null, primeCategoryType: null } } }
})
});
dump.transactionsList = await listResponse.json();
} catch (e) {
dump.transactionsList = { fetchError: e.message };
}

// 2. Raw GetTransactions (hub) pages
let afterCursor = null;
for (let page = 1; page <= maxHubPages; page++) {
if (onProgress) onProgress(`Debug: fetching hub page ${page}/${maxHubPages}...`);
try {
const variables = {
input: {
accountInput: {},
categoryInput: { categoryId: null, primeCategoryType: null },
datePeriodInput: { datePeriod: null },
paginationInput: {}
}
};
if (afterCursor) variables.input.paginationInput.afterCursor = afterCursor;

const hubResponse = await fetch(CONFIG.API_ENDPOINT, {
method: 'POST',
headers: headers,
credentials: 'include',
body: JSON.stringify({
extensions: { persistedQuery: { sha256Hash: CONFIG.TRANSACTIONS_QUERY_HASH, version: 1 } },
operationName: "GetTransactions",
variables: variables
})
});
const hubData = await hubResponse.json();
dump.hubPages.push(hubData);

const pageInfo = hubData.data?.prime?.transactionsHub?.transactionPage?.pageInfo;
if (!pageInfo?.hasNextPage) break;
afterCursor = pageInfo.endCursor;
await new Promise(resolve => setTimeout(resolve, 500));
} catch (e) {
dump.hubPages.push({ fetchError: e.message });
break;
}
}

// Save to file
const blob = new Blob([JSON.stringify(dump, null, 2)], { type: 'application/json' });
const link = document.createElement('a');
link.href = window.URL.createObjectURL(blob);
link.download = `ck_raw_api_debug_${new Date().toISOString().slice(0, 10)}.json`;
link.click();

return {
listCount: dump.transactionsList?.data ? 'ok' : 'failed',
hubPagesFetched: dump.hubPages.length
};
}

// ============================================
// Original DOM-based extraction (Fallback)
// ============================================
Expand Down Expand Up @@ -1179,6 +1254,27 @@ async function captureTransactionsInDateRange(startDate, endDate, fetchAccountNa

// Listener for messages from the popup script
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
if (request.action === 'debugDumpRaw') {
sendResponse({ status: 'started', message: 'Raw API dump started' });

const indicator = document.createElement('div');
indicator.style.cssText = 'position:fixed;top:10px;left:20px;padding:10px 20px;background:rgba(0,0,0,0.8);color:white;border-radius:5px;z-index:9999;font-size:14px;';
indicator.textContent = 'Dumping raw API responses...';
document.body.appendChild(indicator);

debugDumpRawResponses(request.maxHubPages || 5, (msg) => {
indicator.textContent = msg;
}).then((result) => {
indicator.textContent = `Raw dump saved (${result.hubPagesFetched} hub pages).`;
setTimeout(() => indicator.remove(), 5000);
}).catch((error) => {
indicator.textContent = `Raw dump failed: ${error.message}`;
indicator.style.background = 'rgba(180,0,0,0.85)';
setTimeout(() => indicator.remove(), 8000);
});

return true;
}
if (request.action === 'captureTransactions') {
try {
const { startDate, endDate, csvTypes, fetchAccountNames = false, useApi = true, columns } = request;
Expand Down
5 changes: 5 additions & 0 deletions popup.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ <h3 class="section-title">Columns</h3>
<button id="export-btn" class="primary-btn">
Export Transactions
</button>

<button id="debug-raw-btn" class="pill-btn" style="width:100%; margin-top:8px;"
title="Downloads the raw, unprocessed GraphQL responses from both transaction endpoints. Useful for diagnosing wrong/missing fields (e.g. categories) in the CSV export.">
🔧 Debug: Export Raw API JSON
</button>
</main>

<footer>
Expand Down
60 changes: 55 additions & 5 deletions popup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,56 @@ document.querySelectorAll('.pill-btn').forEach(btn => {
});
});

/**
* Send a message to the content script. If it doesn't answer (stale or missing
* content script — e.g. the page was loaded before the extension was updated),
* inject content.js into the tab and retry once.
*/
function sendMessageWithInjection(tabId, message, callback) {
chrome.tabs.sendMessage(tabId, message, (response) => {
if (!chrome.runtime.lastError) {
callback(response, null);
return;
}
console.warn('Content script not responding, injecting fresh copy...', chrome.runtime.lastError.message);
chrome.scripting.executeScript({ target: { tabId }, files: ['content.js'] }, () => {
if (chrome.runtime.lastError) {
callback(null, chrome.runtime.lastError.message);
return;
}
chrome.tabs.sendMessage(tabId, message, (retryResponse) => {
callback(retryResponse, chrome.runtime.lastError ? chrome.runtime.lastError.message : null);
});
});
});
}

// Debug: dump raw API responses to a JSON file
document.getElementById('debug-raw-btn').addEventListener('click', () => {
const btn = document.getElementById('debug-raw-btn');
const originalText = btn.textContent;
btn.textContent = 'Dumping...';
btn.disabled = true;

chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
if (!tabs[0]) {
alert('No active tab found.');
btn.textContent = originalText;
btn.disabled = false;
return;
}
sendMessageWithInjection(tabs[0].id, { action: 'debugDumpRaw', maxHubPages: 5 }, (response, error) => {
if (error) {
alert(`Connection error (${error}): Make sure you are on creditkarma.com and reload the page.`);
}
setTimeout(() => {
btn.textContent = originalText;
btn.disabled = false;
}, 3000);
});
});
});

// Export Logic
document.getElementById('export-btn').addEventListener('click', () => {
const startDate = document.getElementById('start-date').value;
Expand Down Expand Up @@ -90,19 +140,19 @@ document.getElementById('export-btn').addEventListener('click', () => {
return;
}

chrome.tabs.sendMessage(tabs[0].id, {
sendMessageWithInjection(tabs[0].id, {
action: 'captureTransactions',
startDate,
endDate,
useApi,
fetchAccountNames: false, // Legacy override
csvTypes,
columns
}, (response) => {
}, (response, error) => {
// Handle error (e.g. content script not loaded)
if (chrome.runtime.lastError) {
console.error(chrome.runtime.lastError);
alert('Connection error: Make sure you are on the Credit Karma page and reload.');
if (error) {
console.error(error);
alert(`Connection error (${error}): Make sure you are on creditkarma.com and reload the page.`);
resetButton();
return;
}
Expand Down