diff --git a/.changeset/add-configure-preference-upload-command.md b/.changeset/add-configure-preference-upload-command.md new file mode 100644 index 00000000..2980366e --- /dev/null +++ b/.changeset/add-configure-preference-upload-command.md @@ -0,0 +1,10 @@ +--- +'@transcend-io/cli': minor +--- + +Add configure-preference-upload CLI command + +- New `transcend consent configure-preference-upload` command that interactively configures column mappings for preference CSV uploads +- Scans CSV files to discover headers and unique values, then walks through a 6-step wizard (identifiers, identifier names, timestamp, purpose columns, value mappings, metadata) +- Saves a reusable config JSON for fully non-interactive uploads via `upload-preferences` +- Deprecate old parsing functions that use FileMetadataState in favor of new FileFormatState versions diff --git a/.changeset/sdk-preference-format-types.md b/.changeset/sdk-preference-format-types.md new file mode 100644 index 00000000..209f9060 --- /dev/null +++ b/.changeset/sdk-preference-format-types.md @@ -0,0 +1,11 @@ +--- +'@transcend-io/sdk': minor +--- + +Add preference upload types and helpers to SDK + +- Add FileFormatState codec (schema-only CSV column mapping without upload receipts) +- Add RequestUploadReceipts codec (tracks upload progress and results) +- Add loadReferenceData helper (fetches purposes, topics, identifiers in parallel) +- Add getPreferenceIdentifiersFromRow helper (extracts identifiers from a CSV row) +- Add getUniquePreferenceIdentifierNamesFromRow helper (extracts unique identifiers from a CSV row) diff --git a/.cursor/rules/test-colocation.mdc b/.cursor/rules/test-colocation.mdc new file mode 100644 index 00000000..ef1f445e --- /dev/null +++ b/.cursor/rules/test-colocation.mdc @@ -0,0 +1,18 @@ +--- +description: Tests must live next to the code they test +globs: "**/*.test.ts" +alwaysApply: false +--- + +# Test Colocation + +Tests always live in a `tests/` folder next to the source file they cover. + +``` +packages/sdk/src/preference-management/ + getPreferenceIdentifiersFromRow.ts + tests/ + getPreferenceIdentifiersFromRow.test.ts +``` + +When moving a function between packages, move its test file too. diff --git a/packages/cli/src/commands/consent/configure-preference-upload/command.ts b/packages/cli/src/commands/consent/configure-preference-upload/command.ts new file mode 100644 index 00000000..dc9c8534 --- /dev/null +++ b/packages/cli/src/commands/consent/configure-preference-upload/command.ts @@ -0,0 +1,46 @@ +import { buildCommand } from '@stricli/core'; +import { ScopeName } from '@transcend-io/privacy-types'; + +import { + createAuthParameter, + createTranscendUrlParameter, +} from '../../../lib/cli/common-parameters.js'; + +export const configurePreferenceUploadCommand = buildCommand({ + loader: async () => { + const { configurePreferenceUpload } = await import('./impl.js'); + return configurePreferenceUpload; + }, + parameters: { + flags: { + auth: createAuthParameter({ + scopes: [ScopeName.ViewPreferenceStoreSettings, ScopeName.ViewRequestIdentitySettings], + }), + transcendUrl: createTranscendUrlParameter(), + directory: { + kind: 'parsed', + parse: String, + brief: 'Path to the directory of CSV files to scan for column headers and unique values', + }, + schemaFilePath: { + kind: 'parsed', + parse: String, + brief: + 'Path to the config JSON file. Defaults to /../preference-upload-schema.json', + optional: true, + }, + }, + }, + docs: { + brief: 'Interactively configure the column mapping for preference CSV uploads', + fullDescription: `Interactively configure the column mapping for preference CSV uploads. + +Scans ALL CSV files in the given directory to discover every column header +and every unique value per column, then walks through an interactive editor +to build the full mapping config (identifiers, ignored columns, timestamp, +purposes/preferences and their value mappings). + +The resulting config JSON is reused by 'upload-preferences' so subsequent +uploads run fully non-interactively.`, + }, +}); diff --git a/packages/cli/src/commands/consent/configure-preference-upload/impl.ts b/packages/cli/src/commands/consent/configure-preference-upload/impl.ts new file mode 100644 index 00000000..eafb111b --- /dev/null +++ b/packages/cli/src/commands/consent/configure-preference-upload/impl.ts @@ -0,0 +1,353 @@ +import { createReadStream } from 'node:fs'; + +import { PersistedState } from '@transcend-io/persisted-state'; +import { buildTranscendGraphQLClient, FileFormatState, loadReferenceData } from '@transcend-io/sdk'; +import colors from 'colors'; +import { parse as csvParse } from 'csv-parse'; +import inquirer from 'inquirer'; +import * as t from 'io-ts'; + +import type { LocalContext } from '../../../context.js'; +import { doneInputValidation } from '../../../lib/cli/done-input-validation.js'; +import { collectCsvFilesOrExit } from '../../../lib/helpers/collectCsvFilesOrExit.js'; +import { parsePreferenceAndPurposeValuesFromCsv } from '../../../lib/preference-management/parsePreferenceAndPurposeValuesInteractive.js'; +import { parsePreferenceFileFormatFromCsv } from '../../../lib/preference-management/parsePreferenceFileFormatFromCsv.js'; +import { parsePreferenceIdentifiersFromCsv } from '../../../lib/preference-management/parsePreferenceIdentifiersInteractive.js'; +import { readCsv } from '../../../lib/requests/index.js'; +import { logger } from '../../../logger.js'; +import { computeSchemaFile } from '../upload-preferences/artifacts/index.js'; + +export interface ConfigurePreferenceUploadFlags { + auth: string; + transcendUrl: string; + directory: string; + schemaFilePath?: string; +} + +/** + * Scan a single CSV file and collect its column headers plus all unique + * values per column. Uses streaming so large files don't need to be held + * in memory. + * + * @param file - CSV file path to scan + * @returns headers and uniqueValuesByColumn + */ +async function scanOneFile(file: string): Promise<{ + headers: Set; + uniqueValuesByColumn: Record>; +}> { + const headers = new Set(); + const uniqueValuesByColumn: Record> = {}; + + await new Promise((resolve, reject) => { + const parser = createReadStream(file).pipe(csvParse({ columns: true, skip_empty_lines: true })); + parser.on('data', (row: Record) => { + for (const [col, val] of Object.entries(row)) { + headers.add(col); + if (!uniqueValuesByColumn[col]) { + uniqueValuesByColumn[col] = new Set(); + } + const trimmed = (val || '').trim(); + uniqueValuesByColumn[col].add(trimmed); + } + }); + parser.on('end', resolve); + parser.on('error', reject); + }); + + return { headers, uniqueValuesByColumn }; +} + +const SCAN_CONCURRENCY = 25; + +async function scanCsvFiles(files: string[]): Promise<{ + /** Union of all column headers */ + headers: string[]; + /** Map of column name to its unique values (trimmed, non-empty) */ + uniqueValuesByColumn: Record>; +}> { + const allHeaders = new Set(); + const merged: Record> = {}; + let completed = 0; + + const queue = [...files]; + const run = async (): Promise => { + while (queue.length > 0) { + const file = queue.shift()!; + const result = await scanOneFile(file); + for (const h of result.headers) allHeaders.add(h); + for (const [col, vals] of Object.entries(result.uniqueValuesByColumn)) { + if (!merged[col]) merged[col] = new Set(); + for (const v of vals) merged[col].add(v); + } + completed += 1; + if (completed % 25 === 0 || completed === files.length) { + logger.info(colors.green(` Scanned ${completed}/${files.length} files...`)); + } + } + }; + + const workers = Array.from({ length: Math.min(SCAN_CONCURRENCY, files.length) }, () => run()); + await Promise.all(workers); + + return { headers: [...allHeaders], uniqueValuesByColumn: merged }; +} + +/** + * Build synthetic preference rows from the scanned unique values so + * the existing parse functions see every value at least once. + * + * Row count is driven only by `enumColumns` (purpose/preference columns) + * whose unique values actually matter for mapping. High-cardinality + * columns like timestamps or emails are filled with a single sample value. + * + * @param headers - all column headers + * @param uniqueValuesByColumn - unique values per column + * @param enumColumns - columns whose full unique values must be represented + * @returns synthetic rows covering all unique enum values + */ +function buildSyntheticRows( + headers: string[], + uniqueValuesByColumn: Record>, + enumColumns: string[] = [], +): Record[] { + const enumSet = new Set(enumColumns); + const maxRows = Math.max(1, ...enumColumns.map((h) => uniqueValuesByColumn[h]?.size ?? 0)); + const rows: Record[] = []; + for (let i = 0; i < maxRows; i += 1) { + const row: Record = {}; + for (const h of headers) { + const vals = uniqueValuesByColumn[h] ? [...uniqueValuesByColumn[h]] : ['']; + row[h] = enumSet.has(h) ? (vals[i % vals.length] ?? '') : (vals[0] ?? ''); + } + rows.push(row); + } + return rows; +} + +/** + * Interactively configure the column mapping for preference CSV uploads. + * + * Scans ALL CSV files in a directory, discovers every header and unique value, + * then walks the user through mapping identifiers, timestamps, + * purpose/preference value mappings, and metadata columns. + * Saves the result as a reusable config. + * + * @param flags - CLI flags + */ +export async function configurePreferenceUpload( + this: LocalContext, + flags: ConfigurePreferenceUploadFlags, +): Promise { + const { auth, transcendUrl, directory, schemaFilePath } = flags; + + const files = collectCsvFilesOrExit(directory, this); + doneInputValidation(this.process.exit); + + logger.info( + colors.green(`Scanning ${files.length} CSV file(s) for headers and unique values...`), + ); + + const { headers, uniqueValuesByColumn } = await scanCsvFiles(files); + logger.info(colors.green(`Discovered ${headers.length} columns across all files.`)); + + const client = buildTranscendGraphQLClient(transcendUrl, auth); + const { purposes, preferenceTopics, identifiers } = await loadReferenceData(client, { logger }); + + const allIdentifierNames = identifiers.map((id) => id.name); + logger.info( + colors.green( + `Loaded ${purposes.length} purposes, ${preferenceTopics.length} preference topics, ${identifiers.length} identifiers from org.`, + ), + ); + + const schemaFile = computeSchemaFile(schemaFilePath, directory, files[0]); + const initial = { + columnToPurposeName: {}, + lastFetchedAt: new Date().toISOString(), + columnToIdentifier: {}, + } as const; + const schemaState = new PersistedState(schemaFile, FileFormatState, initial); + + // Step 1: select identifier columns + logger.info(colors.green('\n[Step 1/6] Identifier column selection...')); + const existingIdentifierCols = Object.keys(schemaState.getValue('columnToIdentifier')); + let identifierColumns: string[]; + if (existingIdentifierCols.length > 0) { + logger.info( + colors.magenta(`Existing identifier columns: ${existingIdentifierCols.join(', ')}`), + ); + const { reuse } = await inquirer.prompt<{ reuse: boolean }>([ + { + name: 'reuse', + type: 'confirm', + message: `Keep existing identifier column selection? (${existingIdentifierCols.join( + ', ', + )})`, + default: true, + }, + ]); + identifierColumns = reuse + ? existingIdentifierCols + : ( + await inquirer.prompt<{ cols: string[] }>([ + { + name: 'cols', + type: 'checkbox', + message: 'Select columns that are identifiers', + choices: headers, + validate: (v: string[]) => v.length > 0 || 'Select at least one identifier column', + }, + ]) + ).cols; + } else { + identifierColumns = ( + await inquirer.prompt<{ cols: string[] }>([ + { + name: 'cols', + type: 'checkbox', + message: 'Select columns that are identifiers', + choices: headers, + validate: (v: string[]) => v.length > 0 || 'Select at least one identifier column', + }, + ]) + ).cols; + } + + // Step 2: map identifier columns to org identifier names + logger.info( + colors.green(`\n[Step 2/6] Identifier name mapping (validating sample: ${files[0]})...`), + ); + const sampleRows = readCsv(files[0], t.record(t.string, t.string)); + await parsePreferenceIdentifiersFromCsv(sampleRows, { + schemaState, + orgIdentifiers: identifiers, + allowedIdentifierNames: allIdentifierNames, + identifierColumns, + }); + + const identifierCols = Object.keys(schemaState.getValue('columnToIdentifier')); + + // Step 3: select timestamp column + logger.info(colors.green('\n[Step 3/6] Timestamp column selection...')); + const timestampChoices = headers.filter((h) => !identifierCols.includes(h)); + await parsePreferenceFileFormatFromCsv( + [ + Object.fromEntries( + timestampChoices.map((h) => [h, [...(uniqueValuesByColumn[h] ?? [])][0] ?? '']), + ), + ], + schemaState, + ); + + // Step 4: select purpose/preference columns + logger.info(colors.green('\n[Step 4/6] Purpose/preference column selection...')); + const timestampCol = schemaState.getValue('timestampColumn'); + const mappedSoFar = [...identifierCols, ...(timestampCol ? [timestampCol] : [])]; + const remainingColumns = headers.filter((h) => !mappedSoFar.includes(h)); + + const { purposeColumns } = await inquirer.prompt<{ + purposeColumns: string[]; + }>([ + { + name: 'purposeColumns', + type: 'checkbox', + message: 'Select columns that map to purposes/preferences', + choices: remainingColumns, + validate: (v: string[]) => v.length > 0 || 'Select at least one purpose column', + }, + ]); + + const nonPurposeColumns = remainingColumns.filter((h) => !purposeColumns.includes(h)); + + // Step 5: map purpose values + logger.info(colors.green('\n[Step 5/6] Mapping purpose values...')); + const syntheticRows = buildSyntheticRows(headers, uniqueValuesByColumn, purposeColumns); + logger.info( + colors.green( + ` Built ${syntheticRows.length} synthetic rows ` + + `(from ${purposeColumns.length} purpose columns).`, + ), + ); + + await parsePreferenceAndPurposeValuesFromCsv(syntheticRows, schemaState, { + purposeSlugs: purposes.map((p) => p.trackingType), + preferenceTopics, + forceTriggerWorkflows: false, + columnsToIgnore: nonPurposeColumns, + }); + + // Step 6: metadata column selection + logger.info(colors.green('\n[Step 6/6] Metadata column selection...')); + if (nonPurposeColumns.length > 0) { + logger.info( + colors.magenta('\nRemaining unmapped columns:\n' + ` ${nonPurposeColumns.join(', ')}\n`), + ); + + const { metadataColumns } = await inquirer.prompt<{ + metadataColumns: string[]; + }>([ + { + name: 'metadataColumns', + type: 'checkbox', + message: 'Select columns to INCLUDE as metadata ' + '(unselected columns will be ignored)', + choices: nonPurposeColumns, + }, + ]); + + const ignored = nonPurposeColumns.filter((c) => !metadataColumns.includes(c)); + + if (ignored.length > 0) { + schemaState.setValue(ignored, 'columnsToIgnore'); + } + + if (metadataColumns.length > 0) { + const columnToMetadata: Record = {}; + for (const col of metadataColumns) { + columnToMetadata[col] = { key: col }; + } + schemaState.setValue(columnToMetadata, 'columnToMetadata'); + } + + logger.info( + colors.green( + ` Metadata: ${metadataColumns.length > 0 ? metadataColumns.join(', ') : '(none)'}`, + ), + ); + logger.info(colors.green(` Ignored: ${ignored.length > 0 ? ignored.join(', ') : '(none)'}`)); + } + + // Validate completeness + const purposeCols = Object.keys(schemaState.getValue('columnToPurposeName')); + const ignoredCols = schemaState.getValue('columnsToIgnore') ?? []; + const metadataCols = Object.keys(schemaState.getValue('columnToMetadata') ?? {}); + const allMapped = new Set([ + ...identifierCols, + ...purposeCols, + ...ignoredCols, + ...metadataCols, + ...(timestampCol ? [timestampCol] : []), + ]); + const unmapped = headers.filter((h) => !allMapped.has(h)); + if (unmapped.length > 0) { + logger.warn( + colors.yellow( + `Warning: the following columns are not mapped: ${unmapped.join(', ')}. ` + + 'They will cause errors during upload. Re-run this command to fix.', + ), + ); + } + + schemaState.setValue(new Date().toISOString(), 'lastFetchedAt'); + + logger.info(colors.green(`\nConfiguration saved to: ${schemaFile}`)); + logger.info( + colors.green( + ` Identifiers: ${identifierCols.join(', ')}\n` + + ` Timestamp: ${timestampCol || '(none)'}\n` + + ` Purpose columns: ${purposeCols.join(', ')}\n` + + ` Metadata: ${metadataCols.join(', ') || '(none)'}\n` + + ` Ignored: ${ignoredCols.join(', ') || '(none)'}`, + ), + ); +} diff --git a/packages/cli/src/commands/consent/routes.ts b/packages/cli/src/commands/consent/routes.ts index bd24e3d4..b3a8d816 100644 --- a/packages/cli/src/commands/consent/routes.ts +++ b/packages/cli/src/commands/consent/routes.ts @@ -1,6 +1,7 @@ import { buildRouteMap } from '@stricli/core'; import { buildXdiSyncEndpointCommand } from './build-xdi-sync-endpoint/command.js'; +import { configurePreferenceUploadCommand } from './configure-preference-upload/command.js'; import { deletePreferenceRecordsCommand } from './delete-preference-records/command.js'; import { generateAccessTokensCommand } from './generate-access-tokens/command.js'; import { pullConsentMetricsCommand } from './pull-consent-metrics/command.js'; @@ -14,6 +15,7 @@ import { uploadPreferencesCommand } from './upload-preferences/command.js'; export const consentRoutes = buildRouteMap({ routes: { 'build-xdi-sync-endpoint': buildXdiSyncEndpointCommand, + 'configure-preference-upload': configurePreferenceUploadCommand, 'generate-access-tokens': generateAccessTokensCommand, 'pull-consent-metrics': pullConsentMetricsCommand, 'pull-consent-preferences': pullConsentPreferencesCommand, diff --git a/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index e1b8148d..1e0514ef 100644 --- a/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -13,6 +13,7 @@ import { logger } from '../../logger.js'; /** * Parse out the purpose.enabled and preference values from a CSV file * + * @deprecated Use the version in parsePreferenceAndPurposeValuesInteractive.ts which accepts FileFormatState * @param preferences - List of preferences * @param currentState - The current file metadata state for parsing this list * @param options - Options diff --git a/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesInteractive.ts b/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesInteractive.ts new file mode 100644 index 00000000..d6697588 --- /dev/null +++ b/packages/cli/src/lib/preference-management/parsePreferenceAndPurposeValuesInteractive.ts @@ -0,0 +1,274 @@ +import type { PersistedState } from '@transcend-io/persisted-state'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import type { PreferenceTopic } from '@transcend-io/sdk'; +import { FileFormatState } from '@transcend-io/sdk'; +import { mapSeries } from '@transcend-io/utils'; +import { splitCsvToList } from '@transcend-io/utils'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { uniq, difference } from 'lodash-es'; + +import { logger } from '../../logger.js'; + +/** Values that clearly mean "no preference recorded" and should map to null. */ +const NULL_VALUES = new Set(['', 'undefined', 'null', 'none', 'n/a', 'na']); + +const FALSY_VALUES = new Set([ + 'false', + '0', + 'no', + 'n', + 'off', + 'opt-out', + 'optout', + 'opt_out', + 'unsubscribed', +]); + +/** + * Check whether a raw CSV value represents "no data" and should map to null. + * + * @param value - raw CSV cell value + * @returns true when the value should be treated as null (no preference) + */ +function looksNull(value: string): boolean { + return NULL_VALUES.has(value.trim().toLowerCase()); +} + +/** + * Infer a sensible Y/n default for a purpose/preference value prompt. + * + * @param value - raw CSV cell value + * @returns true when the value looks like "opted-in" + */ +function looksOptedIn(value: string): boolean { + return !FALSY_VALUES.has(value.trim().toLowerCase()) && !looksNull(value); +} + +/** + * Parse out the purpose.enabled and preference values from a CSV file + * + * @param preferences - List of preferences + * @param schemaState - The schema state to use for parsing the file + * @param options - Options + * @returns The updated file metadata state + */ +export async function parsePreferenceAndPurposeValuesFromCsv( + preferences: Record[], + schemaState: PersistedState, + { + purposeSlugs, + preferenceTopics, + forceTriggerWorkflows, + columnsToIgnore, + nonInteractive = false, + }: { + /** The purpose slugs that are allowed to be updated */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; + /** Force workflow triggers */ + forceTriggerWorkflows: boolean; + /** Columns to ignore in the CSV file */ + columnsToIgnore: string[]; + /** When true, throw instead of prompting (for worker processes) */ + nonInteractive?: boolean; + }, +): Promise> { + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + const timestampCol = schemaState.getValue('timestampColumn'); + const otherColumns = difference(columnNames, [ + ...Object.keys(schemaState.getValue('columnToIdentifier')), + ...(timestampCol ? [timestampCol] : []), + ...columnsToIgnore, + ...Object.keys(schemaState.getValue('columnToMetadata') ?? {}), + ]); + if (otherColumns.length === 0) { + if (forceTriggerWorkflows) { + return schemaState; + } + throw new Error('No other columns to process'); + } + + const purposeNames = [ + ...purposeSlugs, + ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`), + ]; + + await mapSeries(otherColumns, async (col) => { + const uniqueValues = uniq(preferences.map((x) => x[col] ?? '')); + + const currentPurposeMapping = schemaState.getValue('columnToPurposeName'); + let purposeMapping = currentPurposeMapping[col]; + if (purposeMapping) { + logger.info( + colors.magenta(`Column "${col}" is associated with purpose "${purposeMapping.purpose}"`), + ); + } else { + if (nonInteractive) { + throw new Error( + `Column "${col}" has no purpose mapping in the config. ` + + "Run 'transcend consent configure-preference-upload' to update the config.", + ); + } + + const { purposeName } = await inquirer.prompt<{ + purposeName: string; + }>([ + { + name: 'purposeName', + message: `Choose the purpose that column ${col} is associated with`, + type: 'list', + default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])), + choices: purposeNames, + }, + ]); + const [purposeSlug, preferenceSlug] = purposeName.split('->'); + purposeMapping = { + purpose: purposeSlug, + preference: preferenceSlug || null, + valueMapping: {}, + }; + } + + await mapSeries(uniqueValues, async (value) => { + if (purposeMapping.valueMapping[value] !== undefined) { + logger.info( + colors.magenta( + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, + ), + ); + return; + } + + if (looksNull(value)) { + logger.info( + colors.magenta( + `Value "${value || '(empty)'}" for column "${col}" → null (no preference)`, + ), + ); + purposeMapping.valueMapping[value] = null as unknown as boolean; + return; + } + + if (nonInteractive) { + throw new Error( + `Value "${value}" for column "${col}" has no mapping in the config. ` + + "Run 'transcend consent configure-preference-upload' to update the config.", + ); + } + + if (purposeMapping.preference === null) { + const { purposeValue } = await inquirer.prompt<{ + purposeValue: string; + }>([ + { + name: 'purposeValue', + message: `Map value "${value}" for purpose "${purposeMapping.purpose}"`, + type: 'list', + choices: [ + { name: 'true (opted in)', value: 'true' }, + { name: 'false (opted out)', value: 'false' }, + { name: 'null (skip / no preference)', value: 'null' }, + ], + default: looksOptedIn(value) ? 'true' : 'false', + }, + ]); + purposeMapping.valueMapping[value] = + purposeValue === 'null' ? (null as unknown as boolean) : purposeValue === 'true'; + } + + if (purposeMapping.preference !== null) { + const preferenceTopic = preferenceTopics.find((x) => x.slug === purposeMapping.preference); + if (!preferenceTopic) { + logger.error(colors.red(`Preference topic "${purposeMapping.preference}" not found`)); + return; + } + const preferenceOptions = preferenceTopic.preferenceOptionValues.map(({ slug }) => slug); + + if (preferenceTopic.type === PreferenceTopicType.Boolean) { + const { preferenceValue } = await inquirer.prompt<{ + preferenceValue: string; + }>([ + { + name: 'preferenceValue', + message: `Map value "${value}" for preference "${preferenceTopic.slug}" (${purposeMapping.purpose})`, + type: 'list', + choices: [ + { name: 'true (opted in)', value: 'true' }, + { name: 'false (opted out)', value: 'false' }, + { name: 'null (skip / no preference)', value: 'null' }, + ], + default: looksOptedIn(value) ? 'true' : 'false', + }, + ]); + purposeMapping.valueMapping[value] = + preferenceValue === 'null' ? (null as unknown as boolean) : preferenceValue === 'true'; + return; + } + + if (preferenceTopic.type === PreferenceTopicType.Select) { + const choices = [ + ...preferenceOptions.map((o) => ({ name: o, value: o })), + { name: '(null — skip / no preference)', value: '__null__' }, + ]; + const { preferenceValue } = await inquirer.prompt<{ + preferenceValue: string; + }>([ + { + name: 'preferenceValue', + message: `Map value "${value}" for preference "${preferenceTopic.slug}" (${purposeMapping.purpose})`, + type: 'list', + choices, + default: preferenceOptions.find((x) => x === value), + }, + ]); + purposeMapping.valueMapping[value] = + preferenceValue === '__null__' + ? (null as unknown as boolean) + : (preferenceValue as unknown as boolean); + return; + } + + if (preferenceTopic.type === PreferenceTopicType.MultiSelect) { + const parsedValues = splitCsvToList(value); + await mapSeries(parsedValues, async (parsedValue) => { + if (purposeMapping.valueMapping[parsedValue] !== undefined) { + return; + } + const msChoices = [ + ...preferenceOptions.map((o) => ({ name: o, value: o })), + { + name: '(null — skip / no preference)', + value: '__null__', + }, + ]; + const { preferenceValue } = await inquirer.prompt<{ + preferenceValue: string; + }>([ + { + name: 'preferenceValue', + message: `Map token "${parsedValue}" for preference "${preferenceTopic.slug}" (${purposeMapping.purpose})`, + type: 'list', + choices: msChoices, + default: preferenceOptions.find((x) => x === parsedValue), + }, + ]); + purposeMapping.valueMapping[parsedValue] = + preferenceValue === '__null__' + ? (null as unknown as boolean) + : (preferenceValue as unknown as boolean); + }); + return; + } + + throw new Error(`Unknown preference topic type: ${preferenceTopic.type}`); + } + }); + currentPurposeMapping[col] = purposeMapping; + schemaState.setValue(currentPurposeMapping, 'columnToPurposeName'); + }); + + return schemaState; +} diff --git a/packages/cli/src/lib/preference-management/parsePreferenceFileFormatFromCsv.ts b/packages/cli/src/lib/preference-management/parsePreferenceFileFormatFromCsv.ts new file mode 100644 index 00000000..03a3e3b5 --- /dev/null +++ b/packages/cli/src/lib/preference-management/parsePreferenceFileFormatFromCsv.ts @@ -0,0 +1,94 @@ +import type { PersistedState } from '@transcend-io/persisted-state'; +import { FileFormatState } from '@transcend-io/sdk'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { uniq, difference } from 'lodash-es'; + +import { logger } from '../../logger.js'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv.js'; + +export { NONE_PREFERENCE_MAP }; + +/** + * Parse timestamps and other file format mapping from a CSV list of preferences + * + * When timestamp is requested, this script + * ensures that all rows have a valid timestamp. + * + * Error is throw if timestamp is missing + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @param options - Options + * @returns The updated file metadata state + */ +export async function parsePreferenceFileFormatFromCsv( + preferences: Record[], + currentState: PersistedState, + { + nonInteractive = false, + }: { + /** When true, throw instead of prompting */ nonInteractive?: boolean; + } = {}, +): Promise> { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for timestamp + const remainingColumnsForTimestamp = difference(columnNames, [ + ...Object.keys(currentState.getValue('columnToIdentifier')), + ...Object.keys(currentState.getValue('columnToPurposeName')), + ]); + + // Determine the timestamp column to work off of + if (!currentState.getValue('timestampColumn')) { + if (nonInteractive) { + throw new Error( + 'No timestamp column configured. ' + + "Run 'transcend consent configure-preference-upload' to set it.", + ); + } + + const { timestampName } = await inquirer.prompt<{ + /** timestamp name */ + timestampName: string; + }>([ + { + name: 'timestampName', + message: 'Choose the column that will be used as the timestamp of last preference update', + type: 'list', + default: + remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('date')) || + remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('time')) || + remainingColumnsForTimestamp[0], + choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], + }, + ]); + + currentState.setValue(timestampName, 'timestampColumn'); + } + logger.info( + colors.magenta(`Using timestamp column "${currentState.getValue('timestampColumn')}"`), + ); + + // Validate that all rows have valid timestamp + if (currentState.getValue('timestampColumn') !== NONE_PREFERENCE_MAP) { + const timestampColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.getValue('timestampColumn')!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (timestampColumnsMissing.length > 0) { + throw new Error( + `The timestamp column "${currentState.getValue( + 'timestampColumn', + )}" is missing a value for the following rows: ${timestampColumnsMissing.join('\n')}`, + ); + } + logger.info( + colors.magenta( + `The timestamp column "${currentState.getValue('timestampColumn')}" is present for all row`, + ), + ); + } + return currentState; +} diff --git a/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index f99d5448..ff06588b 100644 --- a/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -14,6 +14,7 @@ import { inquirerConfirmBoolean } from '../helpers/index.js'; * Ensures that all rows have a valid identifier * and that all identifiers are unique. * + * @deprecated Use the version in parsePreferenceIdentifiersInteractive.ts which accepts FileFormatState * @param preferences - List of preferences * @param currentState - The current file metadata state for parsing this list * @returns The updated file metadata state diff --git a/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersInteractive.ts b/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersInteractive.ts new file mode 100644 index 00000000..0d943d7e --- /dev/null +++ b/packages/cli/src/lib/preference-management/parsePreferenceIdentifiersInteractive.ts @@ -0,0 +1,159 @@ +import type { PersistedState } from '@transcend-io/persisted-state'; +import type { Identifier } from '@transcend-io/sdk'; +import type { FileFormatState } from '@transcend-io/sdk'; +import Bluebird from 'bluebird'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { uniq, keyBy } from 'lodash-es'; + +import { logger } from '../../logger.js'; +import { inquirerConfirmBoolean } from '../helpers/index.js'; + +const { mapSeries } = Bluebird; + +/* eslint-disable no-param-reassign */ + +/** + * Parse identifiers from a CSV list of preferences + * + * Ensures that all rows have a valid identifier + * and that all identifiers are unique. + * + * @param preferences - List of preferences + * @param options - Options + * @returns The updated file metadata state + */ +export async function parsePreferenceIdentifiersFromCsv( + preferences: Record[], + { + schemaState, + orgIdentifiers, + allowedIdentifierNames, + identifierColumns, + nonInteractive = false, + }: { + /** The current state of the schema metadata */ + schemaState: PersistedState; + /** The list of identifiers configured for the org */ + orgIdentifiers: Identifier[]; + /** The list of identifier names that are allowed for this upload */ + allowedIdentifierNames: string[]; + /** The columns in the CSV that should be used as identifiers */ + identifierColumns: string[]; + /** When true, throw instead of prompting (for worker processes) */ + nonInteractive?: boolean; + }, +): Promise<{ + /** The updated state */ + schemaState: PersistedState; + /** The updated preferences */ + preferences: Record[]; +}> { + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()).filter((col) => + identifierColumns.includes(col), + ); + const orgIdentifiersByName = keyBy(orgIdentifiers, 'name'); + const filteredOrgIdentifiers = allowedIdentifierNames + .map((name) => orgIdentifiersByName[name]) + .filter(Boolean); + if (filteredOrgIdentifiers.length !== allowedIdentifierNames.length) { + const missingIdentifiers = allowedIdentifierNames.filter((name) => !orgIdentifiersByName[name]); + throw new Error(`No identifier configuration found for "${missingIdentifiers.join('","')}"`); + } + if (columnNames.length !== identifierColumns.length) { + const missingColumns = identifierColumns.filter((col) => !columnNames.includes(col)); + throw new Error( + `The following identifier columns are missing from the CSV: "${missingColumns.join('","')}"`, + ); + } + + if ( + filteredOrgIdentifiers.filter((identifier) => identifier.isUniqueOnPreferenceStore).length === 0 + ) { + throw new Error( + 'No unique identifier was provided. Please ensure that at least one ' + + 'of the allowed identifiers is configured as unique on the preference store.', + ); + } + + const currentColumnToIdentifier = schemaState.getValue('columnToIdentifier'); + await mapSeries(identifierColumns, async (col) => { + const identifierMapping = currentColumnToIdentifier[col]; + if (identifierMapping) { + logger.info( + colors.magenta(`Column "${col}" is associated with identifier "${identifierMapping.name}"`), + ); + return; + } + + if (nonInteractive) { + throw new Error( + `Column "${col}" has no identifier mapping in the config. ` + + "Run 'transcend consent configure-preference-upload' to update the config.", + ); + } + + const { identifierName } = await inquirer.prompt<{ + identifierName: string; + }>([ + { + name: 'identifierName', + message: `Choose the identifier name for column "${col}"`, + type: 'list', + default: allowedIdentifierNames.find((x) => x.startsWith(col)), + choices: allowedIdentifierNames, + }, + ]); + currentColumnToIdentifier[col] = { + name: identifierName, + isUniqueOnPreferenceStore: orgIdentifiersByName[identifierName].isUniqueOnPreferenceStore, + }; + }); + schemaState.setValue(currentColumnToIdentifier, 'columnToIdentifier'); + + const uniqueIdentifierColumns = Object.entries(currentColumnToIdentifier) + .filter(([, identifierMapping]) => identifierMapping.isUniqueOnPreferenceStore) + .map(([col]) => col); + + const uniqueIdentifierMissingIndexes = preferences + .map((pref, ind) => (uniqueIdentifierColumns.some((col) => !!pref[col]) ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + + if (uniqueIdentifierMissingIndexes.length > 0) { + const msg = ` + The following rows ${uniqueIdentifierMissingIndexes.join( + ', ', + )} do not have any unique identifier values for the columns "${uniqueIdentifierColumns.join( + '", "', + )}".`; + logger.warn(colors.yellow(msg)); + + if (nonInteractive) { + throw new Error(msg); + } + + const skip = await inquirerConfirmBoolean({ + message: 'Would you like to skip rows missing unique identifiers?', + }); + if (!skip) { + throw new Error(msg); + } + + const previous = preferences.length; + preferences = preferences.filter( + (pref, index) => !uniqueIdentifierMissingIndexes.includes(index), + ); + logger.info( + colors.yellow(`Skipped ${previous - preferences.length} rows missing unique identifiers`), + ); + } + logger.info( + colors.magenta( + `At least one unique identifier column is present for all ${preferences.length} rows.`, + ), + ); + + return { schemaState, preferences }; +} +/* eslint-enable no-param-reassign */ diff --git a/packages/cli/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/packages/cli/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index 8c35653d..64084ddb 100644 --- a/packages/cli/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/packages/cli/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -17,6 +17,7 @@ export const NONE_PREFERENCE_MAP = '[NONE]'; * * Error is throw if timestamp is missing * + * @deprecated Use parsePreferenceFileFormatFromCsv which accepts FileFormatState * @param preferences - List of preferences * @param currentState - The current file metadata state for parsing this list * @returns The updated file metadata state diff --git a/packages/sdk/src/preference-management/codecs.ts b/packages/sdk/src/preference-management/codecs.ts index 2a61a491..761eeabc 100644 --- a/packages/sdk/src/preference-management/codecs.ts +++ b/packages/sdk/src/preference-management/codecs.ts @@ -129,6 +129,34 @@ export const FileMetadataState = t.intersection([ /** Override type */ export type FileMetadataState = t.TypeOf; +/** + * Schema-only state for a preference CSV file format. + * + * Unlike FileMetadataState this does NOT embed upload receipts — it only + * describes how columns map to identifiers, purposes, timestamps, and metadata. + */ +export const FileFormatState = t.intersection([ + t.type({ + /** Maps each CSV column to its purpose/preference definition in Transcend */ + columnToPurposeName: ColumnPurposeMap, + /** ISO 8601 timestamp of when this config was last generated or refreshed */ + lastFetchedAt: t.string, + /** Maps each CSV column to the identifier it represents (e.g. email, userId) */ + columnToIdentifier: ColumnIdentifierMap, + }), + t.partial({ + /** CSV column whose values contain the consent timestamp */ + timestampColumn: t.string, + /** Maps CSV columns to metadata keys stored alongside the preference record */ + columnToMetadata: ColumnMetadataMap, + /** CSV columns that should be skipped during upload */ + columnsToIgnore: t.array(t.string), + }), +]); + +/** Override type */ +export type FileFormatState = t.TypeOf; + /** * This is the type of the receipts that are stored in the file * that is used to track the state of the upload process. @@ -250,6 +278,26 @@ export const PreferenceState = t.type({ /** Override type */ export type PreferenceState = t.TypeOf; +export const RequestUploadReceipts = t.type({ + /** ISO 8601 timestamp of when the receipt file was last written */ + lastFetchedAt: t.string, + /** Updates that can be applied without conflicting with existing preferences */ + pendingSafeUpdates: PendingSafePreferenceUpdates, + /** Updates that conflict with existing preference values and need review */ + pendingConflictUpdates: PendingWithConflictPreferenceUpdates, + /** Rows skipped because their preferences already match the store */ + skippedUpdates: SkippedPreferenceUpdates, + /** Updates that were attempted but failed with an API error */ + failingUpdates: FailingPreferenceUpdates, + /** Updates still queued to be sent to the API */ + pendingUpdates: PreferenceUpdateMap, + /** Updates that have been successfully written to the preference store */ + successfulUpdates: PreferenceUpdateMap, +}); + +/** Override type */ +export type RequestUploadReceipts = t.TypeOf; + export const DeletePreferenceRecordsInput = t.type({ /** Array of consent preference records to delete */ records: t.array( diff --git a/packages/sdk/src/preference-management/getPreferenceIdentifiersFromRow.ts b/packages/sdk/src/preference-management/getPreferenceIdentifiersFromRow.ts new file mode 100644 index 00000000..ea2d3331 --- /dev/null +++ b/packages/sdk/src/preference-management/getPreferenceIdentifiersFromRow.ts @@ -0,0 +1,31 @@ +import type { PreferenceStoreIdentifier } from '@transcend-io/privacy-types'; + +import type { FileFormatState } from './codecs.js'; + +/** + * Extract preference store identifiers from a CSV row based on the column-to-identifier mapping. + * + * @param options - Options + * @returns Array of identifiers for the preference store API + */ +export function getPreferenceIdentifiersFromRow({ + row, + columnToIdentifier, +}: { + /** The current row from CSV file */ + row: Record; + /** The current file metadata state */ + columnToIdentifier: FileFormatState['columnToIdentifier']; +}): PreferenceStoreIdentifier[] { + const identifiers = Object.entries(columnToIdentifier) + .filter(([col]) => !!row[col]) + .map(([col, identifierMapping]) => ({ + name: identifierMapping.name, + value: row[col]!, + })); + return identifiers.sort( + (a, b) => + (a.name === 'email' ? -1 : 0) - (b.name === 'email' ? -1 : 0) || + a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }), + ); +} diff --git a/packages/sdk/src/preference-management/getUniquePreferenceIdentifierNamesFromRow.ts b/packages/sdk/src/preference-management/getUniquePreferenceIdentifierNamesFromRow.ts new file mode 100644 index 00000000..01107d0c --- /dev/null +++ b/packages/sdk/src/preference-management/getUniquePreferenceIdentifierNamesFromRow.ts @@ -0,0 +1,37 @@ +import type { FileFormatState, IdentifierMetadataForPreference } from './codecs.js'; + +/** + * Helper function to get unique identifier name present in a row + * + * @param options - Options + * @param options.row - The current row from CSV file + * @param options.columnToIdentifier - The column to identifier mapping metadata + * @returns The unique identifier names present in the row + */ +export function getUniquePreferenceIdentifierNamesFromRow({ + row, + columnToIdentifier, +}: { + /** The current row from CSV file */ + row: Record; + /** The current file metadata state */ + columnToIdentifier: FileFormatState['columnToIdentifier']; +}): (IdentifierMetadataForPreference & { + /** Column name */ + columnName: string; + /** Value of the identifier in the row */ + value: string; +})[] { + return Object.entries(columnToIdentifier) + .sort( + ([, a], [, b]) => + (a.name === 'email' ? -1 : 0) - (b.name === 'email' ? -1 : 0) || + a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }), + ) + .filter(([col]) => row[col] && columnToIdentifier[col]!.isUniqueOnPreferenceStore) + .map(([col, identifier]) => ({ + ...identifier, + columnName: col, + value: row[col]!, + })); +} diff --git a/packages/sdk/src/preference-management/index.ts b/packages/sdk/src/preference-management/index.ts index e8ab78f7..3de9a53d 100644 --- a/packages/sdk/src/preference-management/index.ts +++ b/packages/sdk/src/preference-management/index.ts @@ -5,6 +5,8 @@ export * from './createPreferenceAccessTokens.js'; export * from './types.js'; export * from './codecs.js'; export * from './getPreferenceMetadataFromRow.js'; +export * from './getPreferenceIdentifiersFromRow.js'; +export * from './getUniquePreferenceIdentifierNamesFromRow.js'; export * from './getPreferenceUpdatesFromRow.js'; export * from './checkIfPendingPreferenceUpdatesAreNoOp.js'; export * from './checkIfPendingPreferenceUpdatesCauseConflict.js'; @@ -19,3 +21,4 @@ export * from './discoverConsentWindow.js'; export * from './fetchConsentPreferences.js'; export * from './getPreferencesForIdentifiers.js'; export * from './fetchConsentPreferencesChunked.js'; +export * from './loadReferenceData.js'; diff --git a/packages/sdk/src/preference-management/loadReferenceData.ts b/packages/sdk/src/preference-management/loadReferenceData.ts new file mode 100644 index 00000000..0fac506e --- /dev/null +++ b/packages/sdk/src/preference-management/loadReferenceData.ts @@ -0,0 +1,34 @@ +import type { Logger } from '@transcend-io/utils'; +import type { GraphQLClient } from 'graphql-request'; + +import { fetchAllIdentifiers, type Identifier } from '../data-inventory/fetchAllIdentifiers.js'; +import { fetchAllPreferenceTopics, type PreferenceTopic } from './fetchAllPreferenceTopics.js'; +import { fetchAllPurposes, type Purpose } from './fetchAllPurposes.js'; + +export interface PreferenceUploadReferenceData { + /** List of purposes in the organization */ + purposes: Purpose[]; + /** List of preference topics in the organization */ + preferenceTopics: PreferenceTopic[]; + /** List of identifiers in the organization */ + identifiers: Identifier[]; +} + +/** + * Load all required reference data for an upload run. + * + * @param client - GraphQL client + * @param options - Options + * @returns Reference data arrays + */ +export async function loadReferenceData( + client: GraphQLClient, + { logger }: { logger: Logger }, +): Promise { + const [purposes, preferenceTopics, identifiers] = await Promise.all([ + fetchAllPurposes(client, { logger }), + fetchAllPreferenceTopics(client, { logger }), + fetchAllIdentifiers(client, { logger }), + ]); + return { purposes, preferenceTopics, identifiers }; +} diff --git a/packages/sdk/src/preference-management/tests/getPreferenceIdentifiersFromRow.test.ts b/packages/sdk/src/preference-management/tests/getPreferenceIdentifiersFromRow.test.ts new file mode 100644 index 00000000..ce94971d --- /dev/null +++ b/packages/sdk/src/preference-management/tests/getPreferenceIdentifiersFromRow.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from 'vitest'; + +import type { FileFormatState } from '../codecs.js'; +import { getPreferenceIdentifiersFromRow } from '../getPreferenceIdentifiersFromRow.js'; + +const columnToIdentifier: FileFormatState['columnToIdentifier'] = { + email_col: { name: 'email', isUniqueOnPreferenceStore: true }, + user_id_col: { name: 'userId', isUniqueOnPreferenceStore: true }, + phone_col: { name: 'phone', isUniqueOnPreferenceStore: false }, +}; + +describe('getPreferenceIdentifiersFromRow', () => { + it('extracts identifiers for all mapped columns with values', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { + email_col: 'alice@example.com', + user_id_col: 'u-123', + phone_col: '+15551234567', + }, + columnToIdentifier, + }); + + expect(result).toEqual([ + { name: 'email', value: 'alice@example.com' }, + { name: 'phone', value: '+15551234567' }, + { name: 'userId', value: 'u-123' }, + ]); + }); + + it('sorts email to the front', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { + user_id_col: 'u-1', + email_col: 'bob@example.com', + }, + columnToIdentifier, + }); + + expect(result[0]).toEqual({ name: 'email', value: 'bob@example.com' }); + expect(result[1]).toEqual({ name: 'userId', value: 'u-1' }); + }); + + it('sorts non-email identifiers alphabetically', () => { + const mapping: FileFormatState['columnToIdentifier'] = { + col_z: { name: 'zeta', isUniqueOnPreferenceStore: false }, + col_a: { name: 'alpha', isUniqueOnPreferenceStore: false }, + col_m: { name: 'mike', isUniqueOnPreferenceStore: false }, + }; + + const result = getPreferenceIdentifiersFromRow({ + row: { col_z: 'z', col_a: 'a', col_m: 'm' }, + columnToIdentifier: mapping, + }); + + expect(result.map((r) => r.name)).toEqual(['alpha', 'mike', 'zeta']); + }); + + it('skips columns with empty string values', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { + email_col: 'alice@example.com', + user_id_col: '', + phone_col: '+15551234567', + }, + columnToIdentifier, + }); + + expect(result).toEqual([ + { name: 'email', value: 'alice@example.com' }, + { name: 'phone', value: '+15551234567' }, + ]); + }); + + it('skips columns missing from the row', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { email_col: 'alice@example.com' }, + columnToIdentifier, + }); + + expect(result).toEqual([{ name: 'email', value: 'alice@example.com' }]); + }); + + it('returns empty array when no columns match', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { unrelated: 'value' }, + columnToIdentifier, + }); + + expect(result).toEqual([]); + }); + + it('returns empty array for empty columnToIdentifier', () => { + const result = getPreferenceIdentifiersFromRow({ + row: { email_col: 'alice@example.com' }, + columnToIdentifier: {}, + }); + + expect(result).toEqual([]); + }); +}); diff --git a/packages/sdk/src/preference-management/tests/getUniquePreferenceIdentifierNamesFromRow.test.ts b/packages/sdk/src/preference-management/tests/getUniquePreferenceIdentifierNamesFromRow.test.ts new file mode 100644 index 00000000..4aa86f50 --- /dev/null +++ b/packages/sdk/src/preference-management/tests/getUniquePreferenceIdentifierNamesFromRow.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect } from 'vitest'; + +import type { FileFormatState } from '../codecs.js'; +import { getUniquePreferenceIdentifierNamesFromRow } from '../getUniquePreferenceIdentifierNamesFromRow.js'; + +const columnToIdentifier: FileFormatState['columnToIdentifier'] = { + email_col: { name: 'email', isUniqueOnPreferenceStore: true }, + user_id_col: { name: 'userId', isUniqueOnPreferenceStore: true }, + phone_col: { name: 'phone', isUniqueOnPreferenceStore: false }, +}; + +describe('getUniquePreferenceIdentifierNamesFromRow', () => { + it('returns only unique identifiers that have values', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { + email_col: 'alice@example.com', + user_id_col: 'u-123', + phone_col: '+15551234567', + }, + columnToIdentifier, + }); + + expect(result).toEqual([ + { + name: 'email', + isUniqueOnPreferenceStore: true, + columnName: 'email_col', + value: 'alice@example.com', + }, + { + name: 'userId', + isUniqueOnPreferenceStore: true, + columnName: 'user_id_col', + value: 'u-123', + }, + ]); + }); + + it('excludes non-unique identifiers', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { phone_col: '+15551234567' }, + columnToIdentifier, + }); + + expect(result).toEqual([]); + }); + + it('sorts email to the front', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { + user_id_col: 'u-1', + email_col: 'bob@example.com', + }, + columnToIdentifier, + }); + + expect(result[0]!.name).toBe('email'); + expect(result[1]!.name).toBe('userId'); + }); + + it('sorts non-email unique identifiers alphabetically', () => { + const mapping: FileFormatState['columnToIdentifier'] = { + col_z: { name: 'zeta', isUniqueOnPreferenceStore: true }, + col_a: { name: 'alpha', isUniqueOnPreferenceStore: true }, + }; + + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { col_z: 'z', col_a: 'a' }, + columnToIdentifier: mapping, + }); + + expect(result.map((r) => r.name)).toEqual(['alpha', 'zeta']); + }); + + it('skips columns with empty string values', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { + email_col: 'alice@example.com', + user_id_col: '', + }, + columnToIdentifier, + }); + + expect(result).toHaveLength(1); + expect(result[0]!.name).toBe('email'); + }); + + it('skips columns missing from the row', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { email_col: 'alice@example.com' }, + columnToIdentifier, + }); + + expect(result).toEqual([ + { + name: 'email', + isUniqueOnPreferenceStore: true, + columnName: 'email_col', + value: 'alice@example.com', + }, + ]); + }); + + it('returns empty array when no columns match', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { unrelated: 'value' }, + columnToIdentifier, + }); + + expect(result).toEqual([]); + }); + + it('returns empty array for empty columnToIdentifier', () => { + const result = getUniquePreferenceIdentifierNamesFromRow({ + row: { email_col: 'alice@example.com' }, + columnToIdentifier: {}, + }); + + expect(result).toEqual([]); + }); +});