diff --git a/packages/varlock-website/src/content/docs/reference/cli-commands.mdx b/packages/varlock-website/src/content/docs/reference/cli-commands.mdx index e904ebc6..e07796bd 100644 --- a/packages/varlock-website/src/content/docs/reference/cli-commands.mdx +++ b/packages/varlock-website/src/content/docs/reference/cli-commands.mdx @@ -238,6 +238,44 @@ You can also set it up manually -- see the [Secrets guide](/guides/secrets/#scan +
+### `varlock audit` ||audit|| + +Scans your source code for environment variable references and compares them against keys defined in your schema. + +This command reports two drift categories: +- **Missing in schema**: key is used in code but not declared in schema +- **Unused in schema**: key is declared in schema but not referenced in code + +Exit codes: +- `0` when schema and code are in sync +- `1` when drift is detected + +```bash +varlock audit [options] +``` + +**Options:** +- `--path` / `-p`: Path to a specific `.env` file or directory to use as the schema entry point + +**Examples:** +```bash +# Audit current project +varlock audit + +# Audit using a specific .env file as schema entry point +varlock audit --path .env.prod + +# Audit using a directory as schema entry point +varlock audit --path ./config +``` + +:::note +When `--path` points to a directory, code scanning is scoped to that directory tree. When it points to a file, scanning is scoped to that file's parent directory. +::: + +
+
### `varlock typegen` ||typegen|| diff --git a/packages/varlock/src/cli/cli-executable.ts b/packages/varlock/src/cli/cli-executable.ts index 4584a4aa..ffb652fc 100644 --- a/packages/varlock/src/cli/cli-executable.ts +++ b/packages/varlock/src/cli/cli-executable.ts @@ -23,6 +23,7 @@ import { commandSpec as explainCommandSpec } from './commands/explain.command'; import { commandSpec as scanCommandSpec } from './commands/scan.command'; import { commandSpec as typegenCommandSpec } from './commands/typegen.command'; import { commandSpec as installPluginCommandSpec } from './commands/install-plugin.command'; +import { commandSpec as auditCommandSpec } from './commands/audit.command'; // import { commandSpec as loginCommandSpec } from './commands/login.command'; // import { commandSpec as pluginCommandSpec } from './commands/plugin.command'; @@ -58,6 +59,7 @@ subCommands.set('explain', buildLazyCommand(explainCommandSpec, async () => awai subCommands.set('help', buildLazyCommand(helpCommandSpec, async () => await import('./commands/help.command'))); subCommands.set('telemetry', buildLazyCommand(telemetryCommandSpec, async () => await import('./commands/telemetry.command'))); subCommands.set('scan', buildLazyCommand(scanCommandSpec, async () => await import('./commands/scan.command'))); +subCommands.set('audit', buildLazyCommand(auditCommandSpec, async () => await import('./commands/audit.command'))); subCommands.set('typegen', buildLazyCommand(typegenCommandSpec, async () => await import('./commands/typegen.command'))); subCommands.set('install-plugin', buildLazyCommand(installPluginCommandSpec, async () => await import('./commands/install-plugin.command'))); // subCommands.set('login', buildLazyCommand(loginCommandSpec, async () => await import('./commands/login.command'))); diff --git a/packages/varlock/src/cli/commands/audit.command.ts b/packages/varlock/src/cli/commands/audit.command.ts new file mode 100644 index 00000000..d20126b1 --- /dev/null +++ b/packages/varlock/src/cli/commands/audit.command.ts @@ -0,0 +1,158 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import ansis from 'ansis'; +import { define } from 'gunshi'; + +import { FileBasedDataSource } from '../../env-graph'; +import { loadVarlockEnvGraph } from '../../lib/load-graph'; +import { checkForNoEnvFiles, checkForSchemaErrors } from '../helpers/error-checks'; +import { type TypedGunshiCommandFn } from '../helpers/gunshi-type-utils'; +import { + scanCodeForEnvVars, + type EnvVarReference, +} from '../helpers/env-var-scanner'; +import { gracefulExit } from 'exit-hook'; +import { diffSchemaAndCodeKeys } from '../helpers/audit-diff'; + +export const commandSpec = define({ + name: 'audit', + description: 'Audit code env var usage against your .env.schema', + args: { + path: { + type: 'string', + short: 'p', + description: 'Path to a specific .env file or directory to use as the schema entry point', + }, + }, + examples: ` +Scans your source code for environment variable references and compares them +to keys defined in your varlock schema. + +Examples: + varlock audit # Audit current project + varlock audit --path .env.prod # Audit using a specific env entry point +`.trim(), +}); + +function formatReference(cwd: string, ref: EnvVarReference): string { + const relPath = path.relative(cwd, ref.filePath); + return `${relPath}:${ref.lineNumber}:${ref.columnNumber}`; +} + +async function getScanRootFromEntryPath(providedEntryPath: string): Promise { + const resolved = path.resolve(providedEntryPath); + try { + const entryStat = await fs.stat(resolved); + if (entryStat.isDirectory()) return resolved; + } catch { + // loadVarlockEnvGraph validates path before this point; fallback keeps behavior predictable + } + + if (providedEntryPath.endsWith('/') || providedEntryPath.endsWith(path.sep)) { + return resolved; + } + return path.dirname(resolved); +} + +function collectStringArgs(input: unknown, out: Array) { + if (Array.isArray(input)) { + for (const entry of input) collectStringArgs(entry, out); + return; + } + if (typeof input !== 'string') return; + + const normalized = input.trim().replace(/^\.\//, '').replace(/[/\\]+$/, ''); + if (!normalized) return; + out.push(normalized); +} + +async function getCustomAuditIgnorePaths(envGraph: any): Promise> { + const rootDecFns = typeof envGraph?.getRootDecFns === 'function' + ? envGraph.getRootDecFns('auditIgnorePaths') + : []; + + const mergedPaths: Array = []; + for (const dec of rootDecFns || []) { + const resolved = await dec.resolve(); + collectStringArgs(resolved?.arr, mergedPaths); + } + + return [...new Set(mergedPaths)]; +} + +export const commandFn: TypedGunshiCommandFn = async (ctx) => { + const providedEntryPath = ctx.values.path as string | undefined; + const envGraph = await loadVarlockEnvGraph({ + entryFilePath: providedEntryPath, + }); + + checkForSchemaErrors(envGraph); + checkForNoEnvFiles(envGraph); + + const schemaScanRoot = (() => { + if (providedEntryPath) { + return undefined; + } + + const rootSource = envGraph.rootDataSource; + if (rootSource instanceof FileBasedDataSource) { + return path.dirname(rootSource.fullPath); + } + return envGraph.basePath ?? process.cwd(); + })(); + + const finalScanRoot = providedEntryPath + ? await getScanRootFromEntryPath(providedEntryPath) + : (schemaScanRoot ?? process.cwd()); + + const customIgnoredPaths = await getCustomAuditIgnorePaths(envGraph); + if (customIgnoredPaths.length > 0) { + console.log(`ℹ️ Skipping custom ignored paths: ${customIgnoredPaths.join(', ')}`); + } + + const scanResult = await scanCodeForEnvVars( + { cwd: finalScanRoot }, + customIgnoredPaths, + ); + const schemaKeys = Object.keys(envGraph.configSchema); + + const diff = diffSchemaAndCodeKeys(schemaKeys, scanResult.keys); + const unusedInSchema: Array = []; + for (const key of diff.unusedInSchema) { + const item = envGraph.configSchema[key]; + const itemDecorators = (item as any)?.decorators as Record | undefined; + const isIgnored = (typeof item?.getDec === 'function' && (item.getDec('auditIgnore') as unknown) === true) + || (itemDecorators?.auditIgnore === true); + if (isIgnored) continue; + unusedInSchema.push(key); + } + + if (diff.missingInSchema.length === 0 && unusedInSchema.length === 0) { + console.log(ansis.green(`✅ Schema and code references are in sync. (scanned ${scanResult.scannedFilesCount} file${scanResult.scannedFilesCount === 1 ? '' : 's'})`)); + gracefulExit(0); + return; + } + + console.error(ansis.red('\n🚨 Schema/code mismatch detected:\n')); + + if (diff.missingInSchema.length > 0) { + console.error(ansis.red(`Missing in schema (${diff.missingInSchema.length}):`)); + for (const key of diff.missingInSchema) { + const refs = scanResult.references.filter((r) => r.key === key).slice(0, 3); + const refPreview = refs.map((r) => formatReference(finalScanRoot, r)).join(', '); + console.error(` - ${ansis.bold(key)}${refPreview ? ansis.dim(` (seen at ${refPreview})`) : ''}`); + } + console.error(''); + } + + if (unusedInSchema.length > 0) { + console.error(ansis.yellow(`Unused in schema (${unusedInSchema.length}):`)); + for (const key of unusedInSchema) { + console.error(` - ${ansis.bold(key)}`); + } + console.error(ansis.dim('(Hint: If this is used by an external tool, add # @auditIgnore to the item)')); + console.error(''); + } + + gracefulExit(1); +}; diff --git a/packages/varlock/src/cli/commands/init.command.ts b/packages/varlock/src/cli/commands/init.command.ts index 483f8701..f50aaa57 100644 --- a/packages/varlock/src/cli/commands/init.command.ts +++ b/packages/varlock/src/cli/commands/init.command.ts @@ -15,11 +15,13 @@ import prompts from '../helpers/prompts'; import { fmt, logLines } from '../helpers/pretty-format'; import { detectRedundantValues, ensureAllItemsExist, inferSchemaUpdates, type DetectedEnvFile, + inferItemDecorators, } from '../helpers/infer-schema'; import { detectJsPackageManager, installJsDependency } from '../helpers/js-package-manager-utils'; import { type TypedGunshiCommandFn } from '../helpers/gunshi-type-utils'; import { findEnvFiles } from '../helpers/find-env-files'; import { tryCatch } from '@env-spec/utils/try-catch'; +import { scanCodeForEnvVars } from '../helpers/env-var-scanner'; export const commandSpec = define({ name: 'init', @@ -104,6 +106,12 @@ export const commandFn: TypedGunshiCommandFn = async (ctx) = exampleFileToConvert = selectedExample; } + let scannedCodeEnvKeys: Array = []; + if (!exampleFileToConvert) { + const scanResult = await scanCodeForEnvVars(); + scannedCodeEnvKeys = scanResult.keys; + } + // update the schema const parsedEnvSchemaFile = exampleFileToConvert?.parsedFile || parseEnvSpecDotEnvFile(''); if (!parsedEnvSchemaFile) throw new Error('expected parsed .env example file'); @@ -131,6 +139,27 @@ export const commandFn: TypedGunshiCommandFn = async (ctx) = // add items we find in other env files, but are missing in the schema/example ensureAllItemsExist(parsedEnvSchemaFile, Object.values(parsedEnvFiles)); + const scannedCodeKeysToAdd = !exampleFileToConvert + ? scannedCodeEnvKeys.filter((key) => !parsedEnvSchemaFile.configItems.find((i) => i.key === key)) + : []; + + // add items we detect in source code if no sample/example file was provided + if (scannedCodeKeysToAdd.length > 0) { + envSpecUpdater.injectFromStr(parsedEnvSchemaFile, [ + '', + '# items added to schema by `varlock init`', + '# detected by scanning your source code for env var references', + '# PLEASE REVIEW THESE!', + '# ---', + '', + ].join('\n'), { location: 'end' }); + + for (const key of scannedCodeKeysToAdd) { + envSpecUpdater.injectFromStr(parsedEnvSchemaFile, `${key}=`); + inferItemDecorators(parsedEnvSchemaFile, key, ''); + } + } + // write new updated schema file const schemaFilePath = path.join(process.cwd(), '.env.schema'); await fs.writeFile(schemaFilePath, parsedEnvSchemaFile.toString()); @@ -142,6 +171,13 @@ export const commandFn: TypedGunshiCommandFn = async (ctx) = `Your ${fmt.fileName(exampleFileToConvert.fileName)} has been used to generate your new ${fmt.fileName('.env.schema')}:`, fmt.filePath(schemaFilePath), ]); + } else if (scannedCodeKeysToAdd.length > 0) { + logLines([ + '', + `Your new ${fmt.fileName('.env.schema')} file has been created from scanned source code references:`, + fmt.filePath(schemaFilePath), + ansis.dim(`Detected ${scannedCodeEnvKeys.length} env var key${scannedCodeEnvKeys.length === 1 ? '' : 's'} in your codebase.`), + ]); } else { logLines([ '', diff --git a/packages/varlock/src/cli/commands/test/audit.command.test.ts b/packages/varlock/src/cli/commands/test/audit.command.test.ts new file mode 100644 index 00000000..8cb5c93e --- /dev/null +++ b/packages/varlock/src/cli/commands/test/audit.command.test.ts @@ -0,0 +1,223 @@ +import path from 'node:path'; +import { + afterEach, beforeEach, describe, expect, test, vi, +} from 'vitest'; + +import { diffSchemaAndCodeKeys } from '../../helpers/audit-diff'; +import { commandFn } from '../audit.command'; + +const { + gracefulExitMock, + loadVarlockEnvGraphMock, + scanCodeForEnvVarsMock, + fsStatMock, +} = vi.hoisted(() => ({ + gracefulExitMock: vi.fn(), + loadVarlockEnvGraphMock: vi.fn(), + scanCodeForEnvVarsMock: vi.fn(), + fsStatMock: vi.fn(), +})); +let consoleLogSpy: ReturnType; +let consoleErrorSpy: ReturnType; + +vi.mock('exit-hook', () => ({ gracefulExit: gracefulExitMock })); +vi.mock('../../../lib/load-graph', () => ({ loadVarlockEnvGraph: loadVarlockEnvGraphMock })); +vi.mock('../../helpers/env-var-scanner', () => ({ scanCodeForEnvVars: scanCodeForEnvVarsMock })); +vi.mock('node:fs/promises', () => ({ default: { stat: fsStatMock } })); +vi.mock('../../helpers/error-checks', () => ({ + checkForNoEnvFiles: vi.fn(), + checkForSchemaErrors: vi.fn(), +})); + +describe('diffSchemaAndCodeKeys', () => { + test('finds missing and unused keys', () => { + const diff = diffSchemaAndCodeKeys( + ['A', 'B', 'C'], + ['B', 'C', 'D', 'E'], + ); + + expect(diff.missingInSchema).toEqual(['D', 'E']); + expect(diff.unusedInSchema).toEqual(['A']); + }); + + test('returns empty diff when in sync', () => { + const diff = diffSchemaAndCodeKeys( + ['API_KEY', 'DB_URL'], + ['DB_URL', 'API_KEY'], + ); + + expect(diff.missingInSchema).toEqual([]); + expect(diff.unusedInSchema).toEqual([]); + }); +}); + +describe('audit command', () => { + beforeEach(() => { + consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + gracefulExitMock.mockReset(); + loadVarlockEnvGraphMock.mockReset(); + scanCodeForEnvVarsMock.mockReset(); + fsStatMock.mockReset(); + fsStatMock.mockRejectedValue(new Error('missing')); + + loadVarlockEnvGraphMock.mockResolvedValue({ + configSchema: { + API_KEY: { getDec: vi.fn().mockReturnValue(undefined) }, + DATABASE_URL: { getDec: vi.fn().mockReturnValue(undefined) }, + }, + getRootDecFns: vi.fn().mockReturnValue([]), + rootDataSource: undefined, + basePath: '/repo', + }); + }); + + afterEach(() => { + consoleLogSpy.mockRestore(); + consoleErrorSpy.mockRestore(); + }); + + test('exits with code 1 when schema drift exists', async () => { + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY', 'MISSING_FROM_SCHEMA'], + references: [ + { + key: 'MISSING_FROM_SCHEMA', + filePath: '/repo/src/index.ts', + lineNumber: 10, + columnNumber: 3, + syntax: 'process.env.member', + }, + ], + scannedFilesCount: 1, + }); + + await commandFn({ values: {} } as any); + + expect(gracefulExitMock).toHaveBeenCalledWith(1); + }); + + test('exits with code 0 when schema and code match', async () => { + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY', 'DATABASE_URL'], + references: [], + scannedFilesCount: 4, + }); + + await commandFn({ values: {} } as any); + + expect(gracefulExitMock).toHaveBeenCalledWith(0); + }); + + test('scans from schema path directory when --path is provided', async () => { + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY', 'DATABASE_URL'], + references: [], + scannedFilesCount: 2, + }); + + await commandFn({ values: { path: './backend/.env.schema' } } as any); + + expect(scanCodeForEnvVarsMock).toHaveBeenCalledWith( + { + cwd: path.resolve('./backend'), + }, + [], + ); + }); + + test('scans from directory path when --path points to dir without trailing slash', async () => { + fsStatMock.mockResolvedValue({ isDirectory: () => true }); + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY', 'DATABASE_URL'], + references: [], + scannedFilesCount: 2, + }); + + await commandFn({ values: { path: './config' } } as any); + + expect(scanCodeForEnvVarsMock).toHaveBeenCalledWith( + { + cwd: path.resolve('./config'), + }, + [], + ); + }); + + test('scans from directory path when --path ends with slash', async () => { + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY', 'DATABASE_URL'], + references: [], + scannedFilesCount: 2, + }); + + await commandFn({ values: { path: './config/' } } as any); + + expect(scanCodeForEnvVarsMock).toHaveBeenCalledWith( + { + cwd: path.resolve('./config/'), + }, + [], + ); + }); + + test('treats # @auditIgnore as suppressed and # @auditIgnore=false as unsuppressed', async () => { + loadVarlockEnvGraphMock.mockResolvedValue({ + configSchema: { + API_KEY: { getDec: vi.fn().mockReturnValue(undefined) }, + IGNORED_UNUSED: { getDec: vi.fn().mockReturnValue(true) }, // # @auditIgnore + EXPLICIT_FALSE_UNUSED: { getDec: vi.fn().mockReturnValue(false) }, // # @auditIgnore=false + }, + getRootDecFns: vi.fn().mockReturnValue([]), + rootDataSource: undefined, + basePath: '/repo', + }); + + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY'], + references: [], + scannedFilesCount: 1, + }); + + await commandFn({ values: {} } as any); + + expect(gracefulExitMock).toHaveBeenCalledWith(1); + const errorOutput = consoleErrorSpy.mock.calls.flat().join('\n'); + expect(errorOutput).toContain('EXPLICIT_FALSE_UNUSED'); + expect(errorOutput).not.toContain('IGNORED_UNUSED'); + expect(errorOutput).toContain('(Hint: If this is used by an external tool, add # @auditIgnore to the item)'); + }); + + test('flattens multiple # @auditIgnorePaths(...) calls and forwards merged excludes to scanner', async () => { + loadVarlockEnvGraphMock.mockResolvedValue({ + configSchema: { + API_KEY: { getDec: vi.fn().mockReturnValue(undefined) }, + }, + getRootDecFns: vi.fn().mockReturnValue([ + { + resolve: vi.fn().mockResolvedValue({ arr: ['e2e', './scripts/'], obj: { unused: 'x' } }), + }, + { + resolve: vi.fn().mockResolvedValue({ arr: [['mocks']], obj: {} }), + }, + ]), + rootDataSource: undefined, + basePath: '/repo', + }); + + scanCodeForEnvVarsMock.mockResolvedValue({ + keys: ['API_KEY'], + references: [], + scannedFilesCount: 1, + }); + + await commandFn({ values: {} } as any); + + expect(consoleLogSpy).toHaveBeenCalledWith('ℹ️ Skipping custom ignored paths: e2e, scripts, mocks'); + expect(scanCodeForEnvVarsMock).toHaveBeenCalledWith( + { cwd: '/repo' }, + ['e2e', 'scripts', 'mocks'], + ); + }); +}); diff --git a/packages/varlock/src/cli/helpers/audit-diff.ts b/packages/varlock/src/cli/helpers/audit-diff.ts new file mode 100644 index 00000000..48ec7dda --- /dev/null +++ b/packages/varlock/src/cli/helpers/audit-diff.ts @@ -0,0 +1,12 @@ +export function diffSchemaAndCodeKeys(schemaKeys: Array, codeKeys: Array) { + const schemaSet = new Set(schemaKeys); + const codeSet = new Set(codeKeys); + + const missingInSchema = [...codeSet].filter((k) => !schemaSet.has(k)).sort((a, b) => a.localeCompare(b)); + const unusedInSchema = [...schemaSet].filter((k) => !codeSet.has(k)).sort((a, b) => a.localeCompare(b)); + + return { + missingInSchema, + unusedInSchema, + }; +} diff --git a/packages/varlock/src/cli/helpers/env-var-scanner.ts b/packages/varlock/src/cli/helpers/env-var-scanner.ts new file mode 100644 index 00000000..9746419f --- /dev/null +++ b/packages/varlock/src/cli/helpers/env-var-scanner.ts @@ -0,0 +1,638 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +export const DEFAULT_IGNORED_DIRS = [ + '.git', + 'node_modules', + 'dist', + 'build', + '.next', + 'vendor', + '.venv', +] as const; + +const DEFAULT_MAX_FILE_SIZE_BYTES = 1024 * 1024; +const DEFAULT_CONCURRENCY = 50; +const ENV_KEY_IDENTIFIER_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const LANGUAGE_BY_EXTENSION: Record = { + '.js': 'js-like', + '.mjs': 'js-like', + '.cjs': 'js-like', + '.jsx': 'js-like', + '.ts': 'js-like', + '.mts': 'js-like', + '.cts': 'js-like', + '.tsx': 'js-like', + '.vue': 'js-like', + '.svelte': 'js-like', + '.astro': 'js-like', + '.mdx': 'js-like', + + '.py': 'python', + '.go': 'go', + '.rb': 'ruby', + '.php': 'php', + '.rs': 'rust', + '.java': 'java', + '.cs': 'csharp', +}; + +type ScannerLanguage = 'js-like' | 'python' | 'go' | 'ruby' | 'php' | 'rust' | 'java' | 'csharp'; + +export type EnvVarSyntax = 'process.env.member' + | 'process.env.bracket' + | 'process.env.destructure' + | 'import.meta.env.member' + | 'import.meta.env.bracket' + | 'import.meta.env.destructure' + | 'ENV.member' + | 'ENV.bracket' + | 'ENV.destructure' + | 'python.environ' + | 'python.getenv' + | 'go.getenv' + | 'ruby.env' + | 'ruby.fetch' + | 'php.getenv' + | 'php._env' + | 'php._server' + | 'rust.getenv' + | 'java.getenv' + | 'csharp.getenv'; + +export interface EnvVarReference { + key: string; + filePath: string; + lineNumber: number; + columnNumber: number; + syntax: EnvVarSyntax; +} + +export interface ScanCodeEnvVarsOptions { + cwd?: string; + concurrency?: number; + maxFileSizeBytes?: number; + // legacy option, treated as additional excludes + ignoredDirs?: Array; +} + +export interface ScanCodeEnvVarsResult { + keys: Array; + references: Array; + scannedFilesCount: number; +} + +interface SimplePattern { + regex: RegExp; + syntax: EnvVarSyntax; +} + +const PATTERNS_BY_LANGUAGE: Record> = { + 'js-like': [ + { + regex: /\bprocess\.env\.([A-Za-z_][A-Za-z0-9_]*)\b/g, + syntax: 'process.env.member', + }, + { + regex: /\bprocess\.env\[\s*['"`]([A-Za-z_][A-Za-z0-9_]*)['"`]\s*\]/g, + syntax: 'process.env.bracket', + }, + { + regex: /\bimport\.meta\.env\.([A-Za-z_][A-Za-z0-9_]*)\b/g, + syntax: 'import.meta.env.member', + }, + { + regex: /\bimport\.meta\.env\[\s*['"`]([A-Za-z_][A-Za-z0-9_]*)['"`]\s*\]/g, + syntax: 'import.meta.env.bracket', + }, + { + regex: /\bENV\.([A-Za-z_][A-Za-z0-9_]*)\b/g, + syntax: 'ENV.member', + }, + { + regex: /\bENV\[\s*['"`]([A-Za-z_][A-Za-z0-9_]*)['"`]\s*\]/g, + syntax: 'ENV.bracket', + }, + ], + python: [ + { + regex: /\bos\.environ\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g, + syntax: 'python.environ', + }, + { + regex: /\bos\.getenv\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]/g, + syntax: 'python.getenv', + }, + ], + go: [ + { + regex: /\bos\.(?:Getenv|LookupEnv)\(\s*"([A-Za-z_][A-Za-z0-9_]*)"\s*\)/g, + syntax: 'go.getenv', + }, + ], + ruby: [ + { + regex: /\bENV\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g, + syntax: 'ruby.env', + }, + { + regex: /\bENV\.fetch\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]/g, + syntax: 'ruby.fetch', + }, + ], + php: [ + { + regex: /\bgetenv\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]/g, + syntax: 'php.getenv', + }, + { + regex: /\$_ENV\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g, + syntax: 'php._env', + }, + { + regex: /\$_SERVER\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g, + syntax: 'php._server', + }, + ], + rust: [ + { + regex: /\bstd::env::(?:var|var_os)\(\s*"([A-Za-z_][A-Za-z0-9_]*)"\s*\)/g, + syntax: 'rust.getenv', + }, + ], + java: [ + { + regex: /\bSystem\.getenv\(\s*"([A-Za-z_][A-Za-z0-9_]*)"\s*\)/g, + syntax: 'java.getenv', + }, + ], + csharp: [ + { + regex: /\bEnvironment\.GetEnvironmentVariable\(\s*"([A-Za-z_][A-Za-z0-9_]*)"\s*\)/g, + syntax: 'csharp.getenv', + }, + ], +}; + +const JS_DESTRUCTURE_PATTERNS: Array<{ regex: RegExp, syntax: EnvVarSyntax }> = [ + { + regex: /\{([^}]*)\}\s*=\s*process\.env\b/g, + syntax: 'process.env.destructure', + }, + { + regex: /\{([^}]*)\}\s*=\s*import\.meta\.env\b/g, + syntax: 'import.meta.env.destructure', + }, + { + regex: /\{([^}]*)\}\s*=\s*ENV\b/g, + syntax: 'ENV.destructure', + }, +]; + +async function discoverSourceFiles(cwd: string, ignoredDirs: Set): Promise> { + const filePaths: Array = []; + const globExcludes = [...ignoredDirs].flatMap((dirName) => [`**/${dirName}`, `**/${dirName}/**`]); + + for await (const relativePath of fs.glob('**/*', { cwd, exclude: globExcludes })) { + const normalizedRelativePath = String(relativePath).replaceAll('\\', '/'); + + const extension = path.extname(normalizedRelativePath).toLowerCase(); + if (!(extension in LANGUAGE_BY_EXTENSION)) continue; + + filePaths.push(path.resolve(cwd, normalizedRelativePath)); + } + return filePaths; +} + +function extractDestructuredKeys(body: string): Array<{ key: string, relativeIndex: number }> { + const found: Array<{ key: string, relativeIndex: number }> = []; + const propPattern = /(^|,)\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::\s*[A-Za-z_][A-Za-z0-9_]*)?\s*(?:=[^,]+)?\s*(?=,|$)/g; + for (const match of body.matchAll(propPattern)) { + const key = match[2]; + if (!key) continue; + const wholeMatch = match[0] || ''; + const keyIndexInWhole = wholeMatch.indexOf(key); + const relativeIndex = (match.index ?? 0) + (keyIndexInWhole >= 0 ? keyIndexInWhole : 0); + found.push({ key, relativeIndex }); + } + return found; +} + +function getNewlineIndices(content: string): Array { + const indices: Array = []; + for (let i = 0; i < content.length; i++) { + if (content.charCodeAt(i) === 10) indices.push(i); + } + return indices; +} + +function indexToLineAndColumn( + content: string, + newlineIndices: Array, + index: number, +): { lineNumber: number, columnNumber: number } { + let lo = 0; + let hi = newlineIndices.length; + + while (lo < hi) { + const mid = lo + Math.floor((hi - lo) / 2); + if (newlineIndices[mid] < index) lo = mid + 1; + else hi = mid; + } + + const lineNumber = lo + 1; + const lineStartIndex = lo === 0 ? 0 : newlineIndices[lo - 1] + 1; + + return { + lineNumber, + columnNumber: index - lineStartIndex + 1, + }; +} + +function buildReference( + filePath: string, + content: string, + newlineIndices: Array, + index: number, + key: string, + syntax: EnvVarSyntax, +): EnvVarReference { + const { lineNumber, columnNumber } = indexToLineAndColumn(content, newlineIndices, index); + return { + filePath, + key, + lineNumber, + columnNumber, + syntax, + }; +} + +function skipQuotedWithoutMask(chars: Array, startIndex: number, quoteChar: '\'' | '"'): number { + let i = startIndex + 1; + while (i < chars.length) { + const ch = chars[i]; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === quoteChar) { + i++; + return i; + } + i++; + } + return i; +} + +function skipTemplateWithoutMask(chars: Array, startIndex: number): number { + let i = startIndex + 1; + while (i < chars.length) { + const ch = chars[i]; + const next = chars[i + 1]; + + if (ch === '\\') { + i += 2; + continue; + } + if (ch === '`') { + i++; + return i; + } + if (ch === '$' && next === '{') { + i += 2; + let depth = 1; + while (i < chars.length && depth > 0) { + if (chars[i] === '\\') { + i += 2; + continue; + } + if (chars[i] === '{') depth++; + else if (chars[i] === '}') depth--; + i++; + } + continue; + } + i++; + } + return i; +} + +function skipAndMaskQuotedString(chars: Array, startIndex: number, quoteChar: '\'' | '"'): number { + let i = startIndex + 1; + while (i < chars.length) { + const ch = chars[i]; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === quoteChar) { + i++; + break; + } + i++; + } + + const endExclusive = i; + const inner = chars.slice(startIndex + 1, Math.max(startIndex + 1, endExclusive - 1)).join(''); + const keepInner = ENV_KEY_IDENTIFIER_REGEX.test(inner); + if (!keepInner) { + for (let idx = startIndex + 1; idx < endExclusive - 1; idx++) { + if (chars[idx] !== '\n') chars[idx] = ' '; + } + } + return endExclusive; +} + +function skipAndMaskTemplateLiteral(chars: Array, startIndex: number): number { + let i = startIndex + 1; + let segmentStart = i; + const literalSegments: Array<{ start: number, endExclusive: number }> = []; + let hasInterpolation = false; + + while (i < chars.length) { + const ch = chars[i]; + const next = chars[i + 1]; + + if (ch === '\\') { + i += 2; + continue; + } + + if (ch === '`') { + literalSegments.push({ start: segmentStart, endExclusive: i }); + i++; + break; + } + + if (ch === '$' && next === '{') { + hasInterpolation = true; + literalSegments.push({ start: segmentStart, endExclusive: i }); + i += 2; + let depth = 1; + while (i < chars.length && depth > 0) { + const exprCh = chars[i]; + const exprNext = chars[i + 1]; + + if (exprCh === '\\') { + i += 2; + continue; + } + + if (exprCh === '\'' || exprCh === '"') { + i = skipQuotedWithoutMask(chars, i, exprCh); + continue; + } + + if (exprCh === '`') { + i = skipTemplateWithoutMask(chars, i); + continue; + } + + if (exprCh === '{') depth++; + else if (exprCh === '}') depth--; + + if (depth === 0) { + i++; + break; + } + + if (exprCh === '/' && exprNext === '/') { + i += 2; + while (i < chars.length && chars[i] !== '\n') i++; + continue; + } + if (exprCh === '/' && exprNext === '*') { + i += 2; + while (i < chars.length) { + if (chars[i] === '*' && chars[i + 1] === '/') { + i += 2; + break; + } + i++; + } + continue; + } + + i++; + } + segmentStart = i; + continue; + } + + i++; + } + + const endExclusive = i; + if (!hasInterpolation) { + const inner = chars.slice(startIndex + 1, Math.max(startIndex + 1, endExclusive - 1)).join(''); + if (!ENV_KEY_IDENTIFIER_REGEX.test(inner)) { + for (let idx = startIndex + 1; idx < endExclusive - 1; idx++) { + if (chars[idx] !== '\n') chars[idx] = ' '; + } + } + return endExclusive; + } + + for (const segment of literalSegments) { + for (let idx = segment.start; idx < segment.endExclusive; idx++) { + if (chars[idx] !== '\n') chars[idx] = ' '; + } + } + + return endExclusive; +} + +function maskCommentsPreserveLayout(content: string, language: ScannerLanguage): string { + const chars = content.split(''); + + const supportsHashComments = language === 'python' || language === 'ruby' || language === 'php'; + const supportsSlashComments = language !== 'python' && language !== 'ruby'; + + let i = 0; + let inLineComment = false; + let inBlockComment = false; + + while (i < chars.length) { + const ch = chars[i]; + const next = chars[i + 1]; + + if (inLineComment) { + if (ch === '\n') { + inLineComment = false; + } else { + chars[i] = ' '; + } + i++; + continue; + } + + if (inBlockComment) { + if (ch === '*' && next === '/') { + chars[i] = ' '; + chars[i + 1] = ' '; + inBlockComment = false; + i += 2; + continue; + } + if (ch !== '\n') chars[i] = ' '; + i++; + continue; + } + + if (supportsSlashComments && ch === '/' && next === '/') { + chars[i] = ' '; + chars[i + 1] = ' '; + inLineComment = true; + i += 2; + continue; + } + if (supportsSlashComments && ch === '/' && next === '*') { + chars[i] = ' '; + chars[i + 1] = ' '; + inBlockComment = true; + i += 2; + continue; + } + if (supportsHashComments && ch === '#') { + chars[i] = ' '; + inLineComment = true; + i++; + continue; + } + + if (ch === '\'') { + i = skipAndMaskQuotedString(chars, i, '\''); + continue; + } + if (ch === '"') { + i = skipAndMaskQuotedString(chars, i, '"'); + continue; + } + if (ch === '`' && (language === 'js-like' || language === 'go')) { + i = skipAndMaskTemplateLiteral(chars, i); + continue; + } + + i++; + } + + return chars.join(''); +} + +async function scanFileForEnvVarReferences( + filePath: string, + maxFileSizeBytes: number, +): Promise> { + let fileStat; + try { + fileStat = await fs.stat(filePath); + } catch { + return []; + } + + if (!fileStat.isFile() || fileStat.size > maxFileSizeBytes) return []; + + let rawContent: string; + try { + rawContent = await fs.readFile(filePath, 'utf-8'); + } catch { + return []; + } + if (!rawContent || rawContent.includes('\0')) return []; + + const extension = path.extname(filePath).toLowerCase(); + const language = LANGUAGE_BY_EXTENSION[extension]; + if (!language) return []; + + const scanContent = maskCommentsPreserveLayout(rawContent, language); + const newlineIndices = getNewlineIndices(scanContent); + const references: Array = []; + + for (const pattern of PATTERNS_BY_LANGUAGE[language]) { + for (const match of scanContent.matchAll(pattern.regex)) { + const key = match[1]; + if (!key) continue; + references.push(buildReference(filePath, scanContent, newlineIndices, match.index ?? 0, key, pattern.syntax)); + } + } + + if (language === 'js-like') { + for (const pattern of JS_DESTRUCTURE_PATTERNS) { + for (const match of scanContent.matchAll(pattern.regex)) { + const body = match[1]; + if (!body) continue; + + const bodyOffset = (match.index ?? 0) + match[0].indexOf(body); + for (const destructured of extractDestructuredKeys(body)) { + references.push( + buildReference( + filePath, + scanContent, + newlineIndices, + bodyOffset + destructured.relativeIndex, + destructured.key, + pattern.syntax, + ), + ); + } + } + } + } + + return references; +} + +async function scanFilesWithLimit( + items: Array, + concurrency: number, + worker: (item: T) => Promise, +): Promise> { + if (items.length === 0) return []; + const results: Array = new Array(items.length); + let nextIndex = 0; + + async function runWorker() { + while (true) { + const currentIndex = nextIndex; + nextIndex++; + if (currentIndex >= items.length) return; + results[currentIndex] = await worker(items[currentIndex]); + } + } + + const workerCount = Math.max(1, Math.min(concurrency, items.length)); + const workers = Array.from({ length: workerCount }, async () => { + await runWorker(); + }); + await Promise.all(workers); + return results; +} + +export async function scanCodeForEnvVars( + options: ScanCodeEnvVarsOptions = {}, + additionalExcludeDirs: Array = [], +): Promise { + const cwd = options.cwd || process.cwd(); + const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY; + const maxFileSizeBytes = options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE_BYTES; + const excludeDirs = new Set([ + ...DEFAULT_IGNORED_DIRS, + ...(options.ignoredDirs ?? []), + ...additionalExcludeDirs, + ]); + + const filePaths = await discoverSourceFiles(cwd, excludeDirs); + const references = await scanFilesWithLimit(filePaths, concurrency, async (filePath) => { + return scanFileForEnvVarReferences(filePath, maxFileSizeBytes); + }); + + const flattenedReferences = references.flat(); + const uniqueKeys = [...new Set(flattenedReferences.map((r) => r.key))].sort((a, b) => a.localeCompare(b)); + + return { + keys: uniqueKeys, + references: flattenedReferences, + scannedFilesCount: filePaths.length, + }; +} diff --git a/packages/varlock/src/cli/helpers/infer-schema.ts b/packages/varlock/src/cli/helpers/infer-schema.ts index 5a252a9e..b25cf1c6 100644 --- a/packages/varlock/src/cli/helpers/infer-schema.ts +++ b/packages/varlock/src/cli/helpers/infer-schema.ts @@ -40,7 +40,7 @@ const EMAIL_REGEX = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+")) const VALID_NUMBER_REGEX = /^(0|([1-9][0-9]*))?(\.[0-9]+)?$/; -function inferItemDecorators(file: ParsedEnvSpecFile, itemKey: string, valueStr: string) { +export function inferItemDecorators(file: ParsedEnvSpecFile, itemKey: string, valueStr = '') { // infer @sensitive let itemIsPublic = false; if (PUBLIC_PREFIXES.some((prefix) => itemKey.startsWith(prefix))) itemIsPublic = true; diff --git a/packages/varlock/src/cli/helpers/test/env-var-scanner.test.ts b/packages/varlock/src/cli/helpers/test/env-var-scanner.test.ts new file mode 100644 index 00000000..e0945c82 --- /dev/null +++ b/packages/varlock/src/cli/helpers/test/env-var-scanner.test.ts @@ -0,0 +1,137 @@ +import { + afterEach, beforeEach, describe, expect, test, +} from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { scanCodeForEnvVars } from '../env-var-scanner'; + +describe('scanCodeForEnvVars', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'varlock-env-scan-')); + }); + + afterEach(() => { + if (tempDir && fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test('detects JS/TS env syntax including destructuring and ENV object', async () => { + fs.writeFileSync(path.join(tempDir, 'index.ts'), [ + 'const a = process.env.API_KEY;', + 'const b = process.env["DATABASE_URL"];', + 'const c = import.meta.env.VITE_PUBLIC_URL;', + 'const d = ENV.SECRET_TOKEN;', + 'const { PORT, NODE_ENV: envName, FEATURE_FLAG = "on" } = process.env;', + 'const { NEXT_PUBLIC_APP_URL } = import.meta.env;', + 'const { VARLOCK_ITEM } = ENV;', + ].join('\n')); + + const result = await scanCodeForEnvVars({ cwd: tempDir }); + + expect(result.keys).toEqual(expect.arrayContaining([ + 'API_KEY', + 'DATABASE_URL', + 'VITE_PUBLIC_URL', + 'SECRET_TOKEN', + 'PORT', + 'NODE_ENV', + 'FEATURE_FLAG', + 'NEXT_PUBLIC_APP_URL', + 'VARLOCK_ITEM', + ])); + }); + + test('detects multi-language env access patterns', async () => { + fs.writeFileSync(path.join(tempDir, 'app.py'), 'import os\nos.getenv("PY_TOKEN")\nos.environ["PY_URL"]\n'); + fs.writeFileSync(path.join(tempDir, 'main.go'), 'package main\nimport "os"\nfunc main(){_ = os.Getenv("GO_KEY"); _ ,_ = os.LookupEnv("GO_OPT") }\n'); + fs.writeFileSync(path.join(tempDir, 'service.rb'), 'ENV["RB_SECRET"]\nENV.fetch("RB_URL")\n'); + fs.writeFileSync(path.join(tempDir, 'index.php'), ' { + fs.writeFileSync(path.join(tempDir, 'comments.ts'), [ + '// process.env.COMMENTED_OUT', + '/* import.meta.env.BLOCKED_OUT */', + 'const fromString = "process.env.INSIDE_STRING";', + 'const fromTemplate = `ENV.IN_TEMPLATE`;', + 'const real = process.env.REAL_ONE;', + 'const fromBracket = process.env["KEPT_KEY"];', + ].join('\n')); + + const result = await scanCodeForEnvVars({ cwd: tempDir }); + + expect(result.keys).toContain('REAL_ONE'); + expect(result.keys).not.toContain('COMMENTED_OUT'); + expect(result.keys).not.toContain('BLOCKED_OUT'); + expect(result.keys).toContain('KEPT_KEY'); + expect(result.keys).not.toContain('INSIDE_STRING'); + expect(result.keys).not.toContain('IN_TEMPLATE'); + }); + + test('ignores go raw-string references while keeping real calls', async () => { + fs.writeFileSync(path.join(tempDir, 'main.go'), [ + 'package main', + 'import "os"', + 'func main() {', + ' _ = `os.Getenv("IN_RAW_STRING")`', + ' _ = os.Getenv("REAL_GO_KEY")', + '}', + ].join('\n')); + + const result = await scanCodeForEnvVars({ cwd: tempDir }); + + expect(result.keys).toContain('REAL_GO_KEY'); + expect(result.keys).not.toContain('IN_RAW_STRING'); + }); + + test('respects ignored directories', async () => { + fs.mkdirSync(path.join(tempDir, 'node_modules'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'node_modules', 'dep.js'), 'process.env.IGNORED_MOD'); + fs.writeFileSync(path.join(tempDir, 'app.ts'), 'process.env.VISIBLE_KEY'); + + const result = await scanCodeForEnvVars({ cwd: tempDir }); + expect(result.keys).toContain('VISIBLE_KEY'); + expect(result.keys).not.toContain('IGNORED_MOD'); + }); + + test('keeps default ignores while adding additional excluded directories', async () => { + fs.mkdirSync(path.join(tempDir, 'node_modules'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'e2e'), { recursive: true }); + + fs.writeFileSync(path.join(tempDir, 'node_modules', 'dep.js'), 'process.env.DEFAULT_IGNORED'); + fs.writeFileSync(path.join(tempDir, 'e2e', 'spec.ts'), 'process.env.CUSTOM_IGNORED'); + fs.writeFileSync(path.join(tempDir, 'app.ts'), 'process.env.VISIBLE_KEY'); + + const result = await scanCodeForEnvVars({ cwd: tempDir }, ['e2e']); + + expect(result.keys).toContain('VISIBLE_KEY'); + expect(result.keys).not.toContain('DEFAULT_IGNORED'); + expect(result.keys).not.toContain('CUSTOM_IGNORED'); + }); +}); diff --git a/packages/varlock/src/env-graph/test/resolvers.test.ts b/packages/varlock/src/env-graph/test/resolvers.test.ts index 0772aef2..3dab2449 100644 --- a/packages/varlock/src/env-graph/test/resolvers.test.ts +++ b/packages/varlock/src/env-graph/test/resolvers.test.ts @@ -71,7 +71,10 @@ function functionValueTests( } else { expect(item.isValid, `Expected item ${key} to be valid`).toBeTruthy(); } - expect(item.resolvedValue).toEqual(expectedValue); + const normalizedResolvedValue = typeof item.resolvedValue === 'string' + ? item.resolvedValue.replaceAll('\r', '') + : item.resolvedValue; + expect(normalizedResolvedValue).toEqual(expectedValue); } } });