diff --git a/README.md b/README.md index e454a1d..1ba0255 100644 --- a/README.md +++ b/README.md @@ -39,27 +39,26 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca ```json { - "FhirServer": { - "BaseUrl": "https://fhirServerBaseUrl", - "CqlOperation": "$cql" - }, - "Build": { - "CqlFileVersion": "1.0.000", - "CqlOutputPath": "./cql", - "testsRunDescription": '', - "testsRunDescription": "Local host test run", - "cqlTranslator": "Java CQFramework Translator", - "cqlTranslatorVersion": "Unknown", - "cqlEngine": "Java CQFramework Engine", + "FhirServer": { + "BaseUrl": "https://fhirServerBaseUrl", + "CqlOperation": "$cql" + }, + "Build": { + "CqlFileVersion": "1.0.000", + "CqlOutputPath": "./cql", + "testsRunDescription": "Local host test run", + "cqlTranslator": "Java CQFramework Translator", + "cqlTranslatorVersion": "Unknown", + "cqlEngine": "Java CQFramework Engine", "cqlEngineVersion": "4.1.0" }, - "Tests": { - "ResultsPath": "./results", - "SkipList": [] - }, - "Debug": { - "QuickTest": true - } + "Tests": { + "ResultsPath": "./results", + "SkipList": [] + }, + "Debug": { + "QuickTest": true + } } ``` @@ -109,6 +108,171 @@ To run only a specified set of tests (and skip all others), add entries to the ` Create your own configuration file and reference it when running the commands. You can use `conf/localhost.json` as a template for a new configuration with your own settings. +### Time Zone Configuration + +The CQL Tests Runner uses two settings to control how **DateTime** values are evaluated: + +- `SERVER_OFFSET_ISO` +- `TimeZoneOffsetPolicy` + +These settings are required because CQL allows **DateTime values without a timezone offset**, and different engines interpret those values differently. Without explicitly setting these, tests involving DateTime comparison and extraction may produce inconsistent results (pass, fail, or null). + +Reference: http://cql.hl7.org/CodeSystem/cql-language-capabilities + +--- + +#### TimeZoneOffsetPolicy (what it means) + +# πŸ”„ Timezone Offset Policy + +## Background + +CQL allows `DateTime` literals to be specified **with or without a timezone offset**. + +For example: + +``` +@2012-04-01T00:00 +``` + +This value does **not include an explicit timezone offset**. + +--- + +## What the CQL Specification Says + +> If no timezone offset is specified, the timezone offset of the evaluation request timestamp is used. + +This means: + +- A `DateTime` literal may omit an offset in its **source representation** +- But at **evaluation time**, the engine must apply an offset +- That offset comes from the **evaluation request timestamp** + +πŸ‘‰ In other words, under CQL semantics: + +- DateTimes are **always evaluated with an effective timezone offset** +- The only question is **which offset is applied** + +--- + +## Why This Capability Exists + +Although the specification is clear, implementations have historically differed in how they handle `DateTime` values without explicit offsets. + +Observed variations include: + +- Applying the evaluation request timestamp offset (spec-compliant behavior) +- Treating the value as offset-less in some operations +- Returning `null` for operations like `timezoneoffset(...)` +- Applying server-local or implicit defaults inconsistently + +This capability exists to explicitly test and document how an engine behaves in these scenarios. + +--- + +## What Is Being Tested + +This capability evaluates how an engine handles `DateTime` values that omit a timezone offset, including: + +- Whether the engine applies the evaluation request timestamp offset +- How functions like `timezoneoffset(...)` behave +- Whether comparisons and arithmetic treat the value consistently + +--- + +## Clarification + +This capability does **not** test whether a `DateTime` β€œhas a timezone or not.” + +Per the CQL specification: + +- A `DateTime` without an explicit offset is still evaluated **as if it has one** +- The offset is derived from the evaluation context (evaluation request timestamp) + +πŸ‘‰ The purpose of this capability is to verify that engines implement this behavior **correctly and consistently**. + +--- + +## Suggested Terminology + +- ❌ β€œDoes the DateTime have a timezone?” +- βœ… β€œHow does the engine determine the effective timezone offset for DateTimes without an explicit offset?” + +--- + +## Key Clarification + +> This capability is testing **implementation consistency**, not specification ambiguity. + + +--- + +# πŸ”— CapabilityTests Alignment & Mapping + +## Capability Definition + +- Capability: `timezone-offset-policy` +- Focus: Handling of DateTime values without explicit timezone offsets + +--- + +## Test Mapping + +### 1. Default Offset Application +- Tests: `timezone-offset-default` +- Verifies: + - Missing offset uses evaluation request timestamp + - No null offset produced + +--- + +### 2. timezoneoffset(...) Behavior +- Tests: `timezoneoffset-from-datetime` +- Verifies: + - Returns evaluation offset when not explicitly provided + +--- + +### 3. Equality / Comparison Consistency +- Tests: `datetime-equality-offset` +- Verifies: + - `@2012-04-01T00:00` equals equivalent explicit-offset value + +--- + +### 4. Arithmetic Behavior +- Tests: `datetime-arithmetic-offset` +- Verifies: + - Duration calculations respect derived offset + +--- + +## Reviewer Traceability + +Spec β†’ Behavior β†’ Test + +- Spec: Offset defaults to evaluation timestamp +- Behavior: Engine applies offset consistently +- Tests: Confirm consistency across functions and operators + +--- + +## Summary + +This capability ensures engines: + +- Correctly apply implicit timezone offsets +- Do not treat DateTimes as offset-less at runtime +- Maintain consistency across operations + + +#### Notes + +- These settings do not change server behavior; they only control how the runner evaluates tests +- If the server declares a policy in metadata, it overrides configuration +- `SERVER_OFFSET_ISO` is only used where explicitly referenced in test expressions + ### Running the tests The CLI now requires a configuration file path as an argument. Run the tests with the following commands: @@ -197,14 +361,16 @@ If using vscode for development, below are some examples for running the tests w ```json { - "type": "node", - "request": "launch", - "name": "Launch Build Command", - "skipFiles": ["/**"], - "program": "${workspaceFolder}/src/bin/cql-tests.ts", - "args": ["build-cql", "conf/localhost.json"], - "runtimeArgs": ["--import", "tsx"] -}, + "type": "node", + "request": "launch", + "name": "Launch Build Command", + "skipFiles": ["/**"], + "program": "${workspaceFolder}/src/bin/cql-tests.ts", + "args": ["build-cql", "conf/localhost.json"], + "runtimeArgs": ["--import", "tsx"] +} +``` +```json { "type": "node", "request": "launch", @@ -215,7 +381,7 @@ If using vscode for development, below are some examples for running the tests w "runtimeArgs": ["--import", "tsx"], "env": { "SERVER_BASE_URL": "http://localhost:3000" - } + } } ``` diff --git a/assets/schema/cql-test-configuration.schema.json b/assets/schema/cql-test-configuration.schema.json index 1ca7624..ab5edf5 100644 --- a/assets/schema/cql-test-configuration.schema.json +++ b/assets/schema/cql-test-configuration.schema.json @@ -64,7 +64,15 @@ "cqlEngineVersion": { "type": "string", "description": "Which version of the CQL engine is used in this test run" - } + }, + "SERVER_OFFSET_ISO": { + "type": "string", + "description": "ISO 8601 formatted date string to offset the server start time" + }, + "TimeZoneOffsetPolicy": { + "type": "string", + "description": "Which timezone offset policy is used in this test run." + } }, "required": ["CqlFileVersion", "CqlOutputPath"], "additionalProperties": false diff --git a/conf/localhost.json b/conf/localhost.json index 1b711a0..1415778 100644 --- a/conf/localhost.json +++ b/conf/localhost.json @@ -11,7 +11,9 @@ "cqlTranslator": "Java CQFramework Translator", "cqlTranslatorVersion": "Unknown", "cqlEngine": "Java CQFramework Engine", - "cqlEngineVersion": "4.1.0" + "cqlEngineVersion": "4.1.0", + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" }, "Debug": { "QuickTest": false diff --git a/src/commands/run-tests-command.ts b/src/commands/run-tests-command.ts index 3f01349..4d0ac5f 100644 --- a/src/commands/run-tests-command.ts +++ b/src/commands/run-tests-command.ts @@ -1,4 +1,4 @@ -import { TestRunner } from '../services/test-runner.js'; +import { TestRunner } from '../services/test-runner'; import { ConfigLoader } from '../conf/config-loader.js'; // Type declaration for CVL loader @@ -54,7 +54,9 @@ export class RunCommand { cqlTranslator: config.Build?.cqlTranslator, cqlTranslatorVersion: config.Build?.cqlTranslatorVersion, cqlEngine: config.Build?.cqlEngine, - cqlEngineVersion: config.Build?.cqlEngineVersion + cqlEngineVersion: config.Build?.cqlEngineVersion, + SERVER_OFFSET_ISO: config.Build?.SERVER_OFFSET_ISO, + TimeZoneOffsetPolicy: config.Build?.TimeZoneOffsetPolicy, }, Tests: { ResultsPath: config.Tests.ResultsPath, diff --git a/src/conf/config-loader.ts b/src/conf/config-loader.ts index f825ed6..c7aabee 100644 --- a/src/conf/config-loader.ts +++ b/src/conf/config-loader.ts @@ -17,6 +17,8 @@ export class ConfigLoader implements Config { cqlTranslatorVersion: string; cqlEngine: string; cqlEngineVersion: string; + SERVER_OFFSET_ISO: string; + TimeZoneOffsetPolicy: string; }; Tests: { ResultsPath: string; @@ -56,14 +58,18 @@ export class ConfigLoader implements Config { process.env.CQL_VERSION || configData.Build?.CqlVersion, testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription, - cqlTranslator: + cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'Unknown', - cqlTranslatorVersion: + cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'Unknown', cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'Unknown', - cqlEngineVersion: - process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown' + cqlEngineVersion: + process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown', + SERVER_OFFSET_ISO: + process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', + TimeZoneOffsetPolicy: + process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', }; this.Tests = { diff --git a/src/cql-engine/cql-engine.ts b/src/cql-engine/cql-engine.ts index 27c6787..fbbdb85 100644 --- a/src/cql-engine/cql-engine.ts +++ b/src/cql-engine/cql-engine.ts @@ -1,5 +1,6 @@ import axios, { AxiosResponse } from 'axios'; import { CQLEngineInfo } from '../models/results-types.js'; +import { response } from 'express'; /** * Represents a CQL Engine. @@ -39,6 +40,7 @@ export class CQLEngine { private baseURL?: string; private metadata?: any; private description?: string; + private _SERVER_OFFSET_ISO?: string; /** * Creates an instance of CQLEngine. @@ -50,16 +52,18 @@ export class CQLEngine { * @param cqlEngineVersion - CQL engine version (optional). */ constructor( - baseURL: string, - cqlPath: string | null = null, - cqlTranslator: string = '', - cqlTranslatorVersion: string = '', - cqlEngine: string = '', - cqlEngineVersion: string = '' - ) { - this._prepareBaseURL(baseURL, cqlPath); - this._setInformationFields(cqlTranslator, cqlTranslatorVersion, cqlEngine, cqlEngineVersion); - } + baseURL: string, + cqlPath: string | null = null, + cqlTranslator: string = '', + cqlTranslatorVersion: string = '', + cqlEngine: string = '', + cqlEngineVersion: string = '', + SERVER_OFFSET_ISO: string = '' + ) { + this._prepareBaseURL(baseURL, cqlPath); + this._setInformationFields(cqlTranslator, cqlTranslatorVersion, cqlEngine, cqlEngineVersion); + this.SERVER_OFFSET_ISO = SERVER_OFFSET_ISO; + } /** * Prepares the base URL. @@ -216,6 +220,14 @@ export class CQLEngine { return this.info?.cqlEngineVersion ?? null; } + get SERVER_OFFSET_ISO(): string { + return this._SERVER_OFFSET_ISO; + } + + set SERVER_OFFSET_ISO(value: string) { + this._SERVER_OFFSET_ISO = value; + } + /** * Converts the CQLEngine object to JSON. * @returns The JSON representation of the CQLEngine object. @@ -230,6 +242,7 @@ export class CQLEngine { cqlTranslatorVersion: this.info.cqlTranslatorVersion || '', cqlEngine: this.info.cqlEngine || '', cqlEngineVersion: this.info.cqlEngineVersion || '', + SERVER_OFFSET_ISO: this.SERVER_OFFSET_ISO || '', }; } @@ -241,7 +254,7 @@ export class CQLEngine { static fromJSON(cqlInfo: CQLEngineInfo): CQLEngine { const engine = new CQLEngine(cqlInfo.apiUrl || '', null, cqlInfo.cqlTranslator, cqlInfo.cqlTranslatorVersion, - cqlInfo.cqlEngine, cqlInfo.cqlEngineVersion); + cqlInfo.cqlEngine, cqlInfo.cqlEngineVersion, cqlInfo.SERVER_OFFSET_ISO); if (cqlInfo?.cqlVersion) { engine.cqlVersion = cqlInfo.cqlVersion; } @@ -257,6 +270,9 @@ export class CQLEngine { if (cqlInfo?.cqlEngineVersion) { engine.cqlEngineVersion = cqlInfo.cqlEngineVersion; } + if (cqlInfo?.SERVER_OFFSET_ISO) { + engine._SERVER_OFFSET_ISO = cqlInfo.SERVER_OFFSET_ISO; + } return engine; } } diff --git a/src/models/config-types.ts b/src/models/config-types.ts index 16c11e3..16c7d2e 100644 --- a/src/models/config-types.ts +++ b/src/models/config-types.ts @@ -27,6 +27,7 @@ export interface Config { cqlTranslatorVersion?: string; cqlEngine?: string; cqlEngineVersion?: string; + SERVER_OFFSET_ISO: string; }; Tests: { ResultsPath: string; diff --git a/src/models/results-types.ts b/src/models/results-types.ts index c1a5740..c263929 100644 --- a/src/models/results-types.ts +++ b/src/models/results-types.ts @@ -28,4 +28,5 @@ export interface CQLEngineInfo { cqlTranslatorVersion: string; cqlEngine: string; cqlEngineVersion: string; + SERVER_OFFSET_ISO: string; } diff --git a/src/models/test-types.ts b/src/models/test-types.ts index d5b907b..821b0de 100644 --- a/src/models/test-types.ts +++ b/src/models/test-types.ts @@ -40,6 +40,7 @@ export interface TestGroup { description?: string; reference?: string; notes?: string; + capability?: CapabilityKV[]; test: Test[]; } @@ -80,6 +81,7 @@ export interface InternalTestResult { invalid?: 'false' | 'true' | 'semantic' | 'undefined'; expression: string; capability?: CapabilityKV[]; + groupCapability?: CapabilityKV[]; SkipMessage?: string; } diff --git a/src/server/config-utils.ts b/src/server/config-utils.ts index d8eb720..dc3ac42 100644 --- a/src/server/config-utils.ts +++ b/src/server/config-utils.ts @@ -57,7 +57,9 @@ export function createConfigFromData(configData: any): ConfigLoader { cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'No configuration provided', cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'No configuration provided', cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'No configuration provided', - cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided' + cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided', + SERVER_OFFSET_ISO: process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', + TimeZoneOffsetPolicy: process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', }; config.Tests = { diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts index b68627d..d8f2874 100644 --- a/src/services/test-runner.ts +++ b/src/services/test-runner.ts @@ -36,7 +36,8 @@ export class TestRunner { build.cqlTranslator ?? '', build.cqlTranslatorVersion ?? '', build.cqlEngine ?? '', - build.cqlEngineVersion ?? '' + build.cqlEngineVersion ?? '', + build.SERVER_OFFSET_ISO ?? '+00:00' ); cqlEngine.cqlVersion = '1.5'; //default value const cqlVersion = config.Build?.CqlVersion; @@ -44,6 +45,15 @@ export class TestRunner { cqlEngine.cqlVersion = cqlVersion; } + await cqlEngine.fetch(); + + const activeTimeZonePolicy = await this.resolveTimeZoneOffsetPolicy( + config, + cqlEngine.apiUrl!, + cqlEngine['metadata'], + options.useAxios + ); + console.log('Resolved timezone policy:', activeTimeZonePolicy); // Load CVL using dynamic import // @ts-ignore const cvlModule = await import('../../cvl/cvl.mjs'); @@ -86,6 +96,8 @@ export class TestRunner { skipMap, onlySet, config, + cqlEngine, + activeTimeZonePolicy, options.useAxios ); results.add(result); @@ -112,6 +124,8 @@ export class TestRunner { skipMap: Map, onlySet: Set, config: ConfigLoader, + cqlEngine: CQLEngine, + activeTimeZonePolicy: string, useAxios: boolean = false ): Promise { const key = `${result.testsName}-${result.groupName}-${result.testName}`; @@ -130,7 +144,7 @@ export class TestRunner { ); return result; } else if (onlySet.size > 0 && !onlySet.has(key)) { - result.SkipMessage = 'Skipped by OnlyList filter'; + result.skipMessage = 'Skipped by OnlyList filter'; result.testStatus = 'skip'; return result; } else if (skipMap.has(key)) { @@ -147,6 +161,16 @@ export class TestRunner { ); return result; } + + const timezonePolicySkipReason = this.shouldSkipTimezonePolicyTest( + result, + activeTimeZonePolicy + ); + if (timezonePolicySkipReason) { + result.skipMessage = timezonePolicySkipReason; + result.testStatus = 'skip'; + return result; + } const data = generateParametersResource(result, config.FhirServer.CqlOperation); try { @@ -157,6 +181,17 @@ export class TestRunner { result.testName ); + this.applyServerOffsetToParameters(data, cqlEngine); + + // Also resolve the placeholder in result.expression so the results + // log the actual expression sent to the server, not the raw template. + if (cqlEngine.SERVER_OFFSET_ISO?.trim()) { + result.expression = this.replaceServerOffsetPlaceholder( + result.expression, + cqlEngine.SERVER_OFFSET_ISO + ); + } + let response: any; if (useAxios) { // Use axios for backward compatibility @@ -226,6 +261,227 @@ export class TestRunner { return result; } + private async resolveTimeZoneOffsetPolicy( + config: ConfigLoader, + apiUrl: string, + serverMetadata?: any, + useAxios: boolean = false + ): Promise { + // + const metadataPolicy = this.extractTimeZonePolicyFromMetadata(serverMetadata); + if (metadataPolicy) { + console.log('Resolved timezone policy from metadata:', metadataPolicy); + return metadataPolicy; + } + + const configuredPolicy = + process.env.TIME_ZONE_OFFSET_POLICY?.trim() || + config.Build?.TimeZoneOffsetPolicy?.trim(); + + if (configuredPolicy) { + console.log('Resolved timezone policy from env/config:', configuredPolicy); + return configuredPolicy; + } + + const probedPolicy = await this.detectTimeZoneOffsetPolicy(apiUrl, useAxios); + if (probedPolicy) { + console.log('Resolved timezone policy from probe:', probedPolicy); + return probedPolicy; + } + + const fallbackPolicy = 'timezone-offset-policy.default-server-offset'; + console.log('Resolved timezone policy from fallback:', fallbackPolicy); + return fallbackPolicy; + } + + private async detectTimeZoneOffsetPolicy( + apiUrl: string, + useAxios: boolean = false + ): Promise { + // order of setting timezone offset policy: metadata -> env/config -> probe -> fallback + const data = { + resourceType: 'Parameters', + parameter: [ + { + name: 'expression', + valueString: 'timezoneoffset from @2012-04-01T00:00', + }, + ], + }; + + let response: any; + + if (useAxios) { + const axios = await import('axios'); + const axiosResponse = await axios.default.post(apiUrl, data, { + headers: { 'Content-Type': 'application/json' }, + }); + response = { + status: axiosResponse.status, + data: axiosResponse.data, + }; + } else { + const fetchResponse = await fetch(apiUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data), + }); + response = { + status: fetchResponse.status, + data: await fetchResponse.json(), + }; + } + + if (response.status !== 200) { + return null; + } + + const extracted = this.extractProbeResult(response.data); + + if (extracted === 'null') { + return 'timezone-offset-policy.no-default-offset'; + } + + if (typeof extracted === 'number') { + return 'timezone-offset-policy.default-server-offset'; + } + + if (typeof extracted === 'string') { + const trimmed = extracted.trim().toLowerCase(); + if (trimmed === 'null') { + return 'timezone-offset-policy.no-default-offset'; + } + if (/^-?\d+$/.test(trimmed)) { + return 'timezone-offset-policy.default-server-offset'; + } + } + + return null; + } + + private extractProbeResult(responseBody: any): any { + const parameter = responseBody?.parameter; + if (!Array.isArray(parameter) || parameter.length === 0) { + return null; + } + + const returnParam = parameter.find((p: any) => p.name === 'return') || parameter[0]; + + if (returnParam.valueInteger !== undefined) { + return returnParam.valueInteger; + } + if (returnParam.valueDecimal !== undefined) { + return returnParam.valueDecimal; + } + if (returnParam.valueString !== undefined) { + return returnParam.valueString; + } + if (returnParam.valueBoolean !== undefined) { + return returnParam.valueBoolean; + } + + return null; + } + + private extractTimeZonePolicyFromMetadata(metadata: any): string | null { + if (!metadata || typeof metadata !== 'object') { + return null; + } + + const policyCodes = [ + 'timezone-offset-policy.no-default-offset', + 'timezone-offset-policy.default-server-offset', + ]; + + function findInObject(obj: any): string | null { + if (!obj || typeof obj !== 'object') { + return null; + } + + if (Array.isArray(obj)) { + for (const item of obj) { + const found = findInObject(item); + if (found) return found; + } + return null; + } + + for (const [key, value] of Object.entries(obj)) { + if (typeof value === 'string' && policyCodes.includes(value)) { + return value; + } + + if (key === 'code' && typeof value === 'string' && policyCodes.includes(value)) { + return value; + } + + if ( + key === 'valueCode' && + typeof value === 'string' && + policyCodes.includes(value) + ) { + return value; + } + + if ( + key === 'valueString' && + typeof value === 'string' && + policyCodes.includes(value) + ) { + return value; + } + + if (typeof value === 'object') { + const found = findInObject(value); + if (found) return found; + } + } + + return null; + } + + return findInObject(metadata); + } + + private applyServerOffsetToParameters(data: any, engine: CQLEngine): void { + const expressionParam = data?.parameter?.find((p: any) => p.name === 'expression'); + if (!expressionParam || typeof expressionParam.valueString !== 'string') { + return; + } + + const offset = engine.SERVER_OFFSET_ISO; + if (typeof offset !== 'string' || offset.trim() === '') { + return; + } + + expressionParam.valueString = this.replaceServerOffsetPlaceholder( + expressionParam.valueString, + offset + ); + } + + private replaceServerOffsetPlaceholder(expression: string, serverOffsetISO: string): string { + return expression.replace(/\{\{SERVER_OFFSET_ISO\}\}/g, serverOffsetISO); + } + + private shouldSkipTimezonePolicyTest(test: any, activeTimeZonePolicy: string): string | null { + const requiredCapabilities = test.capability || []; + + const requiredPolicy = requiredCapabilities.find((c: any) => + c.code?.startsWith('timezone-offset-policy.') + )?.code; + + if (!requiredPolicy) { + return null; + } + + if (requiredPolicy !== activeTimeZonePolicy) { + return `requires ${requiredPolicy} but server is ${activeTimeZonePolicy}`; + } + + return null; + } + private compareVersions(versionA: string | undefined, versionB: string | undefined): number { // Split into numeric parts (e.g., "1.5.2" β†’ [1,5,2]) const partsA = String(versionA ?? '') diff --git a/src/shared/results-shared.ts b/src/shared/results-shared.ts index b910d55..bdc18a3 100644 --- a/src/shared/results-shared.ts +++ b/src/shared/results-shared.ts @@ -20,8 +20,9 @@ export class Result implements InternalTestResult { invalid: 'false' | 'true' | 'semantic' | 'undefined'; expression: string; capability: CapabilityKV[] = []; + groupCapability: CapabilityKV[] = []; - constructor(testsName: string, groupName: string, test: Test) { + constructor(testsName: string, groupName: string, test: Test, groupCapability: CapabilityKV[] = []) { this.testsName = testsName; this.groupName = groupName; this.testName = test.name; @@ -56,7 +57,9 @@ export class Result implements InternalTestResult { this.capability = Array.isArray(test.capability) ? test.capability.map(({ code, value }) => ({ code, value })) : []; - } + this.groupCapability = Array.isArray(groupCapability) + ? groupCapability.map(({ code, value }) => ({ code, value })) + : []; } } export async function generateEmptyResults( @@ -79,7 +82,7 @@ export async function generateEmptyResults( if (test != undefined) { for (const t of test) { console.log(' Test: ' + t.name); - const r = new Result(ts.name, group.name, t); + const r = new Result(ts.name, group.name, t, group.capability || []); results.push(r); groupTests.push(r); } diff --git a/test/results-utils.test.js b/test/results-utils.test.js new file mode 100644 index 0000000..a1b03fd --- /dev/null +++ b/test/results-utils.test.js @@ -0,0 +1,19 @@ +// Author: Preston Lee + +import { expect, test } from 'vitest'; + +import { resultsEqual } from '../src/shared/results-utils.js'; + +test('singleton list does not equal scalar (comparison stays strict)', () => { + expect(resultsEqual(['a'], 'a')).toBe(false); + expect(resultsEqual('a', ['a'])).toBe(false); +}); + +test('equal lists (order-insensitive)', () => { + expect(resultsEqual(['a', 'b'], ['a', 'b'])).toBe(true); + expect(resultsEqual(['a', 'b'], ['b', 'a'])).toBe(false); +}); + +test('nested structures compared key-wise', () => { + expect(resultsEqual({ x: 1 }, { x: 1 })).toBe(true); +}); diff --git a/test/run-DataTransferItemList.test.ts b/test/run-DataTransferItemList.test.ts new file mode 100644 index 0000000..8224723 --- /dev/null +++ b/test/run-DataTransferItemList.test.ts @@ -0,0 +1,352 @@ +import { describe, it, expect, vi, beforeEach, afterEach, Mock } from 'vitest'; +import { TestRunner } from '../src/services/test-runner.js'; +import * as ResultsUtils from '../src/shared/results-utils.js'; + +vi.mock('../src/loaders/test-loader', () => ({ + TestLoader: { load: vi.fn().mockReturnValue([]) }, +})); + +const makeResult = (testsName: string, groupName: string, testName: string) => ({ + testsName, + groupName, + testName, + expression: 'true', + expected: 'true', + invalid: 'false', + capability: [], +}); + +vi.mock('../src/shared/results-shared', () => ({ + generateEmptyResults: vi + .fn() + .mockImplementation(async () => [ + [makeResult('Suite1', 'Group1', 'Test1'), makeResult('Suite1', 'Group1', 'Test2')], + ]), + generateParametersResource: vi.fn().mockReturnValue({}), +})); + +const baseConfig = { + FhirServer: { BaseUrl: 'http://localhost:8080/fhir', CqlOperation: '$cql' }, + Build: {}, + Debug: { QuickTest: false }, + Tests: { ResultsPath: './results', SkipList: [] as any[], OnlyList: [] as any[] }, +}; + +describe('RunTests filtering (TestRunner)', () => { + let runner: TestRunner; + let fetchSpy: Mock; + let resultsEqualSpy: Mock; + beforeEach(() => { + fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + resultsEqualSpy = vi.spyOn(ResultsUtils, 'resultsEqual').mockReturnValue(true); + runner = new TestRunner(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + delete (process.env as any).ONLY_LIST; + delete (process.env as any).SKIP_LIST; + }); + + it('runs all tests when SkipList and OnlyList are empty', async () => { + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 2, + skipCount: 0, + failCount: 0, + errorCount: 0, + }); + }); + + it('skips tests in SkipList', async () => { + const config = { + FhirServer: { BaseUrl: 'http://localhost:8080/fhir', CqlOperation: '$cql' }, + Build: {}, + Debug: { QuickTest: false }, + Tests: { + ResultsPath: './results', + SkipList: [ + { + testsName: 'Suite1', + groupName: 'Group1', + testName: 'Test1', + reason: 'Disabled', + }, + ], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('skip'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('runs only tests in OnlyList (others skipped)', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + // Two tests total in our mock, one should be run (pass), the other skipped + const statuses = results.results.map(r => ({ + key: `${r.testsName}-${r.groupName}-${r.testName}`, + status: r.testStatus, + })); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test2', status: 'pass' }); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test1', status: 'skip' }); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('skips a test present in both OnlyList and SkipList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }], + SkipList: [ + { + testsName: 'Suite1', + groupName: 'Group1', + testName: 'Test2', + reason: 'Disabled', + }, + ], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + expect(test2?.testStatus).toBe('skip'); + expect(test1?.testStatus).toBe('skip'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 0, + skipCount: 2, + failCount: 0, + errorCount: 0, + }); + }); + + it('reports failure for unequal results', async () => { + resultsEqualSpy.mockReturnValueOnce(true).mockReturnValueOnce(false); + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('fail'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 1, + errorCount: 0, + }); + }); + + it('reports failure for HTTP 500 result', async () => { + fetchSpy + // server connectivity + .mockResolvedValueOnce({ + status: 200, + json: () => Promise.resolve({}), + } as Response) + // test 1 + .mockResolvedValueOnce({ + status: 500, + json: () => Promise.resolve({}), + } as Response) + // test 2 and beyond + .mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('fail'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 1, + errorCount: 0, + }); + }); + + it('reports error for thrown error on fetch', async () => { + fetchSpy.mockReset(); + fetchSpy + // server connectivity + .mockResolvedValueOnce({ + status: 200, + json: () => Promise.resolve({}), + } as Response) + // test 1 + .mockRejectedValueOnce(new Error('error')) + // test 2 and beyond + .mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('error'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 0, + errorCount: 1, + }); + }); + + it('SKIP_LIST env var overrides config Tests.SkipList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + // Config SkipList skips Test1, but env will skip Test2 instead + SkipList: [ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test1', reason: 'Cfg' }, + ], + OnlyList: [], + }, + }; + + process.env.SKIP_LIST = JSON.stringify([ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test2', reason: 'Env' }, + ]); + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('skip'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('ONLY_LIST env var overrides config Tests.OnlyList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + // Config says run Test1 + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test1' }], + }, + }; + + // Env says run Test2 instead + process.env.ONLY_LIST = JSON.stringify([ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }, + ]); + + const results = await runner.runTests(config, { useAxios: false }); + const statuses = results.results.map(r => ({ + key: `${r.testsName}-${r.groupName}-${r.testName}`, + status: r.testStatus, + })); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test2', status: 'pass' }); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test1', status: 'skip' }); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); +}); diff --git a/test/value-Map.test.ts b/test/value-Map.test.ts new file mode 100644 index 0000000..f7e0bb1 --- /dev/null +++ b/test/value-Map.test.ts @@ -0,0 +1,44 @@ +// Author: Preston Lee + +import { expect, test } from 'vitest'; + +import { ValueMap } from '../src/extractors/value-map.js'; + +test('singleton collapse unwraps without hint', () => { + const m = new ValueMap(); + m.add('return', 'a'); + expect(m.toResult()).toBe('a'); +}); + +test('singleton collapse keeps array with hint (issue #82)', () => { + const m = new ValueMap(new Set(['return'])); + m.add('return', 'a'); + expect(m.toResult()).toEqual(['a']); +}); + +test('multiple values stay array without special case', () => { + const m = new ValueMap(); + m.add('return', 'a'); + m.add('return', 'b'); + expect(m.toResult()).toEqual(['a', 'b']); +}); + +test('singletonListKeysFromExpected: non-array yields empty key set', () => { + expect([...ValueMap.singletonListKeysFromExpected('a')]).toEqual([]); + expect([...ValueMap.singletonListKeysFromExpected({ a: 1 })]).toEqual([]); +}); + +test('singletonListKeysFromExpected: empty array adds no keys (issue #90)', () => { + expect([...ValueMap.singletonListKeysFromExpected([])]).toEqual([]); +}); + +test('singletonListKeysFromExpected: array adds return', () => { + expect([...ValueMap.singletonListKeysFromExpected(['a'])]).toEqual(['return']); + expect([...ValueMap.singletonListKeysFromExpected([1, 2])]).toEqual(['return']); +}); + +test('singletonListKeysFromExpected: array of arrays adds return and element', () => { + const keys = ValueMap.singletonListKeysFromExpected([[1, 2]]); + expect(keys.has('return')).toBe(true); + expect(keys.has('element')).toBe(true); +});