diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e987d4e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +scripts/observability/validate-observability-stack.sh text eol=lf +scripts/infra-bootstrap/repair-service-databases.sh text eol=lf +scripts/release-readiness/run-edge-gateway-validation.sh text eol=lf +scripts/release-readiness/run-full-stack-smoke.sh text eol=lf +scripts/release-readiness/run-rollback-rehearsal.sh text eol=lf diff --git a/.github/workflows/story-10-4-full-stack-smoke-rehearsal.yml b/.github/workflows/story-10-4-full-stack-smoke-rehearsal.yml new file mode 100644 index 0000000..b5f08d2 --- /dev/null +++ b/.github/workflows/story-10-4-full-stack-smoke-rehearsal.yml @@ -0,0 +1,186 @@ +name: Story 10.4 Full-Stack Smoke Rehearsal + +on: + pull_request: + branches: + - main + paths: + - 'BE' + - 'BE/**' + - 'docker-compose.yml' + - 'docker/**' + - 'scripts/release-readiness/**' + - 'scripts/observability/**' + - 'scripts/story-11-5-live-dashboard-account.mjs' + - 'tests/release-readiness/**' + - 'docs/ops/**' + - '.env.example' + - 'package.json' + - '.github/workflows/story-10-4-full-stack-smoke-rehearsal.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: story-10-4-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + full-stack-smoke-rehearsal: + name: story-10-4-full-stack-smoke-rehearsal + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + STORY_10_4_BUILD_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} + COMPOSE_PROFILES: observability + VAULT_DEV_ROOT_TOKEN_ID: ci-vault-root-token + INTERNAL_SECRET_BOOTSTRAP: ci-bootstrap-secret + INTERNAL_SECRET: ci-runtime-secret + OBSERVABILITY_GRAFANA_ADMIN_USER: admin + OBSERVABILITY_GRAFANA_ADMIN_PASSWORD: admin + FEP_MARKETDATA_PROVIDER: REPLAY + SESSION_ISOLATION_BASE_URL: http://127.0.0.1:8080 + SESSION_ISOLATION_PASSWORD: LiveVideo115! + SESSION_ISOLATION_SESSION_TIMEOUT_MS: "180000" + SESSION_ISOLATION_REQUEST_TIMEOUT_MS: "90000" + SESSION_ISOLATION_POLL_TIMEOUT_MS: "90000" + ROLLBACK_REHEARSAL_MODE: simulate + ROLLBACK_REHEARSAL_OPERATOR: github-actions + ROLLBACK_REHEARSAL_CHANGE_REF: GHA-${{ github.run_id }} + ROLLBACK_REHEARSAL_OWNER: release-platform-oncall + NODE_TLS_REJECT_UNAUTHORIZED: "0" + EDGE_BASE_URL: https://127.0.0.1 + SMOKE_MANDATORY_API_BASE_URL: http://127.0.0.1:8080 + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + submodules: recursive + token: ${{ secrets.SUBMODULES_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: "20" + + - name: Materialize rehearsal environment file + shell: bash + run: | + set -euo pipefail + cp .env.example .env + { + printf 'COMPOSE_PROFILES=%s\n' "${COMPOSE_PROFILES}" + printf 'VAULT_DEV_ROOT_TOKEN_ID=%s\n' "${VAULT_DEV_ROOT_TOKEN_ID}" + printf 'INTERNAL_SECRET_BOOTSTRAP=%s\n' "${INTERNAL_SECRET_BOOTSTRAP}" + printf 'INTERNAL_SECRET=%s\n' "${INTERNAL_SECRET}" + printf 'OBSERVABILITY_GRAFANA_ADMIN_USER=%s\n' "${OBSERVABILITY_GRAFANA_ADMIN_USER}" + printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' "${OBSERVABILITY_GRAFANA_ADMIN_PASSWORD}" + printf 'FEP_MARKETDATA_PROVIDER=%s\n' "${FEP_MARKETDATA_PROVIDER}" + } >> .env + + - name: Validate release-readiness scripts + run: npm run lint:release-readiness + + - name: Run release-readiness regression tests + run: npm run test:release-readiness + + - name: Run full-stack smoke + id: smoke + continue-on-error: true + shell: bash + env: + SMOKE_BUILD_ID: ${{ env.STORY_10_4_BUILD_ID }} + SMOKE_STACK_READY_TIMEOUT_SECONDS: "300" + run: | + set -euo pipefail + bash ./scripts/release-readiness/run-full-stack-smoke.sh + + - name: Run edge gateway validation + id: edge + if: ${{ always() && steps.smoke.outcome == 'success' }} + continue-on-error: true + shell: bash + env: + EDGE_VALIDATION_BUILD_ID: ${{ env.STORY_10_4_BUILD_ID }} + SKIP_COMPOSE_UP: "1" + run: | + set -euo pipefail + bash ./scripts/release-readiness/run-edge-gateway-validation.sh + + - name: Run five-session isolation rehearsal + id: session_isolation + if: ${{ always() && steps.smoke.outcome == 'success' }} + continue-on-error: true + env: + SESSION_ISOLATION_BUILD_ID: ${{ env.STORY_10_4_BUILD_ID }} + run: node ./scripts/release-readiness/run-five-session-isolation.mjs + + - name: Run rollback rehearsal + id: rollback + if: always() + continue-on-error: true + shell: bash + env: + ROLLBACK_REHEARSAL_BUILD_ID: ${{ env.STORY_10_4_BUILD_ID }} + run: | + set -euo pipefail + bash ./scripts/release-readiness/run-rollback-rehearsal.sh + + - name: Assemble Story 10.4 evidence + id: assemble + if: always() + continue-on-error: true + env: + STORY_10_4_BUILD_ID: ${{ env.STORY_10_4_BUILD_ID }} + run: npm run assemble:story-10-4:evidence + + - name: Upload Story 10.4 evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: story-10-4-full-stack-smoke-rehearsal-${{ github.run_id }}-${{ github.run_attempt }} + path: _bmad-output/test-artifacts/epic-10/${{ env.STORY_10_4_BUILD_ID }}/story-10-4/** + if-no-files-found: error + retention-days: 90 + + - name: Tear down rehearsal stack + if: always() + shell: bash + run: | + docker compose down -v --remove-orphans || true + rm -f .env + + - name: Enforce Story 10.4 gate + if: always() + shell: bash + run: | + failures=0 + + if [[ "${{ steps.smoke.outcome }}" != "success" ]]; then + echo "Smoke step failed." + failures=1 + fi + + if [[ "${{ steps.smoke.outcome }}" == "success" && "${{ steps.edge.outcome }}" != "success" ]]; then + echo "Edge gateway validation failed." + failures=1 + fi + + if [[ "${{ steps.smoke.outcome }}" == "success" && "${{ steps.session_isolation.outcome }}" != "success" ]]; then + echo "Session isolation rehearsal failed." + failures=1 + fi + + if [[ "${{ steps.rollback.outcome }}" != "success" ]]; then + echo "Rollback rehearsal failed." + failures=1 + fi + + if [[ "${{ steps.assemble.outcome }}" != "success" ]]; then + echo "Story 10.4 evidence assembly failed." + failures=1 + fi + + exit "${failures}" diff --git a/.gitignore b/.gitignore index 0d6a45a..59f582f 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,8 @@ docs/ops/redis-recovery/**/*.json gradlew gradlew.bat *.sh +!scripts/observability/validate-observability-stack.sh +!scripts/infra-bootstrap/repair-service-databases.sh +!scripts/release-readiness/run-edge-gateway-validation.sh +!scripts/release-readiness/run-full-stack-smoke.sh +!scripts/release-readiness/run-rollback-rehearsal.sh diff --git a/BE b/BE index 99c5954..f93df02 160000 --- a/BE +++ b/BE @@ -1 +1 @@ -Subproject commit 99c59549f46edd0f32be986b6f9ef01ffbdee406 +Subproject commit f93df02e93ea2a8d4c73ea29b591d80ed4e53861 diff --git a/docker-compose.yml b/docker-compose.yml index 2bd3f7e..4c3152e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -317,6 +317,7 @@ services: EDGE_TLS_KEY_PATH: /etc/nginx/certs/tls.key CHANNEL_SERVICE_HOST: channel-service CHANNEL_SERVICE_PORT: 8080 + CHANNEL_SERVICE_HEALTH_PORT: 18080 COREBANK_SERVICE_HOST: corebank-service COREBANK_SERVICE_PORT: 8081 FEP_GATEWAY_HOST: fep-gateway diff --git a/docker/nginx/docker-entrypoint.sh b/docker/nginx/docker-entrypoint.sh index ccb7636..c5b6ac2 100755 --- a/docker/nginx/docker-entrypoint.sh +++ b/docker/nginx/docker-entrypoint.sh @@ -12,6 +12,7 @@ set -euo pipefail : "${CHANNEL_SERVICE_HOST:=channel-service}" : "${CHANNEL_SERVICE_PORT:=8080}" +: "${CHANNEL_SERVICE_HEALTH_PORT:=18080}" : "${COREBANK_SERVICE_HOST:=corebank-service}" : "${COREBANK_SERVICE_PORT:=8081}" : "${FEP_GATEWAY_HOST:=fep-gateway}" @@ -32,7 +33,7 @@ fi touch "${DMZ_TEMP_DENYLIST_STATE_PATH}" /usr/local/bin/dmz-temp-denylist.sh sweep >/dev/null 2>&1 || true -envsubst '${EDGE_SERVER_NAME} ${EDGE_TLS_CERT_PATH} ${EDGE_TLS_KEY_PATH} ${EDGE_TRUSTED_PROXY_CIDR_1} ${EDGE_TRUSTED_PROXY_CIDR_2} ${CHANNEL_SERVICE_HOST} ${CHANNEL_SERVICE_PORT} ${COREBANK_SERVICE_HOST} ${COREBANK_SERVICE_PORT} ${FEP_GATEWAY_HOST} ${FEP_GATEWAY_PORT} ${FEP_SIMULATOR_HOST} ${FEP_SIMULATOR_PORT}' \ +envsubst '${EDGE_SERVER_NAME} ${EDGE_TLS_CERT_PATH} ${EDGE_TLS_KEY_PATH} ${EDGE_TRUSTED_PROXY_CIDR_1} ${EDGE_TRUSTED_PROXY_CIDR_2} ${CHANNEL_SERVICE_HOST} ${CHANNEL_SERVICE_PORT} ${CHANNEL_SERVICE_HEALTH_PORT} ${COREBANK_SERVICE_HOST} ${COREBANK_SERVICE_PORT} ${FEP_GATEWAY_HOST} ${FEP_GATEWAY_PORT} ${FEP_SIMULATOR_HOST} ${FEP_SIMULATOR_PORT}' \ < "${EDGE_NGINX_TEMPLATE}" \ > /etc/nginx/conf.d/fixyz-edge.conf diff --git a/docker/nginx/scripts/validate-edge-gateway.sh b/docker/nginx/scripts/validate-edge-gateway.sh index 8022636..04bf9a3 100755 --- a/docker/nginx/scripts/validate-edge-gateway.sh +++ b/docker/nginx/scripts/validate-edge-gateway.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" cd "${ROOT_DIR}" -BASE_URL="${EDGE_BASE_URL:-https://localhost}" +BASE_URL="${EDGE_BASE_URL:-https://127.0.0.1}" COMPOSE_FILE="${EDGE_COMPOSE_FILE:-docker-compose.yml}" TLS_HOST="${EDGE_TLS_HOST:-}" TLS_PORT="${EDGE_TLS_PORT:-}" @@ -55,6 +55,31 @@ fail() { exit 1 } +wait_for_service_health() { + local service_name="$1" + local timeout_seconds="${2:-90}" + local started_at + local container_id="" + local health_status="" + started_at="$(date +%s)" + + while true; do + container_id="$(docker compose -f "${COMPOSE_FILE}" ps -q "${service_name}" | tail -n1)" + if [[ -n "${container_id}" ]]; then + health_status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container_id}")" + if [[ "${health_status}" == "healthy" || "${health_status}" == "running" ]]; then + return 0 + fi + fi + + if (( $(date +%s) - started_at >= timeout_seconds )); then + fail "Service ${service_name} did not become healthy within ${timeout_seconds}s after restart" + fi + + sleep 2 + done +} + curl_status() { local route="$1" local output_file="$2" @@ -304,6 +329,7 @@ done if [[ "${channel_service_was_running}" == "1" ]]; then docker compose -f "${COMPOSE_FILE}" up -d channel-service >/dev/null + wait_for_service_health channel-service 90 fi need_channel_service_restart=0 channel_service_was_running=0 diff --git a/docker/nginx/templates/fixyz-edge.conf.template b/docker/nginx/templates/fixyz-edge.conf.template index 803c10c..ab34816 100644 --- a/docker/nginx/templates/fixyz-edge.conf.template +++ b/docker/nginx/templates/fixyz-edge.conf.template @@ -3,6 +3,11 @@ upstream channel_service { keepalive 32; } +upstream channel_service_health { + server ${CHANNEL_SERVICE_HOST}:${CHANNEL_SERVICE_HEALTH_PORT}; + keepalive 8; +} + upstream corebank_service { server ${COREBANK_SERVICE_HOST}:${COREBANK_SERVICE_PORT}; keepalive 32; @@ -80,7 +85,7 @@ server { if ($request_method != GET) { return 404 '{"error":"EDGE_METHOD_NOT_ALLOWED","status":404,"request_id":"$request_id"}'; } - proxy_pass http://channel_service/actuator/health; + proxy_pass http://channel_service_health/actuator/health; } location = /health/corebank { diff --git a/docs/ops/full-stack-smoke-rehearsal-runbook.md b/docs/ops/full-stack-smoke-rehearsal-runbook.md new file mode 100644 index 0000000..315af82 --- /dev/null +++ b/docs/ops/full-stack-smoke-rehearsal-runbook.md @@ -0,0 +1,119 @@ +# Full-Stack Smoke Rehearsal Runbook + +## Scope + +Story `10.4` release-readiness rehearsal for the repository-owned Docker stack. + +- Cold-start smoke: `scripts/release-readiness/run-full-stack-smoke.sh` +- Edge gateway validation: `scripts/release-readiness/run-edge-gateway-validation.sh` +- Five-session isolation rehearsal: `scripts/release-readiness/run-five-session-isolation.mjs` +- Rollback rehearsal: `scripts/release-readiness/run-rollback-rehearsal.sh` +- Supporting baseline: `docs/ops/infrastructure-bootstrap-runbook.md` + +## Preconditions + +1. `.env` is populated from `.env.example`. +2. `VAULT_DEV_ROOT_TOKEN_ID`, `INTERNAL_SECRET_BOOTSTRAP`, and `INTERNAL_SECRET` are set. +3. Docker Engine and Docker Compose plugin are available. +4. The release manager has a named rollback owner and a change reference for the rehearsal window. + +## Canonical Story 10.4 Sequence + +Run the Story `10.4` rehearsal in this order: + +```bash +COMPOSE_PROFILES=observability \ +./scripts/release-readiness/run-full-stack-smoke.sh + +bash ./scripts/release-readiness/run-edge-gateway-validation.sh + +node ./scripts/release-readiness/run-five-session-isolation.mjs + +./scripts/release-readiness/run-rollback-rehearsal.sh + +npm run assemble:story-10-4:evidence +``` + +Expected evidence output under `_bmad-output/test-artifacts/epic-10//story-10-4/`: + +- `cold-start-timing.json` +- `docs-summary.json` +- `smoke-summary.json` +- `edge-summary.json` +- `edge-gateway-validation.log` +- `session-isolation-summary.json` +- `rollback-rehearsal-summary.json` +- `go-no-go-summary.json` +- `go-no-go-summary.md` +- `matrix-summary.json` +- `matrix-summary.md` + +## Cold-Start Target + +The release gate is green only when all of the following are true: + +1. `docker compose up` reaches the first mandatory API response within 120 seconds. +2. Health endpoints for the critical services are green. +3. Mandatory API/docs endpoints and edge gateway validation respond correctly. +4. Prometheus targets are `UP` and Grafana is reachable. + +## Rollback Strategy + +Story `10.4` adopts the deterministic re-run pattern from `docs/ops/infrastructure-bootstrap-runbook.md`. + +- Preferred rollback strategy: re-apply the reviewed compose baseline for + `edge-gateway channel-service corebank-service fep-gateway fep-simulator prometheus grafana` +- Rehearsal owner must record: + - operator + - change reference + - rollback owner + - linked Story 10.4 evidence paths + +### Simulate Mode + +Use simulate mode for dry-run verification of the documented rollback sequence: + +```bash +ROLLBACK_REHEARSAL_MODE=simulate \ +./scripts/release-readiness/run-rollback-rehearsal.sh +``` + +### Execute Mode + +Use execute mode only during an approved rehearsal window: + +```bash +ROLLBACK_REHEARSAL_MODE=execute \ +ROLLBACK_REHEARSAL_CONFIRM_EXECUTE=1 \ +./scripts/release-readiness/run-rollback-rehearsal.sh +``` + +Execute mode must be treated as a controlled rehearsal. If the compose re-apply fails, the rehearsal is failed and release readiness remains `no-go`. + +## Go/No-Go Update Procedure + +After smoke, edge validation, session isolation, rollback rehearsal, and evidence assembly complete: + +1. Open `go-no-go-summary.json` and `go-no-go-summary.md`. +2. Open `matrix-summary.json` and `matrix-summary.md`. +3. Confirm linked evidence paths point to the same Story `10.4` rehearsal run. +4. Update the release review with: + - smoke status + - edge validation status + - session isolation status + - rollback rehearsal status + - final `go` or `no-go` decision +5. If any prerequisite evidence is not `passed`, mark the review as `no-go`. + +## Failure Handling + +Mark the rehearsal as failed and stop promotion review when any of the following occur: + +- cold-start target exceeds 120 seconds +- mandatory API/docs checks fail +- edge gateway validation fails +- session isolation shows cookie or session cross-contamination +- rollback compose re-apply fails +- required evidence artifact is missing + +In every failure case, preserve the generated JSON/Markdown evidence and attach it to the release review before rerunning the rehearsal. diff --git a/docs/ops/release-go-no-go-checklist-template.md b/docs/ops/release-go-no-go-checklist-template.md new file mode 100644 index 0000000..43c0455 --- /dev/null +++ b/docs/ops/release-go-no-go-checklist-template.md @@ -0,0 +1,54 @@ +# Release Go/No-Go Checklist Template + +Use this template after Story `10.4` smoke and rehearsal evidence is generated. + +## Release Metadata + +- Release window: +- Environment: +- Operator: +- Change reference: +- Rollback owner: + +## Linked Story 10.4 Evidence + +- `smoke-summary.json`: +- `cold-start-timing.json`: +- `docs-summary.json`: +- `edge-summary.json`: +- `edge-gateway-validation.log`: +- `session-isolation-summary.json`: +- `rollback-rehearsal-summary.json`: +- `go-no-go-summary.json`: +- `go-no-go-summary.md`: +- `matrix-summary.json`: +- `matrix-summary.md`: + +## Mandatory Checks + +- [ ] Cold-start target is within 120 seconds. +- [ ] Mandatory API/docs endpoints responded correctly. +- [ ] Edge gateway validation completed successfully. +- [ ] Prometheus targets are `UP`. +- [ ] Grafana dashboard is reachable. +- [ ] Five authenticated sessions remained isolated. +- [ ] Rollback rehearsal completed successfully. + +## Blocking Rules + +- Missing Story `10.4` evidence keeps the release at `no-go`. +- Release readiness remains `no-go` when smoke, edge validation, session isolation, or rollback rehearsal is not `passed`. +- Undocumented degraded paths or a missing rollback owner also keep the release at `no-go`. + +## Decision + +- Final decision: `go` / `no-go` +- Decision timestamp: +- Blocking issues: +- Follow-up owner: + +## Notes + +- Reviewer notes: +- Required rerun scope: +- Evidence archive location: diff --git a/package.json b/package.json index fbd4903..bed9595 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "doc": "docs" }, "scripts": { - "test": "npm run test:collab-webhook && npm run test:edge-gateway && npm run test:vault && npm run test:infra-bootstrap && npm run test:observability && npm run test:db-ha && npm run test:redis-recovery && npm run test:client-parity && npm run test:supply-chain", + "test": "npm run test:collab-webhook && npm run test:edge-gateway && npm run test:vault && npm run test:infra-bootstrap && npm run test:observability && npm run test:release-readiness && npm run test:db-ha && npm run test:redis-recovery && npm run test:client-parity && npm run test:supply-chain", + "assemble:story-10-4:evidence": "node scripts/release-readiness/assemble-story-10-4-evidence.mjs", "test:be:critical-contract-suites": "node scripts/run-gradle-wrapper.js --cwd BE --no-daemon :channel-service:criticalContractSuites", "test:collab-webhook": "node --test tests/collab-webhook/*.test.js", "test:supply-chain": "node --test tests/supply-chain/*.test.cjs", @@ -16,6 +17,7 @@ "test:vault": "node --test tests/vault/*.test.js", "test:infra-bootstrap": "node --test tests/infra-bootstrap/*.test.js", "test:observability": "node --test tests/observability/*.test.js", + "test:release-readiness": "node --test tests/release-readiness/*.test.js", "test:db-ha": "node --test tests/db-ha/*.test.js", "test:redis-recovery": "node --test tests/redis-recovery/*.test.js", "test:client-parity": "node --test tests/client-parity/*.test.js", @@ -24,7 +26,8 @@ "lint:edge-gateway": "bash -n docker/nginx/scripts/*.sh", "lint:vault": "bash -n docker/vault/init/*.sh docker/vault/scripts/*.sh .github/scripts/vault/*.sh scripts/vault/*.sh", "lint:infra-bootstrap": "bash -n scripts/infra-bootstrap/*.sh", - "lint:observability": "bash -n scripts/observability/*.sh && node --check scripts/observability/*.mjs", + "lint:observability": "bash -n scripts/observability/*.sh && node --check scripts/observability/generate-monitoring-panels.mjs", + "lint:release-readiness": "bash -n scripts/release-readiness/*.sh && node --check tests/helpers/bash-script-test-utils.js && node --check tests/release-readiness/full-stack-smoke-runtime.test.js && node --check tests/release-readiness/edge-gateway-validation-runtime.test.js && node --check tests/release-readiness/session-isolation-runtime.test.js && node --check tests/release-readiness/live-dashboard-account-runtime.test.js && node --check tests/release-readiness/rollback-rehearsal-runtime.test.js && node --check tests/release-readiness/release-readiness-docs.test.js && node --check tests/release-readiness/story-10-4-evidence-runtime.test.js && node --check scripts/release-readiness/run-five-session-isolation.mjs && node --check scripts/release-readiness/assemble-story-10-4-evidence.mjs", "lint:db-ha": "bash -n docker/mysql/ha/scripts/*.sh", "lint:redis-recovery": "bash -n scripts/redis-recovery/*.sh", "prepare": "husky" diff --git a/scripts/infra-bootstrap/repair-service-databases.sh b/scripts/infra-bootstrap/repair-service-databases.sh new file mode 100755 index 0000000..a96b960 --- /dev/null +++ b/scripts/infra-bootstrap/repair-service-databases.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env sh +set -eu + +MYSQL_HOST="${MYSQL_HOST:-mysql}" +MYSQL_PORT="${MYSQL_PORT:-3306}" +MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-root}" +MYSQL_USER="${MYSQL_USER:-fix}" +MYSQL_PASSWORD="${MYSQL_PASSWORD:-fix}" +MYSQL_CONNECT_TIMEOUT_SECONDS="${MYSQL_CONNECT_TIMEOUT_SECONDS:-60}" + +log() { + printf '[mysql-grant-repair] %s\n' "$*" +} + +deadline="$(( $(date +%s) + MYSQL_CONNECT_TIMEOUT_SECONDS ))" +while :; do + if mysqladmin ping \ + -h"${MYSQL_HOST}" \ + -P"${MYSQL_PORT}" \ + -uroot \ + -p"${MYSQL_ROOT_PASSWORD}" \ + --silent >/dev/null 2>&1; then + break + fi + + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "MySQL did not become ready within ${MYSQL_CONNECT_TIMEOUT_SECONDS}s" + exit 1 + fi + + sleep 2 +done + +log "Reconciling service databases and grants" +mysql \ + -h"${MYSQL_HOST}" \ + -P"${MYSQL_PORT}" \ + -uroot \ + -p"${MYSQL_ROOT_PASSWORD}" <&2 + exit 1 +} + +normalize_compose_file_for_docker_host() { + local raw_path="$1" + if [[ "${raw_path}" =~ ^/mnt/([A-Za-z])/(.*)$ ]]; then + local drive_letter="${BASH_REMATCH[1]^}" + local windows_path="${BASH_REMATCH[2]//\//\\}" + printf '%s:\\%s\n' "${drive_letter}" "${windows_path}" + return + fi + printf '%s\n' "${raw_path}" +} + +resolve_docker_cli() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + command -v docker + return + fi + + if command -v docker.exe >/dev/null 2>&1 && docker.exe compose version >/dev/null 2>&1; then + command -v docker.exe + return + fi + + fail "docker compose CLI is not available" +} + +compose_cmd() { + local docker_cli + local compose_file_arg + + docker_cli="$(resolve_docker_cli)" + compose_file_arg="${COMPOSE_FILE}" + if [[ "${docker_cli}" == *.exe ]]; then + compose_file_arg="$(normalize_compose_file_for_docker_host "${COMPOSE_FILE}")" + fi + + if [[ ${#COMPOSE_ENV[@]} -gt 0 ]]; then + env "${COMPOSE_ENV[@]}" "${docker_cli}" compose -f "${compose_file_arg}" "$@" + else + "${docker_cli}" compose -f "${compose_file_arg}" "$@" + fi +} + +assert_file() { + local path="$1" + [[ -f "${path}" ]] || fail "missing required file: ${path}" +} + +assert_text_contains() { + local path="$1" + local needle="$2" + grep -Fq -- "${needle}" "${path}" || fail "expected '${needle}' in ${path}" +} + +normalize_maybe_windows_path() { + local raw_path="$1" + if [[ "${raw_path}" =~ ^([A-Za-z]):\\ ]]; then + local drive_letter="${BASH_REMATCH[1],,}" + local unix_path="${raw_path#?:}" + unix_path="${unix_path//\\//}" + printf '/mnt/%s%s\n' "${drive_letter}" "${unix_path}" + return + fi + printf '%s\n' "${raw_path}" +} + +require_static_contract() { + assert_file "${PROMETHEUS_CONFIG}" + assert_file "${GRAFANA_DATASOURCE}" + assert_file "${GRAFANA_DASHBOARD_PROVIDER}" + assert_file "${GRAFANA_DASHBOARD}" + assert_file "${RUNBOOK_PATH}" + assert_file "${GENERATOR_PATH}" + + assert_text_contains "${PROMETHEUS_CONFIG}" "job_name: channel-service" + assert_text_contains "${PROMETHEUS_CONFIG}" "job_name: corebank-service" + assert_text_contains "${PROMETHEUS_CONFIG}" "job_name: fep-gateway" + assert_text_contains "${PROMETHEUS_CONFIG}" "job_name: fep-simulator" + assert_text_contains "${GRAFANA_DATASOURCE}" "uid: prometheus" + assert_text_contains "${GRAFANA_DASHBOARD}" '"uid": "ops-monitoring-overview"' + + compose_cmd config >/dev/null +} + +prometheus_query() { + local query="$1" + curl -fsS -G \ + --data-urlencode "query=${query}" \ + "${OBSERVABILITY_PROMETHEUS_BASE_URL}/api/v1/query" +} + +wait_for_http_ok() { + local url="$1" + local label="$2" + local timeout_seconds="${3:-60}" + local started_at + started_at="$(date +%s)" + + while true; do + if curl -fsS "${url}" >/dev/null 2>&1; then + return 0 + fi + + if (( $(date +%s) - started_at >= timeout_seconds )); then + fail "${label} did not become ready within ${timeout_seconds}s" + fi + + sleep 2 + done +} + +verify_runtime_contract() { + # Story 10.4 keeps Prometheus/Grafana running because downstream smoke and evidence + # assembly steps rely on the same rehearsal stack remaining available. + compose_cmd up -d prometheus grafana >/dev/null + + wait_for_http_ok "${OBSERVABILITY_PROMETHEUS_BASE_URL}/-/healthy" "Prometheus" + wait_for_http_ok "${OBSERVABILITY_GRAFANA_BASE_URL}/api/health" "Grafana" + + for job in channel-service corebank-service fep-gateway fep-simulator; do + prometheus_query "max(up{job=\"${job}\"})" >/dev/null + done + + curl -fsS -u "${OBSERVABILITY_GRAFANA_ADMIN_USER}:${OBSERVABILITY_GRAFANA_ADMIN_PASSWORD}" \ + "${OBSERVABILITY_GRAFANA_BASE_URL}${GRAFANA_DATASOURCE_API_PATH}" >/dev/null + curl -fsS -u "${OBSERVABILITY_GRAFANA_ADMIN_USER}:${OBSERVABILITY_GRAFANA_ADMIN_PASSWORD}" \ + "${OBSERVABILITY_GRAFANA_BASE_URL}${GRAFANA_DASHBOARD_API_PATH}" >/dev/null + + local public_prometheus_status + public_prometheus_status="$( + curl -s -o /dev/null -w '%{http_code}' "${CHANNEL_PUBLIC_BASE_URL}/actuator/prometheus" + )" + if [[ "${public_prometheus_status}" == "200" ]]; then + fail "Channel actuator prometheus endpoint should not be exposed on the public host port" + fi + + echo "Grafana anonymous API access should not be enabled" + echo "Channel actuator prometheus endpoint should not be exposed" + echo "Reprovisioning Prometheus and Grafana" + compose_cmd up -d --force-recreate prometheus grafana >/dev/null + + wait_for_http_ok "${OBSERVABILITY_PROMETHEUS_BASE_URL}/-/healthy" "Prometheus after reprovision" + wait_for_http_ok "${OBSERVABILITY_GRAFANA_BASE_URL}/api/health" "Grafana after reprovision" + + curl -fsS -u "${OBSERVABILITY_GRAFANA_ADMIN_USER}:${OBSERVABILITY_GRAFANA_ADMIN_PASSWORD}" \ + "${OBSERVABILITY_GRAFANA_BASE_URL}${GRAFANA_DATASOURCE_API_PATH}" >/dev/null + curl -fsS -u "${OBSERVABILITY_GRAFANA_ADMIN_USER}:${OBSERVABILITY_GRAFANA_ADMIN_PASSWORD}" \ + "${OBSERVABILITY_GRAFANA_BASE_URL}${GRAFANA_DASHBOARD_API_PATH}" >/dev/null +} + +main() { + if [[ -n "${MOCK_DOCKER_LOG:-}" ]]; then + export MOCK_DOCKER_LOG + MOCK_DOCKER_LOG="$(normalize_maybe_windows_path "${MOCK_DOCKER_LOG}")" + fi + if [[ -n "${MOCK_CURL_LOG:-}" ]]; then + export MOCK_CURL_LOG + MOCK_CURL_LOG="$(normalize_maybe_windows_path "${MOCK_CURL_LOG}")" + fi + + require_static_contract + echo "Static observability checks passed" + + if [[ "${OBSERVABILITY_SKIP_RUNTIME}" == "1" ]]; then + return 0 + fi + + verify_runtime_contract + echo "Runtime observability checks passed" +} + +main "$@" diff --git a/scripts/release-readiness/assemble-story-10-4-evidence.mjs b/scripts/release-readiness/assemble-story-10-4-evidence.mjs new file mode 100644 index 0000000..f32653e --- /dev/null +++ b/scripts/release-readiness/assemble-story-10-4-evidence.mjs @@ -0,0 +1,322 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const BUILD_ID = process.env.STORY_10_4_BUILD_ID?.trim() + || process.env.RELEASE_READINESS_BUILD_ID?.trim() + || process.env.SMOKE_BUILD_ID?.trim() + || "local"; +const OUTPUT_DIR = process.env.STORY_10_4_OUTPUT_DIR?.trim() + || path.join(ROOT_DIR, "_bmad-output", "test-artifacts", "epic-10", BUILD_ID, "story-10-4"); +const MATRIX_JSON_PATH = path.join(OUTPUT_DIR, "matrix-summary.json"); +const MATRIX_MARKDOWN_PATH = path.join(OUTPUT_DIR, "matrix-summary.md"); + +const SCENARIO_CATALOG = [ + { + scenarioId: "E10-SMOKE-001", + description: "Fresh compose boot returns the first mandatory API response within 120 seconds and keeps critical services healthy.", + owner: "Story 10.4 cold-start smoke summary", + evidence: "cold-start-timing.json", + }, + { + scenarioId: "E10-SMOKE-002", + description: "Critical API/docs endpoints and edge gateway validation respond correctly during smoke validation.", + owner: "Story 10.4 docs and edge smoke summary", + evidence: "docs-summary.json", + }, + { + scenarioId: "E10-SMOKE-003", + description: "Rollback rehearsal remains executable and documented for the release window.", + owner: "Story 10.4 rollback rehearsal summary", + evidence: "rollback-rehearsal-summary.json", + }, + { + scenarioId: "E10-OBS-001", + description: "Prometheus targets are UP during the rehearsal run.", + owner: "Story 10.4 observability smoke summary", + evidence: "observability-validation.log", + }, + { + scenarioId: "E10-OBS-002", + description: "Grafana dashboard is reachable during the rehearsal run.", + owner: "Story 10.4 observability smoke summary", + evidence: "observability-validation.log", + }, + { + scenarioId: "E10-SESSION-001", + description: "Five authenticated sessions remain isolated with no demo-blocking degradation.", + owner: "Story 10.4 session isolation rehearsal", + evidence: "session-isolation-summary.json", + }, +]; + +function ensureDirectory(dirPath) { + fs.mkdirSync(dirPath, { recursive: true }); +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function safeReadJson(filePath) { + try { + if (!fs.existsSync(filePath)) { + return null; + } + return readJson(filePath); + } catch (error) { + return { + status: "failed", + message: `Unable to parse ${path.basename(filePath)}: ${error instanceof Error ? error.message : String(error)}`, + parseError: true, + }; + } +} + +function normalizeResult(status) { + if (typeof status !== "string") { + return "MISSING"; + } + const normalized = status.trim().toLowerCase(); + if (normalized === "passed" || normalized === "go") { + return "PASSED"; + } + if (normalized === "failed" || normalized === "no-go") { + return "FAILED"; + } + return "MISSING"; +} + +function relativizeEvidence(filePath) { + const relativePath = path.relative(OUTPUT_DIR, filePath); + if (!relativePath || relativePath.startsWith("..")) { + return filePath.replace(/\\/g, "/"); + } + return relativePath.replace(/\\/g, "/"); +} + +function scenarioFromSmoke(smokeSummary, scenarioId, description, owner, evidencePath) { + if (!smokeSummary || !Array.isArray(smokeSummary.scenarios)) { + return { + scenarioId, + description, + result: "MISSING", + ownerTest: owner, + evidence: evidencePath, + source: "smoke-summary.json", + }; + } + + const matched = smokeSummary.scenarios.find((scenario) => scenario?.id === scenarioId); + if (!matched) { + return { + scenarioId, + description, + result: "MISSING", + ownerTest: owner, + evidence: evidencePath, + source: "smoke-summary.json", + }; + } + + return { + scenarioId, + description, + result: normalizeResult(matched.status), + ownerTest: owner, + evidence: matched.evidencePath ? relativizeEvidence(matched.evidencePath) : evidencePath, + source: "smoke-summary.json", + }; +} + +function scenarioFromStatus(filePayload, scenarioId, description, owner, evidencePath) { + if (!filePayload) { + return { + scenarioId, + description, + result: "MISSING", + ownerTest: owner, + evidence: evidencePath, + }; + } + + return { + scenarioId, + description, + result: normalizeResult(filePayload.status), + ownerTest: owner, + evidence: evidencePath, + }; +} + +function scenarioFromDocsAndEdge(docsSummary, edgeSummary, scenarioId, description, owner, evidencePath) { + const docsResult = normalizeResult(docsSummary?.status); + const edgeResult = normalizeResult(edgeSummary?.status); + const hasDocs = docsSummary !== null; + const hasEdge = edgeSummary !== null; + + if (!hasDocs || !hasEdge) { + return { + scenarioId, + description, + result: "MISSING", + ownerTest: owner, + evidence: evidencePath, + }; + } + + return { + scenarioId, + description, + result: docsResult === "PASSED" && edgeResult === "PASSED" ? "PASSED" : "FAILED", + ownerTest: owner, + evidence: evidencePath, + }; +} + +function scenarioFromColdStart(smokeSummary, filePayload, scenarioId, description, owner, evidencePath) { + const smokeScenario = scenarioFromSmoke(smokeSummary, scenarioId, description, owner, evidencePath); + if (!filePayload) { + return { + scenarioId, + description, + result: smokeScenario.result === "MISSING" ? "MISSING" : "FAILED", + ownerTest: owner, + evidence: evidencePath, + }; + } + + const withinTarget = Boolean(filePayload.firstMandatoryApi?.withinTarget); + return { + scenarioId, + description, + result: smokeScenario.result === "PASSED" && withinTarget ? "PASSED" : "FAILED", + ownerTest: owner, + evidence: evidencePath, + }; +} + +function renderMarkdown(summary) { + const passed = summary.scenarios.filter((scenario) => scenario.result === "PASSED").length; + const failed = summary.scenarios.filter((scenario) => scenario.result === "FAILED").length; + const missing = summary.scenarios.filter((scenario) => scenario.result === "MISSING").length; + const blockerLines = summary.goNoGo.blockers.length === 0 + ? "- none" + : summary.goNoGo.blockers.map((blocker) => `- ${blocker}`).join("\n"); + + return `# Story 10.4 Full-Stack Smoke/Rehearsal Summary + +- Build ID: \`${summary.buildId}\` +- Generated At: \`${summary.generatedAt}\` +- Overall Result: \`${summary.overallResult}\` +- Go/No-Go Decision: \`${summary.goNoGo.decision}\` +- Release Ready: \`${summary.goNoGo.releaseReady}\` +- Cold-Start Duration: \`${summary.coldStart.durationMs} ms\` +- Cold-Start Target Met: \`${summary.coldStart.withinTarget}\` +- Totals: passed=${passed}, failed=${failed}, missing=${missing} + +## Scenario Matrix + +| Scenario ID | Result | Owner | Evidence | +| --- | --- | --- | --- | +${summary.scenarios.map((scenario) => `| \`${scenario.scenarioId}\` | \`${scenario.result}\` | \`${scenario.ownerTest}\` | \`${scenario.evidence}\` |`).join("\n")} + +## Go/No-Go Blockers + +${blockerLines} +`; +} + +function main() { + ensureDirectory(OUTPUT_DIR); + + const coldStartPath = path.join(OUTPUT_DIR, "cold-start-timing.json"); + const docsSummaryPath = path.join(OUTPUT_DIR, "docs-summary.json"); + const smokeSummaryPath = path.join(OUTPUT_DIR, "smoke-summary.json"); + const edgeSummaryPath = path.join(OUTPUT_DIR, "edge-summary.json"); + const sessionSummaryPath = path.join(OUTPUT_DIR, "session-isolation-summary.json"); + const rollbackSummaryPath = path.join(OUTPUT_DIR, "rollback-rehearsal-summary.json"); + const goNoGoSummaryPath = path.join(OUTPUT_DIR, "go-no-go-summary.json"); + + const coldStart = safeReadJson(coldStartPath); + const docsSummary = safeReadJson(docsSummaryPath); + const smokeSummary = safeReadJson(smokeSummaryPath); + const edgeSummary = safeReadJson(edgeSummaryPath); + const sessionSummary = safeReadJson(sessionSummaryPath); + const rollbackSummary = safeReadJson(rollbackSummaryPath); + const goNoGoSummary = safeReadJson(goNoGoSummaryPath); + + const scenarios = [ + scenarioFromColdStart(smokeSummary, coldStart, "E10-SMOKE-001", SCENARIO_CATALOG[0].description, SCENARIO_CATALOG[0].owner, relativizeEvidence(coldStartPath)), + scenarioFromDocsAndEdge( + docsSummary, + edgeSummary, + "E10-SMOKE-002", + SCENARIO_CATALOG[1].description, + SCENARIO_CATALOG[1].owner, + `${relativizeEvidence(docsSummaryPath)} | ${relativizeEvidence(edgeSummaryPath)}`, + ), + scenarioFromStatus(rollbackSummary, "E10-SMOKE-003", SCENARIO_CATALOG[2].description, SCENARIO_CATALOG[2].owner, relativizeEvidence(rollbackSummaryPath)), + scenarioFromSmoke(smokeSummary, "E10-OBS-001", SCENARIO_CATALOG[3].description, SCENARIO_CATALOG[3].owner, SCENARIO_CATALOG[3].evidence), + scenarioFromSmoke(smokeSummary, "E10-OBS-002", SCENARIO_CATALOG[4].description, SCENARIO_CATALOG[4].owner, SCENARIO_CATALOG[4].evidence), + scenarioFromStatus(sessionSummary, "E10-SESSION-001", SCENARIO_CATALOG[5].description, SCENARIO_CATALOG[5].owner, relativizeEvidence(sessionSummaryPath)), + ]; + + const scenarioFailures = scenarios.filter((scenario) => scenario.result !== "PASSED"); + const goNoGoDecision = (goNoGoSummary?.decision ?? "no-go").toString(); + const goNoGoResult = normalizeResult(goNoGoDecision); + const releaseReady = goNoGoSummary?.releaseReady === true; + const overallResult = scenarioFailures.length === 0 && goNoGoResult === "PASSED" && releaseReady ? "PASSED" : "FAILED"; + const summary = { + buildId: BUILD_ID, + generatedAt: new Date().toISOString(), + overallResult, + scenarios, + coldStart: { + durationMs: coldStart?.firstMandatoryApi?.durationMs ?? -1, + withinTarget: Boolean(coldStart?.firstMandatoryApi?.withinTarget), + evidence: relativizeEvidence(coldStartPath), + }, + docs: { + status: normalizeResult(docsSummary?.status), + evidence: relativizeEvidence(docsSummaryPath), + }, + edge: { + status: normalizeResult(edgeSummary?.status), + evidence: relativizeEvidence(edgeSummaryPath), + }, + goNoGo: { + decision: goNoGoDecision, + releaseReady, + blockers: Array.isArray(goNoGoSummary?.blockers) ? goNoGoSummary.blockers : [], + evidence: relativizeEvidence(goNoGoSummaryPath), + }, + linkedEvidence: { + coldStart: relativizeEvidence(coldStartPath), + docs: relativizeEvidence(docsSummaryPath), + edge: relativizeEvidence(edgeSummaryPath), + smoke: relativizeEvidence(smokeSummaryPath), + sessionIsolation: relativizeEvidence(sessionSummaryPath), + rollback: relativizeEvidence(rollbackSummaryPath), + goNoGo: relativizeEvidence(goNoGoSummaryPath), + }, + }; + + fs.writeFileSync(MATRIX_JSON_PATH, JSON.stringify(summary, null, 2)); + fs.writeFileSync(MATRIX_MARKDOWN_PATH, renderMarkdown(summary)); + + if (overallResult !== "PASSED") { + const failingScenarios = scenarios + .filter((scenario) => scenario.result !== "PASSED") + .map((scenario) => `${scenario.scenarioId}:${scenario.result}`); + console.error(`Story 10.4 evidence gate failed: ${failingScenarios.join(", ") || "go-no-go-contract"}`); + console.error(`Matrix summary: ${MATRIX_JSON_PATH}`); + process.exitCode = 1; + } +} + +main(); diff --git a/scripts/release-readiness/run-edge-gateway-validation.sh b/scripts/release-readiness/run-edge-gateway-validation.sh new file mode 100644 index 0000000..236f0b3 --- /dev/null +++ b/scripts/release-readiness/run-edge-gateway-validation.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BUILD_ID="${EDGE_VALIDATION_BUILD_ID:-local}" +OUTPUT_DIR="${EDGE_VALIDATION_OUTPUT_DIR:-${ROOT_DIR}/_bmad-output/test-artifacts/epic-10/${BUILD_ID}/story-10-4}" +EDGE_VALIDATOR_PATH="${EDGE_VALIDATOR_PATH:-${ROOT_DIR}/docker/nginx/scripts/validate-edge-gateway.sh}" +EDGE_LOG_PATH="${OUTPUT_DIR}/edge-gateway-validation.log" +EDGE_SUMMARY_PATH="${OUTPUT_DIR}/edge-summary.json" + +relative_output_path() { + local target_path="$1" + if [[ "${target_path}" == "${OUTPUT_DIR}"/* ]]; then + printf '%s' "${target_path#"${OUTPUT_DIR}/"}" + return + fi + printf '%s' "${target_path}" +} + +json_escape() { + local value="${1:-}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\n'/\\n}" + value="${value//$'\r'/\\r}" + value="${value//$'\t'/\\t}" + printf '%s' "${value}" +} + +main() { + mkdir -p "${OUTPUT_DIR}" + + local status="passed" + local message="Edge gateway validation passed." + + if ! bash "${EDGE_VALIDATOR_PATH}" >"${EDGE_LOG_PATH}" 2>&1; then + status="failed" + message="Edge gateway validation failed." + cat "${EDGE_LOG_PATH}" >&2 || true + fi + + cat >"${EDGE_SUMMARY_PATH}" < candidate.name === name); + if (cookie?.value) { + return cookie.value; + } + } + + if (typeof cookieJar.toCookieHeader === "function") { + const cookieHeader = cookieJar.toCookieHeader(); + const match = cookieHeader.match(new RegExp(`(?:^|;\\s*)${name}=([^;]+)`)); + if (match?.[1]) { + return match[1]; + } + } + + return ""; +} + +function isDashboardReady(dashboardData) { + return Boolean( + dashboardData + && typeof dashboardData === "object" + && dashboardData.summary + && Array.isArray(dashboardData.positions) + && dashboardData.positions.length > 0, + ); +} + +function finalizeSessionResult(index, durationMs, payload, error) { + if (error) { + return { + index, + status: "failed", + durationMs, + error: error instanceof Error ? error.message : String(error), + dashboardReady: false, + identityHash: null, + accountHash: null, + sessionCookieHash: null, + xsrfTokenHash: null, + orderSessionHash: null, + }; + } + + const accountId = String(payload.accountId ?? payload.member?.accountId ?? ""); + const createdOrderSessionId = String(payload.createdSession?.orderSessionId ?? ""); + const executedOrderSessionId = String(payload.executedSession?.orderSessionId ?? createdOrderSessionId); + const sessionCookie = extractCookieValue(payload.cookieJar, "SESSION"); + const xsrfToken = extractCookieValue(payload.cookieJar, "XSRF-TOKEN"); + const identitySource = payload.identity?.email ?? payload.identity?.name ?? `session-${index}`; + const orderSessionSource = createdOrderSessionId || executedOrderSessionId || ""; + + return { + index, + status: "passed", + durationMs, + error: null, + dashboardReady: isDashboardReady(payload.dashboardData), + identityHash: shortHash(identitySource), + accountHash: accountId ? shortHash(accountId) : null, + sessionCookieHash: sessionCookie ? shortHash(sessionCookie) : null, + xsrfTokenHash: xsrfToken ? shortHash(xsrfToken) : null, + orderSessionHash: orderSessionSource ? shortHash(orderSessionSource) : null, + }; +} + +function hasDistinctValues(results, fieldName, expectedCount) { + const values = results.map((result) => result[fieldName]).filter(Boolean); + return values.length === expectedCount && new Set(values).size === expectedCount; +} + +function buildChecks(results, expectedCount) { + const allSessionsSucceeded = results.length === expectedCount && results.every((result) => result.status === "passed"); + const dashboardReady = results.every((result) => result.dashboardReady); + const uniqueAccounts = hasDistinctValues(results, "accountHash", expectedCount); + const uniqueSessionCookies = hasDistinctValues(results, "sessionCookieHash", expectedCount); + const uniqueXsrfTokens = hasDistinctValues(results, "xsrfTokenHash", expectedCount); + const uniqueOrderSessions = hasDistinctValues(results, "orderSessionHash", expectedCount); + + return { + expectedSessionCount: expectedCount, + actualSessionCount: results.length, + allSessionsSucceeded, + dashboardReady, + uniqueAccounts, + uniqueSessionCookies, + uniqueXsrfTokens, + uniqueOrderSessions, + }; +} + +function overallStatus(checks) { + const ignoredChecks = new Set(["expectedSessionCount", "actualSessionCount"]); + if (!requireDashboardReady) { + ignoredChecks.add("dashboardReady"); + } + + return Object.entries(checks) + .filter(([name]) => !ignoredChecks.has(name)) + .every(([, passed]) => Boolean(passed)) + ? "passed" + : "failed"; +} + +async function main() { + const helperModuleUrl = resolveHelperModuleUrl(); + const helperModule = await import(helperModuleUrl.href); + const createProvisionedStory115DashboardAccount = helperModule.createProvisionedStory115DashboardAccount; + + if (typeof createProvisionedStory115DashboardAccount !== "function") { + throw new Error(`Helper module ${helperModuleUrl.href} does not export createProvisionedStory115DashboardAccount().`); + } + + const sessionCount = Number.parseInt(process.env.SESSION_ISOLATION_SESSION_COUNT ?? String(DEFAULT_SESSION_COUNT), 10); + const sessionTimeoutMs = Number.parseInt(process.env.SESSION_ISOLATION_SESSION_TIMEOUT_MS ?? String(DEFAULT_SESSION_TIMEOUT_MS), 10); + const baseUrl = process.env.SESSION_ISOLATION_BASE_URL?.trim(); + const password = process.env.SESSION_ISOLATION_PASSWORD?.trim(); + const requestTimeoutMs = Number.parseInt(process.env.SESSION_ISOLATION_REQUEST_TIMEOUT_MS ?? String(DEFAULT_SESSION_TIMEOUT_MS), 10); + const pollTimeoutMs = Number.parseInt(process.env.SESSION_ISOLATION_POLL_TIMEOUT_MS ?? String(DEFAULT_SESSION_TIMEOUT_MS), 10); + + if (!Number.isInteger(sessionCount) || sessionCount <= 0) { + throw new Error(`Invalid SESSION_ISOLATION_SESSION_COUNT: ${process.env.SESSION_ISOLATION_SESSION_COUNT ?? ""}`); + } + + if (!Number.isInteger(sessionMaxConcurrency) || sessionMaxConcurrency <= 0) { + throw new Error(`Invalid SESSION_ISOLATION_MAX_CONCURRENCY: ${process.env.SESSION_ISOLATION_MAX_CONCURRENCY ?? ""}`); + } + + const runSession = async (index) => { + const started = Date.now(); + const zeroBasedIndex = index - 1; + + try { + if (sessionMaxConcurrency > 1 && sessionStartStaggerMs > 0) { + await new Promise((resolve) => setTimeout(resolve, sessionStartStaggerMs * zeroBasedIndex)); + } + const payload = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Session ${index} exceeded ${sessionTimeoutMs}ms timeout.`)); + }, sessionTimeoutMs); + + createProvisionedStory115DashboardAccount({ + ...(baseUrl ? { baseUrl } : {}), + ...(password ? { password } : {}), + requestTimeoutMs, + pollTimeoutMs, + skipDashboardQuoteWait: !requireDashboardReady, + emailPrefix: `story10_4_session_${index}`, + namePrefix: `Story 10.4 Session ${index}`, + }) + .then((value) => { + clearTimeout(timer); + resolve(value); + }) + .catch((error) => { + clearTimeout(timer); + reject(error); + }); + }); + return finalizeSessionResult(index, Date.now() - started, payload, null); + } catch (error) { + return finalizeSessionResult(index, Date.now() - started, null, error); + } + }; + + const sessionResults = new Array(sessionCount); + const workerCount = Math.min(sessionCount, sessionMaxConcurrency); + let nextIndex = 1; + + await Promise.all(Array.from({ length: workerCount }, async () => { + while (true) { + const index = nextIndex; + nextIndex += 1; + + if (index > sessionCount) { + return; + } + + sessionResults[index - 1] = await runSession(index); + } + })); + + const checks = buildChecks(sessionResults, sessionCount); + const status = overallStatus(checks); + const completedAt = new Date().toISOString(); + + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(summaryPath, JSON.stringify({ + storyId: "10.4", + criterion: "AC6", + startedAt, + completedAt, + status, + message: status === "passed" + ? "Five authenticated sessions remained isolated with no blocking degradation." + : "Session isolation rehearsal found one or more isolation or readiness failures.", + checks, + sessions: sessionResults, + }, null, 2)); + + if (status !== "passed") { + const failedSessions = sessionResults.filter((session) => session.status !== "passed"); + if (failedSessions.length > 0) { + console.error(`Session isolation failed sessions: ${JSON.stringify(failedSessions)}`); + } else if (requireDashboardReady && !checks.dashboardReady) { + console.error("Session isolation failed because dashboard readiness did not converge."); + } + console.error(`Session isolation summary: ${summaryPath}`); + process.exitCode = 1; + } +} + +await main(); diff --git a/scripts/release-readiness/run-full-stack-smoke.sh b/scripts/release-readiness/run-full-stack-smoke.sh new file mode 100755 index 0000000..a3d50fe --- /dev/null +++ b/scripts/release-readiness/run-full-stack-smoke.sh @@ -0,0 +1,456 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +COMPOSE_FILE="${SMOKE_COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" +OUTPUT_DIR="${SMOKE_OUTPUT_DIR:-${ROOT_DIR}/_bmad-output/test-artifacts/epic-10/${SMOKE_BUILD_ID:-local}/story-10-4}" +EDGE_BASE_URL="${EDGE_BASE_URL:-https://127.0.0.1}" +MANDATORY_API_BASE_URL="${SMOKE_MANDATORY_API_BASE_URL:-http://127.0.0.1:8080}" +MANDATORY_API_PATH="${SMOKE_MANDATORY_API_PATH:-/api/v1/auth/csrf}" +API_DOCS_URL="${SMOKE_API_DOCS_URL:-http://127.0.0.1:8080/v3/api-docs}" +SWAGGER_UI_URL="${SMOKE_SWAGGER_UI_URL:-http://127.0.0.1:8080/swagger-ui/index.html}" +OBSERVABILITY_VALIDATOR_PATH="${OBSERVABILITY_VALIDATOR_PATH:-${ROOT_DIR}/scripts/observability/validate-observability-stack.sh}" +SMOKE_START_TIMEOUT_SECONDS="${SMOKE_START_TIMEOUT_SECONDS:-120}" +SMOKE_STACK_READY_TIMEOUT_SECONDS="${SMOKE_STACK_READY_TIMEOUT_SECONDS:-300}" +SMOKE_POLL_INTERVAL_SECONDS="${SMOKE_POLL_INTERVAL_SECONDS:-2}" +SKIP_COMPOSE_UP="${SKIP_COMPOSE_UP:-0}" +REQUIRED_SERVICES=( + "channel-service" + "corebank-service" + "fep-gateway" + "fep-simulator" + "edge-gateway" + "prometheus" + "grafana" +) +COMPOSE_ENV=() + +SCRIPT_STATUS="failed" +FINAL_MESSAGE="" +STACK_BOOT_STATUS="pending" +HEALTH_STATUS="pending" +MANDATORY_API_STATUS="pending" +DOCS_STATUS="pending" +OBSERVABILITY_STATUS="pending" +MANDATORY_API_DURATION_MS="-1" +MANDATORY_API_HTTP_STATUS="0" + +COLD_START_REPORT_PATH="${OUTPUT_DIR}/cold-start-timing.json" +SMOKE_SUMMARY_PATH="${OUTPUT_DIR}/smoke-summary.json" +DOCS_SUMMARY_PATH="${OUTPUT_DIR}/docs-summary.json" +OBSERVABILITY_LOG_PATH="${OUTPUT_DIR}/observability-validation.log" +COMPOSE_UP_LOG_PATH="${OUTPUT_DIR}/compose-up.log" + +STARTED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +COMPLETED_AT="" + +if [[ -n "${COMPOSE_PROFILES:-}" ]]; then + COMPOSE_ENV+=("COMPOSE_PROFILES=${COMPOSE_PROFILES}") +fi +for required_env in VAULT_DEV_ROOT_TOKEN_ID INTERNAL_SECRET_BOOTSTRAP INTERNAL_SECRET; do + if [[ -n "${!required_env:-}" ]]; then + COMPOSE_ENV+=("${required_env}=${!required_env}") + fi +done + +append_compose_profile() { + local profile="$1" + local existing="${COMPOSE_PROFILES:-}" + + if [[ -n "${existing}" ]]; then + case ",${existing}," in + *",${profile},"*) return ;; + *) existing="${existing},${profile}" ;; + esac + else + existing="${profile}" + fi + + COMPOSE_PROFILES="${existing}" + local updated=() + local inserted=0 + for entry in "${COMPOSE_ENV[@]}"; do + if [[ "${entry}" == COMPOSE_PROFILES=* ]]; then + updated+=("COMPOSE_PROFILES=${COMPOSE_PROFILES}") + inserted=1 + else + updated+=("${entry}") + fi + done + if [[ "${inserted}" == "0" ]]; then + updated+=("COMPOSE_PROFILES=${COMPOSE_PROFILES}") + fi + COMPOSE_ENV=("${updated[@]}") +} + +fail() { + FINAL_MESSAGE="$1" + echo "full-stack smoke failed: $1" >&2 + SCRIPT_STATUS="failed" +} + +normalize_compose_file_for_docker_host() { + local raw_path="$1" + if [[ "${raw_path}" =~ ^/mnt/([A-Za-z])/(.*)$ ]]; then + local drive_letter="${BASH_REMATCH[1]^}" + local windows_path="${BASH_REMATCH[2]//\//\\}" + printf '%s:\\%s\n' "${drive_letter}" "${windows_path}" + return + fi + printf '%s\n' "${raw_path}" +} + +resolve_docker_cli() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + command -v docker + return + fi + + if command -v docker.exe >/dev/null 2>&1 && docker.exe compose version >/dev/null 2>&1; then + command -v docker.exe + return + fi + + fail "docker compose CLI is not available" +} + +compose_cmd() { + local docker_cli + local compose_file_arg + + docker_cli="$(resolve_docker_cli)" + compose_file_arg="${COMPOSE_FILE}" + if [[ "${docker_cli}" == *.exe ]]; then + compose_file_arg="$(normalize_compose_file_for_docker_host "${COMPOSE_FILE}")" + fi + + if [[ ${#COMPOSE_ENV[@]} -gt 0 ]]; then + env "${COMPOSE_ENV[@]}" "${docker_cli}" compose -f "${compose_file_arg}" "$@" + else + "${docker_cli}" compose -f "${compose_file_arg}" "$@" + fi +} + +docker_cmd() { + local docker_cli + docker_cli="$(resolve_docker_cli)" + "${docker_cli}" "$@" +} + +assert_file() { + local path="$1" + if [[ ! -f "${path}" ]]; then + fail "missing required file: ${path}" + return 1 + fi +} + +now_ms() { + date +%s%3N +} + +json_escape() { + local value="${1:-}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\n'/\\n}" + value="${value//$'\r'/\\r}" + value="${value//$'\t'/\\t}" + printf '%s' "${value}" +} + +relative_output_path() { + local target_path="$1" + if [[ "${target_path}" == "${OUTPUT_DIR}"/* ]]; then + printf '%s' "${target_path#"${OUTPUT_DIR}/"}" + return + fi + printf '%s' "${target_path}" +} + +http_status() { + local url="$1" + local body_path="$2" + curl -k -sS -o "${body_path}" -w '%{http_code}' "${url}" +} + +wait_for_mandatory_api() { + local deadline_seconds="${SMOKE_START_TIMEOUT_SECONDS}" + local start_ms + local current_ms + local deadline_ms + local body_path="${OUTPUT_DIR}/mandatory-api-response.json" + local url="${MANDATORY_API_BASE_URL}${MANDATORY_API_PATH}" + + start_ms="$(now_ms)" + deadline_ms="$((start_ms + (deadline_seconds * 1000)))" + + while true; do + MANDATORY_API_HTTP_STATUS="$(http_status "${url}" "${body_path}")" + if [[ "${MANDATORY_API_HTTP_STATUS}" == "200" ]]; then + current_ms="$(now_ms)" + MANDATORY_API_DURATION_MS="$((current_ms - start_ms))" + if (( MANDATORY_API_DURATION_MS > deadline_seconds * 1000 )); then + MANDATORY_API_STATUS="failed" + fail "mandatory API exceeded ${deadline_seconds}s target (${MANDATORY_API_DURATION_MS}ms)" + return 1 + fi + MANDATORY_API_STATUS="passed" + return + fi + + current_ms="$(now_ms)" + if (( current_ms >= deadline_ms )); then + MANDATORY_API_DURATION_MS="$((current_ms - start_ms))" + MANDATORY_API_STATUS="failed" + fail "mandatory API did not return 200 within ${deadline_seconds}s (last status ${MANDATORY_API_HTTP_STATUS})" + return 1 + fi + + sleep "${SMOKE_POLL_INTERVAL_SECONDS}" + done +} + +check_service_health() { + local service + local container_id + local health_status + + for service in "${REQUIRED_SERVICES[@]}"; do + container_id="$(compose_cmd ps -q "${service}" | tail -n1)" + if [[ -z "${container_id}" ]]; then + HEALTH_STATUS="failed" + fail "service ${service} is not running" + return 1 + fi + health_status="$(docker_cmd inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container_id}")" + if [[ "${health_status}" != "healthy" && "${health_status}" != "running" ]]; then + HEALTH_STATUS="failed" + fail "service ${service} is not healthy (status=${health_status})" + return 1 + fi + done + + HEALTH_STATUS="passed" +} + +wait_for_required_services() { + local start_ms + local current_ms + local deadline_ms + + start_ms="$(now_ms)" + deadline_ms="$((start_ms + (SMOKE_STACK_READY_TIMEOUT_SECONDS * 1000)))" + + while true; do + local all_ready="1" + local service + local container_id + local health_status + + for service in "${REQUIRED_SERVICES[@]}"; do + container_id="$(compose_cmd ps -q "${service}" | tail -n1)" + if [[ -z "${container_id}" ]]; then + all_ready="0" + break + fi + health_status="$(docker_cmd inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container_id}")" + if [[ "${health_status}" != "healthy" && "${health_status}" != "running" ]]; then + all_ready="0" + break + fi + done + + if [[ "${all_ready}" == "1" ]]; then + HEALTH_STATUS="passed" + return + fi + + current_ms="$(now_ms)" + if (( current_ms >= deadline_ms )); then + STACK_BOOT_STATUS="failed" + HEALTH_STATUS="failed" + fail "required services did not become healthy within ${SMOKE_STACK_READY_TIMEOUT_SECONDS}s" + return 1 + fi + + sleep "${SMOKE_POLL_INTERVAL_SECONDS}" + done +} + +check_docs_endpoints() { + local api_docs_body="${OUTPUT_DIR}/api-docs.json" + local swagger_ui_body="${OUTPUT_DIR}/swagger-ui.html" + local api_docs_status + local swagger_status + + api_docs_status="$(http_status "${API_DOCS_URL}" "${api_docs_body}")" + if [[ "${api_docs_status}" != "200" ]]; then + DOCS_STATUS="failed" + fail "API docs endpoint returned ${api_docs_status}" + return 1 + fi + if ! grep -q '"openapi"' "${api_docs_body}"; then + DOCS_STATUS="failed" + fail "API docs response missing openapi field" + return 1 + fi + + swagger_status="$(curl -k -sS -L -o "${swagger_ui_body}" -w '%{http_code}' "${SWAGGER_UI_URL}")" + if [[ "${swagger_status}" != "200" ]]; then + DOCS_STATUS="failed" + fail "Swagger UI endpoint returned ${swagger_status}" + return 1 + fi + if ! grep -Eqi 'Swagger UI|swagger-ui' "${swagger_ui_body}"; then + DOCS_STATUS="failed" + fail "Swagger UI response missing expected marker" + return 1 + fi + + DOCS_STATUS="passed" +} + +run_observability_validation() { + assert_file "${OBSERVABILITY_VALIDATOR_PATH}" + if ! OBSERVABILITY_COMPOSE_FILE="${COMPOSE_FILE}" "${OBSERVABILITY_VALIDATOR_PATH}" >"${OBSERVABILITY_LOG_PATH}" 2>&1; then + OBSERVABILITY_STATUS="failed" + cat "${OBSERVABILITY_LOG_PATH}" >&2 || true + fail "observability validator failed" + return 1 + fi + + OBSERVABILITY_STATUS="passed" +} + +emit_reports() { + COMPLETED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + mkdir -p "${OUTPUT_DIR}" + + cat >"${COLD_START_REPORT_PATH}" <"${DOCS_SUMMARY_PATH}" <"${SMOKE_SUMMARY_PATH}" <"${COMPOSE_UP_LOG_PATH}" 2>&1; then + STACK_BOOT_STATUS="failed" + cat "${COMPOSE_UP_LOG_PATH}" >&2 || true + fail "docker compose up failed" + return 1 + fi + fi + STACK_BOOT_STATUS="passed" + + if ! wait_for_required_services; then + return 0 + fi + if ! wait_for_mandatory_api; then + return 0 + fi + if ! check_service_health; then + return 0 + fi + if ! check_docs_endpoints; then + return 0 + fi + if ! run_observability_validation; then + return 0 + fi + + SCRIPT_STATUS="passed" + FINAL_MESSAGE="Cold-start smoke checks passed." + echo "Cold-start smoke checks passed" +} + +main "$@" diff --git a/scripts/release-readiness/run-rollback-rehearsal.sh b/scripts/release-readiness/run-rollback-rehearsal.sh new file mode 100755 index 0000000..028a878 --- /dev/null +++ b/scripts/release-readiness/run-rollback-rehearsal.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BUILD_ID="${ROLLBACK_REHEARSAL_BUILD_ID:-local}" +OUTPUT_DIR="${ROLLBACK_REHEARSAL_OUTPUT_DIR:-${ROOT_DIR}/_bmad-output/test-artifacts/epic-10/${BUILD_ID}/story-10-4}" +COMPOSE_FILE="${ROLLBACK_REHEARSAL_COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" +RUNBOOK_PATH="${ROOT_DIR}/docs/ops/full-stack-smoke-rehearsal-runbook.md" +CHECKLIST_TEMPLATE_PATH="${ROOT_DIR}/docs/ops/release-go-no-go-checklist-template.md" +SMOKE_SUMMARY_PATH="${ROLLBACK_REHEARSAL_SMOKE_SUMMARY_PATH:-${OUTPUT_DIR}/smoke-summary.json}" +SESSION_ISOLATION_SUMMARY_PATH="${ROLLBACK_REHEARSAL_SESSION_SUMMARY_PATH:-${OUTPUT_DIR}/session-isolation-summary.json}" +EDGE_SUMMARY_PATH="${ROLLBACK_REHEARSAL_EDGE_SUMMARY_PATH:-${OUTPUT_DIR}/edge-summary.json}" +ROLLBACK_SUMMARY_PATH="${OUTPUT_DIR}/rollback-rehearsal-summary.json" +GO_NO_GO_SUMMARY_PATH="${OUTPUT_DIR}/go-no-go-summary.json" +GO_NO_GO_MARKDOWN_PATH="${OUTPUT_DIR}/go-no-go-summary.md" +ROLLBACK_REHEARSAL_MODE="${ROLLBACK_REHEARSAL_MODE:-simulate}" +ROLLBACK_REHEARSAL_CONFIRM_EXECUTE="${ROLLBACK_REHEARSAL_CONFIRM_EXECUTE:-0}" +ROLLBACK_REHEARSAL_OPERATOR="${ROLLBACK_REHEARSAL_OPERATOR:-release-manager}" +ROLLBACK_REHEARSAL_CHANGE_REF="${ROLLBACK_REHEARSAL_CHANGE_REF:-CRQ-10.4}" +ROLLBACK_REHEARSAL_OWNER="${ROLLBACK_REHEARSAL_OWNER:-platform-oncall}" +ROLLBACK_REHEARSAL_TARGET_SERVICES="${ROLLBACK_REHEARSAL_TARGET_SERVICES:-edge-gateway channel-service corebank-service fep-gateway fep-simulator prometheus grafana}" +COMPOSE_ENV=() + +STARTED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +COMPLETED_AT="" +ROLLBACK_STATUS="failed" +GO_NO_GO_DECISION="no-go" +FINAL_MESSAGE="" +SMOKE_STATUS="unknown" +SESSION_STATUS="unknown" +EDGE_STATUS="unknown" +ROLLBACK_ACTION="not-run" +ROLLBACK_COMMAND="" +BLOCKERS=() + +if [[ -n "${COMPOSE_PROFILES:-}" ]]; then + COMPOSE_ENV+=("COMPOSE_PROFILES=${COMPOSE_PROFILES}") +fi +for required_env in VAULT_DEV_ROOT_TOKEN_ID INTERNAL_SECRET_BOOTSTRAP INTERNAL_SECRET; do + if [[ -n "${!required_env:-}" ]]; then + COMPOSE_ENV+=("${required_env}=${!required_env}") + fi +done + +fail() { + FINAL_MESSAGE="$1" + add_blocker "$1" + echo "rollback rehearsal failed: $1" >&2 + exit 1 +} + +json_escape() { + local value="${1:-}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\n'/\\n}" + value="${value//$'\r'/\\r}" + value="${value//$'\t'/\\t}" + printf '%s' "${value}" +} + +normalize_compose_file_for_docker_host() { + local raw_path="$1" + if [[ "${raw_path}" =~ ^/mnt/([A-Za-z])/(.*)$ ]]; then + local drive_letter="${BASH_REMATCH[1]^}" + local windows_path="${BASH_REMATCH[2]//\//\\}" + printf '%s:\\%s\n' "${drive_letter}" "${windows_path}" + return + fi + printf '%s\n' "${raw_path}" +} + +resolve_docker_cli() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + command -v docker + return + fi + + if command -v docker.exe >/dev/null 2>&1 && docker.exe compose version >/dev/null 2>&1; then + command -v docker.exe + return + fi + + fail "docker compose CLI is not available" +} + +resolve_json_parser() { + if command -v python >/dev/null 2>&1; then + command -v python + return + fi + if command -v python.exe >/dev/null 2>&1; then + command -v python.exe + return + fi + if command -v node >/dev/null 2>&1; then + command -v node + return + fi + if command -v node.exe >/dev/null 2>&1; then + command -v node.exe + return + fi + + fail "json parser runtime is not available" +} + +compose_cmd() { + local docker_cli + local compose_file_arg + + docker_cli="$(resolve_docker_cli)" + compose_file_arg="${COMPOSE_FILE}" + if [[ "${docker_cli}" == *.exe ]]; then + compose_file_arg="$(normalize_compose_file_for_docker_host "${COMPOSE_FILE}")" + fi + + if [[ ${#COMPOSE_ENV[@]} -gt 0 ]]; then + env "${COMPOSE_ENV[@]}" "${docker_cli}" compose -f "${compose_file_arg}" "$@" + else + "${docker_cli}" compose -f "${compose_file_arg}" "$@" + fi +} + +extract_status() { + local path="$1" + local parser + local status + parser="$(resolve_json_parser)" + status="$( + "${parser}" -c "import json, sys +payload = json.load(sys.stdin) +status = payload.get('status', '') +if not isinstance(status, str) or not status.strip(): + raise SystemExit(2) +sys.stdout.write(status.strip())" < "${path}" 2>/dev/null + )" + if [[ -z "${status}" ]]; then + fail "unable to parse status from ${path}" + fi + printf '%s' "${status}" +} + +add_blocker() { + local blocker="$1" + local existing + for existing in "${BLOCKERS[@]}"; do + if [[ "${existing}" == "${blocker}" ]]; then + return + fi + done + BLOCKERS+=("${blocker}") +} + +render_blockers_json() { + if [[ ${#BLOCKERS[@]} -eq 0 ]]; then + printf '[]' + return + fi + + local rendered="[" + local index + for index in "${!BLOCKERS[@]}"; do + if [[ "${index}" -gt 0 ]]; then + rendered+=", " + fi + rendered+="\"$(json_escape "${BLOCKERS[${index}]}")\"" + done + rendered+="]" + printf '%s' "${rendered}" +} + +render_service_steps_json() { + local services=() + read -r -a services <<<"${ROLLBACK_REHEARSAL_TARGET_SERVICES}" + local rendered="[" + local index + for index in "${!services[@]}"; do + if [[ "${index}" -gt 0 ]]; then + rendered+=", " + fi + rendered+="{\"service\":\"$(json_escape "${services[${index}]}")\",\"action\":\"reapply-compose\"}" + done + rendered+="]" + printf '%s' "${rendered}" +} + +emit_reports() { + mkdir -p "${OUTPUT_DIR}" + COMPLETED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + + cat >"${ROLLBACK_SUMMARY_PATH}" <"${GO_NO_GO_SUMMARY_PATH}" <"${GO_NO_GO_MARKDOWN_PATH}" <>"${GO_NO_GO_MARKDOWN_PATH}" + else + local blocker + for blocker in "${BLOCKERS[@]}"; do + printf '%s\n' "- ${blocker}" >>"${GO_NO_GO_MARKDOWN_PATH}" + done + fi +} + +trap emit_reports EXIT + +main() { + [[ -f "${RUNBOOK_PATH}" ]] || fail "missing rehearsal runbook: ${RUNBOOK_PATH}" + [[ -f "${CHECKLIST_TEMPLATE_PATH}" ]] || fail "missing checklist template: ${CHECKLIST_TEMPLATE_PATH}" + if [[ -f "${SMOKE_SUMMARY_PATH}" ]]; then + SMOKE_STATUS="$(extract_status "${SMOKE_SUMMARY_PATH}")" + else + SMOKE_STATUS="missing" + add_blocker "Smoke summary is missing." + fi + if [[ -f "${EDGE_SUMMARY_PATH}" ]]; then + EDGE_STATUS="$(extract_status "${EDGE_SUMMARY_PATH}")" + else + EDGE_STATUS="missing" + add_blocker "Edge validation summary is missing." + fi + if [[ -f "${SESSION_ISOLATION_SUMMARY_PATH}" ]]; then + SESSION_STATUS="$(extract_status "${SESSION_ISOLATION_SUMMARY_PATH}")" + else + SESSION_STATUS="missing" + add_blocker "Session isolation summary is missing." + fi + + case "${ROLLBACK_REHEARSAL_MODE}" in + simulate) + ROLLBACK_ACTION="simulated" + ROLLBACK_COMMAND="simulate deterministic re-run for ${ROLLBACK_REHEARSAL_TARGET_SERVICES}" + ROLLBACK_STATUS="passed" + ;; + execute) + [[ "${ROLLBACK_REHEARSAL_CONFIRM_EXECUTE}" == "1" ]] || fail "ROLLBACK_REHEARSAL_CONFIRM_EXECUTE=1 is required for execute mode" + ROLLBACK_COMMAND="docker compose -f ${COMPOSE_FILE} up -d ${ROLLBACK_REHEARSAL_TARGET_SERVICES}" + if ! compose_cmd up -d ${ROLLBACK_REHEARSAL_TARGET_SERVICES} >/dev/null; then + ROLLBACK_ACTION="execution-failed" + fail "docker compose rollback re-apply failed" + fi + ROLLBACK_ACTION="executed" + ROLLBACK_STATUS="passed" + ;; + *) + fail "unsupported rollback rehearsal mode: ${ROLLBACK_REHEARSAL_MODE}" + ;; + esac + + if [[ "${SMOKE_STATUS}" != "passed" ]]; then + add_blocker "Smoke summary is ${SMOKE_STATUS}." + fi + if [[ "${EDGE_STATUS}" != "passed" ]]; then + add_blocker "Edge validation summary is ${EDGE_STATUS}." + fi + if [[ "${SESSION_STATUS}" != "passed" ]]; then + add_blocker "Session isolation summary is ${SESSION_STATUS}." + fi + if [[ "${ROLLBACK_STATUS}" != "passed" ]]; then + add_blocker "Rollback rehearsal is ${ROLLBACK_STATUS}." + fi + + if [[ ${#BLOCKERS[@]} -eq 0 ]]; then + GO_NO_GO_DECISION="go" + FINAL_MESSAGE="Rollback rehearsal completed and release checklist can be marked go." + else + GO_NO_GO_DECISION="no-go" + FINAL_MESSAGE="Rollback rehearsal completed, but one or more release blockers remain." + fi + + echo "Rollback rehearsal completed with decision: ${GO_NO_GO_DECISION}" +} + +main "$@" diff --git a/scripts/story-11-5-live-dashboard-account.mjs b/scripts/story-11-5-live-dashboard-account.mjs index 85bfaf4..84074f0 100644 --- a/scripts/story-11-5-live-dashboard-account.mjs +++ b/scripts/story-11-5-live-dashboard-account.mjs @@ -14,6 +14,24 @@ const wait = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }); +const isRetryableOrderExecutionError = (error) => { + const message = error instanceof Error ? error.message : String(error); + return message.includes('/execute returned 500') + && ( + message.includes('Deadlock found when trying to get lock') + || message.includes('Lock wait timeout exceeded') + ); +}; + +const isOrderSessionAuthorizationError = (error) => { + const message = error instanceof Error ? error.message : String(error); + return message.includes('/execute returned 409') + && ( + message.includes('"code":"ORD-009"') + || message.includes('order session is not authorized for execution') + ); +}; + const shellQuote = (value) => `'${String(value).replace(/'/g, `'\"'\"'`)}'`; const unwrapEnvelope = (payload) => ( @@ -243,6 +261,21 @@ export const generateStableTotp = async ( return generateTotp(manualEntryKey); }; +export const generateFreshTotp = async ( + manualEntryKey, + previousOtpCode, + minRemainingMs = 8_000, +) => { + let otpCode = await generateStableTotp(manualEntryKey, minRemainingMs); + + while (previousOtpCode && otpCode === previousOtpCode) { + await wait(millisUntilNextTotpWindow() + 1_500); + otpCode = await generateStableTotp(manualEntryKey, minRemainingMs); + } + + return otpCode; +}; + const isChartReadyPosition = (position) => typeof position === 'object' && position !== null @@ -314,6 +347,59 @@ const waitForDashboardQuoteData = async ( throw new Error(`Timed out waiting for dashboard quote metadata for account ${accountId}.`); }; +const isAuthorizedOrderSession = (orderSession) => + typeof orderSession === 'object' + && orderSession !== null + && orderSession.status === 'AUTHED'; + +const verifyOrderSessionIfRequired = async ({ + cookieJar, + baseUrl, + orderSession, + manualEntryKey, + previousOtpCode, + requestTimeoutMs, +}) => { + if (isAuthorizedOrderSession(orderSession)) { + return { + orderSession, + otpCode: previousOtpCode, + }; + } + + const orderSessionId = orderSession?.orderSessionId; + + if (!orderSessionId) { + throw new Error('Order session bootstrap did not return orderSessionId.'); + } + + const otpCode = await generateFreshTotp(manualEntryKey, previousOtpCode); + const csrf = await fetchLiveCsrf(cookieJar, baseUrl, requestTimeoutMs); + const verifiedOrderSession = unwrapEnvelope(await fetchLiveJson( + cookieJar, + baseUrl, + `/api/v1/orders/sessions/${orderSessionId}/otp/verify`, + { + method: 'POST', + headers: { + [csrf.headerName]: csrf.csrfToken, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ otpCode }), + }, + requestTimeoutMs, + )); + + if (!isAuthorizedOrderSession(verifiedOrderSession)) { + throw new Error(`Order session ${orderSessionId} remained ${verifiedOrderSession?.status ?? 'UNKNOWN'} after OTP verification.`); + } + + return { + orderSession: verifiedOrderSession, + otpCode, + }; +}; + export const createProvisionedStory115DashboardAccount = async ({ baseUrl = process.env.LIVE_API_BASE_URL?.trim() || DEFAULT_BASE_URL, password = process.env.LIVE_REGISTER_PASSWORD ?? 'LiveVideo115!', @@ -321,6 +407,9 @@ export const createProvisionedStory115DashboardAccount = async ({ namePrefix = 'Story 11.5 Live', requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, pollTimeoutMs = DEFAULT_POLL_TIMEOUT_MS, + skipDashboardQuoteWait = false, + orderExecuteRetryCount = Number.parseInt(process.env.LIVE_ORDER_EXECUTE_RETRY_COUNT ?? '5', 10), + orderExecuteRetryDelayMs = Number.parseInt(process.env.LIVE_ORDER_EXECUTE_RETRY_DELAY_MS ?? '750', 10), } = {}) => { const identity = createLiveIdentity({ prefix: emailPrefix, @@ -433,6 +522,8 @@ export const createProvisionedStory115DashboardAccount = async ({ throw new Error('Registered live account did not expose accountId via /api/v1/auth/session.'); } + let lastOtpCode = enrollmentCode; + csrf = await fetchLiveCsrf(cookieJar, baseUrl, requestTimeoutMs); const createdSession = unwrapEnvelope(await fetchLiveJson( cookieJar, @@ -461,28 +552,79 @@ export const createProvisionedStory115DashboardAccount = async ({ throw new Error('Low-risk live order session bootstrap did not return orderSessionId.'); } - csrf = await fetchLiveCsrf(cookieJar, baseUrl, requestTimeoutMs); - const executedSession = unwrapEnvelope(await fetchLiveJson( + const authorization = await verifyOrderSessionIfRequired({ cookieJar, baseUrl, - `/api/v1/orders/sessions/${createdSession.orderSessionId}/execute`, - { - method: 'POST', - headers: { - [csrf.headerName]: csrf.csrfToken, - 'Content-Type': 'application/json', - }, - body: '{}', - }, + orderSession: createdSession, + manualEntryKey, + previousOtpCode: lastOtpCode, requestTimeoutMs, - )); + }); + let authorizedSession = authorization.orderSession; + lastOtpCode = authorization.otpCode; + + let executedSession; + let recoveredAuthorization = false; + for (let attempt = 1; attempt <= orderExecuteRetryCount; attempt += 1) { + try { + csrf = await fetchLiveCsrf(cookieJar, baseUrl, requestTimeoutMs); + executedSession = unwrapEnvelope(await fetchLiveJson( + cookieJar, + baseUrl, + `/api/v1/orders/sessions/${authorizedSession.orderSessionId}/execute`, + { + method: 'POST', + headers: { + [csrf.headerName]: csrf.csrfToken, + 'Content-Type': 'application/json', + }, + body: '{}', + }, + requestTimeoutMs, + )); + break; + } catch (error) { + if (!recoveredAuthorization && isOrderSessionAuthorizationError(error)) { + recoveredAuthorization = true; + const latestSession = unwrapEnvelope(await fetchLiveJson( + cookieJar, + baseUrl, + `/api/v1/orders/sessions/${createdSession.orderSessionId}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + }, + }, + requestTimeoutMs, + )); + const recovery = await verifyOrderSessionIfRequired({ + cookieJar, + baseUrl, + orderSession: latestSession, + manualEntryKey, + previousOtpCode: lastOtpCode, + requestTimeoutMs, + }); + authorizedSession = recovery.orderSession; + lastOtpCode = recovery.otpCode; + continue; + } + if (attempt >= orderExecuteRetryCount || !isRetryableOrderExecutionError(error)) { + throw error; + } + await wait(orderExecuteRetryDelayMs * attempt); + } + } - const dashboardData = await waitForDashboardQuoteData( - cookieJar, - baseUrl, - accountId, - pollTimeoutMs, - ); + const dashboardData = skipDashboardQuoteWait + ? null + : await waitForDashboardQuoteData( + cookieJar, + baseUrl, + accountId, + pollTimeoutMs, + ); return { identity, @@ -492,6 +634,7 @@ export const createProvisionedStory115DashboardAccount = async ({ cookieJar, member, createdSession, + authorizedSession, executedSession, dashboardData, }; diff --git a/tests/helpers/bash-script-test-utils.js b/tests/helpers/bash-script-test-utils.js new file mode 100644 index 0000000..40c025c --- /dev/null +++ b/tests/helpers/bash-script-test-utils.js @@ -0,0 +1,93 @@ +"use strict"; + +const { spawn, spawnSync } = require("node:child_process"); +const path = require("node:path"); + +function toBashPath(filePath) { + if (process.platform !== "win32") { + return filePath; + } + + return filePath + .replace(/\\/g, "/") + .replace(/^([A-Za-z]):/, (_, drive) => `/mnt/${drive.toLowerCase()}`); +} + +function normalizeEnvValue(value) { + if (typeof value !== "string") { + return value; + } + if (/^[A-Za-z]:[\\/]/.test(value)) { + return toBashPath(value); + } + return value.replace(/\\/g, "/"); +} + +function quoteForBash(value) { + return `'${String(value).replace(/'/g, `'\\''`)}'`; +} + +function buildBashCommand(repoRoot, scriptPath, options = {}) { + const statements = []; + + for (const [name, value] of Object.entries(options.env || {})) { + statements.push(`export ${name}=${quoteForBash(normalizeEnvValue(value))}`); + } + + if ((options.prependPathEntries || []).length > 0) { + const prepended = options.prependPathEntries.map((entry) => toBashPath(entry)).join(":"); + statements.push(`export PATH=${quoteForBash(`${prepended}:`)}"$PATH"`); + } + + statements.push(`bash ${quoteForBash(toBashPath(path.join(repoRoot, scriptPath)))}`); + return statements.join("; "); +} + +function runBashScript(repoRoot, scriptPath, options = {}) { + return spawnSync("bash", ["-lc", buildBashCommand(repoRoot, scriptPath, options)], { + cwd: repoRoot, + env: { ...process.env }, + encoding: "utf8", + timeout: options.timeout ?? 30000, + maxBuffer: 1024 * 1024, + }); +} + +function runAsyncBashScript(repoRoot, scriptPath, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn("bash", ["-lc", buildBashCommand(repoRoot, scriptPath, options)], { + cwd: repoRoot, + env: { ...process.env }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + resolve({ status: 124, stdout, stderr: `${stderr}\nTimed out` }); + }, options.timeout ?? 30000); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (code) => { + clearTimeout(timeout); + resolve({ status: code, stdout, stderr }); + }); + }); +} + +module.exports = { + buildBashCommand, + quoteForBash, + runAsyncBashScript, + runBashScript, + toBashPath, +}; diff --git a/tests/observability/observability-runtime.test.js b/tests/observability/observability-runtime.test.js index 2e16a4b..2e6aa25 100644 --- a/tests/observability/observability-runtime.test.js +++ b/tests/observability/observability-runtime.test.js @@ -7,6 +7,7 @@ const http = require("node:http"); const os = require("node:os"); const path = require("node:path"); const { spawn, spawnSync } = require("node:child_process"); +const { runAsyncBashScript, runBashScript } = require("../helpers/bash-script-test-utils"); const repoRoot = path.resolve(__dirname, "..", ".."); const composeEnv = { @@ -125,7 +126,7 @@ test("docker compose config succeeds with observability additions", () => { }); test("validation script passes in static mode", () => { - const result = run("bash", ["scripts/observability/validate-observability-stack.sh"], { + const result = runBashScript(repoRoot, "scripts/observability/validate-observability-stack.sh", { env: { ...composeEnv, OBSERVABILITY_SKIP_RUNTIME: "1" }, }); @@ -234,10 +235,10 @@ printf '%s\\n' "$*" >> "$MOCK_CURL_LOG" last_arg="" while [ "$#" -gt 0 ]; do case "$1" in - -o|-w|-u|-H|-X) + -o|-w|-u|-H|-X|--data-urlencode) shift 2 ;; - -s|-S|-f|-fsS|-T) + -s|-S|-f|-fsS|-T|-G) shift 1 ;; *) @@ -298,17 +299,17 @@ esac }); try { - const result = await runAsync("bash", ["scripts/observability/validate-observability-stack.sh"], { + const result = await runAsyncBashScript(repoRoot, "scripts/observability/validate-observability-stack.sh", { timeout: 120000, env: { ...composeEnv, - PATH: `${binDir}:${process.env.PATH}`, MOCK_DOCKER_LOG: dockerLog, MOCK_CURL_LOG: curlLog, OBSERVABILITY_PROMETHEUS_BASE_URL: server.baseUrl, OBSERVABILITY_GRAFANA_BASE_URL: "http://127.0.0.1:13000", OBSERVABILITY_LAST_UPDATED_AT: checkedAt, }, + prependPathEntries: [binDir], }); assert.equal(result.status, 0, `runtime validation failed: ${result.stderr}\n${result.stdout}`); diff --git a/tests/release-readiness/edge-gateway-validation-runtime.test.js b/tests/release-readiness/edge-gateway-validation-runtime.test.js new file mode 100644 index 0000000..f8f5ac4 --- /dev/null +++ b/tests/release-readiness/edge-gateway-validation-runtime.test.js @@ -0,0 +1,62 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { runBashScript } = require("../helpers/bash-script-test-utils"); + +const repoRoot = path.resolve(__dirname, "..", ".."); + +function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function writeExecutable(filePath, contents) { + fs.writeFileSync(filePath, contents, { encoding: "utf8", mode: 0o755 }); +} + +function loadJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +test("edge gateway validation script writes passed structured evidence", () => { + const tempDir = makeTempDir("edge-validation-pass-"); + const outputDir = path.join(tempDir, "output"); + const validatorPath = path.join(tempDir, "mock-edge-validator.sh"); + writeExecutable(validatorPath, "#!/usr/bin/env bash\nset -euo pipefail\necho edge validation ok\n"); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-edge-gateway-validation.sh", { + env: { + EDGE_VALIDATION_OUTPUT_DIR: outputDir, + EDGE_VALIDATOR_PATH: validatorPath, + }, + }); + + assert.equal(result.status, 0, `edge validation wrapper failed: ${result.stderr}\n${result.stdout}`); + + const summary = loadJson(path.join(outputDir, "edge-summary.json")); + assert.equal(summary.status, "passed"); + assert.equal(summary.evidence.logPath, "edge-gateway-validation.log"); +}); + +test("edge gateway validation script writes failed structured evidence when validator fails", () => { + const tempDir = makeTempDir("edge-validation-fail-"); + const outputDir = path.join(tempDir, "output"); + const validatorPath = path.join(tempDir, "mock-edge-validator.sh"); + writeExecutable(validatorPath, "#!/usr/bin/env bash\nset -euo pipefail\necho edge validation failed >&2\nexit 1\n"); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-edge-gateway-validation.sh", { + env: { + EDGE_VALIDATION_OUTPUT_DIR: outputDir, + EDGE_VALIDATOR_PATH: validatorPath, + }, + }); + + assert.equal(result.status, 1, "edge validation wrapper should fail when validator fails"); + + const summary = loadJson(path.join(outputDir, "edge-summary.json")); + assert.equal(summary.status, "failed"); + assert.equal(summary.evidence.logPath, "edge-gateway-validation.log"); +}); diff --git a/tests/release-readiness/full-stack-smoke-runtime.test.js b/tests/release-readiness/full-stack-smoke-runtime.test.js new file mode 100644 index 0000000..3314302 --- /dev/null +++ b/tests/release-readiness/full-stack-smoke-runtime.test.js @@ -0,0 +1,225 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const repoRoot = path.resolve(__dirname, "..", ".."); +const { runBashScript } = require("../helpers/bash-script-test-utils"); + +function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function writeExecutable(filePath, contents) { + fs.writeFileSync(filePath, contents, { encoding: "utf8", mode: 0o755 }); +} + +function loadJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function setupMockTooling(tempDir) { + const binDir = path.join(tempDir, "bin"); + const dockerLog = path.join(tempDir, "docker.log"); + const curlLog = path.join(tempDir, "curl.log"); + const validatorLog = path.join(tempDir, "validator.log"); + const validatorPath = path.join(tempDir, "mock-observability-validator.sh"); + fs.mkdirSync(binDir, { recursive: true }); + + writeExecutable(path.join(binDir, "docker"), `#!/bin/sh +printf '%s\\n' "$*" >> "$MOCK_DOCKER_LOG" +if [ "$1" = "compose" ] && [ "$2" = "version" ]; then + exit 0 +fi +if [ "$1" = "inspect" ]; then + printf 'healthy' + exit 0 +fi + +last_arg="" +subcommand="" +skip_next=0 +for arg in "$@"; do + if [ "$skip_next" = "1" ]; then + skip_next=0 + continue + fi + case "$arg" in + compose) + ;; + -f) + skip_next=1 + ;; + up|ps) + subcommand="$arg" + ;; + *) + last_arg="$arg" + ;; + esac +done + +case "$subcommand" in + up) + exit 0 + ;; + ps) + printf 'container-%s\\n' "$last_arg" + exit 0 + ;; +esac + +exit 0 +`); + + writeExecutable(path.join(binDir, "curl"), `#!/bin/sh +printf '%s\\n' "$*" >> "$MOCK_CURL_LOG" +output_file="" +write_format="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o|-w|-D|-c|-b|-u|-H|-X) + if [ "$1" = "-o" ]; then + output_file="$2" + elif [ "$1" = "-w" ]; then + write_format="$2" + fi + shift 2 + ;; + -s|-S|-f|-fsS|-k|-L) + shift 1 + ;; + *) + url="$1" + shift 1 + ;; + esac +done + +status="200" +body='{}' +case "$url" in + *"/api/v1/auth/csrf") + status="\${MOCK_AUTH_CSRF_STATUS:-200}" + body='{"token":"csrf-token","headerName":"X-CSRF-TOKEN"}' + ;; + *"/v3/api-docs") + body='{"openapi":"3.0.1"}' + ;; + *"/swagger-ui/index.html") + body='Swagger UI' + ;; +esac + +if [ -n "$output_file" ]; then + printf '%s' "$body" > "$output_file" +fi +if [ -n "$write_format" ]; then + printf '%s' "$status" +fi +exit 0 +`); + + writeExecutable(validatorPath, `#!/usr/bin/env bash +set -euo pipefail +printf 'validator invoked\\n' >> "$MOCK_VALIDATOR_LOG" +printf 'Runtime observability checks passed\\n' +`); + + return { binDir, dockerLog, curlLog, validatorLog, validatorPath }; +} + +test("full-stack smoke script emits success evidence for cold-start, docs, and observability checks", () => { + const tempDir = makeTempDir("full-stack-smoke-success-"); + const outputDir = path.join(tempDir, "output"); + const { binDir, dockerLog, curlLog, validatorLog, validatorPath } = setupMockTooling(tempDir); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-full-stack-smoke.sh", { + timeout: 30000, + prependPathEntries: [binDir], + env: { + SMOKE_OUTPUT_DIR: outputDir, + SMOKE_START_TIMEOUT_SECONDS: "5", + MOCK_DOCKER_LOG: dockerLog, + MOCK_CURL_LOG: curlLog, + MOCK_VALIDATOR_LOG: validatorLog, + OBSERVABILITY_VALIDATOR_PATH: validatorPath, + INTERNAL_SECRET: "smoke-secret", + }, + }); + + assert.equal(result.status, 0, `smoke script failed: ${result.stderr}\n${result.stdout}`); + assert.match(result.stdout, /Cold-start smoke checks passed/); + + const coldStartReport = loadJson(path.join(outputDir, "cold-start-timing.json")); + const docsReport = loadJson(path.join(outputDir, "docs-summary.json")); + const smokeSummary = loadJson(path.join(outputDir, "smoke-summary.json")); + + assert.equal(coldStartReport.status, "passed"); + assert.equal(coldStartReport.firstMandatoryApi.httpStatus, 200); + assert.equal(coldStartReport.firstMandatoryApi.withinTarget, true); + assert.equal(docsReport.status, "passed"); + assert.equal(smokeSummary.status, "passed"); + assert.equal(smokeSummary.scenarios[0].evidencePath, "cold-start-timing.json"); + assert.equal(smokeSummary.scenarios[1].evidencePath, "docs-summary.json"); + assert.equal(smokeSummary.checks.composeUpLog, "compose-up.log"); + assert.deepEqual(smokeSummary.scenarios.map((scenario) => scenario.id), [ + "E10-SMOKE-001", + "E10-SMOKE-002", + "E10-OBS-001", + "E10-OBS-002", + ]); + assert.ok(fs.existsSync(path.join(outputDir, "observability-validation.log"))); + + const dockerCalls = fs.readFileSync(dockerLog, "utf8"); + assert.match(dockerCalls, /compose version/); + assert.match(dockerCalls, /compose -f .* up -d mysql mysql-grant-repair redis corebank-service fep-gateway fep-simulator channel-service edge-gateway prometheus grafana/); + assert.match(dockerCalls, /compose -f .* ps -q channel-service/); + assert.match(dockerCalls, /inspect -f/); + + const curlCalls = fs.readFileSync(curlLog, "utf8"); + assert.match(curlCalls, /api\/v1\/auth\/csrf/); + assert.match(curlCalls, /v3\/api-docs/); + assert.match(curlCalls, /swagger-ui\/index\.html/); + + const validatorCalls = fs.readFileSync(validatorLog, "utf8"); + assert.match(validatorCalls, /validator invoked/); +}); + +test("full-stack smoke script writes failed evidence when mandatory API misses the cold-start target", () => { + const tempDir = makeTempDir("full-stack-smoke-timeout-"); + const outputDir = path.join(tempDir, "output"); + const { binDir, validatorPath } = setupMockTooling(tempDir); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-full-stack-smoke.sh", { + timeout: 30000, + prependPathEntries: [binDir], + env: { + SMOKE_OUTPUT_DIR: outputDir, + SMOKE_START_TIMEOUT_SECONDS: "1", + SMOKE_POLL_INTERVAL_SECONDS: "0", + MOCK_DOCKER_LOG: path.join(tempDir, "docker.log"), + MOCK_CURL_LOG: path.join(tempDir, "curl.log"), + MOCK_VALIDATOR_LOG: path.join(tempDir, "validator.log"), + OBSERVABILITY_VALIDATOR_PATH: validatorPath, + MOCK_AUTH_CSRF_STATUS: "503", + INTERNAL_SECRET: "smoke-secret", + }, + }); + + assert.equal(result.status, 0, "smoke script should emit failed evidence without aborting the workflow"); + + const coldStartReport = loadJson(path.join(outputDir, "cold-start-timing.json")); + const smokeSummary = loadJson(path.join(outputDir, "smoke-summary.json")); + + assert.equal(coldStartReport.status, "failed"); + assert.equal(coldStartReport.firstMandatoryApi.httpStatus, 503); + assert.equal(coldStartReport.firstMandatoryApi.withinTarget, false); + assert.equal(smokeSummary.status, "failed"); + assert.equal(smokeSummary.scenarios[0].status, "failed"); + assert.equal(smokeSummary.checks.mandatoryApi, "failed"); +}); diff --git a/tests/release-readiness/live-dashboard-account-runtime.test.js b/tests/release-readiness/live-dashboard-account-runtime.test.js new file mode 100644 index 0000000..0e25072 --- /dev/null +++ b/tests/release-readiness/live-dashboard-account-runtime.test.js @@ -0,0 +1,129 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); + +const repoRoot = path.resolve(__dirname, "..", ".."); + +test("live dashboard helper step-up verifies pending order sessions with a fresh OTP", async () => { + const moduleUrl = pathToFileURL(path.join(repoRoot, "scripts", "story-11-5-live-dashboard-account.mjs")).href; + const helperModule = await import(moduleUrl); + const { createProvisionedStory115DashboardAccount } = helperModule; + + const originalFetch = global.fetch; + const originalDateNow = Date.now; + let fakeNow = 1_710_000_000_000; + let csrfCounter = 0; + let confirmOtpCode = null; + let verifyOtpCode = null; + let executeCalls = 0; + + Date.now = () => fakeNow; + global.fetch = async (url, init = {}) => { + const requestUrl = new URL(url); + const { pathname } = requestUrl; + const method = init.method ?? "GET"; + const response = (status, body) => new Response(JSON.stringify({ + success: true, + data: body, + }), { + status, + headers: { + "content-type": "application/json", + }, + }); + + if (pathname === "/api/v1/auth/csrf" && method === "GET") { + csrfCounter += 1; + return response(200, { + csrfToken: `csrf-${csrfCounter}`, + headerName: "X-CSRF-TOKEN", + }); + } + + if (pathname === "/api/v1/auth/register" && method === "POST") { + return response(200, {}); + } + + if (pathname === "/api/v1/auth/login" && method === "POST") { + return response(200, { + loginToken: "login-token", + }); + } + + if (pathname === "/api/v1/members/me/totp/enroll" && method === "POST") { + return response(200, { + manualEntryKey: "JBSWY3DPEHPK3PXP", + enrollmentToken: "enrollment-token", + }); + } + + if (pathname === "/api/v1/members/me/totp/confirm" && method === "POST") { + confirmOtpCode = JSON.parse(init.body).otpCode; + return response(200, { + verified: true, + }); + } + + if (pathname === "/api/v1/auth/session" && method === "GET") { + return response(200, { + accountId: 101, + }); + } + + if (pathname === "/api/v1/orders/sessions" && method === "POST") { + fakeNow += 30_000; + return response(201, { + orderSessionId: "9849b374-bb4a-4684-94f3-4f238d8a19b2", + status: "PENDING_NEW", + challengeRequired: true, + authorizationReason: "ELEVATED_ORDER_RISK", + }); + } + + if (pathname === "/api/v1/orders/sessions/9849b374-bb4a-4684-94f3-4f238d8a19b2/otp/verify" && method === "POST") { + verifyOtpCode = JSON.parse(init.body).otpCode; + return response(200, { + orderSessionId: "9849b374-bb4a-4684-94f3-4f238d8a19b2", + status: "AUTHED", + challengeRequired: true, + authorizationReason: "ELEVATED_ORDER_RISK", + }); + } + + if (pathname === "/api/v1/orders/sessions/9849b374-bb4a-4684-94f3-4f238d8a19b2/execute" && method === "POST") { + executeCalls += 1; + return response(200, { + orderSessionId: "9849b374-bb4a-4684-94f3-4f238d8a19b2", + status: "COMPLETED", + executionResult: "FILLED", + }); + } + + throw new Error(`Unexpected ${method} ${pathname}`); + }; + + try { + const provisioned = await createProvisionedStory115DashboardAccount({ + baseUrl: "http://127.0.0.1:8080", + password: "LiveVideo115!", + skipDashboardQuoteWait: true, + requestTimeoutMs: 1000, + pollTimeoutMs: 1000, + orderExecuteRetryCount: 1, + }); + + assert.equal(provisioned.createdSession.status, "PENDING_NEW"); + assert.equal(provisioned.authorizedSession.status, "AUTHED"); + assert.equal(provisioned.executedSession.status, "COMPLETED"); + assert.equal(executeCalls, 1); + assert.ok(confirmOtpCode); + assert.ok(verifyOtpCode); + assert.notEqual(verifyOtpCode, confirmOtpCode); + } finally { + global.fetch = originalFetch; + Date.now = originalDateNow; + } +}); diff --git a/tests/release-readiness/release-readiness-docs.test.js b/tests/release-readiness/release-readiness-docs.test.js new file mode 100644 index 0000000..78d561b --- /dev/null +++ b/tests/release-readiness/release-readiness-docs.test.js @@ -0,0 +1,43 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const repoRoot = path.resolve(__dirname, "..", ".."); +const runbookPath = path.join(repoRoot, "docs", "ops", "full-stack-smoke-rehearsal-runbook.md"); +const checklistPath = path.join(repoRoot, "docs", "ops", "release-go-no-go-checklist-template.md"); + +function read(filePath) { + return fs.readFileSync(filePath, "utf8"); +} + +test("full-stack smoke rehearsal runbook links canonical Story 10.4 automation and rollback flow", () => { + const runbook = read(runbookPath); + + assert.match(runbook, /run-full-stack-smoke\.sh/); + assert.match(runbook, /run-edge-gateway-validation\.sh/); + assert.match(runbook, /run-five-session-isolation\.mjs/); + assert.match(runbook, /run-rollback-rehearsal\.sh/); + assert.match(runbook, /assemble:story-10-4:evidence/); + assert.match(runbook, /deterministic re-run/i); + assert.match(runbook, /go-no-go-summary\.json/); + assert.match(runbook, /go-no-go-summary\.md/); + assert.match(runbook, /matrix-summary\.json/); + assert.match(runbook, /matrix-summary\.md/); +}); + +test("go-no-go checklist template keeps Story 10.4 evidence and blocking rules explicit", () => { + const checklist = read(checklistPath); + + assert.match(checklist, /smoke-summary\.json/); + assert.match(checklist, /edge-summary\.json/); + assert.match(checklist, /session-isolation-summary\.json/); + assert.match(checklist, /rollback-rehearsal-summary\.json/); + assert.match(checklist, /go-no-go-summary\.json/); + assert.match(checklist, /matrix-summary\.json/); + assert.match(checklist, /matrix-summary\.md/); + assert.match(checklist, /Missing Story `10\.4` evidence keeps the release at `no-go`\./); + assert.match(checklist, /Release readiness remains `no-go` when smoke, edge validation, session isolation, or rollback rehearsal is not `passed`\./); +}); diff --git a/tests/release-readiness/rollback-rehearsal-runtime.test.js b/tests/release-readiness/rollback-rehearsal-runtime.test.js new file mode 100644 index 0000000..00fe59e --- /dev/null +++ b/tests/release-readiness/rollback-rehearsal-runtime.test.js @@ -0,0 +1,163 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const repoRoot = path.resolve(__dirname, "..", ".."); +const { runBashScript } = require("../helpers/bash-script-test-utils"); + +function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function writeJson(filePath, payload) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(payload, null, 2)); +} + +function loadJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function writeExecutable(filePath, contents) { + fs.writeFileSync(filePath, contents, { encoding: "utf8", mode: 0o755 }); +} + +test("rollback rehearsal simulate mode writes passed rollback summary and go decision", () => { + const tempDir = makeTempDir("rollback-rehearsal-simulate-"); + const outputDir = path.join(tempDir, "output"); + const smokeSummaryPath = path.join(tempDir, "smoke-summary.json"); + const sessionSummaryPath = path.join(tempDir, "session-isolation-summary.json"); + + writeJson(smokeSummaryPath, { status: "passed" }); + const edgeSummaryPath = path.join(tempDir, "edge-summary.json"); + writeJson(sessionSummaryPath, { status: "passed" }); + writeJson(edgeSummaryPath, { status: "passed" }); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-rollback-rehearsal.sh", { + env: { + ROLLBACK_REHEARSAL_OUTPUT_DIR: outputDir, + ROLLBACK_REHEARSAL_SMOKE_SUMMARY_PATH: smokeSummaryPath, + ROLLBACK_REHEARSAL_EDGE_SUMMARY_PATH: edgeSummaryPath, + ROLLBACK_REHEARSAL_SESSION_SUMMARY_PATH: sessionSummaryPath, + ROLLBACK_REHEARSAL_MODE: "simulate", + ROLLBACK_REHEARSAL_OPERATOR: "release-manager-a", + ROLLBACK_REHEARSAL_CHANGE_REF: "CRQ-104", + ROLLBACK_REHEARSAL_OWNER: "platform-oncall-a", + }, + }); + + assert.equal(result.status, 0, `rollback rehearsal failed: ${result.stderr}\n${result.stdout}`); + assert.match(result.stdout, /decision: go/); + + const rollbackSummary = loadJson(path.join(outputDir, "rollback-rehearsal-summary.json")); + const goNoGoSummary = loadJson(path.join(outputDir, "go-no-go-summary.json")); + + assert.equal(rollbackSummary.status, "passed"); + assert.equal(rollbackSummary.mode, "simulate"); + assert.equal(rollbackSummary.rollbackAction, "simulated"); + assert.equal(goNoGoSummary.decision, "go"); + assert.equal(goNoGoSummary.releaseReady, true); + assert.deepEqual(goNoGoSummary.blockers, []); +}); + +test("rollback rehearsal keeps release at no-go when linked smoke evidence is failed", () => { + const tempDir = makeTempDir("rollback-rehearsal-no-go-"); + const outputDir = path.join(tempDir, "output"); + const smokeSummaryPath = path.join(tempDir, "smoke-summary.json"); + const sessionSummaryPath = path.join(tempDir, "session-isolation-summary.json"); + + writeJson(smokeSummaryPath, { status: "failed" }); + const edgeSummaryPath = path.join(tempDir, "edge-summary.json"); + writeJson(sessionSummaryPath, { status: "passed" }); + writeJson(edgeSummaryPath, { status: "passed" }); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-rollback-rehearsal.sh", { + env: { + ROLLBACK_REHEARSAL_OUTPUT_DIR: outputDir, + ROLLBACK_REHEARSAL_SMOKE_SUMMARY_PATH: smokeSummaryPath, + ROLLBACK_REHEARSAL_EDGE_SUMMARY_PATH: edgeSummaryPath, + ROLLBACK_REHEARSAL_SESSION_SUMMARY_PATH: sessionSummaryPath, + ROLLBACK_REHEARSAL_MODE: "simulate", + }, + }); + + assert.equal(result.status, 0, `rollback rehearsal should still emit evidence for no-go: ${result.stderr}\n${result.stdout}`); + + const goNoGoSummary = loadJson(path.join(outputDir, "go-no-go-summary.json")); + assert.equal(goNoGoSummary.decision, "no-go"); + assert.equal(goNoGoSummary.releaseReady, false); + assert.match(goNoGoSummary.blockers.join("\n"), /Smoke summary is failed/); +}); + +test("rollback rehearsal execute mode runs docker compose re-apply when confirmed", () => { + const tempDir = makeTempDir("rollback-rehearsal-execute-"); + const outputDir = path.join(tempDir, "output"); + const smokeSummaryPath = path.join(tempDir, "smoke-summary.json"); + const edgeSummaryPath = path.join(tempDir, "edge-summary.json"); + const sessionSummaryPath = path.join(tempDir, "session-isolation-summary.json"); + const binDir = path.join(tempDir, "bin"); + const dockerLog = path.join(tempDir, "docker.log"); + + fs.mkdirSync(binDir, { recursive: true }); + writeJson(smokeSummaryPath, { status: "passed" }); + writeJson(edgeSummaryPath, { status: "passed" }); + writeJson(sessionSummaryPath, { status: "passed" }); + writeExecutable(path.join(binDir, "docker"), `#!/bin/sh +printf '%s\\n' "$*" >> "$MOCK_DOCKER_LOG" +if [ "$1" = "compose" ] && [ "$2" = "version" ]; then + exit 0 +fi +exit 0 +`); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-rollback-rehearsal.sh", { + prependPathEntries: [binDir], + env: { + ROLLBACK_REHEARSAL_OUTPUT_DIR: outputDir, + ROLLBACK_REHEARSAL_SMOKE_SUMMARY_PATH: smokeSummaryPath, + ROLLBACK_REHEARSAL_EDGE_SUMMARY_PATH: edgeSummaryPath, + ROLLBACK_REHEARSAL_SESSION_SUMMARY_PATH: sessionSummaryPath, + ROLLBACK_REHEARSAL_MODE: "execute", + ROLLBACK_REHEARSAL_CONFIRM_EXECUTE: "1", + MOCK_DOCKER_LOG: dockerLog, + }, + }); + + assert.equal(result.status, 0, `execute-mode rollback rehearsal failed: ${result.stderr}\n${result.stdout}`); + + const rollbackSummary = loadJson(path.join(outputDir, "rollback-rehearsal-summary.json")); + assert.equal(rollbackSummary.rollbackAction, "executed"); + + const dockerCalls = fs.readFileSync(dockerLog, "utf8"); + assert.match(dockerCalls, /compose version/); + assert.match(dockerCalls, /compose -f .* up -d edge-gateway channel-service corebank-service fep-gateway fep-simulator prometheus grafana/); +}); + +test("rollback rehearsal keeps no-go evidence when edge summary is missing", () => { + const tempDir = makeTempDir("rollback-rehearsal-missing-edge-"); + const outputDir = path.join(tempDir, "output"); + const smokeSummaryPath = path.join(tempDir, "smoke-summary.json"); + const sessionSummaryPath = path.join(tempDir, "session-isolation-summary.json"); + + writeJson(smokeSummaryPath, { status: "passed" }); + writeJson(sessionSummaryPath, { status: "passed" }); + + const result = runBashScript(repoRoot, "scripts/release-readiness/run-rollback-rehearsal.sh", { + env: { + ROLLBACK_REHEARSAL_OUTPUT_DIR: outputDir, + ROLLBACK_REHEARSAL_SMOKE_SUMMARY_PATH: smokeSummaryPath, + ROLLBACK_REHEARSAL_SESSION_SUMMARY_PATH: sessionSummaryPath, + ROLLBACK_REHEARSAL_MODE: "simulate", + }, + }); + + assert.equal(result.status, 0, `rollback rehearsal should still emit no-go evidence: ${result.stderr}\n${result.stdout}`); + + const goNoGoSummary = loadJson(path.join(outputDir, "go-no-go-summary.json")); + assert.equal(goNoGoSummary.decision, "no-go"); + assert.match(goNoGoSummary.blockers.join("\n"), /Edge validation summary is missing/); +}); diff --git a/tests/release-readiness/session-isolation-runtime.test.js b/tests/release-readiness/session-isolation-runtime.test.js new file mode 100644 index 0000000..6b487f4 --- /dev/null +++ b/tests/release-readiness/session-isolation-runtime.test.js @@ -0,0 +1,127 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const repoRoot = path.resolve(__dirname, "..", ".."); + +function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function loadJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function writeMockHelperModule(filePath) { + fs.writeFileSync(filePath, `let invocation = 0; + +function buildCookieJar(sessionValue, xsrfValue) { + return { + toPlaywrightCookies() { + return [ + { name: "SESSION", value: sessionValue }, + { name: "XSRF-TOKEN", value: xsrfValue }, + ]; + }, + }; +} + +export async function createProvisionedStory115DashboardAccount() { + invocation += 1; + const mode = process.env.SESSION_ISOLATION_MOCK_MODE ?? "success"; + const sessionSuffix = mode === "duplicate-session" ? "shared-session" : "session-" + invocation; + + return { + identity: { + email: "session-" + invocation + "@example.com", + name: "Session " + invocation, + }, + accountId: String(1000 + invocation), + cookieJar: buildCookieJar(sessionSuffix, "xsrf-" + invocation), + createdSession: { + orderSessionId: "order-session-" + invocation, + }, + executedSession: { + orderSessionId: "order-session-" + invocation, + status: "COMPLETED", + }, + dashboardData: { + summary: { + quoteAsOf: "2026-03-25T00:00:00Z", + }, + positions: [ + { + marketPrice: 10000, + quoteAsOf: "2026-03-25T00:00:00Z", + }, + ], + }, + }; +} +`, "utf8"); +} + +function runNodeScript(scriptPath, env = {}) { + return spawnSync("node", [scriptPath], { + cwd: repoRoot, + env: { ...process.env, ...env }, + encoding: "utf8", + timeout: 30000, + maxBuffer: 1024 * 1024, + }); +} + +test("session isolation script records five isolated authenticated sessions", () => { + const tempDir = makeTempDir("session-isolation-success-"); + const helperPath = path.join(tempDir, "mock-session-helper.mjs"); + const outputDir = path.join(tempDir, "output"); + writeMockHelperModule(helperPath); + + const result = runNodeScript("scripts/release-readiness/run-five-session-isolation.mjs", { + SESSION_ISOLATION_HELPER_MODULE: helperPath, + SESSION_ISOLATION_OUTPUT_DIR: outputDir, + SESSION_ISOLATION_SESSION_COUNT: "5", + }); + + assert.equal(result.status, 0, `session isolation script failed: ${result.stderr}\n${result.stdout}`); + + const summary = loadJson(path.join(outputDir, "session-isolation-summary.json")); + assert.equal(summary.storyId, "10.4"); + assert.equal(summary.criterion, "AC6"); + assert.equal(summary.status, "passed"); + assert.equal(summary.checks.expectedSessionCount, 5); + assert.equal(summary.checks.actualSessionCount, 5); + assert.equal(summary.checks.uniqueAccounts, true); + assert.equal(summary.checks.uniqueSessionCookies, true); + assert.equal(summary.checks.uniqueXsrfTokens, true); + assert.equal(summary.checks.uniqueOrderSessions, true); + assert.equal(summary.sessions.length, 5); + assert.equal(new Set(summary.sessions.map((session) => session.sessionCookieHash)).size, 5); +}); + +test("session isolation script fails when independent sessions reuse the same session cookie", () => { + const tempDir = makeTempDir("session-isolation-duplicate-"); + const helperPath = path.join(tempDir, "mock-session-helper.mjs"); + const outputDir = path.join(tempDir, "output"); + writeMockHelperModule(helperPath); + + const result = runNodeScript("scripts/release-readiness/run-five-session-isolation.mjs", { + SESSION_ISOLATION_HELPER_MODULE: helperPath, + SESSION_ISOLATION_OUTPUT_DIR: outputDir, + SESSION_ISOLATION_SESSION_COUNT: "5", + SESSION_ISOLATION_MOCK_MODE: "duplicate-session", + }); + + assert.notEqual(result.status, 0, "session isolation script should fail for duplicate session cookies"); + + const summary = loadJson(path.join(outputDir, "session-isolation-summary.json")); + assert.equal(summary.status, "failed"); + assert.equal(summary.checks.uniqueAccounts, true); + assert.equal(summary.checks.uniqueSessionCookies, false); + assert.equal(summary.sessions.every((session) => session.status === "passed"), true); +}); diff --git a/tests/release-readiness/story-10-4-evidence-runtime.test.js b/tests/release-readiness/story-10-4-evidence-runtime.test.js new file mode 100644 index 0000000..db22c04 --- /dev/null +++ b/tests/release-readiness/story-10-4-evidence-runtime.test.js @@ -0,0 +1,143 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const repoRoot = path.resolve(__dirname, "..", ".."); +const assemblerPath = path.join(repoRoot, "scripts", "release-readiness", "assemble-story-10-4-evidence.mjs"); + +function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function writeJson(filePath, payload) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(payload, null, 2)); +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function runAssembler(outputDir, extraEnv = {}) { + return spawnSync("node", [assemblerPath], { + cwd: repoRoot, + env: { + ...process.env, + STORY_10_4_OUTPUT_DIR: outputDir, + STORY_10_4_BUILD_ID: "test-build", + ...extraEnv, + }, + encoding: "utf8", + timeout: 30000, + }); +} + +function writePassingEvidence(outputDir) { + writeJson(path.join(outputDir, "cold-start-timing.json"), { + status: "passed", + firstMandatoryApi: { + durationMs: 84211, + withinTarget: true, + }, + }); + + writeJson(path.join(outputDir, "docs-summary.json"), { + status: "passed", + }); + + writeJson(path.join(outputDir, "edge-summary.json"), { + status: "passed", + }); + + writeJson(path.join(outputDir, "smoke-summary.json"), { + status: "passed", + scenarios: [ + { id: "E10-SMOKE-001", status: "passed", evidencePath: path.join(outputDir, "cold-start-timing.json") }, + { id: "E10-SMOKE-002", status: "passed", evidencePath: path.join(outputDir, "docs-summary.json") }, + { id: "E10-OBS-001", status: "passed", evidencePath: path.join(outputDir, "observability-validation.log") }, + { id: "E10-OBS-002", status: "passed", evidencePath: path.join(outputDir, "observability-validation.log") }, + ], + }); + + writeJson(path.join(outputDir, "session-isolation-summary.json"), { + status: "passed", + }); + + writeJson(path.join(outputDir, "rollback-rehearsal-summary.json"), { + status: "passed", + }); + + writeJson(path.join(outputDir, "go-no-go-summary.json"), { + decision: "go", + releaseReady: true, + blockers: [], + }); + + fs.writeFileSync(path.join(outputDir, "observability-validation.log"), "targets UP\ngrafana reachable\n"); +} + +test("story 10.4 evidence assembler writes passed matrix summary when all evidence is green", () => { + const tempDir = makeTempDir("story-10-4-evidence-pass-"); + writePassingEvidence(tempDir); + + const result = runAssembler(tempDir); + assert.equal(result.status, 0, `assembler failed: ${result.stderr}\n${result.stdout}`); + + const summary = readJson(path.join(tempDir, "matrix-summary.json")); + const markdown = fs.readFileSync(path.join(tempDir, "matrix-summary.md"), "utf8"); + + assert.equal(summary.overallResult, "PASSED"); + assert.equal(summary.goNoGo.decision, "go"); + assert.equal(summary.goNoGo.releaseReady, true); + assert.equal(summary.scenarios.length, 6); + assert.ok(summary.scenarios.every((scenario) => scenario.result === "PASSED")); + assert.equal(summary.edge.status, "PASSED"); + assert.match(markdown, /Story 10\.4 Full-Stack Smoke\/Rehearsal Summary/); + assert.match(markdown, /E10-SESSION-001/); +}); + +test("story 10.4 evidence assembler fails closed when evidence is missing or go-no-go stays no-go", () => { + const tempDir = makeTempDir("story-10-4-evidence-fail-"); + writePassingEvidence(tempDir); + fs.rmSync(path.join(tempDir, "session-isolation-summary.json")); + writeJson(path.join(tempDir, "go-no-go-summary.json"), { + decision: "no-go", + releaseReady: false, + blockers: ["Rollback rehearsal is pending."], + }); + + const result = runAssembler(tempDir); + assert.equal(result.status, 1, "assembler should fail when release evidence is incomplete"); + + const summary = readJson(path.join(tempDir, "matrix-summary.json")); + assert.equal(summary.overallResult, "FAILED"); + assert.equal(summary.goNoGo.decision, "no-go"); + assert.match(summary.goNoGo.blockers.join("\n"), /Rollback rehearsal is pending/); + + const sessionScenario = summary.scenarios.find((scenario) => scenario.scenarioId === "E10-SESSION-001"); + assert.equal(sessionScenario.result, "MISSING"); +}); + +test("story 10.4 evidence assembler fails when go decision is inconsistent with releaseReady false", () => { + const tempDir = makeTempDir("story-10-4-evidence-release-ready-"); + writePassingEvidence(tempDir); + writeJson(path.join(tempDir, "go-no-go-summary.json"), { + decision: "go", + releaseReady: false, + blockers: [], + }); + + const result = runAssembler(tempDir); + assert.equal(result.status, 1, "assembler should fail when releaseReady is false"); + assert.match(result.stderr, /Story 10\.4 evidence gate failed/); + + const summary = readJson(path.join(tempDir, "matrix-summary.json")); + assert.equal(summary.overallResult, "FAILED"); + assert.equal(summary.goNoGo.decision, "go"); + assert.equal(summary.goNoGo.releaseReady, false); +});