From e649636fae85ef2f2ac33ec5d36531e79d8675ea Mon Sep 17 00:00:00 2001 From: rogu3bear Date: Wed, 1 Jul 2026 02:10:22 -0500 Subject: [PATCH] fix previews duplicate cleanup Repeated safe plan refreshes can leave several active preview receipts for the same request, which makes maildesk sender-domain readiness look noisier than it is. Add a cfctl previews purge-duplicate-active subcommand that groups active trusted receipts by lane plus policy/request/target fingerprints, keeps the newest receipt, and removes only older local artifacts. Expired and legacy cleanup remain separate paths. Extend the static and public contract checks plus README/runbook/runtime-policy docs so the new cleanup stays discoverable and remains clearly outside Cloudflare mutation. Risk is limited to local ignored preview inventory cleanup; ack-plan writes are unchanged. --- README.md | 3 + commands/cfctl.sh | 96 ++++++++++++++++++++++++- docs/runbooks/cfctl.md | 2 + docs/runtime-policy.md | 2 +- scripts/verify_public_contract.sh | 4 ++ scripts/verify_static_contract.sh | 116 ++++++++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d52330..962a612 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,7 @@ CF_TOKEN_LANE=global CFCTL_PUBLIC_CONTRACT_ZONE=example.com ./scripts/verify_pub cfctl previews cfctl previews purge-expired cfctl previews purge-inactive-legacy +cfctl previews purge-duplicate-active cfctl locks cfctl locks clear-stale cfctl admin authorizations @@ -297,6 +298,8 @@ cfctl admin revoke-backend --path `bootstrap_required`, while configured-but-unhealthy lanes remain unsafe. - `cfctl previews purge-inactive-legacy` removes only legacy preview receipts without complete trust metadata; active trusted previews are not targeted. +- `cfctl previews purge-duplicate-active` removes older active preview receipts + when a newer trusted receipt exists for the same lane, target, request, and policy. - Direct API wrappers: account inventory, DNS, Access, tunnels, email routing, targeted writes. - `cfctl wrangler ...` via [scripts/cf_wrangler.sh](scripts/cf_wrangler.sh): wrapped wrangler with cfctl logs, artifacts, and preview gating. - `cfctl cloudflared ...` via [scripts/cf_cloudflared.sh](scripts/cf_cloudflared.sh): wrapped cloudflared with the same envelope. diff --git a/commands/cfctl.sh b/commands/cfctl.sh index 809ea18..24da27e 100644 --- a/commands/cfctl.sh +++ b/commands/cfctl.sh @@ -46,6 +46,7 @@ Usage: cfctl previews cfctl previews purge-expired cfctl previews purge-inactive-legacy + cfctl previews purge-duplicate-active cfctl locks cfctl locks clear-stale cfctl env run [--lane dev|global] -- [args...] @@ -80,7 +81,7 @@ Verb intent: surfaces List supported surfaces with read/write support, lane fit, and desired-state support. docs Show the compact official Cloudflare docs bank and tracked incoming capabilities. standards Show the canonical configuration standards for this runtime, one surface, or a workspace audit. - previews Inspect actionable, legacy, and expired preview receipts, and purge expired ones. + previews Inspect actionable, legacy, and duplicate preview receipts, and purge safe local preview artifacts. locks Inspect write locks and clear stale ones. env Run an external argv command with lane-derived Cloudflare auth and redacted output. ownership Read and verify the checked-in Cloudflare ownership authority registry. @@ -122,6 +123,7 @@ Examples: cfctl previews cfctl previews purge-expired cfctl previews purge-inactive-legacy + cfctl previews purge-duplicate-active cfctl locks cfctl locks clear-stale cfctl env run --lane dev -- env @@ -898,6 +900,9 @@ cfctl_preview_rows_json() { preview_expires_at: (.trust.preview_expires_at // null), lock_mode: (.trust.lock_mode // null), lock_key: (.trust.lock_key // null), + policy_fingerprint: (.trust.policy_fingerprint // null), + request_fingerprint: (.trust.request_fingerprint // null), + target_fingerprint: (.trust.target_fingerprint // null), trust_complete: $trust_complete, expired: $expired } @@ -965,6 +970,80 @@ cfctl_preview_purge_inactive_legacy_json() { ' } +cfctl_preview_purge_duplicate_active_json() { + local previews_json + local duplicate_json + local results='[]' + local candidate + local path + local removed + + previews_json="$(cfctl_preview_rows_json)" + duplicate_json="$( + jq ' + [ + .previews[] + | select(.expired != true and .trust_complete == true) + | . + { + duplicate_key: ([ + (.action // ""), + (.surface // ""), + (.operation // ""), + (.lane // ""), + (.policy_fingerprint // ""), + (.request_fingerprint // ""), + (.target_fingerprint // "") + ] | @json) + } + ] + | group_by(.duplicate_key) + | map( + select(length > 1) + | (sort_by((.generated_at // ""), (.artifact_path // ""))) as $ordered + | { + duplicate_key: ($ordered[0].duplicate_key // ""), + kept_artifact_path: ($ordered[-1].artifact_path // null), + kept_operation_id: ($ordered[-1].operation_id // null), + removed: $ordered[0:-1] + } + ) as $groups + | { + duplicate_group_count: ($groups | length), + candidates: ($groups | map(.removed[]) ) + } + ' <<< "${previews_json}" + )" + + while IFS= read -r candidate; do + [[ -n "${candidate}" ]] || continue + path="$(jq -r '.artifact_path // empty' <<< "${candidate}")" + [[ -n "${path}" ]] || continue + removed="false" + if [[ -f "${path}" ]]; then + rm -f "${path}" + removed="true" + fi + results="$( + jq \ + --argjson candidate "${candidate}" \ + --argjson removed "${removed}" \ + '. + [($candidate + {removed: $removed})]' \ + <<< "${results}" + )" + done < <(jq -c '.candidates[]' <<< "${duplicate_json}") + + jq -n \ + --argjson duplicate_json "${duplicate_json}" \ + --argjson results "${results}" \ + ' + { + purged_count: ($results | map(select(.removed == true)) | length), + duplicate_group_count: ($duplicate_json.duplicate_group_count // 0), + results: $results + } + ' +} + cfctl_lock_rows_json() { local lock_health_json @@ -1541,12 +1620,27 @@ cfctl_handle_previews() { "${purge_json}" \ "" ;; + purge-duplicate-active) + purge_json="$(cfctl_preview_purge_duplicate_active_json)" + cfctl_emit_result \ + "true" \ + "previews" \ + "runtime" \ + "runtime" \ + "true" \ + '{"state":"not_applicable","basis":"preview_cleanup","errors":[],"request":null,"status_code":null,"permission_family":"Cloudflare API"}' \ + '{"state":"not_applicable"}' \ + "$(jq '{purged_count: .purged_count, duplicate_group_count: .duplicate_group_count}' <<< "${purge_json}")" \ + "${purge_json}" \ + "" + ;; -h|--help|help) cat <<'EOF' Usage: cfctl previews cfctl previews purge-expired cfctl previews purge-inactive-legacy + cfctl previews purge-duplicate-active EOF ;; *) diff --git a/docs/runbooks/cfctl.md b/docs/runbooks/cfctl.md index 3dead96..76ec870 100644 --- a/docs/runbooks/cfctl.md +++ b/docs/runbooks/cfctl.md @@ -47,6 +47,7 @@ CF_TOKEN_LANE=global CFCTL_PUBLIC_CONTRACT_ZONE=example.com ./scripts/verify_pub cfctl previews cfctl previews purge-expired cfctl previews purge-inactive-legacy +cfctl previews purge-duplicate-active cfctl locks cfctl locks clear-stale cfctl env run --lane dev -- env @@ -130,6 +131,7 @@ CF_TOKEN_LANE=global cfctl apply edge.certificate order --zone example.com --hos - `previews` lists actionable, legacy, and expired preview receipts - `previews purge-expired` removes expired preview receipts only - `previews purge-inactive-legacy` removes only legacy preview receipts that lack complete trust metadata +- `previews purge-duplicate-active` removes older active preview receipts only when a newer trusted receipt exists for the same lane, target, request, and policy - `locks` lists active write locks and their stale/orphaned state - `locks clear-stale` removes stale/orphaned locks only - `env run --lane dev -- [args...]` runs an external argv command with lane-derived Cloudflare auth, strips parent lane secrets, redacts child output, and records a runtime artifact diff --git a/docs/runtime-policy.md b/docs/runtime-policy.md index 60a09e5..1cd8b9e 100644 --- a/docs/runtime-policy.md +++ b/docs/runtime-policy.md @@ -30,7 +30,7 @@ cfctl guide ... - Preview-required operations must run with `--plan` first. - The reviewed preview emits an `operation_id`. - The real mutation must repeat the command with `--ack-plan `. -- Use `cfctl previews` to inspect preview receipts, `cfctl previews purge-expired` to remove expired ones, and `cfctl previews purge-inactive-legacy` to remove only legacy receipts without complete trust metadata. +- Use `cfctl previews` to inspect preview receipts, `cfctl previews purge-expired` to remove expired ones, `cfctl previews purge-inactive-legacy` to remove only legacy receipts without complete trust metadata, and `cfctl previews purge-duplicate-active` to remove older trusted active receipts only when a newer receipt exists for the same lane, target, request, and policy. - Use `cfctl locks` to inspect write locks and `cfctl locks clear-stale` to remove only stale/orphaned locks. - Destructive paths still require explicit confirmation such as `--confirm delete`. diff --git a/scripts/verify_public_contract.sh b/scripts/verify_public_contract.sh index a9f0075..d2d90b4 100755 --- a/scripts/verify_public_contract.sh +++ b/scripts/verify_public_contract.sh @@ -108,6 +108,10 @@ cleanup_legacy_previews_json="$(run_json success "previews purge-inactive-legacy assert_artifact_exists "previews purge-inactive-legacy" "${cleanup_legacy_previews_json}" assert_json "previews purge-inactive-legacy" '.ok == true and .action == "previews" and (.summary.purged_count // 0) >= 0' "${cleanup_legacy_previews_json}" +cleanup_duplicate_previews_json="$(run_json success "previews purge-duplicate-active" "${CFCTL}" previews purge-duplicate-active)" +assert_artifact_exists "previews purge-duplicate-active" "${cleanup_duplicate_previews_json}" +assert_json "previews purge-duplicate-active" '.ok == true and .action == "previews" and (.summary.purged_count // 0) >= 0 and (.summary.duplicate_group_count // 0) >= 0' "${cleanup_duplicate_previews_json}" + cleanup_locks_json="$(run_json success "locks clear-stale" "${CFCTL}" locks clear-stale)" assert_artifact_exists "locks clear-stale" "${cleanup_locks_json}" assert_json "locks clear-stale" '.ok == true and .action == "locks"' "${cleanup_locks_json}" diff --git a/scripts/verify_static_contract.sh b/scripts/verify_static_contract.sh index 08d035f..ab971d2 100755 --- a/scripts/verify_static_contract.sh +++ b/scripts/verify_static_contract.sh @@ -128,6 +128,117 @@ for surface_module in \ bash -n "${surface_module}" done +preview_dedupe_json="$( + ROOT_DIR="${ROOT_DIR}" bash <<'BASH' +set -euo pipefail + +tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/cfctl-preview-dedupe.XXXXXX")" +cleanup_tmp_root() { + local base + base="$(basename "${tmp_root}")" + if [[ "${base}" == cfctl-preview-dedupe.* && -d "${tmp_root}" ]]; then + rm -rf -- "${tmp_root}" + fi +} +trap cleanup_tmp_root EXIT + +# shellcheck disable=SC1091 +source "${ROOT_DIR}/lib/runtime/cfctl.sh" +# shellcheck disable=SC1091 +source "${ROOT_DIR}/commands/cfctl.sh" + +CF_REPO_ROOT="${tmp_root}" +preview_dir="${CF_REPO_ROOT}/var/inventory/runtime" +mkdir -p "${preview_dir}" + +write_preview() { + local path="$1" + local operation_id="$2" + local generated_at="$3" + local request_fingerprint="$4" + local target_fingerprint="$5" + local policy_fingerprint="$6" + local expires_at="$7" + + jq -n \ + --arg operation_id "${operation_id}" \ + --arg generated_at "${generated_at}" \ + --arg request_fingerprint "${request_fingerprint}" \ + --arg target_fingerprint "${target_fingerprint}" \ + --arg policy_fingerprint "${policy_fingerprint}" \ + --arg expires_at "${expires_at}" \ + '{ + generated_at: $generated_at, + ok: true, + action: "apply", + surface: "sender_domain", + operation: "enable", + operation_id: $operation_id, + auth: {lane: "global"}, + summary: {plan_mode: true}, + trust: { + lane: "global", + policy_fingerprint: $policy_fingerprint, + request_fingerprint: $request_fingerprint, + target_fingerprint: $target_fingerprint, + preview_expires_at: $expires_at + } + }' > "${path}" +} + +write_preview "${preview_dir}/duplicate-old.json" "op-old" "2026-01-01T00:00:00Z" "request-a" "target-a" "policy-a" "2099-01-01T00:00:00Z" +write_preview "${preview_dir}/duplicate-new.json" "op-new" "2026-01-02T00:00:00Z" "request-a" "target-a" "policy-a" "2099-01-01T00:00:00Z" +write_preview "${preview_dir}/distinct-active.json" "op-distinct" "2026-01-01T12:00:00Z" "request-b" "target-a" "policy-a" "2099-01-01T00:00:00Z" +write_preview "${preview_dir}/duplicate-expired.json" "op-expired" "2026-01-03T00:00:00Z" "request-a" "target-a" "policy-a" "2000-01-01T00:00:00Z" +jq -n '{ + generated_at: "2026-01-01T00:00:00Z", + ok: true, + action: "apply", + surface: "sender_domain", + operation: "enable", + operation_id: "op-legacy", + auth: {lane: "global"}, + summary: {plan_mode: true} +}' > "${preview_dir}/legacy.json" + +purge_json="$(cfctl_preview_purge_duplicate_active_json)" + +jq -n \ + --argjson purge "${purge_json}" \ + --arg duplicate_old "${preview_dir}/duplicate-old.json" \ + --arg duplicate_new "${preview_dir}/duplicate-new.json" \ + --arg distinct_active "${preview_dir}/distinct-active.json" \ + --arg duplicate_expired "${preview_dir}/duplicate-expired.json" \ + --arg legacy "${preview_dir}/legacy.json" \ + --argjson duplicate_old_exists "$([[ -f "${preview_dir}/duplicate-old.json" ]] && echo true || echo false)" \ + --argjson duplicate_new_exists "$([[ -f "${preview_dir}/duplicate-new.json" ]] && echo true || echo false)" \ + --argjson distinct_active_exists "$([[ -f "${preview_dir}/distinct-active.json" ]] && echo true || echo false)" \ + --argjson duplicate_expired_exists "$([[ -f "${preview_dir}/duplicate-expired.json" ]] && echo true || echo false)" \ + --argjson legacy_exists "$([[ -f "${preview_dir}/legacy.json" ]] && echo true || echo false)" \ + '{ + purge: $purge, + files: { + duplicate_old: {path: $duplicate_old, exists: $duplicate_old_exists}, + duplicate_new: {path: $duplicate_new, exists: $duplicate_new_exists}, + distinct_active: {path: $distinct_active, exists: $distinct_active_exists}, + duplicate_expired: {path: $duplicate_expired, exists: $duplicate_expired_exists}, + legacy: {path: $legacy, exists: $legacy_exists} + } + }' +BASH +)" +jq -e ' + .purge.purged_count == 1 + and .purge.duplicate_group_count == 1 + and (.purge.results | length) == 1 + and (.purge.results[0].operation_id == "op-old") + and (.files.duplicate_old.exists == false) + and (.files.duplicate_new.exists == true) + and (.files.distinct_active.exists == true) + and (.files.duplicate_expired.exists == true) + and (.files.legacy.exists == true) +' <<< "${preview_dedupe_json}" >/dev/null || die "preview duplicate active purge assertion failed" + python3 "${ROOT_DIR}/scripts/render_capabilities_doc.py" --check "${ROOT_DIR}/docs/capabilities.md" >/dev/null python3 "${ROOT_DIR}/scripts/verify_permission_catalog.py" >/dev/null python3 -m py_compile "${ROOT_DIR}/scripts/cf_maildesk_cf_lifecycle.py" @@ -793,10 +904,12 @@ assert_contains "cfctl prompt hostname" "For \`hostname\`, treat \`verify\`, \`d assert_contains "cfctl prompt maildesk" "For \`maildesk-cf\`, treat \`init\`, \`verify\`, \`snapshot\`, \`diff\`, \`plan\`, and \`provision --plan\` as read-only composite evidence flows" "${ROOT_DIR}/CFCTL_PROMPT.md" assert_contains "cfctl prompt wrapper gating" "For \`wrangler\` and \`cloudflared\`, treat clearly read-only subcommands as direct wrapped executions" "${ROOT_DIR}/CFCTL_PROMPT.md" assert_contains "cfctl preview inactive legacy cleanup command" "purge-inactive-legacy" "${ROOT_DIR}/commands/cfctl.sh" +assert_contains "cfctl preview duplicate active cleanup command" "purge-duplicate-active" "${ROOT_DIR}/commands/cfctl.sh" assert_contains "readme wrapper examples" "cfctl wrangler --version" "${ROOT_DIR}/README.md" assert_contains "readme env run" "cfctl env run --lane dev -- [args...]" "${ROOT_DIR}/README.md" assert_contains "readme env run argv secrecy" "do not pass secrets as command-line arguments" "${ROOT_DIR}/README.md" assert_contains "readme inactive legacy preview cleanup" "cfctl previews purge-inactive-legacy" "${ROOT_DIR}/README.md" +assert_contains "readme duplicate active preview cleanup" "cfctl previews purge-duplicate-active" "${ROOT_DIR}/README.md" assert_contains "readme source-live boundary" "Source Config Vs Live State" "${ROOT_DIR}/README.md" assert_contains "readme hostname lifecycle" "Hostname lifecycle" "${ROOT_DIR}/README.md" assert_contains "readme maildesk lifecycle" "maildesk-cf lifecycle" "${ROOT_DIR}/README.md" @@ -819,6 +932,7 @@ assert_contains "auth runbook env run" "CF_SHARED_ENV_FILE=/Users/star/dev/.env assert_contains "auth runbook env run argv secrecy" "Because argv is evidence, do not pass secrets as command" "${ROOT_DIR}/docs/runbooks/auth-and-env.md" assert_contains "runtime policy env run" "\`cfctl env run\` strips parent lane secrets" "${ROOT_DIR}/docs/runtime-policy.md" assert_contains "runbook inactive legacy preview cleanup" "previews purge-inactive-legacy" "${ROOT_DIR}/docs/runbooks/cfctl.md" +assert_contains "runbook duplicate active preview cleanup" "previews purge-duplicate-active" "${ROOT_DIR}/docs/runbooks/cfctl.md" assert_contains "runbook audit log read" "cfctl list audit.log" "${ROOT_DIR}/docs/runbooks/cfctl.md" assert_contains "runbook hostname lifecycle" "cfctl hostname verify --file state/hostname/example.yaml" "${ROOT_DIR}/docs/runbooks/cfctl.md" assert_contains "runbook maildesk lifecycle" "cfctl maildesk-cf verify --file state/maildesk-cf/example.json" "${ROOT_DIR}/docs/runbooks/cfctl.md" @@ -829,6 +943,7 @@ assert_contains "runbook standards audit source evidence" "standards audit\` is assert_contains "config standards compatibility freshness" "Compatibility-date freshness is intentionally advisory" "${ROOT_DIR}/docs/config-standards.md" assert_contains "config standards canonical notes" "\`canonical_warning_count\`" "${ROOT_DIR}/docs/config-standards.md" assert_contains "runtime policy inactive legacy preview cleanup" "cfctl previews purge-inactive-legacy" "${ROOT_DIR}/docs/runtime-policy.md" +assert_contains "runtime policy duplicate active preview cleanup" "cfctl previews purge-duplicate-active" "${ROOT_DIR}/docs/runtime-policy.md" assert_contains "capabilities operable note" "This table is the operable runtime surface." "${ROOT_DIR}/docs/capabilities.md" assert_contains "capabilities generated note" "_Generated from \`catalog/surfaces.json\` and \`catalog/runtime.json\`." "${ROOT_DIR}/docs/capabilities.md" assert_contains "capabilities module column" "| Surface | Read | Can | Apply | Verify | Desired State | Standards | Docs Topics | Module |" "${ROOT_DIR}/docs/capabilities.md" @@ -899,6 +1014,7 @@ assert_contains "contract workflow manual secret gate" "Live Cloudflare contract assert_contains "contract workflow scheduled secret skip" "Skipping scheduled live Cloudflare contract" "${ROOT_DIR}/.github/workflows/cfctl-contract.yml" assert_contains "contract workflow protected environment" "environment: cfctl-live" "${ROOT_DIR}/.github/workflows/cfctl-contract.yml" assert_contains "public contract inactive legacy preview cleanup" "previews purge-inactive-legacy" "${ROOT_DIR}/scripts/verify_public_contract.sh" +assert_contains "public contract duplicate active preview cleanup" "previews purge-duplicate-active" "${ROOT_DIR}/scripts/verify_public_contract.sh" assert_contains "permission doctrine source" "Cloudflare API token permissions are resource-scoped" "${ROOT_DIR}/docs/permission-doctrine.md" assert_contains "permission doctrine environment" "cfctl-live" "${ROOT_DIR}/docs/permission-doctrine.md" assert_contains "permission doctrine bootstrap creator" "Account API Tokens Write" "${ROOT_DIR}/docs/permission-doctrine.md"