Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 50 additions & 8 deletions .github/scripts/Invoke-GovernanceContract.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,54 @@ if ([string]::IsNullOrWhiteSpace($env:GH_TOKEN)) {
throw 'GH token is required. Set GH_ADMIN_TOKEN (preferred) or WORKFLOW_BOT_TOKEN/GH_TOKEN/GITHUB_TOKEN.'
}

$requiredContexts = @(
'CI Pipeline',
'Workspace Installer Contract',
'Reproducibility Contract',
'Provenance Contract'
)
function ConvertTo-BoolOrDefault {
param(
[Parameter()][AllowNull()][string]$Value = '',
[Parameter()][bool]$Default = $false
)

if ([string]::IsNullOrWhiteSpace([string]$Value)) {
return $Default
}

try {
return [System.Convert]::ToBoolean([string]$Value)
} catch {
$normalized = ([string]$Value).Trim().ToLowerInvariant()
if (@('1', 'yes', 'y', 'on') -contains $normalized) {
return $true
}
if (@('0', 'no', 'n', 'off') -contains $normalized) {
return $false
}
return $Default
}
}

$enableSelfHostedContracts = ConvertTo-BoolOrDefault -Value ([string]$env:ENABLE_SELF_HOSTED_CONTRACTS) -Default $false
$requirePullRequestReviews = ConvertTo-BoolOrDefault -Value ([string]$env:GOVERNANCE_REQUIRE_PR_REVIEWS) -Default $false
$requiredContexts = [System.Collections.Generic.List[string]]::new()
foreach ($context in @(
'CI Pipeline',
'Integration Gate',
'Release Race Hardening Drill'
)) {
if (-not $requiredContexts.Contains([string]$context)) {
[void]$requiredContexts.Add([string]$context)
}
}

if ($enableSelfHostedContracts) {
foreach ($context in @(
'Workspace Installer Contract',
'Reproducibility Contract',
'Provenance Contract'
)) {
if (-not $requiredContexts.Contains([string]$context)) {
[void]$requiredContexts.Add([string]$context)
}
}
}

$endpoint = "repos/$RepoSlug/branches/$([uri]::EscapeDataString($Branch))/protection"
$response = & gh api $endpoint 2>&1
Expand All @@ -68,13 +110,13 @@ if ($null -ne $protection.required_status_checks -and $null -ne $protection.requ
$actualContexts = @($protection.required_status_checks.contexts)
}

foreach ($context in $requiredContexts) {
foreach ($context in @($requiredContexts)) {
if ($actualContexts -notcontains $context) {
$issues += "missing required status context: $context"
}
}

if ($null -eq $protection.required_pull_request_reviews) {
if ($requirePullRequestReviews -and $null -eq $protection.required_pull_request_reviews) {
$issues += 'required_pull_request_reviews is not enabled'
}

Expand Down
16 changes: 13 additions & 3 deletions .github/workflows/branch-protection-drift-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,20 @@ jobs:
- name: Checkout
uses: actions/checkout@v4

- name: Validate workflow bot token
shell: pwsh
env:
WORKFLOW_BOT_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
run: |
$ErrorActionPreference = 'Stop'
if ([string]::IsNullOrWhiteSpace($env:WORKFLOW_BOT_TOKEN)) {
throw "workflow_bot_token_missing: Required secret WORKFLOW_BOT_TOKEN is not configured. Add a token with repository administration read/write access."
}

- name: Verify release branch-protection policy
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
run: |
$ErrorActionPreference = 'Stop'
$reportPath = Join-Path $env:RUNNER_TEMP 'branch-protection-drift-report.json'
Expand All @@ -44,7 +54,7 @@ jobs:
if: failure()
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
REPOSITORY: ${{ github.repository }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
Expand Down Expand Up @@ -78,7 +88,7 @@ jobs:
if: success()
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
REPOSITORY: ${{ github.repository }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/governance-contract.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jobs:
GH_ADMIN_TOKEN: ${{ secrets.GH_ADMIN_TOKEN }}
WORKFLOW_BOT_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
GH_TOKEN: ${{ github.token }}
ENABLE_SELF_HOSTED_CONTRACTS: ${{ vars.ENABLE_SELF_HOSTED_CONTRACTS }}
run: |
pwsh -NoProfile -File ./.github/scripts/Invoke-GovernanceContract.ps1 `
-RepoSlug '${{ github.repository }}' `
Expand Down
16 changes: 13 additions & 3 deletions .github/workflows/release-guardrails-autoremediate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,20 @@ jobs:
- name: Checkout
uses: actions/checkout@v4

- name: Validate workflow bot token
shell: pwsh
env:
WORKFLOW_BOT_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
run: |
$ErrorActionPreference = 'Stop'
if ([string]::IsNullOrWhiteSpace($env:WORKFLOW_BOT_TOKEN)) {
throw "workflow_bot_token_missing: Required secret WORKFLOW_BOT_TOKEN is not configured. Add a token with repository administration read/write and actions write scopes."
}

- name: Execute release guardrails auto-remediation
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
run: |
$ErrorActionPreference = 'Stop'
$reportPath = Join-Path $env:RUNNER_TEMP 'release-guardrails-autoremediate-report.json'
Expand Down Expand Up @@ -104,7 +114,7 @@ jobs:
if: failure()
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
REPOSITORY: ${{ github.repository }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
Expand Down Expand Up @@ -137,7 +147,7 @@ jobs:
if: success()
shell: pwsh
env:
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN != '' && secrets.WORKFLOW_BOT_TOKEN || github.token }}
GH_TOKEN: ${{ secrets.WORKFLOW_BOT_TOKEN }}
REPOSITORY: ${{ github.repository }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
Expand Down
7 changes: 6 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,12 @@ Build and gate lanes must run in isolated workspaces on every run (`D:\dev` pref
- Race-hardening gate must fail when latest successful drill evidence is missing/stale, `reason_code != drill_passed`, or collision evidence is absent.
- `.github/workflows/branch-protection-drift-check.yml` must run `scripts/Test-ReleaseBranchProtectionPolicy.ps1` and maintain incident lifecycle title `Branch Protection Drift Alert`.
- `.github/workflows/release-guardrails-autoremediate.yml` must run `scripts/Invoke-ReleaseGuardrailsSelfHealing.ps1` and maintain incident lifecycle title `Release Guardrails Auto-Remediation Alert`.
- Branch-protection query/apply workflows must use `WORKFLOW_BOT_TOKEN` when available, with deterministic fallback to `github.token`.
- Branch-protection query/apply workflows must require repository secret `WORKFLOW_BOT_TOKEN` and fail fast with deterministic `workflow_bot_token_missing` when the secret is not configured.
- `scripts/Set-ReleaseBranchProtectionPolicy.ps1` is the deterministic apply path for required-check drift repair.
- Branch-protection query classifier reason codes must remain explicit:
- `branch_protection_query_failed`
- `branch_protection_authentication_missing`
- `branch_protection_authz_denied`
- Guardrails self-healing policy must remain explicit under `ops_control_plane_policy.self_healing.guardrails`:
- `remediation_workflow`
- `race_drill_workflow`
Expand All @@ -329,6 +333,7 @@ Build and gate lanes must run in isolated workspaces on every run (`D:\dev` pref
- `remediation_execution_failed`
- `remediation_verify_failed`
- `guardrails_self_heal_runtime_error`
- Guardrails report must include `remediation_hints` when status is fail and auto-remediation cannot fully recover.
- Race-hardening drill reason codes must remain explicit:
- `drill_passed`
- `control_plane_collision_not_observed`
Expand Down
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,11 @@ It runs `scripts/Test-ReleaseRaceHardeningGate.ps1` and fails when:
- `integration/*`

Use `scripts/Set-ReleaseBranchProtectionPolicy.ps1` to deterministically apply/repair required check contracts.
Branch-protection workflows prefer `WORKFLOW_BOT_TOKEN` when available and deterministically fall back to `github.token`.
Branch-protection workflows require repository secret `WORKFLOW_BOT_TOKEN` and fail fast with `workflow_bot_token_missing` when absent.
Branch-protection query failures remain deterministic with classified reason codes:
- `branch_protection_query_failed`
- `branch_protection_authentication_missing`
- `branch_protection_authz_denied`

`release-guardrails-autoremediate.yml` is scheduled hourly and supports manual dispatch. It runs `scripts/Invoke-ReleaseGuardrailsSelfHealing.ps1` to:
- evaluate branch-protection drift and release race-hardening freshness in one pass
Expand All @@ -521,6 +525,7 @@ Branch-protection workflows prefer `WORKFLOW_BOT_TOKEN` when available and deter
- `remediation_execution_failed`
- `remediation_verify_failed`
- `guardrails_self_heal_runtime_error`
- include `remediation_hints` in the report when guardrails cannot self-heal (for token/authz and stale drill guidance)

Guardrails policy is codified in `installer_contract.release_client.ops_control_plane_policy.self_healing.guardrails`:
- `remediation_workflow`
Expand Down
12 changes: 10 additions & 2 deletions docs/runbooks/release-ops-incident-response.md
Original file line number Diff line number Diff line change
Expand Up @@ -320,8 +320,14 @@ gh workflow run branch-protection-drift-check.yml -R LabVIEW-Community-CI-CD/lab
```

Token policy for branch-protection workflows:
- prefer repository secret `WORKFLOW_BOT_TOKEN`
- deterministic fallback to `github.token` when the secret is unavailable
- require repository secret `WORKFLOW_BOT_TOKEN`
- workflows fail fast with `workflow_bot_token_missing` when the secret is unavailable
- token must include repository administration permissions for branch-protection GraphQL read/apply operations

Branch-protection query failure reason codes:
- `branch_protection_query_failed`
- `branch_protection_authentication_missing`
- `branch_protection_authz_denied`

Local policy verify:

Expand Down Expand Up @@ -373,6 +379,8 @@ Deterministic guardrails reason codes:
- `remediation_verify_failed`
- `guardrails_self_heal_runtime_error`

When `reason_code=no_automatable_action` or `reason_code=remediation_verify_failed`, inspect `remediation_hints` in `release-guardrails-autoremediate-report.json` for deterministic next actions.

Guardrails incident title:
- `Release Guardrails Auto-Remediation Alert`

Expand Down
41 changes: 41 additions & 0 deletions scripts/Invoke-ReleaseGuardrailsSelfHealing.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,32 @@ function Format-ReasonCodeSet {
return [string]::Join(',', @($normalized))
}

function Get-GuardrailsRemediationHints {
param(
[Parameter()][string[]]$BranchReasonCodes = @(),
[Parameter()][string[]]$RaceReasonCodes = @()
)

$hints = [System.Collections.Generic.List[string]]::new()
$normalizedBranchReasons = ConvertTo-StringArray -Value $BranchReasonCodes
$normalizedRaceReasons = ConvertTo-StringArray -Value $RaceReasonCodes

if (@($normalizedBranchReasons) -contains 'branch_protection_authentication_missing') {
[void]$hints.Add('Configure WORKFLOW_BOT_TOKEN (or GH_TOKEN) with repository administration read/write permissions before rerunning guardrails remediation.')
}
if (@($normalizedBranchReasons) -contains 'branch_protection_authz_denied') {
[void]$hints.Add('Token lacks sufficient repository administration permissions for branch-protection GraphQL operations; rotate/replace WORKFLOW_BOT_TOKEN and rerun.')
}
if (@($normalizedBranchReasons) -contains 'branch_protection_query_failed' -and @($hints).Count -eq 0) {
[void]$hints.Add('Review branch-protection query connectivity/authentication in GitHub Actions logs, then rerun guardrails remediation.')
}
if (@($normalizedRaceReasons) -contains 'drill_run_stale') {
[void]$hints.Add('Dispatch release-race-hardening-drill.yml and confirm a fresh successful run is available before re-evaluating guardrails.')
}

return @($hints)
}

function Test-ContainsAnyReasonCode {
param(
[Parameter()][string[]]$Source = @(),
Expand Down Expand Up @@ -255,6 +281,7 @@ $report = [ordered]@{
status = 'fail'
reason_code = ''
message = ''
remediation_hints = @()
initial_assessment = $null
remediation_attempts = @()
final_assessment = $null
Expand Down Expand Up @@ -468,7 +495,14 @@ try {
$report.reason_code = 'no_automatable_action'
$finalBranchReasons = Get-ReasonCodesFromReport -Report $currentBranchAssessment.report
$finalRaceReasons = Get-ReasonCodesFromReport -Report $currentRaceAssessment.report
$report.remediation_hints = @(
Get-GuardrailsRemediationHints -BranchReasonCodes @($finalBranchReasons) -RaceReasonCodes @($finalRaceReasons)
)
$hintText = if (@($report.remediation_hints).Count -gt 0) { " remediation_hints=$([string]::Join(' | ', @($report.remediation_hints)))" } else { '' }
$report.message = "No automatable remediation path. branch_reason_codes=$(Format-ReasonCodeSet -ReasonCodes $finalBranchReasons) race_reason_codes=$(Format-ReasonCodeSet -ReasonCodes $finalRaceReasons)"
if (-not [string]::IsNullOrWhiteSpace($hintText)) {
$report.message = "$($report.message)$hintText"
}
} elseif ($executionFailureCount -gt 0) {
$report.status = 'fail'
$report.reason_code = 'remediation_execution_failed'
Expand All @@ -478,7 +512,14 @@ try {
$report.reason_code = 'remediation_verify_failed'
$finalBranchReasons = Get-ReasonCodesFromReport -Report $currentBranchAssessment.report
$finalRaceReasons = Get-ReasonCodesFromReport -Report $currentRaceAssessment.report
$report.remediation_hints = @(
Get-GuardrailsRemediationHints -BranchReasonCodes @($finalBranchReasons) -RaceReasonCodes @($finalRaceReasons)
)
$hintText = if (@($report.remediation_hints).Count -gt 0) { " remediation_hints=$([string]::Join(' | ', @($report.remediation_hints)))" } else { '' }
$report.message = "Guardrails remain failing after bounded remediation. branch_reason_codes=$(Format-ReasonCodeSet -ReasonCodes $finalBranchReasons) race_reason_codes=$(Format-ReasonCodeSet -ReasonCodes $finalRaceReasons)"
if (-not [string]::IsNullOrWhiteSpace($hintText)) {
$report.message = "$($report.message)$hintText"
}
}
}
}
Expand Down
58 changes: 57 additions & 1 deletion scripts/Test-ReleaseBranchProtectionPolicy.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,54 @@ function Test-RuleContract {
}
}

function Resolve-QueryFailureReasonCodes {
param(
[Parameter()][string]$MessageText = '',
[Parameter()][bool]$GhTokenPresent = $false
)

$resolved = [System.Collections.Generic.List[string]]::new()
[void]$resolved.Add('branch_protection_query_failed')

if (-not $GhTokenPresent) {
[void]$resolved.Add('branch_protection_authentication_missing')
return @($resolved)
}

$normalized = ([string]$MessageText).ToLowerInvariant()
$authnTokens = @(
'authentication required',
'requires authentication',
'http 401',
'gh auth login',
'not logged into any hosts',
'bad credentials'
)
$authzTokens = @(
'resource not accessible by integration',
'must have admin rights',
'requires admin access',
'repository administration',
'insufficient permissions'
)

foreach ($token in $authnTokens) {
if ($normalized.Contains([string]$token)) {
[void]$resolved.Add('branch_protection_authentication_missing')
break
}
}

foreach ($token in $authzTokens) {
if ($normalized.Contains([string]$token)) {
[void]$resolved.Add('branch_protection_authz_denied')
break
}
}

return @($resolved)
}

$reasonCodes = [System.Collections.Generic.List[string]]::new()

$MainRequiredContexts = Normalize-RequiredContexts -Values @($MainRequiredContexts)
Expand All @@ -150,6 +198,9 @@ $report = [ordered]@{
main_rule = $null
integration_rule = $null
}
auth_context = [ordered]@{
gh_token_present = -not [string]::IsNullOrWhiteSpace([string]$env:GH_TOKEN)
}
}

try {
Expand Down Expand Up @@ -228,7 +279,12 @@ query($owner:String!, $name:String!) {
}
catch {
if ($reasonCodes.Count -eq 0) {
Add-ReasonCode -Target $reasonCodes -ReasonCode 'branch_protection_query_failed'
$queryFailureReasons = Resolve-QueryFailureReasonCodes `
-MessageText ([string]$_.Exception.Message) `
-GhTokenPresent ([bool]$report.auth_context.gh_token_present)
foreach ($reasonCode in @($queryFailureReasons)) {
Add-ReasonCode -Target $reasonCodes -ReasonCode ([string]$reasonCode)
}
}
$report.status = 'fail'
$report.reason_codes = @($reasonCodes)
Expand Down
Loading