diff --git a/.github/workflows/E2E.yaml b/.github/workflows/E2E.yaml index c87e79a183..1e67100606 100644 --- a/.github/workflows/E2E.yaml +++ b/.github/workflows/E2E.yaml @@ -232,6 +232,37 @@ jobs: $allScenarios = @(Get-ChildItem -Path (Join-Path $ENV:GITHUB_WORKSPACE "e2eTests/scenarios/*/runtest.ps1") | ForEach-Object { $_.Directory.Name }) $filteredScenarios = $allScenarios | Where-Object { $scenario = $_; $scenariosFilter | ForEach-Object { $scenario -like $_ } } + # Load disabled scenarios from config file (optional) + $disabledScenariosConfigPath = Join-Path $ENV:GITHUB_WORKSPACE "e2eTests/disabled-scenarios.json" + $disabledScenariosConfig = @() + if (Test-Path -Path $disabledScenariosConfigPath) { + $disabledScenariosContent = Get-Content -Path $disabledScenariosConfigPath -Encoding UTF8 -Raw + if (-not [string]::IsNullOrWhiteSpace($disabledScenariosContent)) { + $disabledScenariosConfig = $disabledScenariosContent | ConvertFrom-Json + } + } + else { + Write-Host "No disabled-scenarios.json found; proceeding with all scenarios enabled." + } + $disabledScenarios = @() + if ($disabledScenariosConfig -and $disabledScenariosConfig.Count -gt 0) { + $disabledScenarios = @($disabledScenariosConfig | ForEach-Object { $_.scenario }) + } + Write-Host "Disabled scenarios from config: $($disabledScenarios -join ', ')" + + # Filter out disabled scenarios + $scenariosBeforeDisabledFilter = $filteredScenarios + $beforeFilter = $filteredScenarios.Count + $filteredScenarios = $filteredScenarios | Where-Object { $disabledScenarios -notcontains $_ } + $afterFilter = $filteredScenarios.Count + if ($beforeFilter -ne $afterFilter) { + Write-Host "Filtered out $($beforeFilter - $afterFilter) disabled scenario(s)" + $disabledScenariosConfig | Where-Object { ($scenariosBeforeDisabledFilter -contains $_.scenario) -and ($filteredScenarios -notcontains $_.scenario) } | ForEach-Object { + Write-Host " - $($_.scenario): $($_.reason)" + } + } + Write-Host "Scenarios to run: $($filteredScenarios -join ', ')" + $scenariosJson = @{ "matrix" = @{ "include" = @($filteredScenarios | ForEach-Object { @{ "Scenario" = $_ } }) diff --git a/e2eTests/disabled-scenarios.json b/e2eTests/disabled-scenarios.json new file mode 100644 index 0000000000..f95fda8561 --- /dev/null +++ b/e2eTests/disabled-scenarios.json @@ -0,0 +1,6 @@ +[ + { + "scenario": "FederatedCredentials", + "reason": "Azure resource migration work in progress" + } +] diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 68a4c3b155..a3c2e2ce69 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -328,6 +328,150 @@ function SetRepositorySecret { gh secret set $name -b $value --repo $repository } +function CleanupWorkflowRuns { + Param( + [Parameter(Mandatory = $true)] + [string] $repository + ) + + Write-Host -ForegroundColor Yellow "`nCleaning up workflow runs in $repository" + + RefreshToken -repository $repository + + # Get all workflow runs with pagination + $page = 1 + $totalDeleted = 0 + do { + $runs = invoke-gh api "/repos/$repository/actions/runs?per_page=100&page=$page" -silent -returnValue | ConvertFrom-Json + + if ($runs.workflow_runs.Count -eq 0) { + break + } + + Write-Host "Processing page $page with $($runs.workflow_runs.Count) workflow runs..." + foreach ($run in $runs.workflow_runs) { + try { + Write-Host "Deleting run $($run.id) ($($run.name) - $($run.status))" + invoke-gh api /repos/$repository/actions/runs/$($run.id) --method DELETE -silent | Out-Null + $totalDeleted++ + } + catch { + Write-Host "Warning: Failed to delete run $($run.id): $_" + } + } + $page++ + } while ($runs.workflow_runs.Count -gt 0) + + if ($totalDeleted -eq 0) { + Write-Host "No workflow runs found to delete" + } + else { + Write-Host "Cleanup completed. Deleted $totalDeleted workflow run(s)" + } +} + +<# +.SYNOPSIS + Resets a repository to match the content of a source repository. + +.DESCRIPTION + Clones the target repository, fetches content from a source repository, and performs a hard reset + followed by a force push. This preserves the repository identity while resetting its content to + match the source repository. Useful for ensuring deterministic state in end-to-end tests. + +.PARAMETER repository + The full name of the target repository to reset in the format "owner/repo" (e.g., "microsoft/AL-Go"). + +.PARAMETER sourceRepository + The full name of the source repository to copy content from in the format "owner/repo". + +.PARAMETER branch + The branch name to reset. Defaults to "main". + +.EXAMPLE + ResetRepositoryToSource -repository "microsoft/test-repo" -sourceRepository "microsoft/source-repo" -branch "main" +#> +function ResetRepositoryToSource { + Param( + [Parameter(Mandatory = $true)] + [string] $repository, + [Parameter(Mandatory = $true)] + [string] $sourceRepository, + [string] $branch = "main" + ) + + Write-Host -ForegroundColor Yellow "`nResetting repository $repository to match $sourceRepository" + + RefreshToken -repository $repository + + # Clone the repository locally if not already in it + $tempPath = [System.IO.Path]::GetTempPath() + $repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString()) + New-Item $repoPath -ItemType Directory | Out-Null + + Push-Location $repoPath + try { + Write-Host "Cloning $repository..." + try { + invoke-gh repo clone $repository . + } + catch { + throw "Failed to clone repository $repository`: $_" + } + + # Fetch the source repository content + Write-Host "Fetching source repository $sourceRepository..." + try { + invoke-git remote add source "https://github.com/$sourceRepository.git" + } + catch { + throw "Failed to add remote source $sourceRepository`: $_" + } + + try { + invoke-git fetch source $branch --quiet + } + catch { + throw "Failed to fetch branch $branch from source $sourceRepository`: $_" + } + + # Reset the current branch to match the source + Write-Host "Resetting $branch to match source/$branch..." + try { + invoke-git checkout $branch --quiet + } + catch { + throw "Failed to checkout branch $branch`: $_" + } + + try { + invoke-git reset --hard "source/$branch" --quiet + } + catch { + throw "Failed to reset branch $branch to source/$branch`: $_" + } + + # Force push to update the repository + Write-Host "Force pushing changes..." + try { + invoke-git push origin $branch --force --quiet + } + catch { + throw "Failed to force push changes to $repository`: $_" + } + + Write-Host "Repository reset completed successfully" + } + catch { + Write-Host "Error resetting repository: $_" + throw + } + finally { + Pop-Location + Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue + } +} + function CreateNewAppInFolder { Param( [string] $folder, diff --git a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 index 3aad793fe8..502d86fa3e 100644 --- a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 +++ b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 @@ -29,18 +29,23 @@ Write-Host -ForegroundColor Yellow @' # This test uses the bcsamples-bingmaps.appsource repository and will deliver a new build of the app to AppSource. # The bcsamples-bingmaps.appsource repository is setup to use an Azure KeyVault for secrets and app signing. # -# During the test, the bcsamples-bingmaps.appsource repository will be copied to a new repository called tmp-bingmaps.appsource. -# tmp-bingmaps.appsource has access to the same Azure KeyVault as bcsamples-bingmaps.appsource using federated credentials. +# The test requires a stable temporary repository called e2e-bingmaps.appsource that must be manually created +# with federated credentials configured before running this test. +# This is required because federated credentials no longer work with repository name-based matching, +# so the repository must remain stable to maintain the federated credential configuration. +# e2e-bingmaps.appsource has access to the same Azure KeyVault as bcsamples-bingmaps.appsource using federated credentials. # The bcSamples-bingmaps.appsource repository is setup for continuous delivery to AppSource -# tmp-bingmaps.appsource also has access to the Entra ID app registration for delivering to AppSource using federated credentials. +# e2e-bingmaps.appsource also has access to the Entra ID app registration for delivering to AppSource using federated credentials. # This test will deliver another build of the latest app version already delivered to AppSource (without go-live) # # This test tests the following scenario: # -# - Create a new repository called tmp-bingmaps.appsource (based on bcsamples-bingmaps.appsource) -# - Update AL-Go System Files in branch main in tmp-bingmaps.appsource -# - Update version numbers in app.json in tmp-bingmaps.appsource in order to not be lower than the version number in AppSource (and not be higher than the next version from bcsamples-bingmaps.appsource) -# - Wait for CI/CD in branch main in repository tmp-bingmaps.appsource +# - Verify that the repository e2e-bingmaps.appsource exists (error out if not) +# - Reset the repository to match bcsamples-bingmaps.appsource for deterministic state +# - Clean up old workflow runs to ensure proper workflow tracking +# - Update AL-Go System Files in branch main in e2e-bingmaps.appsource +# - Update version numbers in app.json in e2e-bingmaps.appsource in order to not be lower than the version number in AppSource (and not be higher than the next version from bcsamples-bingmaps.appsource) +# - Wait for CI/CD in branch main in repository e2e-bingmaps.appsource # - Check that artifacts are created and signed # - Check that the app is delivered to AppSource '@ @@ -55,40 +60,122 @@ if ($linux) { Remove-Module e2eTestHelper -ErrorAction SilentlyContinue Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking -$repository = "$githubOwner/tmp-bingmaps.appsource" +$repository = "$githubOwner/e2e-bingmaps.appsource" $template = "https://github.com/$appSourceTemplate" $sourceRepository = 'microsoft/bcsamples-bingmaps.appsource' # E2E test will create a copy of this repository -# Create temp repository from sourceRepository +# Setup authentication and repository SetTokenAndRepository -github:$github -githubOwner $githubOwner -appId $e2eAppId -appKey $e2eAppKey -repository $repository -gh api repos/$repository --method HEAD -if ($LASTEXITCODE -eq 0) { - Write-Host "Repository $repository already exists. Deleting it." - gh repo delete $repository --yes | Out-Host - Start-Sleep -Seconds 30 +# Check if the repository already exists +# This repository must exist with federated credentials already configured +try { + invoke-gh api repos/$repository --method HEAD -silent | Out-Null } +catch { + throw "Repository $repository does not exist. The repository must be created manually with federated credentials configured before running this test." +} + +# Repository exists - reuse it and reset to source state +# This is required because federated credentials no longer work with repository name-based matching, +# so the repository must remain stable across test runs +Write-Host "Repository $repository exists. Reusing and resetting to match source." -CreateAlGoRepository ` - -github:$github ` - -template "https://github.com/$sourceRepository" ` - -repository $repository ` - -addRepoSettings @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow" } +# Reset the repository to match the source repository +ResetRepositoryToSource -repository $repository -sourceRepository $sourceRepository -branch 'main' +# Clean up workflow runs to ensure proper workflow tracking +CleanupWorkflowRuns -repository $repository + +# Always set/update secrets (they may have changed or repo may have been reset) SetRepositorySecret -repository $repository -name 'Azure_Credentials' -value $azureCredentials +# Capture previous workflow run IDs before making any changes +# This follows the established pattern from CommitAndPush function +Write-Host "Capturing previous workflow runs..." +$previousRuns = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json +$previousRunIds = $previousRuns.workflow_runs | Where-Object { $_.event -eq 'push' } | Select-Object -ExpandProperty id +if ($previousRunIds) { + Write-Host "Previous run IDs: $($previousRunIds -join ', ')" +} +else { + Write-Host "No previous runs found" +} + +# Re-apply the custom repository settings that were lost during reset +$tempPath = [System.IO.Path]::GetTempPath() +$repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString()) +New-Item $repoPath -ItemType Directory | Out-Null +Push-Location $repoPath +try { + Write-Host "Re-applying repository settings..." + invoke-gh repo clone $repository . + $repoSettingsFile = ".github\AL-Go-Settings.json" + if (Test-Path $repoSettingsFile) { + Add-PropertiesToJsonFile -path $repoSettingsFile -properties @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow"} + invoke-git add $repoSettingsFile + invoke-git commit -m "Update repository settings for test" --quiet + invoke-git push --quiet + Write-Host "Settings push completed. This will trigger a CI/CD workflow run." + } + else { + Write-Host "Warning: .github\AL-Go-Settings.json not found after cloning. Settings may not be applied correctly." + } +} +finally { + Pop-Location + Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue +} + +# Wait a moment for the settings push workflow to be registered, then update baseline +Write-Host "Updating baseline workflow runs to include the settings push workflow..." +Start-Sleep -Seconds 5 +$updatedRuns = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json +$previousRunIds = @($updatedRuns.workflow_runs | Select-Object -First 50 | Select-Object -ExpandProperty id) +Write-Host "Updated baseline now includes $($previousRunIds.Count) workflow run(s)" + # Upgrade AL-Go System Files to test version -RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $template -repository $repository | Out-Null +RunUpdateAlGoSystemFiles -directCommit -wait -repository $repository | Out-Null + +# Wait for CI/CD workflow to start (triggered by Update AL-Go System Files) +# This follows the established pattern from CommitAndPush: poll until a new run appears +Write-Host "Waiting for CI/CD workflow to start (triggered by Update AL-Go System Files push)..." + +# Poll for new workflow run that wasn't in the previous list +$maxAttempts = 60 # 10 minutes maximum wait (60 * 10 seconds) +$attempts = 0 +$run = $null + +while ($attempts -lt $maxAttempts) { + Start-Sleep -Seconds 10 + $attempts++ + + $currentRuns = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json + # Find new push workflow runs that weren't in our previous list + $newRuns = $currentRuns.workflow_runs | Where-Object { + $_.event -eq 'push' -and $previousRunIds -notcontains $_.id + } + + if ($newRuns) { + # Get the most recent new run (first one, since API returns newest first) + $run = $newRuns | Select-Object -First 1 + Write-Host "Found new CI/CD workflow run: $($run.id) (created at $($run.created_at))" + break + } + + Write-Host "Waiting for new CI/CD workflow run to appear (attempt $attempts/$maxAttempts)..." +} + +if (-not $run) { + throw "Error: Timeout waiting for CI/CD workflow run to start after Update AL-Go System Files completed." +} -# Wait for CI/CD to complete -Start-Sleep -Seconds 60 -$runs = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json -$run = $runs.workflow_runs | Select-Object -First 1 +Write-Host "Waiting for CI/CD workflow run $($run.id) to complete..." WaitWorkflow -repository $repository -runid $run.id -noError -# The CI/CD workflow should fail because the version number of the app in thie repository is lower than the version number in AppSource +# The CI/CD workflow should fail because the version number of the app in the repository is lower than the version number in AppSource # Reason being that major.minor from the original bcsamples-bingmaps.appsource is the same and the build number in the newly created repository is lower than the one in AppSource -# This error is expected we will grab the version number from AppSource, add one to revision number (by switching to versioningstrategy 3 in the tmp repo) and use it in the next run +# This error is expected we will grab the version number from AppSource, add one to revision number (by switching to versioningstrategy 3 in the e2e-bingmaps.appsource repo) and use it in the next run $MatchArr = Test-LogContainsFromRun -repository $repository -runid $run.id -jobName 'Deliver to AppSource' -stepName 'Deliver' -expectedText '(?m)^.*The new version number \((\d+(?:\.\d+){3})\) is lower than the existing version number \((\d+(?:\.\d+){3})\) in Partner Center.*$' -isRegEx $appSourceVersion = [System.Version]$MatchArr[2] $newVersion = [System.Version]::new($appSourceVersion.Major, $appSourceVersion.Minor, $appSourceVersion.Build, 0)