From 1695ac042bb179a1b705bfac36404f044fdc6f0b Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 13:15:01 -0400 Subject: [PATCH 001/103] Add repo-init and repo-update CI commands Introduce repo-init and repo-update commands to manage repository CI configuration. repo-init provides a one-time interactive wizard (or flags) to write ci.provider into .genesis/config and generate .genesis/ci scaffold files (integrations.yml, targets.yml, resources.yml, ci-overrides). repo-update is an idempotent updater that can run non-interactively (apply only provided flags) or launch a pre-populated wizard; it can patch integrations.yml in-place and preserves existing scaffold edits. Implements helper routines for defaults, prompts, file writing, and scaffold templates, and adds unit tests covering config writes and scaffold behavior. --- bin/genesis | 70 ++++ lib/Genesis/Commands/Repo.pm | 320 +++++++++++++++++++ t/unit-tests/genesis_commands_repo_ci-core.t | 233 ++++++++++++++ 3 files changed, 623 insertions(+) create mode 100644 t/unit-tests/genesis_commands_repo_ci-core.t diff --git a/bin/genesis b/bin/genesis index 38ff5e19..2bc2440a 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1308,6 +1308,76 @@ define_command("kit-provider", { } }); +# }}} +# genesis repo-init - one-time CI provider setup {{{ +define_command("repo-init", { + summary => "Initialize CI configuration for this Genesis deployment repository.", + usage => "repo-init [--ci-provider PROVIDER] [--git-uri URI] [--git-branch BRANCH] [--vault-url URL] [--pipeline-name NAME]", + description => + "One-time setup that writes a #C{ci:} section to #C{.genesis/config} and ". + "generates the #C{.genesis/ci/} scaffold files required by the pipeline ". + "compiler.\n". + "\n". + "Errors if CI is already configured for this repository. Use ". + "#C{genesis repo-update} to modify an existing configuration.\n". + "\n". + "When required flags are omitted an interactive wizard collects them.", + function_group => Genesis::Commands::REPOSITORY, + scope => 'repo', + option_group => Genesis::Commands::REPO_OPTIONS, + options => [ + 'ci-provider|P=s' => + "CI provider to configure: #C{concourse} (default), ". + "#C{github-actions}, or #C{none}.", + + 'git-uri|g=s' => + "Git repository URI (e.g. #C{git\@github.com:org/repo.git}).", + + 'git-branch|b=s' => + "Default branch for the source-control integration. Defaults to #C{main}.", + + 'vault-url|V=s' => + "Vault URL for the secrets integration ". + "(e.g. #C{https://vault.example.com:8200}).", + + 'pipeline-name|n=s' => + "Name written into #C{pipeline.yml} metadata. Defaults to the ". + "deployment type.", + ], +}, 'Genesis::Commands::Repo::repo_init'); +# }}} +# genesis repo-update - idempotent CI config update {{{ +define_command("repo-update", { + summary => "Update CI configuration for this Genesis deployment repository.", + usage => "repo-update [--ci-provider PROVIDER] [--git-uri URI] [--git-branch BRANCH] [--vault-url URL] [--pipeline-name NAME]", + description => + "Idempotent counterpart to #C{genesis repo-init}. Updates the CI ". + "configuration in #C{.genesis/config} and regenerates missing scaffold ". + "files in #C{.genesis/ci/}.\n". + "\n". + "When called with flags, only the specified values are changed; ". + "everything else is left as-is. A bare invocation (no flags) launches ". + "an interactive wizard pre-populated with the current configuration.", + function_group => Genesis::Commands::REPOSITORY, + scope => 'repo', + option_group => Genesis::Commands::REPO_OPTIONS, + options => [ + 'ci-provider|P=s' => + "CI provider to configure: #C{concourse}, #C{github-actions}, or #C{none}.", + + 'git-uri|g=s' => + "Git repository URI.", + + 'git-branch|b=s' => + "Default branch for the source-control integration.", + + 'vault-url|V=s' => + "Vault URL for the secrets integration.", + + 'pipeline-name|n=s' => + "Name written into #C{pipeline.yml} metadata.", + ], +}, 'Genesis::Commands::Repo::repo_update'); # }}} # }}} diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 4b662767..2e207c27 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -7,6 +7,7 @@ use Genesis; use Genesis::Commands; use Genesis::Top; use Genesis::Kit::Provider; +use Genesis::UI; use Cwd qw/getcwd abs_path/; use File::Basename qw/basename/; @@ -238,5 +239,324 @@ sub kit_provider { info(" Kits: %s\n\n", $kit_list) if $info{status} eq "ok"; } +sub repo_init { + my %options = %{get_options()}; + command_usage(1) if @_; + + my $top = Genesis::Top->new('.', no_vault => 1); + + bail( + "CI provider already configured (#C{%s}).\n". + "Use #C{genesis repo-update} to modify the existing configuration.", + $top->config->get('ci.provider') + ) if $top->config->has('ci.provider'); + + my $cfg = _ci_wizard(\%options, $top, _empty_ci_defaults($top)); + _write_ci_config($top, $cfg); + + info( + "\n#G{CI configuration initialized}\n". + " Provider : #C{%s}\n". + " Config : #C{.genesis/config}\n". + " Scaffold : #C{.genesis/ci/}\n\n". + "Add environment targets to #C{.genesis/ci/targets.yml}, then run\n". + "#C{genesis pipeline-apply} to deploy the pipeline.\n", + $cfg->{ci_provider} + ); + exit 0; +} + +sub repo_update { + my %options = %{get_options()}; + command_usage(1) if @_; + + my $top = Genesis::Top->new('.', no_vault => 1); + + my @flag_keys = grep { $_ ne 'yes' } keys %options; + + if (@flag_keys) { + # Non-interactive: apply only the provided flags, leave everything else alone + _apply_ci_flags(\%options, $top); + } else { + # Bare invocation: full wizard with existing values pre-populated + my $cfg = _ci_wizard(\%options, $top, _existing_ci_defaults($top)); + _write_ci_config($top, $cfg, update => 1); + } + + info( + "\n#G{CI configuration updated}\n". + " Provider : #C{%s}\n". + " Config : #C{.genesis/config}\n". + " Scaffold : #C{.genesis/ci/}\n", + $top->config->get('ci.provider') // '(none)' + ); + exit 0; +} + +### Private helpers ########################################################### + +# _empty_ci_defaults - blank defaults for repo_init wizard +sub _empty_ci_defaults { + my ($top) = @_; + return { + ci_provider => 'concourse', + git_uri => '', + git_branch => 'main', + vault_url => '', + pipeline_name => $top->type, + }; +} + +# _existing_ci_defaults - load current values for repo_update wizard +sub _existing_ci_defaults { + my ($top) = @_; + + my $defaults = _empty_ci_defaults($top); + $defaults->{ci_provider} = $top->config->get('ci.provider') + if $top->config->has('ci.provider'); + + my $ci_dir = $top->path('.genesis/ci'); + if (-f "$ci_dir/integrations.yml") { + eval { + my $raw = slurp("$ci_dir/integrations.yml"); + # Extract vault url from YAML without full spruce evaluation + if ($raw =~ /^\s{0,2}vault:\s*\n.*?\n\s{2,4}url:\s*(\S+)/ms) { + $defaults->{vault_url} = $1; + } elsif ($raw =~ /url:\s*(\S+)/) { + $defaults->{vault_url} = $1; + } + if ($raw =~ /uri:\s*(\S+)/) { + $defaults->{git_uri} = $1; + } + if ($raw =~ /default_branch:\s*(\S+)/) { + $defaults->{git_branch} = $1; + } + }; + } + + if (-f "$ci_dir/pipeline.yml") { + eval { + my $raw = slurp("$ci_dir/pipeline.yml"); + if ($raw =~ /name:\s*(\S+)/) { + $defaults->{pipeline_name} = $1; + } + }; + } + + return $defaults; +} + +# _ci_wizard - prompt for any config values not supplied as flags +sub _ci_wizard { + my ($options, $top, $defaults) = @_; + + my $ci_provider = $options->{'ci-provider'} // do { + prompt_for_choice( + "CI provider:", + [qw(concourse github-actions none)], + $defaults->{ci_provider}, + ); + }; + + my $pipeline_name = $options->{'pipeline-name'} // do { + prompt_for_line( + "Pipeline name:", + "pipeline name", + $defaults->{pipeline_name}, + ); + }; + + my $git_uri = $options->{'git-uri'} // do { + prompt_for_line( + "Git repository URI (e.g. git\@github.com:org/repo.git):", + "git uri", + $defaults->{git_uri}, + ); + }; + + my $git_branch = $options->{'git-branch'} // do { + prompt_for_line( + "Default branch:", + "branch", + $defaults->{git_branch}, + ); + }; + + my $vault_url = $options->{'vault-url'} // do { + prompt_for_line( + "Vault URL (e.g. https://vault.example.com:8200):", + "vault url", + $defaults->{vault_url}, + ); + }; + + return { + ci_provider => $ci_provider, + pipeline_name => $pipeline_name, + git_uri => $git_uri, + git_branch => $git_branch, + vault_url => $vault_url, + }; +} + +# _apply_ci_flags - non-interactive partial update (repo_update with flags) +sub _apply_ci_flags { + my ($options, $top) = @_; + + if (exists $options->{'ci-provider'}) { + my $provider = $options->{'ci-provider'}; + bail( + "Unknown CI provider '#R{%s}'. Valid values: concourse, github-actions, none.", + $provider + ) unless grep { $_ eq $provider } qw(concourse github-actions none); + $top->config->set('ci.provider', $provider, 1); + } + + my $ci_dir = $top->path('.genesis/ci'); + mkdir_or_fail($ci_dir) unless -d $ci_dir; + + # Update integrations.yml in-place if any integration flags were given + my @integration_flags = grep { exists $options->{$_} } qw(git-uri git-branch vault-url); + if (@integration_flags && -f "$ci_dir/integrations.yml") { + my $raw = slurp("$ci_dir/integrations.yml"); + if (exists $options->{'vault-url'}) { + my $v = $options->{'vault-url'}; + $raw =~ s/^(\s{0,4}url:\s*)\S+/$1$v/m; + } + if (exists $options->{'git-uri'}) { + my $v = $options->{'git-uri'}; + $raw =~ s/^(\s{0,4}uri:\s*)\S+/$1$v/m; + } + if (exists $options->{'git-branch'}) { + my $v = $options->{'git-branch'}; + $raw =~ s/^(\s{0,4}default_branch:\s*)\S+/$1$v/m; + } + mkfile_or_fail("$ci_dir/integrations.yml", $raw); + } elsif (@integration_flags) { + # integrations.yml doesn't exist yet - need full defaults to write it + my $defaults = _existing_ci_defaults($top); + $defaults->{'vault-url'} = $options->{'vault-url'} if exists $options->{'vault-url'}; + $defaults->{'git-uri'} = $options->{'git-uri'} if exists $options->{'git-uri'}; + $defaults->{'git-branch'} = $options->{'git-branch'} if exists $options->{'git-branch'}; + _write_integrations($ci_dir, $defaults); + } + + # Write provider override scaffold if provider changed and file is absent + if (exists $options->{'ci-provider'} && $options->{'ci-provider'} ne 'none') { + _write_if_absent("$ci_dir/ci-overrides-$options->{'ci-provider'}.yml", + _overrides_scaffold($options->{'ci-provider'})); + } + + # Ensure other scaffold stubs exist + _write_if_absent("$ci_dir/targets.yml", _targets_scaffold()); + _write_if_absent("$ci_dir/resources.yml", _resources_scaffold()); +} + +# _write_ci_config - write config key and scaffold directory for init/full update +sub _write_ci_config { + my ($top, $cfg, %opts) = @_; + + my $provider = $cfg->{ci_provider}; + + $top->config->set('ci.provider', $provider, 1); + + my $ci_dir = $top->path('.genesis/ci'); + mkdir_or_fail($ci_dir) unless -d $ci_dir; + + # integrations.yml — always write (contains wizard-collected values) + _write_integrations($ci_dir, $cfg); + + # Remaining scaffold files: write only if absent (preserve manual edits on update) + _write_if_absent("$ci_dir/targets.yml", _targets_scaffold()); + _write_if_absent("$ci_dir/resources.yml", _resources_scaffold()); + if ($provider ne 'none') { + _write_if_absent("$ci_dir/ci-overrides-${provider}.yml", + _overrides_scaffold($provider)); + } +} + +# _write_integrations - write .genesis/ci/integrations.yml from collected config +sub _write_integrations { + my ($ci_dir, $cfg) = @_; + + my $vault_url = $cfg->{vault_url} // $cfg->{'vault-url'} // ''; + my $git_uri = $cfg->{git_uri} // $cfg->{'git-uri'} // ''; + my $git_branch = $cfg->{git_branch} // $cfg->{'git-branch'} // 'main'; + + mkfile_or_fail("$ci_dir/integrations.yml", <<"YAML"); +--- +vault: + url: $vault_url + namespace: genesis + auth: + type: approle + role_id: ((vault-role-id)) + secret_id: ((vault-secret-id)) + options: + tls_verify: true + +source_control: + provider: github + uri: $git_uri + default_branch: $git_branch + root: "." + auth: + type: ssh-key + private_key: ((github-deploy-key)) + commit_author: + name: Concourse Bot + email: ci\@example.com +YAML +} + +sub _write_if_absent { + my ($path, $content) = @_; + mkfile_or_fail($path, $content) unless -f $path; +} + +sub _targets_scaffold { + return <<'YAML'; +--- +# targets.yml - BOSH director targets for pipeline deployments. +# Add one stanza per environment. The key name must match the environment +# name used in pipeline topology declarations (genesis.pipeline.prior_env). +# +# Example: +# sandbox: +# name: sandbox +# alias: us-west-1-sandbox +# type: bosh-director +# tags: [] +# connection: +# url: https://bosh.sandbox.example.com:25555 +# auth: +# type: basic +# client_id: admin +# client_secret: ((sandbox-bosh-password)) +# ca_cert: ((sandbox-bosh-ca)) +targets: {} +YAML +} + +sub _resources_scaffold { + return <<'YAML'; +--- +# resources.yml - Additional CI resources. +# Add custom Concourse resource definitions or GitHub Actions inputs here. +resources: [] +YAML +} + +sub _overrides_scaffold { + my ($provider) = @_; + return <<"YAML"; +--- +# ci-overrides-${provider}.yml +# Spruce-merged over generated pipeline YAML after provider output. +# Supports all spruce operators: (( grab )), (( inject )), (( append )), etc. +# Leave empty or remove this file to use the default generated pipeline. +YAML +} + 1; # vim: fdm=marker:foldlevel=0:noet diff --git a/t/unit-tests/genesis_commands_repo_ci-core.t b/t/unit-tests/genesis_commands_repo_ci-core.t new file mode 100644 index 00000000..0bec1ef8 --- /dev/null +++ b/t/unit-tests/genesis_commands_repo_ci-core.t @@ -0,0 +1,233 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use lib 't'; +use helper; + +use Genesis; +use Genesis::Config; +use_ok 'Genesis::Commands::Repo'; + +my $tmp = workdir(); + +### Helpers ################################################################### + +# Minimal .genesis/config that looks like a Genesis repo +sub make_repo { + my ($dir, %opts) = @_; + my $genesis_dir = "$dir/.genesis"; + mkdir_or_fail($genesis_dir); + + my $content = "---\ncreator_version: 3.0.0\ndeployment_type: test-kit\nversion: 2\n"; + if ($opts{ci_provider}) { + $content .= "ci:\n provider: $opts{ci_provider}\n"; + } + mkfile_or_fail("$genesis_dir/config", $content); + return $dir; +} + +### Tests ##################################################################### + +subtest 'repo_init writes ci.provider to .genesis/config' => sub { + my $dir = workdir("repo-init-1"); + make_repo($dir); + + # Call the private _write_ci_config logic directly via a Genesis::Config + # object so we test the config writing without needing a full Top object. + my $config = Genesis::Config->new("$dir/.genesis/config", 1); + $config->set('ci.provider', 'concourse', 1); + + my $config2 = Genesis::Config->new("$dir/.genesis/config"); + ok $config2->has('ci.provider'), "ci.provider key written"; + is $config2->get('ci.provider'), 'concourse', "ci.provider value is 'concourse'"; +}; + +subtest 'repo_init scaffold: targets.yml created' => sub { + my $dir = workdir("repo-init-scaffold"); + make_repo($dir); + mkdir_or_fail("$dir/.genesis/ci"); + + Genesis::Commands::Repo::_write_if_absent( + "$dir/.genesis/ci/targets.yml", + Genesis::Commands::Repo::_targets_scaffold() + ); + + ok -f "$dir/.genesis/ci/targets.yml", "targets.yml exists"; + my $content = slurp("$dir/.genesis/ci/targets.yml"); + like $content, qr/targets:\s*\{\}/, "contains empty targets map"; + like $content, qr/bosh-director/, "contains bosh-director comment"; +}; + +subtest 'repo_init scaffold: resources.yml created' => sub { + my $dir = workdir("repo-init-resources"); + make_repo($dir); + mkdir_or_fail("$dir/.genesis/ci"); + + Genesis::Commands::Repo::_write_if_absent( + "$dir/.genesis/ci/resources.yml", + Genesis::Commands::Repo::_resources_scaffold() + ); + + ok -f "$dir/.genesis/ci/resources.yml", "resources.yml exists"; + my $content = slurp("$dir/.genesis/ci/resources.yml"); + like $content, qr/resources:\s*\[\]/, "contains empty resources list"; +}; + +subtest 'repo_init scaffold: ci-overrides file created' => sub { + my $dir = workdir("repo-init-overrides"); + make_repo($dir); + mkdir_or_fail("$dir/.genesis/ci"); + + Genesis::Commands::Repo::_write_if_absent( + "$dir/.genesis/ci/ci-overrides-concourse.yml", + Genesis::Commands::Repo::_overrides_scaffold('concourse') + ); + + ok -f "$dir/.genesis/ci/ci-overrides-concourse.yml", "override file exists"; + my $content = slurp("$dir/.genesis/ci/ci-overrides-concourse.yml"); + like $content, qr/ci-overrides-concourse/, "contains provider name in header"; + like $content, qr/spruce/i, "references spruce in comment"; +}; + +subtest 'repo_init scaffold: integrations.yml written with provided values' => sub { + my $dir = workdir("repo-init-integrations"); + make_repo($dir); + my $ci_dir = "$dir/.genesis/ci"; + mkdir_or_fail($ci_dir); + + my $cfg = { + ci_provider => 'concourse', + pipeline_name => 'my-pipeline', + git_uri => 'git@github.com:example/repo.git', + git_branch => 'main', + vault_url => 'https://vault.example.com:8200', + }; + Genesis::Commands::Repo::_write_integrations($ci_dir, $cfg); + + ok -f "$ci_dir/integrations.yml", "integrations.yml exists"; + my $content = slurp("$ci_dir/integrations.yml"); + like $content, qr|https://vault\.example\.com:8200|, "vault url present"; + like $content, qr|git\@github\.com:example/repo\.git|, "git uri present"; + like $content, qr/default_branch:\s*main/, "branch present"; +}; + +subtest '_write_if_absent: does not overwrite existing file' => sub { + my $dir = workdir("repo-init-absent"); + make_repo($dir); + mkdir_or_fail("$dir/.genesis/ci"); + + mkfile_or_fail("$dir/.genesis/ci/targets.yml", "---\ncustom: true\n"); + + Genesis::Commands::Repo::_write_if_absent( + "$dir/.genesis/ci/targets.yml", + Genesis::Commands::Repo::_targets_scaffold() + ); + + my $content = slurp("$dir/.genesis/ci/targets.yml"); + like $content, qr/custom: true/, "existing content preserved"; + unlike $content, qr/targets: \{\}/, "scaffold not written over existing file"; +}; + +subtest 'repo_init guard: errors if ci.provider already set' => sub { + my $dir = workdir("repo-init-guard"); + make_repo($dir, ci_provider => 'concourse'); + + # We test the guard condition in isolation by reading the config directly + my $config = Genesis::Config->new("$dir/.genesis/config"); + ok $config->has('ci.provider'), "guard precondition: ci.provider is set"; + + # The actual bail() in repo_init would fire here; we just verify the + # config state that triggers it rather than calling the full command + # (which requires a Top object and exits via bail). + is $config->get('ci.provider'), 'concourse', "guard sees correct provider value"; +}; + +subtest 'repo_update --ci-provider: _apply_ci_flags updates only provider' => sub { + my $dir = workdir("repo-update-provider"); + make_repo($dir, ci_provider => 'concourse'); + my $ci_dir = "$dir/.genesis/ci"; + mkdir_or_fail($ci_dir); + + # Pre-populate integrations.yml with known content + mkfile_or_fail("$ci_dir/integrations.yml", <<'YAML'); +--- +vault: + url: https://vault.example.com:8200 +source_control: + uri: git@github.com:org/repo.git + default_branch: main +YAML + + my $config = Genesis::Config->new("$dir/.genesis/config", 1); + + # Simulate what _apply_ci_flags does for --ci-provider only + $config->set('ci.provider', 'github-actions', 1); + + my $config2 = Genesis::Config->new("$dir/.genesis/config"); + is $config2->get('ci.provider'), 'github-actions', "provider updated to github-actions"; + + # integrations.yml should be unchanged + my $integrations = slurp("$ci_dir/integrations.yml"); + like $integrations, qr|https://vault\.example\.com:8200|, "integrations.yml untouched"; +}; + +subtest 'repo_update _apply_ci_flags: patches integrations.yml fields in place' => sub { + my $dir = workdir("repo-update-integrations"); + make_repo($dir, ci_provider => 'concourse'); + my $ci_dir = "$dir/.genesis/ci"; + mkdir_or_fail($ci_dir); + + mkfile_or_fail("$ci_dir/integrations.yml", <<'YAML'); +--- +vault: + url: https://old-vault.example.com:8200 +source_control: + uri: git@github.com:org/old-repo.git + default_branch: master +YAML + + # Simulate _apply_ci_flags with --vault-url and --git-branch + my $raw = slurp("$ci_dir/integrations.yml"); + my $new_vault = 'https://new-vault.example.com:8200'; + my $new_branch = 'main'; + $raw =~ s/^(\s{0,4}url:\s*)\S+/$1$new_vault/m; + $raw =~ s/^(\s{0,4}default_branch:\s*)\S+/$1$new_branch/m; + mkfile_or_fail("$ci_dir/integrations.yml", $raw); + + my $content = slurp("$ci_dir/integrations.yml"); + like $content, qr|https://new-vault\.example\.com:8200|, "vault url updated"; + like $content, qr/default_branch:\s*main/, "branch updated to main"; + unlike $content, qr/master/, "old branch removed"; + like $content, qr|git\@github\.com:org/old-repo\.git|, "git uri unchanged"; +}; + +subtest '_existing_ci_defaults: reads values from integrations.yml' => sub { + my $dir = workdir("repo-defaults"); + make_repo($dir, ci_provider => 'concourse'); + my $ci_dir = "$dir/.genesis/ci"; + mkdir_or_fail($ci_dir); + + mkfile_or_fail("$ci_dir/integrations.yml", <<'YAML'); +--- +vault: + url: https://vault.corp.example.com:8200 + namespace: genesis +source_control: + uri: git@github.com:corp/deployments.git + default_branch: trunk +YAML + + # Test the extraction logic from _existing_ci_defaults in isolation + my $raw = slurp("$ci_dir/integrations.yml"); + my ($vault_url, $git_uri, $git_branch); + ($vault_url) = $raw =~ /url:\s*(\S+)/; + ($git_uri) = $raw =~ /uri:\s*(\S+)/; + ($git_branch) = $raw =~ /default_branch:\s*(\S+)/; + + is $vault_url, 'https://vault.corp.example.com:8200', "vault url extracted"; + is $git_uri, 'git@github.com:corp/deployments.git', "git uri extracted"; + is $git_branch, 'trunk', "branch extracted"; +}; + +done_testing; From c80277f7a162dcdf17a058edbff8addf57bfdf9e Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:03:25 -0400 Subject: [PATCH 002/103] Add commit/no-commit options to init Add two passthrough options for the init command (--commit and --no-commit) and update repo initialization flow to show a summary of staged files (git diff --cached --stat). If --commit is passed the initial state is committed automatically; if --no-commit is passed the commit is skipped; otherwise the user is prompted (default yes). Preserves existing failure handling and the original commit message (Initial Genesis Repo). --- bin/genesis | 6 ++++++ lib/Genesis/Commands/Repo.pm | 27 +++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bin/genesis b/bin/genesis index 2bc2440a..25fd51e0 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1234,6 +1234,12 @@ define_command("init", { "the '-deployments' suffix, but the option is still available for ". "backward compatibility by setting the #Y{legacy_repo_suffix} option in ". "the Genesis configuration file \$HOME/.genesis/config.yml to true.", + + "commit" => + "Automatically commit the initial state without prompting.", + + "no-commit" => + "Stage files and show the summary but skip the commit entirely.", ], option_passthrough => 1, arguments => [ diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 2e207c27..ca870d8b 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -118,8 +118,31 @@ sub init { run({ onfailure => "Failed to initialize a git repository in $human_root/" }, 'git init && git add .'); - run({ onfailure => "Failed to commit initial Genesis repository in $human_root/" }, - 'git commit -m "Initial Genesis Repo"'); + # Show a summary of what was staged + my ($stat) = run({}, 'git diff --cached --stat'); + if ($stat && $stat =~ /\S/) { + info "\n#G{Files staged for initial commit:}"; + for my $line (split /\n/, $stat) { + info " %s", $line; + } + info ""; + } + + my $do_commit; + if ($options{commit}) { + $do_commit = 1; + } elsif ($options{'no-commit'}) { + $do_commit = 0; + } else { + $do_commit = prompt_for_boolean( + "Commit initial state? [y|n]", "y" + ); + } + + if ($do_commit) { + run({ onfailure => "Failed to commit initial Genesis repository in $human_root/" }, + 'git commit -m "Initial Genesis Repo"'); + } }; my $err = $@; popd; From 540bbfd877ec6f340df74c0d21bcbe1be5b6eb60 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:27:51 -0400 Subject: [PATCH 003/103] Improve pipeline status order and repo CI parsing Pipeline: fix color tag in pause-pipeline error message and change status listing to follow the pipeline AST workflow order when available (index jobs by name, build ordered list from workflow stage order, append any missing jobs alphabetically). Repo: add warnings when CI provider is not configured (repo_update and _apply_ci_flags), include all provided flags for non-interactive updates, and improve parsing of integrations.yml to extract vault.url only from the vault: block to avoid false matches. Also simplify _write_ci_config signature. These changes tighten CI config detection and make pipeline status output more predictable. --- lib/Genesis/Commands/Pipeline.pm | 25 +++++++++++++++++++++---- lib/Genesis/Commands/Repo.pm | 30 +++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/Genesis/Commands/Pipeline.pm b/lib/Genesis/Commands/Pipeline.pm index be67c427..c4415d09 100644 --- a/lib/Genesis/Commands/Pipeline.pm +++ b/lib/Genesis/Commands/Pipeline.pm @@ -61,7 +61,7 @@ sub apply { my $target = $opts->{target} || $layout || $name; my ($out, $rc) = run('fly -t $1 pause-pipeline -p $2', $target, $name); - bail("Could not pause #c{%s} pipeline: %s", $name, $out) + bail("Could not pause #C{%s} pipeline: %s", $name, $out) unless $rc == 0 || $out =~ /pipeline '.*' not found/; my $yes = $opts->{yes} ? ' -n ' : ''; @@ -232,12 +232,29 @@ sub status { output "#G{Pipeline}: #C{%s} (#Yi{target}: %s)", $name, $target; output ""; + # Index jobs by name for ordered lookup + my %job_by_name = map { $_->{name} => $_ } @$jobs; + + # Use AST pipeline order when available; fall back to alphabetical + my @ordered_names; + for my $wf_name ($ast->workflow_names) { + my @stage_order = eval { $ast->workflow_stage_order($wf_name) }; + if (@stage_order) { + my $nodes = ($ast->workflows->{$wf_name} || {})->{graph}{nodes} || {}; + push @ordered_names, map { $nodes->{$_}{alias} || $_ } @stage_order; + } + } + # Append any jobs from fly that didn't appear in the AST (e.g. update-pipeline) + my %seen = map { $_ => 1 } @ordered_names; + push @ordered_names, sort grep { !$seen{$_} } keys %job_by_name; + my $col_w = 40; output " %-${col_w}s %-10s %s", "Environment", "Status", "Notes"; output " %s %s %s", '-' x $col_w, '-' x 10, '-' x 20; - for my $job (sort { $a->{name} cmp $b->{name} } @$jobs) { - next if $filter_env && $job->{name} ne $filter_env; + for my $job_name (@ordered_names) { + next if $filter_env && $job_name ne $filter_env; + my $job = $job_by_name{$job_name} or next; my $status = _job_status_label($job); my @notes; @@ -245,7 +262,7 @@ sub status { push @notes, 'errored' if ($job->{finished_build} || {})->{status} eq 'errored'; output " %-${col_w}s %-10s %s", - $job->{name}, + $job_name, $status, join(', ', @notes) || ''; } diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index ca870d8b..2291667d 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -295,7 +295,14 @@ sub repo_update { my $top = Genesis::Top->new('.', no_vault => 1); - my @flag_keys = grep { $_ ne 'yes' } keys %options; + unless ($top->config->has('ci.provider')) { + warning( + "CI provider is not configured for this repository.\n". + "Use #C{genesis repo-init} to set up CI from scratch." + ); + } + + my @flag_keys = keys %options; if (@flag_keys) { # Non-interactive: apply only the provided flags, leave everything else alone @@ -342,12 +349,14 @@ sub _existing_ci_defaults { if (-f "$ci_dir/integrations.yml") { eval { my $raw = slurp("$ci_dir/integrations.yml"); - # Extract vault url from YAML without full spruce evaluation - if ($raw =~ /^\s{0,2}vault:\s*\n.*?\n\s{2,4}url:\s*(\S+)/ms) { - $defaults->{vault_url} = $1; - } elsif ($raw =~ /url:\s*(\S+)/) { - $defaults->{vault_url} = $1; + # Extract vault.url: capture only within the vault: block, stopping + # before the next top-level key (no leading spaces) to avoid matching + # url: keys in other sections. + if ($raw =~ /^vault:\s*\n((?:[ \t]+[^\n]*\n)*)/m) { + my $vault_block = $1; + $defaults->{vault_url} = $1 if $vault_block =~ /url:\s*(\S+)/; } + # source_control.uri and default_branch are unique keys in our schema if ($raw =~ /uri:\s*(\S+)/) { $defaults->{git_uri} = $1; } @@ -426,6 +435,13 @@ sub _ci_wizard { sub _apply_ci_flags { my ($options, $top) = @_; + unless ($top->config->has('ci.provider') || exists $options->{'ci-provider'}) { + warning( + "CI provider not configured and --ci-provider not given.\n". + "Run #C{genesis repo-init} to perform initial CI setup." + ); + } + if (exists $options->{'ci-provider'}) { my $provider = $options->{'ci-provider'}; bail( @@ -477,7 +493,7 @@ sub _apply_ci_flags { # _write_ci_config - write config key and scaffold directory for init/full update sub _write_ci_config { - my ($top, $cfg, %opts) = @_; + my ($top, $cfg) = @_; my $provider = $cfg->{ci_provider}; From 413ad8f1d7873bd0aaa7a5711e080f98ba659450 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:33:18 -0400 Subject: [PATCH 004/103] Include referenced envs; regex & minor fixes Make ASTBuilder._build_from_env_files two-pass: collect genesis.pipeline data, include env files that are referenced as prior_env even if they lack a pipeline block, and build edges from the collected prior_env map. Add tests covering inclusion/exclusion of referenced envs and default pipeline flags. Improve Layout pattern handling by constructing a safe regex via quotemeta and splitting on '*' for proper glob semantics. Remove a redundant self->{layout} assignment in Concourse parse, normalize pipeline pause bail messages to use #C, remove an unnecessary update => 1 flag when writing CI config, and simplify a YAML key detection conditional. --- lib/Genesis/CI/Compiler/ASTBuilder.pm | 44 ++++++++++++------ .../CI/Compiler/Providers/Concourse.pm | 1 - lib/Genesis/CI/Layout.pm | 5 +- lib/Genesis/Commands/Pipelines.pm | 4 +- lib/Genesis/Commands/Repo.pm | 2 +- t/ci-compiler.t | 46 +++++++++++++++++++ 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/lib/Genesis/CI/Compiler/ASTBuilder.pm b/lib/Genesis/CI/Compiler/ASTBuilder.pm index 23f424f6..c47191cd 100644 --- a/lib/Genesis/CI/Compiler/ASTBuilder.pm +++ b/lib/Genesis/CI/Compiler/ASTBuilder.pm @@ -378,18 +378,38 @@ sub _build_workflow_graph { sub _build_from_env_files { my ($self, $dir) = @_; - my (%nodes, %prior_envs); - opendir(my $dh, $dir) or return ({}, []); my @files = sort grep { /\.ya?ml$/i && -f "$dir/$_" } readdir($dh); closedir $dh; + # Pass 1: collect genesis.pipeline data and note which files exist + my (%pipeline_data, %file_exists); for my $file (@files) { - my $pipeline_data = _read_genesis_pipeline_keys("$dir/$file"); - next unless %$pipeline_data; - (my $env = $file) =~ s/\.ya?ml$//i; + $file_exists{$env} = 1; + my $data = _read_genesis_pipeline_keys("$dir/$file"); + $pipeline_data{$env} = $data if %$data; + } + + # Identify envs referenced as prior_env — these are pipeline entrypoints + # that may have no genesis.pipeline block themselves (Phase C design). + my %referenced_upstream; + for my $env (keys %pipeline_data) { + my $upstream = $pipeline_data{$env}{prior_env} or next; + $referenced_upstream{$upstream} = 1; + } + # Pass 2: build nodes for envs that either have pipeline data OR are + # referenced as upstream by another env AND have a file in the directory. + my %nodes; + my %prior_env_map; + my %envs_to_include = ( + %pipeline_data, + map { $_ => {} } grep { $file_exists{$_} } keys %referenced_upstream, + ); + + for my $env (sort keys %envs_to_include) { + my $data = $pipeline_data{$env} || {}; $nodes{$env} = { stage_name => $env, target_name => $env, @@ -397,17 +417,15 @@ sub _build_from_env_files { genesis_env => $env, auto => 0, type => 'deployment', - require_pr => _truthy($pipeline_data->{require_pr}), - manual => _truthy($pipeline_data->{manual}), + require_pr => _truthy($data->{require_pr}), + manual => _truthy($data->{manual}), }; - - $prior_envs{$env} = $pipeline_data->{prior_env} - if $pipeline_data->{prior_env}; + $prior_env_map{$env} = $data->{prior_env} if $data->{prior_env}; } my @edges; - for my $env (sort keys %prior_envs) { - my $upstream = $prior_envs{$env}; + for my $env (sort keys %prior_env_map) { + my $upstream = $prior_env_map{$env}; push @edges, { from => $upstream, to => $env } if exists $nodes{$upstream}; } @@ -445,7 +463,7 @@ sub _read_genesis_pipeline_keys { # Top-level key resets context if ($line =~ /^([a-zA-Z_][a-zA-Z0-9_]*):\s*(.*)$/) { my $key = $1; - $in_genesis = ($key eq 'genesis') ? 1 : 0; + $in_genesis = $key eq 'genesis' ? 1 : 0; $in_pipeline = 0; next; } diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 4290907f..83b507a4 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -101,7 +101,6 @@ sub parse { $self->{ast} = $ast; $self->{config} = $parsed; - $self->{layout} = $self->{layout}; return $self; } diff --git a/lib/Genesis/CI/Layout.pm b/lib/Genesis/CI/Layout.pm index f174e8cf..90cd2fcc 100644 --- a/lib/Genesis/CI/Layout.pm +++ b/lib/Genesis/CI/Layout.pm @@ -113,8 +113,9 @@ sub parse { my @expansion_pool = $known ? @$known : @envs; my %auto_envs; for my $pattern (@auto_patterns) { - my $regex = $pattern; - $regex =~ s/\*/.*/g; + # Build a safe regex: escape metacharacters in literal segments, + # then replace \* (escaped asterisk) with .* for glob semantics. + my $regex = join('.*', map { quotemeta($_) } split(/\*/, $pattern, -1)); $regex = qr/^$regex$/; my $matched = 0; diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 1d07602c..aeb2dec2 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -72,7 +72,7 @@ sub repipe { 'fly -t $1 pause-pipeline -p $2', get_options->{target}, $pipeline->{pipeline}{name} ); - bail("Could not pause #c{%s} pipeline: $out", $pipeline->{pipeline}{name}) + bail("Could not pause #C{%s} pipeline: $out", $pipeline->{pipeline}{name}) unless $rc == 0 || $out =~ /pipeline '.*' not found/; my $yes = get_options->{yes} ? ' -n ' : ''; @@ -481,7 +481,7 @@ sub _repipe_compiled { 'fly -t $1 pause-pipeline -p $2', get_options->{target}, $name ); - bail("Could not pause #c{%s} pipeline: $out", $name) + bail("Could not pause #C{%s} pipeline: $out", $name) unless $rc == 0 || $out =~ /pipeline '.*' not found/; my $yes = get_options->{yes} ? ' -n ' : ''; diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 2291667d..86c620b8 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -310,7 +310,7 @@ sub repo_update { } else { # Bare invocation: full wizard with existing values pre-populated my $cfg = _ci_wizard(\%options, $top, _existing_ci_defaults($top)); - _write_ci_config($top, $cfg, update => 1); + _write_ci_config($top, $cfg); } info( diff --git a/t/ci-compiler.t b/t/ci-compiler.t index 24d1696c..d4030d80 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -555,6 +555,52 @@ subtest 'ASTBuilder - _build_from_env_files: prior_env referencing unknown env i is scalar(@$edges), 0, "no edge added for unknown prior_env"; }; +subtest 'ASTBuilder - _build_from_env_files: entrypoint with no pipeline block is included when referenced' => sub { + my $tmp = tempdir(CLEANUP => 1); + + # lab.yml — pipeline entrypoint; Phase C convention writes NO pipeline block + open my $fh, '>', "$tmp/lab.yml" or die $!; + print $fh "---\ngenesis:\n env: lab\n"; + close $fh; + + # nonprod.yml — references lab as prior_env + open $fh, '>', "$tmp/nonprod.yml" or die $!; + print $fh "---\ngenesis:\n pipeline:\n prior_env: lab\n"; + close $fh; + + my $builder = Genesis::CI::Compiler::ASTBuilder->new(); + my ($nodes, $edges) = $builder->_build_from_env_files($tmp); + + ok exists $nodes->{lab}, "lab node present despite no genesis.pipeline block"; + ok exists $nodes->{nonprod}, "nonprod node present"; + is scalar(@$edges), 1, "one edge"; + is $edges->[0]{from}, 'lab', "edge from: lab"; + is $edges->[0]{to}, 'nonprod', "edge to: nonprod"; + is $nodes->{lab}{require_pr}, 0, "lab require_pr defaults to 0"; + is $nodes->{lab}{manual}, 0, "lab manual defaults to 0"; +}; + +subtest 'ASTBuilder - _build_from_env_files: unreferenced files without pipeline block excluded' => sub { + my $tmp = tempdir(CLEANUP => 1); + + # infra.yml — genesis block but no pipeline sub-key, not referenced + open my $fh, '>', "$tmp/infra.yml" or die $!; + print $fh "---\ngenesis:\n env: infra\n"; + close $fh; + + # lab.yml — has pipeline block and references nothing + open $fh, '>', "$tmp/lab.yml" or die $!; + print $fh "---\ngenesis:\n pipeline:\n require_pr: false\n"; + close $fh; + + my $builder = Genesis::CI::Compiler::ASTBuilder->new(); + my ($nodes, $edges) = $builder->_build_from_env_files($tmp); + + ok exists $nodes->{lab}, "lab included (has pipeline block)"; + ok !exists $nodes->{infra}, "infra excluded (no pipeline block, not referenced)"; + is scalar(@$edges), 0, "no edges"; +}; + subtest 'ASTBuilder - _build_from_env_files: non-existent dir returns empty' => sub { my $builder = Genesis::CI::Compiler::ASTBuilder->new(); my ($nodes, $edges) = $builder->_build_from_env_files('/does/not/exist/xyz'); From 87c2d8741c97c9a30c4f1295a42f546b0707f6ff Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:00:14 -0400 Subject: [PATCH 005/103] Safely handle globs and empty pipeline Replace naive glob-to-regex substitutions with a split/quotemeta approach so '*' and '?' become '.*' and '.' while other characters are escaped. Apply this safer pattern building in AST (resources_matching/targets_matching), PipelineProvider::matches_pattern, and Validator cross-reference checks to avoid false positives from regex metacharacters (e.g. dots). Treat an empty pipeline hash as env-file-topology mode and skip pipeline validation. Remove a redundant empty else in Legacy and add tests covering glob behavior and the empty-pipeline validation case. --- lib/Genesis/CI/Compiler/AST.pm | 12 ++-- lib/Genesis/CI/Compiler/PipelineProvider.pm | 6 +- lib/Genesis/CI/Compiler/Validator.pm | 12 +++- lib/Genesis/CI/Legacy.pm | 1 - t/ci-compiler.t | 65 +++++++++++++++++++++ 5 files changed, 83 insertions(+), 13 deletions(-) diff --git a/lib/Genesis/CI/Compiler/AST.pm b/lib/Genesis/CI/Compiler/AST.pm index e077e40c..0a2b5570 100644 --- a/lib/Genesis/CI/Compiler/AST.pm +++ b/lib/Genesis/CI/Compiler/AST.pm @@ -132,9 +132,9 @@ sub trigger_names { sub resources_matching { my ($self, $pattern) = @_; - my $regex = $pattern; - $regex =~ s/\*/.*/g; - $regex =~ s/\?/./g; + my $regex = join('', map { + $_ eq '*' ? '.*' : $_ eq '?' ? '.' : quotemeta($_) + } split(/([*?])/, $pattern, -1)); $regex = qr/^$regex$/; my @matching; @@ -149,9 +149,9 @@ sub resources_matching { sub targets_matching { my ($self, $pattern) = @_; - my $regex = $pattern; - $regex =~ s/\*/.*/g; - $regex =~ s/\?/./g; + my $regex = join('', map { + $_ eq '*' ? '.*' : $_ eq '?' ? '.' : quotemeta($_) + } split(/([*?])/, $pattern, -1)); $regex = qr/^$regex$/; my @matching; diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 7f753f6f..73ffb37f 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -148,9 +148,9 @@ sub topological_sort { sub matches_pattern { my ($self, $name, $pattern) = @_; - my $regex = $pattern; - $regex =~ s/\*/.*/g; - $regex =~ s/\?/./g; + my $regex = join('', map { + $_ eq '*' ? '.*' : $_ eq '?' ? '.' : quotemeta($_) + } split(/([*?])/, $pattern, -1)); return $name =~ /^$regex$/; } diff --git a/lib/Genesis/CI/Compiler/Validator.pm b/lib/Genesis/CI/Compiler/Validator.pm index 9e1573ca..c3fe7568 100644 --- a/lib/Genesis/CI/Compiler/Validator.pm +++ b/lib/Genesis/CI/Compiler/Validator.pm @@ -452,6 +452,10 @@ sub _validate_pipeline_section { return; } + # Empty pipeline section means env-file-topology mode (no pipeline.yml). + # Topology is derived from genesis.pipeline.* in env files; nothing to validate here. + return unless %$pipeline; + # Metadata if ($pipeline->{metadata}) { $self->_error("'metadata.name' is required") @@ -611,12 +615,14 @@ sub _validate_cross_references { next unless $trigger->{pattern}; my $pattern = $trigger->{pattern}; - my $regex = $pattern; - $regex =~ s/\*/.*/g; + my $regex = join('', map { + $_ eq '*' ? '.*' : $_ eq '?' ? '.' : quotemeta($_) + } split(/([*?])/, $pattern, -1)); + $regex = qr/^$regex$/; my $matched = 0; for my $target_name (keys %$targets) { - if ($target_name =~ /^$regex$/) { + if ($target_name =~ $regex) { $matched = 1; last; } diff --git a/lib/Genesis/CI/Legacy.pm b/lib/Genesis/CI/Legacy.pm index ad295649..035cce64 100644 --- a/lib/Genesis/CI/Legacy.pm +++ b/lib/Genesis/CI/Legacy.pm @@ -276,7 +276,6 @@ sub validate_pipeline { push @errors, "Unrecognized `pipeline.email.smtp.$_' key found." unless m/^(host|port|username|password)$/; } - } else { } } } diff --git a/t/ci-compiler.t b/t/ci-compiler.t index d4030d80..a950a079 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2012,6 +2012,71 @@ subtest 'Compiler - override lookup uses ci_dir' => sub { "override in wrong directory is not applied"; }; +### ============================================================ ### +### AST - glob metacharacter safety +### ============================================================ ### + +subtest 'AST - targets_matching escapes regex metacharacters in literal segments' => sub { + my $ast = Genesis::CI::Compiler::AST->new( + targets => { + 'aws.dev-sandbox' => { type => 'bosh-director' }, + 'awsXdev-sandbox' => { type => 'bosh-director' }, # should NOT match 'aws.dev-*' + 'aws.dev-prod' => { type => 'bosh-director' }, + }, + ); + + my @matched = $ast->targets_matching('aws.dev-*'); + is scalar(@matched), 2, "targets_matching 'aws.dev-*' matches 2 (dot is literal)"; + + @matched = $ast->targets_matching('awsXdev-*'); + is scalar(@matched), 1, "targets_matching 'awsXdev-*' matches only 1 (no false positive)"; + + @matched = $ast->targets_matching('aws.dev-sandbox'); + is scalar(@matched), 1, "exact match with dot finds exactly 1"; +}; + +subtest 'AST - resources_matching escapes regex metacharacters in literal segments' => sub { + my $ast = Genesis::CI::Compiler::AST->new( + resources => { + 'git.repo' => { type => 'git' }, + 'gitXrepo' => { type => 'git' }, # should NOT match 'git.repo' + 'git.config' => { type => 'git' }, + }, + ); + + my @matched = $ast->resources_matching('git.repo'); + is scalar(@matched), 1, "resources_matching 'git.repo' matches only 1 (dot is literal)"; + + @matched = $ast->resources_matching('git.*'); + is scalar(@matched), 2, "resources_matching 'git.*' matches 2 git.* entries"; +}; + +### ============================================================ ### +### Validator - env-file-topology mode (no pipeline.yml) +### ============================================================ ### + +subtest 'Validator - multi-file without pipeline.yml passes validation' => sub { + my $v = Genesis::CI::Compiler::Validator->new(); + + # When no pipeline.yml exists, the parser sets pipeline => {}. + # The validator must not require 'workflows' in this case. + $v->validate({ + _source_format => 'multi-file', + pipeline => {}, # empty: no pipeline.yml, topology from env files + integrations => { + vault => { url => 'https://vault.example.com' }, + source_control => { provider => 'github', repository => 'org/repo' }, + }, + targets => { + sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } }, + }, + }); + + ok !$v->has_errors, + "empty pipeline section (env-file-topology mode) passes without 'workflows required' error" + or diag join("\n", @{$v->errors}); +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu From 3d5a9643d6977d67e40d8c2645cb314581bfca3b Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:50:29 -0400 Subject: [PATCH 006/103] Support ci: in .genesis/config (inline CI config) Add support for embedding CI configuration in the repo config (ci: in .genesis/config) and wire it into the compiler pipeline. Genesis::CI::Compiler now registers itself as owner of the ci section, exposes can_compile_from_genesis_config(), and validates the ci: structure via validate_config_section(). Parser selection order was updated to prefer .genesis/ci/, then inline ci:, then legacy ci.yml; a new _parse_genesis_config() normalizes inline data to the same shape as multi-file parsing. Top.pm gains a registry for delegated config-section handlers and delegates validation to registered owners; the repo schema marks ci as opaque. Genesis::Config accepts opaque types (passthrough). Commands updated to detect inline config and preserve backward compatibility. New tests exercise detection, parsing, validation, and the registration mechanism. --- lib/Genesis/CI/Compiler.pm | 48 +++++++ lib/Genesis/CI/Compiler/Parser.pm | 54 ++++++- lib/Genesis/Commands/Pipelines.pm | 18 ++- lib/Genesis/Config.pm | 3 + lib/Genesis/Top.pm | 33 +++++ t/ci-compiler.t | 229 ++++++++++++++++++++++++++++++ 6 files changed, 377 insertions(+), 8 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 6a272402..4d33a08f 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -9,6 +9,14 @@ use Genesis::CI::Compiler::ScriptDiscovery; use Genesis::CI::Compiler::ASTBuilder; use Genesis::CI::Compiler::PipelineDescriptor; +# Register as the owner of the ci: section in .genesis/config. +# This runs at module load time so Top.pm's _validate_config() can +# delegate ci: section validation to us when we're loaded. +{ + require Genesis::Top; + Genesis::Top->register_config_section('ci', __PACKAGE__); +} + ### Constructor {{{ # new - create a new compiler instance {{{ @@ -145,6 +153,46 @@ sub can_compile_from_env_files { ); } +# }}} +# can_compile_from_genesis_config - detect if ci: section exists in .genesis/config {{{ +# +# Returns true when $top has a Genesis::Config with a ci: key, meaning +# CI configuration is embedded inline in .genesis/config rather than in +# separate files under .genesis/ci/. +sub can_compile_from_genesis_config { + my ($class, $top) = @_; + return 0 unless $top && $top->can('config'); + return 0 unless eval { $top->config->has('ci') }; + return 1; +} + +# }}} +# validate_config_section - validate the ci: section delegated by Top.pm {{{ +# +# Called by Top::_validate_config() when this module is loaded and a ci: key +# exists in .genesis/config. Performs structural validation; detailed +# cross-reference checks happen later in Compiler::Validator during compile(). +sub validate_config_section { + my ($class, $data, $top) = @_; + + return unless defined $data; + bail("'ci' configuration in .genesis/config must be a hash") + unless ref($data) eq 'HASH'; + + bail("'ci.targets' is required and must be a non-empty hash") + unless ref($data->{targets}) eq 'HASH' && %{$data->{targets}}; + + bail("'ci.integrations' is required and must be a hash") + unless ref($data->{integrations}) eq 'HASH'; + + bail("'ci.integrations.vault' must be a hash if present") + if defined $data->{integrations}{vault} + && ref($data->{integrations}{vault}) ne 'HASH'; + + bail("'ci.integrations.source_control' is required and must be a hash") + unless ref($data->{integrations}{source_control}) eq 'HASH'; +} + # }}} # }}} ### Internal Methods {{{ diff --git a/lib/Genesis/CI/Compiler/Parser.pm b/lib/Genesis/CI/Compiler/Parser.pm index 4d4532d1..bba9cf89 100644 --- a/lib/Genesis/CI/Compiler/Parser.pm +++ b/lib/Genesis/CI/Compiler/Parser.pm @@ -27,9 +27,14 @@ sub new { sub parse { my ($self) = @_; - # Determine parsing mode: multi-file (.genesis/ci/) or legacy single file + # Determine parsing mode, in priority order: + # 1. .genesis/ci/ directory (multi-file) + # 2. .genesis/config ci: key (inline genesis-config) + # 3. Legacy ci.yml file (single-file legacy) if ($self->{ci_dir} && -d $self->{ci_dir}) { return $self->_parse_multi_file($self->{ci_dir}); + } elsif ($self->{top} && eval { $self->{top}->config->has('ci') }) { + return $self->_parse_genesis_config($self->{top}->config->get('ci')); } elsif ($self->{file} && -f $self->{file}) { return $self->_parse_legacy_file($self->{file}); } elsif ($self->{ci_dir}) { @@ -37,7 +42,8 @@ sub parse { } elsif ($self->{file}) { bail("CI configuration file '%s' not found", $self->{file}); } else { - bail("No CI configuration file or directory specified"); + bail("No CI configuration found: no .genesis/ci/ directory, no ci: section ". + "in .genesis/config, and no ci.yml file"); } } @@ -99,6 +105,50 @@ sub _parse_multi_file { return \%parsed; } +# }}} +# }}} +### Genesis Config Parser {{{ + +# _parse_genesis_config - parse the ci: section from .genesis/config {{{ +# +# Maps the ci: sub-keys directly to the same normalized structure produced by +# _parse_multi_file(), so all downstream stages (Validator, ASTBuilder, etc.) +# are format-agnostic. +# +# Expected ci: structure (all optional except targets + integrations): +# +# ci: +# targets: { name: { type, connection, ... } } # required +# integrations: { vault: {...}, source_control: {...} } # required +# pipeline: { workflows: {...}, ... } # optional +# scripts: { script_id: { path, ... } } # optional +# provider_config: { concourse: {...} } # optional +sub _parse_genesis_config { + my ($self, $data) = @_; + + my %parsed; + + $parsed{pipeline} = $data->{pipeline} || {}; + $parsed{targets} = $data->{targets} || {}; + $parsed{integrations} = $data->{integrations} || {}; + $parsed{scripts} = $data->{scripts} || {}; + $parsed{provider_config} = $data->{provider_config} || {}; + + # When no pipeline section is provided, workflow topology is derived from + # genesis.pipeline.* keys in environment YAML files (same as multi-file + # with no pipeline.yml). Point ASTBuilder at the repo root. + unless (%{$parsed{pipeline}}) { + $parsed{env_dir} = $self->{top} ? $self->{top}->path() : '.'; + } + + $parsed{_source_format} = 'genesis-config'; + $parsed{_source_path} = $self->{top} + ? $self->{top}->path('.genesis/config') + : '.genesis/config'; + + return \%parsed; +} + # }}} # }}} ### Legacy Single-File Parser {{{ diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index aeb2dec2..110d933c 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -578,14 +578,20 @@ sub _compile_pipeline { my %compiler_opts = (top => $top); - # Detect configuration source - my $ci_dir = '.genesis/ci'; - if (-d $ci_dir && -f "$ci_dir/pipeline.yml") { + # Detect configuration source — priority order: + # 1. .genesis/ci/ directory with targets.yml or pipeline.yml (multi-file) + # 2. ci: section in .genesis/config (genesis-config, Phase E) + # 3. Legacy ci.yml / --config file (backward compat) + my $ci_dir = $top->path('.genesis/ci'); + if (-d $ci_dir && (-f "$ci_dir/pipeline.yml" || -f "$ci_dir/targets.yml")) { $compiler_opts{ci_dir} = $ci_dir; - info("Using multi-file configuration from #C{%s/}", $ci_dir); + info("Using multi-file CI configuration from #C{.genesis/ci/}"); + } elsif (Genesis::CI::Compiler->can_compile_from_genesis_config($top)) { + # Parser reads ci: from $top->config — no ci_dir or file needed + info("Using inline CI configuration from #C{.genesis/config}"); } else { - $compiler_opts{file} = get_options->{config} || 'ci.yml'; - info("Using legacy configuration from #C{%s}", $compiler_opts{file}); + $compiler_opts{file} = get_options->{config} || $top->path('ci.yml'); + info("Using legacy CI configuration from #C{%s}", $compiler_opts{file}); } my $compiler = Genesis::CI::Compiler->new(%compiler_opts); diff --git a/lib/Genesis/Config.pm b/lib/Genesis/Config.pm index 00d57932..e767234f 100644 --- a/lib/Genesis/Config.pm +++ b/lib/Genesis/Config.pm @@ -590,6 +590,9 @@ sub _validate_key { if (defined($value)) { push @errors, "#R{$key}: expected null, not #ri{".($value ? $value : "")."}"; } + } elsif ($type eq 'opaque') { + # Passthrough — any value accepted, sub-keys not validated here. + # Used for config sections delegated to other modules (see Top::register_config_section). } elsif ($type eq 'any') { # Do nothing } else { diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 14e2a2ca..5b5ae043 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -19,6 +19,27 @@ use Genesis::Config; use Cwd (); use File::Path qw/rmtree/; +### Config Section Delegation Registry {{{ +# Modules may register themselves as handlers for specific top-level keys in +# .genesis/config. Top.pm owns the core schema; registered handlers own their +# section's schema and validation. This pattern is reusable for any future +# section beyond ci:. +# +# Genesis::Top->register_config_section('ci', 'Genesis::CI::Compiler'); +# +# The handler class must implement: +# validate_config_section($data, $top) # called after core schema validation + +my %_config_section_handlers; + +# register_config_section - register a module as owner of a config section {{{ +sub register_config_section { + my ($class, $section, $handler) = @_; + $_config_section_handlers{$section} = $handler; +} + +# }}} +# }}} ### Class Methods {{{ # _build - common construction logic for bare Genesis::Top object {{{ @@ -1154,6 +1175,14 @@ sub _validate_config { } elsif ($config_version == 2){ $self->config->validate($self->_repo_config_schema()); + + # Delegate validation of registered sections to their owning modules + for my $section (sort keys %_config_section_handlers) { + next unless $self->config->has($section); + my $handler = $_config_section_handlers{$section}; + $handler->validate_config_section($self->config->get($section), $self) + if $handler->can('validate_config_section'); + } } else { bail "Genesis deployment repo configuration version $config_version is not supported"; } @@ -1307,6 +1336,10 @@ sub _repo_config_schema { envvar => 'GENESIS_CONFIRM_RELEASE_OVERRIDES', description => 'Confirm release overrides' }, + ci => { + type => 'opaque', + description => 'CI pipeline configuration (owned by Genesis::CI::Compiler)', + }, }; } diff --git a/t/ci-compiler.t b/t/ci-compiler.t index a950a079..dede486c 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2077,6 +2077,235 @@ subtest 'Validator - multi-file without pipeline.yml passes validation' => sub { or diag join("\n", @{$v->errors}); }; +### ============================================================ ### +### Phase E: Genesis Config Delegation Tests +### ============================================================ ### + +# Minimal mock objects for testing without loading Genesis::Top / Genesis::Config. +# We only need duck-typed config->has/get and top->path/config. + +{ + package MockConfig; + sub new { + my ($class, %data) = @_; + return bless { data => \%data }, $class; + } + sub has { my ($self, $k) = @_; return exists $self->{data}{$k} } + sub get { my ($self, $k) = @_; return $self->{data}{$k} } +} + +{ + package MockTop; + sub new { + my ($class, %opts) = @_; + return bless { config => $opts{config}, base => $opts{base} || '/fake' }, $class; + } + sub config { $_[0]->{config} } + sub path { + my ($self, $rel) = @_; + return defined $rel ? "$self->{base}/$rel" : $self->{base}; + } +} + +my $_ci_data = { + targets => { + sandbox => { + type => 'bosh-director', + connection => { url => 'https://bosh.sandbox.example.com' }, + }, + prod => { + type => 'bosh-director', + connection => { url => 'https://bosh.prod.example.com' }, + }, + }, + integrations => { + vault => { + url => 'https://vault.example.com', + auth => { + role_id => { secret_ref => 'secret/ci:role_id' }, + secret_id => { secret_ref => 'secret/ci:secret_id' }, + }, + }, + source_control => { + provider => 'github', + repository => 'org/repo', + auth => { type => 'ssh-key', private_key => { secret_ref => 'secret/ci:private_key' } }, + }, + }, + pipeline => { + workflows => { + deploy => { + type => 'deploy', + stages => [qw(sandbox prod)], + }, + }, + }, +}; + +subtest 'Compiler - can_compile_from_genesis_config: false without top' => sub { + ok !Genesis::CI::Compiler->can_compile_from_genesis_config(undef), + "returns false when top is undef"; + ok !Genesis::CI::Compiler->can_compile_from_genesis_config( + bless({}, 'NoConfigMethods') + ), "returns false when top has no config method"; +}; + +subtest 'Compiler - can_compile_from_genesis_config: detects ci: in config' => sub { + my $top_with_ci = MockTop->new( + config => MockConfig->new(ci => $_ci_data), + ); + ok(Genesis::CI::Compiler->can_compile_from_genesis_config($top_with_ci), + "returns true when top->config has ci: key"); + + my $top_without_ci = MockTop->new( + config => MockConfig->new(deployment_type => 'cf'), + ); + ok(!Genesis::CI::Compiler->can_compile_from_genesis_config($top_without_ci), + "returns false when top->config has no ci: key"); +}; + +subtest 'Compiler - validate_config_section: accepts valid ci structure' => sub { + eval { Genesis::CI::Compiler->validate_config_section($_ci_data, undef) }; + ok !$@, "valid ci structure passes without error" or diag $@; +}; + +subtest 'Compiler - validate_config_section: rejects missing targets' => sub { + my $bad = { %$_ci_data, targets => {} }; # empty targets + eval { Genesis::CI::Compiler->validate_config_section($bad, undef) }; + like $@, qr/ci\.targets.*required/i, "empty targets triggers error"; + + my $no_targets = { %$_ci_data }; + delete $no_targets->{targets}; + eval { Genesis::CI::Compiler->validate_config_section($no_targets, undef) }; + like $@, qr/ci\.targets.*required/i, "missing targets triggers error"; +}; + +subtest 'Compiler - validate_config_section: rejects missing source_control' => sub { + my $bad = { + %$_ci_data, + integrations => { vault => { url => 'https://vault' } }, # no source_control + }; + eval { Genesis::CI::Compiler->validate_config_section($bad, undef) }; + like $@, qr/source_control.*required/i, "missing source_control triggers error"; +}; + +subtest 'Parser - genesis-config: produces correct normalized structure' => sub { + my $top = MockTop->new( + config => MockConfig->new(ci => $_ci_data), + base => '/myrepo', + ); + + my $parser = Genesis::CI::Compiler::Parser->new(top => $top); + my $parsed = eval { $parser->parse() }; + ok !$@, "parse() succeeds with genesis-config source" or diag $@; + + is $parsed->{_source_format}, 'genesis-config', "_source_format is 'genesis-config'"; + like $parsed->{_source_path}, qr{\.genesis/config$}, "_source_path ends with .genesis/config"; + + ok ref($parsed->{targets}) eq 'HASH', "targets is a hash"; + ok exists $parsed->{targets}{sandbox}, "sandbox target present"; + ok exists $parsed->{targets}{prod}, "prod target present"; + + ok ref($parsed->{integrations}) eq 'HASH', "integrations is a hash"; + ok $parsed->{integrations}{vault}{url}, "vault url present"; + + ok ref($parsed->{pipeline}) eq 'HASH', "pipeline is a hash"; + ok $parsed->{pipeline}{workflows}, "workflows present (from ci.pipeline)"; + + # env_dir must NOT be set when a pipeline section is provided + ok(!$parsed->{env_dir}, + "env_dir not set when pipeline section is provided"); +}; + +subtest 'Parser - genesis-config: sets env_dir when no pipeline section' => sub { + my $ci_no_pipeline = { %$_ci_data }; + delete $ci_no_pipeline->{pipeline}; + + my $top = MockTop->new( + config => MockConfig->new(ci => $ci_no_pipeline), + base => '/myrepo', + ); + + my $parser = Genesis::CI::Compiler::Parser->new(top => $top); + my $parsed = eval { $parser->parse() }; + ok !$@, "parse() succeeds when ci has no pipeline key" or diag $@; + + ok $parsed->{env_dir}, "env_dir is set when no pipeline section"; + is $parsed->{env_dir}, '/myrepo', "env_dir is top->path()"; + is_deeply $parsed->{pipeline}, {}, "pipeline is empty hash (env-file topology mode)"; +}; + +subtest 'Parser - genesis-config: fallback order (ci_dir missing, file missing)' => sub { + my $top = MockTop->new( + config => MockConfig->new(ci => $_ci_data), + ); + + # Neither ci_dir nor file exist; should fall through to genesis-config + my $parser = Genesis::CI::Compiler::Parser->new( + ci_dir => '/nonexistent/ci', + file => '/nonexistent/ci.yml', + top => $top, + ); + my $parsed = eval { $parser->parse() }; + ok !$@, "falls through to genesis-config when ci_dir and file are absent" or diag $@; + is $parsed->{_source_format}, 'genesis-config', + "_source_format is genesis-config after fallthrough"; +}; + +subtest 'Validator - genesis-config format routes to multi-file validation' => sub { + my $v = Genesis::CI::Compiler::Validator->new(); + + $v->validate({ + _source_format => 'genesis-config', + pipeline => { + workflows => { + deploy => { + type => 'deployment', + stages => [ + { name => 'sandbox' }, + { name => 'prod' }, + ], + }, + }, + }, + integrations => { + vault => { url => 'https://vault.example.com' }, + source_control => { provider => 'github', repository => 'org/repo' }, + }, + targets => { + sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } }, + prod => { type => 'bosh-director', connection => { url => 'https://bosh' } }, + }, + scripts => {}, + provider_config => {}, + }); + + ok !$v->has_errors, + "genesis-config format passes multi-file validation" + or diag join("\n", @{$v->errors}); +}; + +subtest 'Top - register_config_section stores handler' => sub { + # Verify the registration mechanism works by calling it directly + # (Compiler.pm registered 'ci' when it was loaded at the top of this file) + # We test by creating a custom handler for a synthetic section. + + { + package FakeHandler; + our $called = 0; + sub validate_config_section { $called = 1 } + } + + Genesis::Top->register_config_section('_test_section_', 'FakeHandler'); + + # Calling validate_config_section through the registry requires _validate_config + # to run, which needs a real repo. We just verify registration succeeded by + # checking the handler is retrievable. + ok $FakeHandler::called == 0, "handler not yet called (no config loaded)"; + Genesis::Top->register_config_section('_test_section_', 'FakeHandler'); + ok 1, "re-registering same section does not error"; +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu From a2e86dc2bd05f2f19be4e0e398db5871334f4ca7 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 14 Apr 2026 15:16:05 -0700 Subject: [PATCH 007/103] Add genesis repo-init command New repo-init replaces init with three enhancements: - Subrepo detection: auto-detects git repo, creates subdirectory without .git. --sub/--no-sub flags. - Vault selection: --skip-vault defers config, or interactive selection from safe targets. - CI provider: --ci-provider flag (scaffold TBD). Also adds --force to replace existing directories, skip_vault support in Top.pm, and phased execution pattern (_parse, _validate, _execute, _report). Includes 41 integration tests and 10 validation tests covering all option combinations. --- bin/genesis | 26 ++- lib/Genesis/Commands/Repo.pm | 306 ++++++++++++++++++++++++++ lib/Genesis/Top.pm | 8 +- t/unit-tests/genesis-cli.t | 411 +++++++++++++++++++++++++++++++++++ 4 files changed, 746 insertions(+), 5 deletions(-) diff --git a/bin/genesis b/bin/genesis index 25fd51e0..ea65000a 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1191,13 +1191,15 @@ define_command("remove-secrets", { ### Repository Management commands {{{ # genesis init - initialize a new Genesis repository {{{ -define_command("init", { +define_command("repo-init", { summary => "Initialize a new Genesis deployment repository.", - usage => "init [-k KIT/VERSION|-l path/to/local/kit] [-d directory] [--vault target] [name]", + usage => "repo-init [-k KIT] [] [name]", + alias => 'init', description => "Create a new Genesis deployment repository, specific to the given kit type.", function_group => Genesis::Commands::REPOSITORY, scope => 'empty', + no_vault => 1, option_group => Genesis::Commands::REPO_OPTIONS, options => [ "kit|k=s" => @@ -1240,6 +1242,26 @@ define_command("init", { "no-commit" => "Stage files and show the summary but skip the commit entirely.", + + 'sub!' => + "Control whether the new repo is created as a subdirectory of the ". + "current git repository (no separate .git). If not specified and ". + "the current directory is inside a git repo, subdirectory mode is ". + "used automatically.", + + 'force|F' => + "If the target directory already exists, remove it and recreate. ". + "Without this flag, you will be prompted to confirm replacement.", + + 'skip-vault' => + "Defer vault configuration. The repo will be created without a ". + "secrets provider. You must configure one via #C{genesis secrets-provider} ". + "before creating environments.", + + 'ci-provider=s' => + "CI provider for pipeline automation: 'concourse', 'github-actions', ". + "or 'manual'. When specified, writes the ci: section to .genesis/config ". + "and generates the .genesis/ci/ scaffold.", ], option_passthrough => 1, arguments => [ diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 86c620b8..a60ef584 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -5,6 +5,7 @@ use warnings; use Genesis; use Genesis::Commands; +use Genesis::Term qw/in_controlling_terminal/; use Genesis::Top; use Genesis::Kit::Provider; use Genesis::UI; @@ -14,6 +15,311 @@ use File::Basename qw/basename/; use File::Path qw/rmtree/; use JSON::PP qw/encode_json/; +sub repo_init { + _repo_init_parse(); + _repo_init_validate(); + my $result = _repo_init_execute(); + _repo_init_report($result); + exit 0; +} + +sub _repo_init_execute { + my %opts = %{get_options()}; + my $name = $opts{_name}; + my $dir = $opts{_dir}; + + # Resolve link-dev-kit to absolute path before we potentially chdir + my $abs_dev_target; + if ($opts{'link-dev-kit'}) { + $abs_dev_target = abs_path($opts{'link-dev-kit'}); + bail("Link target '%s' cannot be found!", $opts{'link-dev-kit'}) + unless $abs_dev_target; + } + + # Check git author config + if ($ENV{GIT_AUTHOR_NAME}) { + $ENV{GIT_COMMITTER_NAME} ||= $ENV{GIT_AUTHOR_NAME}; + } else { + run({ onfailure => 'Please setup git: git config --global user.name "Your Name"' }, + 'git config user.name'); + } + if ($ENV{GIT_AUTHOR_EMAIL}) { + $ENV{GIT_COMMITTER_EMAIL} ||= $ENV{GIT_AUTHOR_EMAIL}; + } else { + run({ onfailure => 'Please setup git: git config --global user.email your@email.com' }, + 'git config user.email'); + } + + # Handle existing directory (before any interactive prompts) + my %create_opts = %opts; + if (-e $dir) { + if ($opts{force}) { + rmtree $dir; + } elsif (in_controlling_terminal && Genesis::UI::prompt_for_boolean( + "Directory #C{$dir} already exists. Replace it? [y|n] ", 0 + )) { + rmtree $dir; + } else { + bail("Cannot create repository: directory #C{$dir} already exists. Use -F to force replacement."); + } + } + + # Subrepo detection + my $in_git_repo = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null'); + my $use_subdir = $opts{sub}; + if ($in_git_repo && !defined($use_subdir)) { + $use_subdir = 1; + } + + # Vault selection + if ($opts{'skip-vault'}) { + $create_opts{skip_vault} = 1; + delete $create_opts{vault}; + delete $create_opts{'skip-vault'}; + } elsif (!$opts{vault}) { + # Interactive vault selection + my $vault = _select_vault_target(); + if ($vault) { + $create_opts{vault} = $vault->{name}; + } else { + $create_opts{skip_vault} = 1; + } + } + + # Kits path handling + my $kit_path; + if (exists($create_opts{'kits-path'})) { + $kit_path = abs_path($create_opts{'kits-path'} // $ENV{HOME}.'/.genesis/kits'); + mkdir_or_fail($kit_path) unless -d $kit_path; + delete $create_opts{'kits-path'}; + } + + # Remove repo-init specific options before passing to Top->create + my $ci_provider = delete $create_opts{'ci-provider'}; + delete $create_opts{force}; + delete $create_opts{'skip-vault'}; + delete $create_opts{'sub'}; + delete $create_opts{_name}; + delete $create_opts{_dir}; + + # Create the repo via Top->create + my $top = Genesis::Top->create('.', $name, %create_opts, kits_path => $kit_path); + $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0); + + my $root = $top->path; + my $human_root = humanize_path($root); + my $kit_desc = ""; + + pushd($root); + eval { + # Kit setup + if ($abs_dev_target) { + symlink_or_fail($abs_dev_target, "./dev"); + $kit_desc = "linked to kit at #C{$abs_dev_target}"; + } elsif ($opts{kit}) { + my $kit_file; + if ($opts{kit} =~ m#(?:.*/)?([^/]+)-\d+\.\d+\.\d+(?:-rc\.?\d+)?\.t(?:ar\.)?gz#) { + bail("Local compiled kit file %s not found", $opts{kit}) unless -f $opts{kit}; + $kit_file = $opts{kit}; + } + if ($kit_file) { + my $target = $top->path(".genesis/kits"); + mkdir_or_fail($target); + my $abs_src = $kit_file =~ m#^/# ? $kit_file : abs_path($ENV{GENESIS_CALLER_DIR}."/".$kit_file); + copy_or_fail($abs_src, $target); + $kit_desc = "using locally provided compiled kit #C{$kit_file}"; + } else { + my ($kit_name, $kit_version) = $top->download_kit($opts{kit}); + $kit_desc = "using the #C{$kit_name/$kit_version} kit"; + } + } else { + mkdir_or_fail("./dev"); + $kit_desc = "with an empty development kit in #C{$human_root/dev}"; + } + + # CI provider scaffold + if ($ci_provider) { + _create_ci_scaffold($top, $ci_provider); + } + + # Git init (skip if subdirectory of existing repo) + unless ($use_subdir) { + run({ onfailure => "Failed to initialize git in $human_root/" }, + 'git init && git add .'); + run({ onfailure => "Failed to commit initial repository in $human_root/" }, + 'git commit -m "Initial Genesis Repo"'); + } + }; + my $err = $@; + popd; + if ($err) { + debug("removing incomplete Genesis repository at #C{$root} due to failed creation"); + rmtree $root; + bail $err; + } + + return { + root => $root, + human_root => $human_root, + name => $name, + kit_desc => $kit_desc, + ci_provider => $ci_provider, + vault_skipped => $create_opts{skip_vault} ? 1 : 0, + vault => $create_opts{skip_vault} ? undef : ($top->vault ? $top->vault->url : undef), + submodule => $use_subdir, + }; +} + +sub _repo_init_report { + my ($result) = @_; + + my @details; + push @details, " - $result->{kit_desc}" if $result->{kit_desc}; + + if ($result->{vault}) { + my $vault_desc = "using vault at #C{$result->{vault}}"; + push @details, " - $vault_desc"; + } else { + push @details, " - #Y{vault not configured} (use #C{genesis secrets-provider} to set)"; + } + + if ($result->{ci_provider}) { + push @details, " - CI provider: #C{$result->{ci_provider}}"; + } + + if ($result->{submodule}) { + push @details, " - created as subdirectory (no separate git)"; + } + + info "\nInitialized Genesis repository in #C{%s}\n%s\n", + $result->{human_root}, join("\n", @details); +} + +sub _select_vault_target { + # Ask if user wants to configure vault now or skip + my $configure = Genesis::UI::prompt_for_boolean( + "Would you like to configure a secrets vault now? [y|n] ", 1 + ); + return undef unless $configure; + + # Use the existing interactive vault selector from Service::Vault::Remote + require Service::Vault::Remote; + my $vault = eval { Service::Vault::Remote->target(undef) }; + return $vault; +} + +sub _create_ci_scaffold { + my ($top, $provider) = @_; + + # Write ci: section to .genesis/config + $top->config->set('ci', { + enabled => Genesis::Config::TRUE, + provider => $provider, + pipeline => { + name => $top->config->get('deployment_type'), + }, + }); + $top->config->save; + + # Create .genesis/ci/ scaffold + my $ci_dir = $top->path(".genesis/ci"); + mkdir_or_fail($ci_dir); + + mkfile_or_fail("$ci_dir/targets.yml", <<'TARGETS'); +--- +# BOSH director connection info (per environment) +# Fill in for each environment that will be deployed via pipeline. +# +# Example: +# my-env: +# url: https://bosh.example.com:25555 +# ca_cert: (( vault "secret/bosh/ssl:ca" )) +# username: admin +# password: (( vault "secret/bosh/admin:password" )) +TARGETS + + mkfile_or_fail("$ci_dir/resources.yml", <<'RESOURCES'); +--- +# Abstract resource declarations +# These are translated to provider-specific format during compilation. +RESOURCES + + mkfile_or_fail("$ci_dir/ci-overrides-${provider}.yml", <<'OVERRIDES'); +--- +# Post-provider output overrides +# Merged over provider output after compilation. +# Use this for provider-specific customizations. +OVERRIDES +} + +sub _repo_init_parse { + my %options; + Genesis::Kit::Provider->parse_opts(\@_, \%options); + append_options(%options); + return 1; +} + +sub _repo_init_validate { + my %opts = %{get_options()}; + my @args = get_args(); + my $name = $args[0]; + + # Derive name from kit if not specified + unless ($name) { + if ($opts{kit} && $opts{kit} !~ m#\.t(?:ar\.)?gz$#) { + ($name = $opts{kit}) =~ s|/.*||; + } elsif ($opts{'link-dev-kit'}) { + $name = basename($opts{'link-dev-kit'}); + } + } + + # Must have a name or a kit to derive it from + bail( + "You must specify a deployment name, a kit (-k), or a dev link target (-l)." + ) unless $name; + + # --kit and --link-dev-kit are mutually exclusive + bail( + "You can only specify one of kit (-k) or link to a kit (-l)." + ) if $opts{kit} && $opts{'link-dev-kit'}; + + # --vault and --skip-vault are mutually exclusive + bail( + "Cannot specify both --vault and --skip-vault." + ) if $opts{vault} && $opts{'skip-vault'}; + + # --ci-provider must be a valid value + if ($opts{'ci-provider'}) { + my @valid = qw(concourse github-actions manual); + bail( + "Invalid --ci-provider '%s'. Must be one of: %s", + $opts{'ci-provider'}, join(', ', @valid) + ) unless grep { $_ eq $opts{'ci-provider'} } @valid; + } + + # Store derived values back into options for execution phase + my $dir = $opts{directory} || $name; + option_defaults( + _name => $name, + _dir => $dir, + ); + + # Summarize intent + my @plan; + push @plan, "kit: #C{$opts{kit}}" if $opts{kit}; + push @plan, "kit: dev link to #C{$opts{'link-dev-kit'}}" if $opts{'link-dev-kit'}; + push @plan, "kit: #Yi{empty dev directory}" unless $opts{kit} || $opts{'link-dev-kit'}; + push @plan, "vault: #C{$opts{vault}}" if $opts{vault}; + push @plan, "vault: #Yi{deferred}" if $opts{'skip-vault'}; + push @plan, "vault: #Yi{will prompt}" unless $opts{vault} || $opts{'skip-vault'}; + push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; + info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; + info " %s", $_ for @plan; + info ""; + + return 1; +} + sub init { my %options; # FIXME: The following might work, but it may need some tweaking as it used to diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 5b5ae043..19930c70 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -175,10 +175,12 @@ sub create { } } - # Only set vault configuration if not using no_vault (and only allow no_vault in tests) - if ($opts{no_vault}) { + # Set vault configuration if available + if ($opts{no_vault} || $opts{skip_vault}) { + # no_vault: test contexts only + # skip_vault: user explicitly deferred via --skip-vault (legacy mode) bail("no_vault option can only be used in test contexts") - if $ENV{GENESIS_COMMAND}; + if $opts{no_vault} && $ENV{GENESIS_COMMAND}; } else { $self->config->set('secrets_provider', { url => $self->vault->url, diff --git a/t/unit-tests/genesis-cli.t b/t/unit-tests/genesis-cli.t index b83cc82f..76f5e0ea 100644 --- a/t/unit-tests/genesis-cli.t +++ b/t/unit-tests/genesis-cli.t @@ -77,6 +77,7 @@ subtest 'genesis terminate' => sub { $args[0] = mock 'Genesis::Env' => { name => 'my-env', type => 'my-type', + deployment_change_reason_required_size_policy => 0, notify => sub { my ($self, $msg) = @_; info("[my-env/my-type] ".$msg); @@ -126,6 +127,7 @@ subtest 'genesis terminate' => sub { $args[0] = mock 'Genesis::Env' => { name => 'my-env', type => 'my-type', + deployment_change_reason_required_size_policy => 0, notify => sub { my ($self, $msg) = @_; info("[my-env/my-type] ".$msg); @@ -176,6 +178,7 @@ subtest 'genesis terminate' => sub { $args[0] = mock 'Genesis::Env' => { name => 'my-env', type => 'my-type', + deployment_change_reason_required_size_policy => 0, notify => sub { my ($self, $msg) = @_; info("[my-env/my-type] ".$msg); @@ -256,4 +259,412 @@ EOF }; +subtest 'genesis repo-init' => sub { + plan tests => 17; + + ok(has_command('repo-init'), "repo-init command is registered"); + ok(is_equivalent_command('init' => 'repo-init'), "init is an alias for repo-init"); + like(command_properties('repo-init')->{usage}, qr/repo-init.*\[.*options/, "usage string mentions repo-init and options"); + + is(command_properties('repo-init')->{function_group}, Genesis::Commands::REPOSITORY, "repo-init belongs to repository group"); + is(command_properties('repo-init')->{scope}, 'empty', "repo-init has empty scope (no existing repo needed)"); + + my %opts = command_properties('repo-init')->{options}->@*; + + # Existing init options preserved + ok(exists $opts{'kit|k=s'}, "repo-init has --kit option"); + ok(exists $opts{'link-dev-kit|l=s'}, "repo-init has --link-dev-kit option"); + ok(exists $opts{'directory|d=s'}, "repo-init has --directory option"); + ok(exists $opts{'vault=s'}, "repo-init has --vault option"); + + # New options + ok(exists $opts{'sub!'}, "repo-init has toggleable --sub option for subrepo detection"); + ok(exists $opts{'skip-vault'}, "repo-init has --skip-vault option to defer vault config"); + ok(exists $opts{'ci-provider=s'}, "repo-init has --ci-provider option"); + ok(exists $opts{'force|F'}, "repo-init has --force option"); + + not_ok(command_properties('repo-init')->{deprecated}, "repo-init is not deprecated"); + + my $subref = $Genesis::Commands::RUN{'repo-init'}; + is(ref($subref), 'CODE', "repo-init command has a subroutine reference"); + cmp_deeply(scalar(closed_over($subref)), { + '$fn' => \'Genesis::Commands::Repo::repo_init', + '$fn_require' => \'Genesis/Commands/Repo.pm', + '$name' => \'repo-init', + }, "repo-init command routes to Repo::repo_init"); + + # init alias resolves to repo-init via GENESIS_COMMANDS + is($Genesis::Commands::GENESIS_COMMANDS{init}, 'repo-init', + "init alias resolves to repo-init"); +}; + +subtest 'repo-init option processing' => sub { + plan tests => 18; + + # Basic invocation with kit and name + prepare_command('repo-init', '-k', 'bosh', 'my-bosh'); + build_command_environment; + my %opts = %{get_options()}; + my @args = get_args(); + is($opts{kit}, 'bosh', "kit option parsed correctly"); + is($args[0], 'my-bosh', "name argument passed through"); + + # With --skip-vault + prepare_command('repo-init', '-k', 'bosh', '--skip-vault', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + ok($opts{'skip-vault'}, "--skip-vault flag is set"); + ok(!$opts{vault}, "vault is not set when skip-vault used"); + + # With --vault + prepare_command('repo-init', '-k', 'bosh', '--vault', 'my-vault', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + is($opts{vault}, 'my-vault', "--vault value parsed correctly"); + ok(!$opts{'skip-vault'}, "skip-vault not set when vault specified"); + + # With --ci-provider + prepare_command('repo-init', '-k', 'cf', '--ci-provider', 'concourse', 'my-cf'); + build_command_environment; + %opts = %{get_options()}; + is($opts{'ci-provider'}, 'concourse', "--ci-provider concourse parsed"); + + prepare_command('repo-init', '-k', 'cf', '--ci-provider', 'github-actions', 'my-cf'); + build_command_environment; + %opts = %{get_options()}; + is($opts{'ci-provider'}, 'github-actions', "--ci-provider github-actions parsed"); + + prepare_command('repo-init', '-k', 'cf', '--ci-provider', 'manual', 'my-cf'); + build_command_environment; + %opts = %{get_options()}; + is($opts{'ci-provider'}, 'manual', "--ci-provider manual parsed"); + + # With --sub + prepare_command('repo-init', '-k', 'bosh', '--sub', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + is($opts{sub}, 1, "--sub flag sets sub to true"); + + # With --no-sub + prepare_command('repo-init', '-k', 'bosh', '--no-sub', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + is($opts{sub}, 0, "--no-sub flag sets sub to false"); + + # Without --sub (not specified) + prepare_command('repo-init', '-k', 'bosh', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + ok(!defined($opts{sub}), "sub is undefined when not specified"); + + # With --directory + prepare_command('repo-init', '-k', 'bosh', '-d', '/tmp/my-repo', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + is($opts{directory}, '/tmp/my-repo', "--directory parsed correctly"); + + # With --link-dev-kit + prepare_command('repo-init', '-l', '/path/to/dev-kit', 'my-dev'); + build_command_environment; + %opts = %{get_options()}; + is($opts{'link-dev-kit'}, '/path/to/dev-kit', "--link-dev-kit parsed correctly"); + ok(!$opts{kit}, "kit not set when link-dev-kit used"); + + # Name defaults from kit when not specified + prepare_command('repo-init', '-k', 'shield'); + build_command_environment; + @args = get_args(); + is(scalar(@args), 0, "no positional args when name not specified"); + + # All options together + prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'concourse', '--skip-vault', '--sub', '-d', '/tmp/test', 'my-bosh'); + build_command_environment; + %opts = %{get_options()}; + @args = get_args(); + is($opts{kit}, 'bosh', "kit parsed in combined invocation"); + is($args[0], 'my-bosh', "name parsed in combined invocation"); +}; + +subtest 'repo-init validation' => sub { + plan tests => 10; + + require Genesis::Commands::Repo; + delete $ENV{GENESIS_IGNORE_EVAL}; # Allow bail() to die instead of exit + + # Valid cases + prepare_command('repo-init', '-k', 'bosh', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: kit + name passes"; + + prepare_command('repo-init', '-k', 'bosh', '--skip-vault', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: skip-vault passes"; + + prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'concourse', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: ci-provider concourse passes"; + + prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'manual', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: ci-provider manual passes"; + + prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'github-actions', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: ci-provider github-actions passes"; + + # Name derived from kit when not specified + prepare_command('repo-init', '-k', 'shield'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + lives_ok { Genesis::Commands::Repo::_repo_init_validate() } + "valid: name derived from kit"; + + # Invalid: --vault and --skip-vault together + prepare_command('repo-init', '-k', 'bosh', '--vault', 'my-vault', '--skip-vault', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + throws_ok { Genesis::Commands::Repo::_repo_init_validate() } + qr/Cannot specify both --vault and --skip-vault/, + "rejects --vault with --skip-vault"; + + # Invalid: --kit and --link-dev-kit together + prepare_command('repo-init', '-k', 'bosh', '-l', '/path/to/kit', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + throws_ok { Genesis::Commands::Repo::_repo_init_validate() } + qr/only specify one of kit.*or link/i, + "rejects --kit with --link-dev-kit"; + + # Invalid: no name, no kit, no link-dev-kit + prepare_command('repo-init'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + throws_ok { Genesis::Commands::Repo::_repo_init_validate() } + qr/must specify a deployment name/i, + "rejects empty invocation"; + + # Invalid: bad ci-provider value + prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'jenkins', 'my-bosh'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + throws_ok { Genesis::Commands::Repo::_repo_init_validate() } + qr/Invalid --ci-provider 'jenkins'/, + "rejects invalid ci-provider value"; +}; + +subtest 'repo-init execution (integration)' => sub { + plan tests => 41; + + require Genesis::Commands::Repo; + local $Genesis::VERSION = '3.2.0-rc2'; + delete $ENV{GENESIS_IGNORE_EVAL}; + $ENV{GIT_AUTHOR_NAME} = 'Test User'; + $ENV{GIT_AUTHOR_EMAIL} = 'test@example.com'; + $ENV{GIT_COMMITTER_NAME} = 'Test User'; + $ENV{GIT_COMMITTER_EMAIL} = 'test@example.com'; + + # Test 1: Basic --sub --skip-vault creates directory without .git + my $basedir = workdir('repo-init-test-1'); + pushd($basedir); + # Init a git repo so --sub detection works + run('git init 2>/dev/null'); + + prepare_command('repo-init', '-k', 'bosh', '--sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir/bosh", "bosh directory created"); + ok(-d "$basedir/bosh/.genesis", ".genesis directory created"); + ok(-f "$basedir/bosh/.genesis/config", ".genesis/config created"); + ok(!-e "$basedir/bosh/.git", "no .git in subdirectory mode"); + ok(-d "$basedir/bosh/.genesis/kits", ".genesis/kits directory created"); + ok(-d "$basedir/bosh/.genesis/bin", ".genesis/bin directory created (embedded genesis)"); + + # Verify config has no secrets_provider + my $config_text = slurp("$basedir/bosh/.genesis/config"); + unlike($config_text, qr/secrets_provider/, "config has no secrets_provider when skip-vault"); + like($config_text, qr/deployment_type: bosh/, "config has correct deployment_type"); + + # Verify result hash + is($result->{name}, 'bosh', "result name is bosh"); + ok($result->{vault_skipped}, "result vault_skipped is true"); + ok(!$result->{vault}, "result vault is undef"); + ok($result->{submodule}, "result submodule is true"); + popd; + + # Test 2: --no-sub creates .git + my $basedir2 = workdir('repo-init-test-2'); + pushd($basedir2); + run('git init 2>/dev/null'); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result2 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir2/bosh/.git", ".git created in --no-sub mode"); + ok(!$result2->{submodule}, "result submodule is false"); + popd; + + # Test 3: --force replaces existing directory + my $basedir3 = workdir('repo-init-test-3'); + pushd($basedir3); + mkdir_or_fail("$basedir3/bosh"); + mkfile_or_fail("$basedir3/bosh/sentinel.txt", "should be removed"); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '-F'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result3 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir3/bosh/.genesis", ".genesis created after force replace"); + ok(!-f "$basedir3/bosh/sentinel.txt", "old sentinel file removed by force"); + popd; + + # Test 4: Name derived from kit + my $basedir4 = workdir('repo-init-test-4'); + pushd($basedir4); + + prepare_command('repo-init', '-k', 'shield', '--no-sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result4 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir4/shield/.genesis", "directory named from kit"); + is($result4->{name}, 'shield', "name derived from kit"); + popd; + + # Test 5: Existing directory without --force bails (non-interactive) + my $basedir5 = workdir('repo-init-test-5'); + pushd($basedir5); + mkdir_or_fail("$basedir5/bosh"); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + throws_ok { + Genesis::Commands::Repo::_repo_init_execute(); + } qr/already exists.*-F/i, + "bails on existing directory in non-interactive mode"; + popd; + + # Test 6: Custom --directory + my $basedir6 = workdir('repo-init-test-6'); + pushd($basedir6); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '-d', 'my-custom-dir'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result6 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir6/my-custom-dir/.genesis", "custom directory created via --directory"); + ok(!-e "$basedir6/bosh", "default 'bosh' directory not created when --directory used"); + my $cfg6 = slurp("$basedir6/my-custom-dir/.genesis/config"); + like($cfg6, qr/deployment_type: bosh/, "config deployment_type is still bosh despite custom dir name"); + popd; + + # Test 7: Custom name different from kit + my $basedir7 = workdir('repo-init-test-7'); + pushd($basedir7); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', 'my-boshen'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result7 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir7/my-boshen/.genesis", "directory uses custom name, not kit name"); + ok(!-e "$basedir7/bosh", "default 'bosh' directory not created"); + is($result7->{name}, 'my-boshen', "result name matches custom name"); + my $cfg7 = slurp("$basedir7/my-boshen/.genesis/config"); + like($cfg7, qr/deployment_type: my-boshen/, "config deployment_type uses custom name"); + popd; + + # Test 8: --link-dev-kit creates symlink + my $basedir8 = workdir('repo-init-test-8'); + my $devkit_dir = workdir('repo-init-test-8-devkit'); + mkdir_or_fail("$devkit_dir/hooks"); + mkfile_or_fail("$devkit_dir/kit.yml", "name: testkit\nversion: 0.0.1\n"); + pushd($basedir8); + + prepare_command('repo-init', '-l', $devkit_dir, '--no-sub', '--skip-vault', 'my-devkit'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result8 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir8/my-devkit/.genesis", ".genesis created for linked dev kit"); + ok(-l "$basedir8/my-devkit/dev", "dev is a symlink"); + is(Cwd::abs_path(readlink("$basedir8/my-devkit/dev")), Cwd::abs_path($devkit_dir), "dev symlink points to correct target"); + like($result8->{kit_desc}, qr/linked.*kit/i, "result kit_desc mentions linked kit"); + popd; + + # Test 9: Auto-detect git repo → subdirectory mode (no --sub flag) + my $basedir9a = workdir('repo-init-test-9a'); + pushd($basedir9a); + run('git init 2>/dev/null'); + + prepare_command('repo-init', '-k', 'bosh', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result9a = Genesis::Commands::Repo::_repo_init_execute(); + + ok(!-e "$basedir9a/bosh/.git", "auto-detected git repo: no .git created"); + ok($result9a->{submodule}, "auto-detected git repo: result submodule is true"); + popd; + + # Test 10: Outside git repo without --sub → standalone with .git + my $basedir10 = workdir('repo-init-test-10'); + pushd($basedir10); + # Do NOT run git init — this is not a git repo + + prepare_command('repo-init', '-k', 'bosh', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result10 = Genesis::Commands::Repo::_repo_init_execute(); + + ok(-d "$basedir10/bosh/.git", "outside git repo: .git created"); + ok(!$result10->{submodule}, "outside git repo: result submodule is false"); + popd; + + # Test 11: Config values are correct across scenarios (detailed config validation) + my $basedir11 = workdir('repo-init-test-11'); + pushd($basedir11); + + prepare_command('repo-init', '-k', 'cf', '--no-sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result11 = Genesis::Commands::Repo::_repo_init_execute(); + + my $cfg9 = slurp("$basedir11/cf/.genesis/config"); + like($cfg9, qr/deployment_type: cf/, "cf config has deployment_type: cf"); + like($cfg9, qr/version: 2/, "config has version: 2"); + like($cfg9, qr/creator_version: 3\.2\.0-rc2/, "config has correct creator_version"); + like($cfg9, qr/minimum_version: 3\.2\.0-rc2/, "config has minimum_version"); + like($cfg9, qr/manifest_store: exodus/, "config has manifest_store: exodus"); + unlike($cfg9, qr/secrets_provider/, "config has no secrets_provider when skip-vault"); + unlike($cfg9, qr/kit_provider/, "config has no kit_provider (using default genesis-community)"); + popd; +}; + done_testing; From 6a656253d48b98c8dbafcce32a1e410b0bbdcfd9 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 14 Apr 2026 15:32:29 -0700 Subject: [PATCH 008/103] Add CI provider scaffold to repo-init --ci-provider writes ci: section to .genesis/config and creates empty .genesis/ci/ directory. No scaffold files generated -- targets resolved from vault exodus data at pipeline compile time. Tests for all three providers (concourse, github-actions, manual) and absence case. Total 55 integration tests. --- lib/Genesis/Commands/Repo.pm | 29 ++------------- t/unit-tests/genesis-cli.t | 68 +++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index a60ef584..9cfd1bd1 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -221,35 +221,10 @@ sub _create_ci_scaffold { }); $top->config->save; - # Create .genesis/ci/ scaffold + # Create .genesis/ci/ directory for future user overrides my $ci_dir = $top->path(".genesis/ci"); mkdir_or_fail($ci_dir); - - mkfile_or_fail("$ci_dir/targets.yml", <<'TARGETS'); ---- -# BOSH director connection info (per environment) -# Fill in for each environment that will be deployed via pipeline. -# -# Example: -# my-env: -# url: https://bosh.example.com:25555 -# ca_cert: (( vault "secret/bosh/ssl:ca" )) -# username: admin -# password: (( vault "secret/bosh/admin:password" )) -TARGETS - - mkfile_or_fail("$ci_dir/resources.yml", <<'RESOURCES'); ---- -# Abstract resource declarations -# These are translated to provider-specific format during compilation. -RESOURCES - - mkfile_or_fail("$ci_dir/ci-overrides-${provider}.yml", <<'OVERRIDES'); ---- -# Post-provider output overrides -# Merged over provider output after compilation. -# Use this for provider-specific customizations. -OVERRIDES + mkfile_or_fail("$ci_dir/.keep", ""); } sub _repo_init_parse { diff --git a/t/unit-tests/genesis-cli.t b/t/unit-tests/genesis-cli.t index 76f5e0ea..a0e16aab 100644 --- a/t/unit-tests/genesis-cli.t +++ b/t/unit-tests/genesis-cli.t @@ -463,7 +463,7 @@ subtest 'repo-init validation' => sub { }; subtest 'repo-init execution (integration)' => sub { - plan tests => 41; + plan tests => 55; require Genesis::Commands::Repo; local $Genesis::VERSION = '3.2.0-rc2'; @@ -665,6 +665,72 @@ subtest 'repo-init execution (integration)' => sub { unlike($cfg9, qr/secrets_provider/, "config has no secrets_provider when skip-vault"); unlike($cfg9, qr/kit_provider/, "config has no kit_provider (using default genesis-community)"); popd; + + # Test 12: --ci-provider concourse writes config and creates ci dir + my $basedir12 = workdir('repo-init-test-12'); + pushd($basedir12); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '--ci-provider', 'concourse'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result12 = Genesis::Commands::Repo::_repo_init_execute(); + + my $cfg12 = slurp("$basedir12/bosh/.genesis/config"); + like($cfg12, qr/ci:/, "concourse: config has ci: section"); + like($cfg12, qr/provider: concourse/, "concourse: ci.provider set"); + like($cfg12, qr/enabled: true/, "concourse: ci.enabled is true"); + like($cfg12, qr/name: bosh/, "concourse: ci.pipeline.name matches deployment type"); + ok(-d "$basedir12/bosh/.genesis/ci", "concourse: .genesis/ci/ directory created"); + is($result12->{ci_provider}, 'concourse', "concourse: result ci_provider correct"); + popd; + + # Test 13: --ci-provider manual + my $basedir13 = workdir('repo-init-test-13'); + pushd($basedir13); + + prepare_command('repo-init', '-k', 'cf', '--no-sub', '--skip-vault', '--ci-provider', 'manual'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result13 = Genesis::Commands::Repo::_repo_init_execute(); + + my $cfg13 = slurp("$basedir13/cf/.genesis/config"); + like($cfg13, qr/provider: manual/, "manual: ci.provider set"); + ok(-d "$basedir13/cf/.genesis/ci", "manual: .genesis/ci/ directory created"); + is($result13->{ci_provider}, 'manual', "manual: result ci_provider correct"); + popd; + + # Test 14: --ci-provider github-actions + my $basedir14 = workdir('repo-init-test-14'); + pushd($basedir14); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '--ci-provider', 'github-actions'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result14 = Genesis::Commands::Repo::_repo_init_execute(); + + my $cfg14 = slurp("$basedir14/bosh/.genesis/config"); + like($cfg14, qr/provider: github-actions/, "github-actions: ci.provider set"); + is($result14->{ci_provider}, 'github-actions', "github-actions: result ci_provider correct"); + popd; + + # Test 15: No --ci-provider → no ci: in config, no .genesis/ci/ + my $basedir15 = workdir('repo-init-test-15'); + pushd($basedir15); + + prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + build_command_environment; + Genesis::Commands::Repo::_repo_init_parse(); + Genesis::Commands::Repo::_repo_init_validate(); + my $result15 = Genesis::Commands::Repo::_repo_init_execute(); + + my $cfg15 = slurp("$basedir15/bosh/.genesis/config"); + unlike($cfg15, qr/^ci:/m, "no-provider: config has no ci: section"); + ok(!-d "$basedir15/bosh/.genesis/ci", "no-provider: no .genesis/ci/ directory"); + ok(!$result15->{ci_provider}, "no-provider: result ci_provider is undef"); + popd; }; done_testing; From 500d6c50fe555e8bee3779128e57e3d3e5c26c2e Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 14 Apr 2026 15:49:47 -0700 Subject: [PATCH 009/103] Nest CI provider config under provider.type Provider-specific fields live under ci.provider hash (type, target, team, etc.) instead of flat ci.provider string. Avoids key collisions and groups provider config naturally. --- lib/Genesis/Commands/Repo.pm | 4 +++- t/unit-tests/genesis-cli.t | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 9cfd1bd1..3e9a9807 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -214,7 +214,9 @@ sub _create_ci_scaffold { # Write ci: section to .genesis/config $top->config->set('ci', { enabled => Genesis::Config::TRUE, - provider => $provider, + provider => { + type => $provider, + }, pipeline => { name => $top->config->get('deployment_type'), }, diff --git a/t/unit-tests/genesis-cli.t b/t/unit-tests/genesis-cli.t index a0e16aab..df13afea 100644 --- a/t/unit-tests/genesis-cli.t +++ b/t/unit-tests/genesis-cli.t @@ -678,7 +678,7 @@ subtest 'repo-init execution (integration)' => sub { my $cfg12 = slurp("$basedir12/bosh/.genesis/config"); like($cfg12, qr/ci:/, "concourse: config has ci: section"); - like($cfg12, qr/provider: concourse/, "concourse: ci.provider set"); + like($cfg12, qr/type: concourse/, "concourse: ci.provider.type set"); like($cfg12, qr/enabled: true/, "concourse: ci.enabled is true"); like($cfg12, qr/name: bosh/, "concourse: ci.pipeline.name matches deployment type"); ok(-d "$basedir12/bosh/.genesis/ci", "concourse: .genesis/ci/ directory created"); @@ -696,7 +696,7 @@ subtest 'repo-init execution (integration)' => sub { my $result13 = Genesis::Commands::Repo::_repo_init_execute(); my $cfg13 = slurp("$basedir13/cf/.genesis/config"); - like($cfg13, qr/provider: manual/, "manual: ci.provider set"); + like($cfg13, qr/type: manual/, "manual: ci.provider.type set"); ok(-d "$basedir13/cf/.genesis/ci", "manual: .genesis/ci/ directory created"); is($result13->{ci_provider}, 'manual', "manual: result ci_provider correct"); popd; @@ -712,7 +712,7 @@ subtest 'repo-init execution (integration)' => sub { my $result14 = Genesis::Commands::Repo::_repo_init_execute(); my $cfg14 = slurp("$basedir14/bosh/.genesis/config"); - like($cfg14, qr/provider: github-actions/, "github-actions: ci.provider set"); + like($cfg14, qr/type: github-actions/, "github-actions: ci.provider.type set"); is($result14->{ci_provider}, 'github-actions', "github-actions: result ci_provider correct"); popd; From 565768a21dbf69b5edeaccb78c26dbd0358b29c7 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 14 Apr 2026 16:02:49 -0700 Subject: [PATCH 010/103] Only embed genesis when CI provider is set Pipeline needs bundled genesis binary; repos without CI provider do not. Also updated test to verify .genesis/bin absent without provider, present with. --- lib/Genesis/Commands/Repo.pm | 2 +- t/unit-tests/genesis-cli.t | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 3e9a9807..e6464a3d 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -104,7 +104,7 @@ sub _repo_init_execute { # Create the repo via Top->create my $top = Genesis::Top->create('.', $name, %create_opts, kits_path => $kit_path); - $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0); + $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider; my $root = $top->path; my $human_root = humanize_path($root); diff --git a/t/unit-tests/genesis-cli.t b/t/unit-tests/genesis-cli.t index df13afea..976af7cf 100644 --- a/t/unit-tests/genesis-cli.t +++ b/t/unit-tests/genesis-cli.t @@ -463,7 +463,7 @@ subtest 'repo-init validation' => sub { }; subtest 'repo-init execution (integration)' => sub { - plan tests => 55; + plan tests => 56; require Genesis::Commands::Repo; local $Genesis::VERSION = '3.2.0-rc2'; @@ -490,7 +490,7 @@ subtest 'repo-init execution (integration)' => sub { ok(-f "$basedir/bosh/.genesis/config", ".genesis/config created"); ok(!-e "$basedir/bosh/.git", "no .git in subdirectory mode"); ok(-d "$basedir/bosh/.genesis/kits", ".genesis/kits directory created"); - ok(-d "$basedir/bosh/.genesis/bin", ".genesis/bin directory created (embedded genesis)"); + ok(!-d "$basedir/bosh/.genesis/bin", "no .genesis/bin without CI provider"); # Verify config has no secrets_provider my $config_text = slurp("$basedir/bosh/.genesis/config"); @@ -682,6 +682,7 @@ subtest 'repo-init execution (integration)' => sub { like($cfg12, qr/enabled: true/, "concourse: ci.enabled is true"); like($cfg12, qr/name: bosh/, "concourse: ci.pipeline.name matches deployment type"); ok(-d "$basedir12/bosh/.genesis/ci", "concourse: .genesis/ci/ directory created"); + ok(-d "$basedir12/bosh/.genesis/bin", "concourse: genesis binary embedded for pipeline"); is($result12->{ci_provider}, 'concourse', "concourse: result ci_provider correct"); popd; From b62c5930110cad699a6f87ba45694e27e1fbadbf Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 15 Apr 2026 12:20:28 -0700 Subject: [PATCH 011/103] Refactor repo-init validation and test fixtures Merge parse+validate into single validation phase with proper ordering: check options, validate kit source, check git config, detect git repo, check existing directory, prompt for vault, summarize. Directory deletion deferred to execute phase. Use absolute paths for directory checks. Kit provider resolution in validation for early fail. Tests use local dev-kit links and kit tarballs from t/ fixtures instead of network downloads. Filed FWT-921 for pre-existing provider check bug. --- lib/Genesis/Commands/Repo.pm | 358 +++++++++++++++++++++-------------- t/unit-tests/genesis-cli.t | 135 ++++++------- 2 files changed, 275 insertions(+), 218 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index e6464a3d..1f2ecd10 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -15,25 +15,102 @@ use File::Basename qw/basename/; use File::Path qw/rmtree/; use JSON::PP qw/encode_json/; +# ============================================================================== +# repo-init (replaces init) +# ============================================================================== + sub repo_init { - _repo_init_parse(); _repo_init_validate(); my $result = _repo_init_execute(); _repo_init_report($result); exit 0; } -sub _repo_init_execute { +# -- Phase 1: Validation (parse + validate + gather + prompt) ------------------ +# +# Order within validation: +# 1. Parse options and derive values (fast, no side effects) +# 2. Check invalid option combinations (fast bail) +# 3. Gather data: resolve kit provider, check kit availability, detect git repo +# 4. Check for destructive prerequisites (existing directory) +# 5. Prompt for missing info (vault selection) +# 6. Summarize intent +# +sub _repo_init_validate { my %opts = %{get_options()}; - my $name = $opts{_name}; - my $dir = $opts{_dir}; + my @args = get_args(); - # Resolve link-dev-kit to absolute path before we potentially chdir - my $abs_dev_target; - if ($opts{'link-dev-kit'}) { - $abs_dev_target = abs_path($opts{'link-dev-kit'}); - bail("Link target '%s' cannot be found!", $opts{'link-dev-kit'}) - unless $abs_dev_target; + # --- 1. Parse and derive --- + + my $name = $args[0]; + my $kit_file; + + # Check if kit is a local file path + if ($opts{kit} && $opts{kit} =~ m#(?:.*/)?([^/]+)-\d+\.\d+\.\d+(?:-rc\.?\d+)?\.t(?:ar\.)?gz#) { + $kit_file = $opts{kit}; + $name = $1 unless $name; + } + + # Derive name from kit or link-dev-kit if not specified + unless ($name) { + if ($opts{kit} && !$kit_file) { + ($name = $opts{kit}) =~ s|/.*||; + } elsif ($opts{'link-dev-kit'}) { + $name = basename($opts{'link-dev-kit'}); + } + } + + my $dir = $opts{directory} || $name; + my $parent_dir = abs_path(getcwd()); + my $target_path = "$parent_dir/$dir"; + + # --- 2. Check invalid option combinations --- + + bail( + "You must specify a deployment name, a kit (-k), or a dev link target (-l)." + ) unless $name; + + bail( + "You can only specify one of kit (-k) or link to a kit (-l)." + ) if $opts{kit} && $opts{'link-dev-kit'}; + + bail( + "Cannot specify both --vault and --skip-vault." + ) if $opts{vault} && $opts{'skip-vault'}; + + if ($opts{'ci-provider'}) { + my @valid = qw(concourse github-actions manual); + bail( + "Invalid --ci-provider '%s'. Must be one of: %s", + $opts{'ci-provider'}, join(', ', @valid) + ) unless grep { $_ eq $opts{'ci-provider'} } @valid; + } + + # --- 3. Gather data: validate kit source, detect git repo --- + + # Validate kit source exists before any destructive actions + my ($resolved_kit_name, $resolved_kit_version); + if ($kit_file) { + bail("Local compiled kit file '%s' not found.", $kit_file) + unless -f $kit_file; + } elsif ($opts{'link-dev-kit'}) { + bail("Dev kit link target '%s' not found.", $opts{'link-dev-kit'}) + unless -e $opts{'link-dev-kit'}; + } elsif ($opts{kit}) { + # Remote kit — build provider and resolve name/version + ($resolved_kit_name, $resolved_kit_version) = split('/', $opts{kit}, 2); + + my %provider_opts; + Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); + my $provider = eval { Genesis::Kit::Provider->init(%provider_opts) }; + bail("Could not initialize kit provider: %s", $@) if $@; + + # Resolve version — also validates the kit exists on the provider + unless ($resolved_kit_version && $resolved_kit_version ne 'latest') { + $resolved_kit_version = eval { $provider->latest_version_of($resolved_kit_name) }; + bail("Kit '%s' not found or no versions available from %s.", + $resolved_kit_name, $provider->label) unless $resolved_kit_version; + } } # Check git author config @@ -50,60 +127,122 @@ sub _repo_init_execute { 'git config user.email'); } - # Handle existing directory (before any interactive prompts) - my %create_opts = %opts; - if (-e $dir) { + # Detect git repo for subdirectory mode + my $in_git_repo = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null'); + my $use_subdir = $opts{sub}; + if ($in_git_repo && !defined($use_subdir)) { + $use_subdir = 1; + } + + # --- 4. Check destructive prerequisites --- + + my $replace_existing = 0; + if (-e $target_path) { if ($opts{force}) { - rmtree $dir; + $replace_existing = 1; } elsif (in_controlling_terminal && Genesis::UI::prompt_for_boolean( "Directory #C{$dir} already exists. Replace it? [y|n] ", 0 )) { - rmtree $dir; + $replace_existing = 1; } else { bail("Cannot create repository: directory #C{$dir} already exists. Use -F to force replacement."); } } - # Subrepo detection - my $in_git_repo = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null'); - my $use_subdir = $opts{sub}; - if ($in_git_repo && !defined($use_subdir)) { - $use_subdir = 1; - } + # --- 5. Prompt for missing info --- - # Vault selection + my $vault_target; if ($opts{'skip-vault'}) { - $create_opts{skip_vault} = 1; - delete $create_opts{vault}; - delete $create_opts{'skip-vault'}; - } elsif (!$opts{vault}) { - # Interactive vault selection + # explicitly skipped + } elsif ($opts{vault}) { + $vault_target = $opts{vault}; + } else { my $vault = _select_vault_target(); - if ($vault) { - $create_opts{vault} = $vault->{name}; - } else { - $create_opts{skip_vault} = 1; - } + $vault_target = $vault->{name} if $vault; } + # --- 6. Store derived values and summarize intent --- + + option_defaults( + _name => $name, + _dir => $dir, + _parent_dir => $parent_dir, + _target_path => $target_path, + _kit_file => $kit_file, + _use_subdir => $use_subdir, + _vault_target => $vault_target, + _replace_existing => $replace_existing, + _resolved_kit_name => $resolved_kit_name, + _resolved_kit_version => $resolved_kit_version, + ); + + my @plan; + if ($resolved_kit_name) { + push @plan, "kit: #C{$resolved_kit_name/$resolved_kit_version}"; + } elsif ($kit_file) { + push @plan, "kit: #C{$kit_file} (local)"; + } elsif ($opts{'link-dev-kit'}) { + push @plan, "kit: dev link to #C{$opts{'link-dev-kit'}}"; + } else { + push @plan, "kit: #Yi{empty dev directory}"; + } + push @plan, "vault: #C{$vault_target}" if $vault_target; + push @plan, "vault: #Yi{deferred}" unless $vault_target; + push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; + push @plan, "subdirectory: #C{yes} (no separate git)" if $use_subdir; + info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; + info " %s", $_ for @plan; + info ""; + + return 1; +} + +# -- Phase 2: Execution ------------------------------------------------------- +# +# All validation is complete. This phase only does work — no prompts, no bails +# on user input. Failures here are unexpected errors. +# +sub _repo_init_execute { + my %opts = %{get_options()}; + my $name = $opts{_name}; + my $dir = $opts{_dir}; + my $parent_dir = $opts{_parent_dir}; + my $target_path = $opts{_target_path}; + my $kit_file = $opts{_kit_file}; + my $use_subdir = $opts{_use_subdir}; + my $vault_target = $opts{_vault_target}; # undef = skip vault + my $replace_existing = $opts{_replace_existing}; + my $ci_provider = $opts{'ci-provider'}; + + # Remove existing directory if validation approved it + if ($replace_existing && -e $target_path) { + rmtree $target_path; + } + + # Resolve link-dev-kit to absolute path before we potentially chdir + my $abs_dev_target; + if ($opts{'link-dev-kit'}) { + $abs_dev_target = abs_path($opts{'link-dev-kit'}); + } + + # Build create options for Top->create + my %create_opts; + if ($vault_target) { + $create_opts{vault} = $vault_target; + } else { + $create_opts{skip_vault} = 1; + } + $create_opts{directory} = $opts{directory} if $opts{directory}; + # Kits path handling my $kit_path; - if (exists($create_opts{'kits-path'})) { - $kit_path = abs_path($create_opts{'kits-path'} // $ENV{HOME}.'/.genesis/kits'); + if (exists($opts{'kits-path'})) { + $kit_path = abs_path($opts{'kits-path'} // $ENV{HOME}.'/.genesis/kits'); mkdir_or_fail($kit_path) unless -d $kit_path; - delete $create_opts{'kits-path'}; } - # Remove repo-init specific options before passing to Top->create - my $ci_provider = delete $create_opts{'ci-provider'}; - delete $create_opts{force}; - delete $create_opts{'skip-vault'}; - delete $create_opts{'sub'}; - delete $create_opts{_name}; - delete $create_opts{_dir}; - # Create the repo via Top->create - my $top = Genesis::Top->create('.', $name, %create_opts, kits_path => $kit_path); + my $top = Genesis::Top->create($parent_dir, $name, %create_opts, kits_path => $kit_path); $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider; my $root = $top->path; @@ -116,22 +255,15 @@ sub _repo_init_execute { if ($abs_dev_target) { symlink_or_fail($abs_dev_target, "./dev"); $kit_desc = "linked to kit at #C{$abs_dev_target}"; + } elsif ($kit_file) { + my $target = $top->path(".genesis/kits"); + mkdir_or_fail($target); + my $abs_src = $kit_file =~ m#^/# ? $kit_file : abs_path($ENV{GENESIS_CALLER_DIR}."/".$kit_file); + copy_or_fail($abs_src, $target); + $kit_desc = "using locally provided compiled kit #C{$kit_file}"; } elsif ($opts{kit}) { - my $kit_file; - if ($opts{kit} =~ m#(?:.*/)?([^/]+)-\d+\.\d+\.\d+(?:-rc\.?\d+)?\.t(?:ar\.)?gz#) { - bail("Local compiled kit file %s not found", $opts{kit}) unless -f $opts{kit}; - $kit_file = $opts{kit}; - } - if ($kit_file) { - my $target = $top->path(".genesis/kits"); - mkdir_or_fail($target); - my $abs_src = $kit_file =~ m#^/# ? $kit_file : abs_path($ENV{GENESIS_CALLER_DIR}."/".$kit_file); - copy_or_fail($abs_src, $target); - $kit_desc = "using locally provided compiled kit #C{$kit_file}"; - } else { - my ($kit_name, $kit_version) = $top->download_kit($opts{kit}); - $kit_desc = "using the #C{$kit_name/$kit_version} kit"; - } + my ($kit_name, $kit_version) = $top->download_kit($opts{kit}); + $kit_desc = "using the #C{$kit_name/$kit_version} kit"; } else { mkdir_or_fail("./dev"); $kit_desc = "with an empty development kit in #C{$human_root/dev}"; @@ -159,17 +291,19 @@ sub _repo_init_execute { } return { - root => $root, - human_root => $human_root, - name => $name, - kit_desc => $kit_desc, - ci_provider => $ci_provider, - vault_skipped => $create_opts{skip_vault} ? 1 : 0, - vault => $create_opts{skip_vault} ? undef : ($top->vault ? $top->vault->url : undef), - submodule => $use_subdir, + root => $root, + human_root => $human_root, + name => $name, + kit_desc => $kit_desc, + ci_provider => $ci_provider, + vault_skipped => $vault_target ? 0 : 1, + vault => $vault_target ? ($top->vault ? $top->vault->url : $vault_target) : undef, + submodule => $use_subdir, }; } +# -- Phase 3: Report ---------------------------------------------------------- + sub _repo_init_report { my ($result) = @_; @@ -177,8 +311,7 @@ sub _repo_init_report { push @details, " - $result->{kit_desc}" if $result->{kit_desc}; if ($result->{vault}) { - my $vault_desc = "using vault at #C{$result->{vault}}"; - push @details, " - $vault_desc"; + push @details, " - using vault at #C{$result->{vault}}"; } else { push @details, " - #Y{vault not configured} (use #C{genesis secrets-provider} to set)"; } @@ -195,14 +328,14 @@ sub _repo_init_report { $result->{human_root}, join("\n", @details); } +# -- Helpers ------------------------------------------------------------------- + sub _select_vault_target { - # Ask if user wants to configure vault now or skip my $configure = Genesis::UI::prompt_for_boolean( "Would you like to configure a secrets vault now? [y|n] ", 1 ); return undef unless $configure; - # Use the existing interactive vault selector from Service::Vault::Remote require Service::Vault::Remote; my $vault = eval { Service::Vault::Remote->target(undef) }; return $vault; @@ -211,7 +344,6 @@ sub _select_vault_target { sub _create_ci_scaffold { my ($top, $provider) = @_; - # Write ci: section to .genesis/config $top->config->set('ci', { enabled => Genesis::Config::TRUE, provider => { @@ -223,89 +355,23 @@ sub _create_ci_scaffold { }); $top->config->save; - # Create .genesis/ci/ directory for future user overrides my $ci_dir = $top->path(".genesis/ci"); mkdir_or_fail($ci_dir); mkfile_or_fail("$ci_dir/.keep", ""); } -sub _repo_init_parse { - my %options; - Genesis::Kit::Provider->parse_opts(\@_, \%options); - append_options(%options); - return 1; -} - -sub _repo_init_validate { - my %opts = %{get_options()}; - my @args = get_args(); - my $name = $args[0]; - - # Derive name from kit if not specified - unless ($name) { - if ($opts{kit} && $opts{kit} !~ m#\.t(?:ar\.)?gz$#) { - ($name = $opts{kit}) =~ s|/.*||; - } elsif ($opts{'link-dev-kit'}) { - $name = basename($opts{'link-dev-kit'}); - } - } - - # Must have a name or a kit to derive it from - bail( - "You must specify a deployment name, a kit (-k), or a dev link target (-l)." - ) unless $name; - # --kit and --link-dev-kit are mutually exclusive - bail( - "You can only specify one of kit (-k) or link to a kit (-l)." - ) if $opts{kit} && $opts{'link-dev-kit'}; - - # --vault and --skip-vault are mutually exclusive - bail( - "Cannot specify both --vault and --skip-vault." - ) if $opts{vault} && $opts{'skip-vault'}; - - # --ci-provider must be a valid value - if ($opts{'ci-provider'}) { - my @valid = qw(concourse github-actions manual); - bail( - "Invalid --ci-provider '%s'. Must be one of: %s", - $opts{'ci-provider'}, join(', ', @valid) - ) unless grep { $_ eq $opts{'ci-provider'} } @valid; - } - - # Store derived values back into options for execution phase - my $dir = $opts{directory} || $name; - option_defaults( - _name => $name, - _dir => $dir, - ); - - # Summarize intent - my @plan; - push @plan, "kit: #C{$opts{kit}}" if $opts{kit}; - push @plan, "kit: dev link to #C{$opts{'link-dev-kit'}}" if $opts{'link-dev-kit'}; - push @plan, "kit: #Yi{empty dev directory}" unless $opts{kit} || $opts{'link-dev-kit'}; - push @plan, "vault: #C{$opts{vault}}" if $opts{vault}; - push @plan, "vault: #Yi{deferred}" if $opts{'skip-vault'}; - push @plan, "vault: #Yi{will prompt}" unless $opts{vault} || $opts{'skip-vault'}; - push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; - info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; - info " %s", $_ for @plan; - info ""; - - return 1; -} +# ============================================================================== +# Legacy init (preserved for backward compatibility, aliased to repo-init) +# ============================================================================== sub init { my %options; - # FIXME: The following might work, but it may need some tweaking as it used to - # run before the regular option parser. Genesis::Kit::Provider->parse_opts(\@_, \%options); append_options(%options); %options = %{get_options()}; - command_usage(1) if @_ > 1; # name is now optional if kit specified + command_usage(1) if @_ > 1; command_usage(1, "You can only specify one of kit (-k) or link to a kit (-L)") if scalar(grep {$_ =~ /^(?:kit|link-dev-kit)$/} keys %options) > 1; @@ -438,8 +504,12 @@ sub init { exit 0; } +# ============================================================================== +# Other repo commands +# ============================================================================== + sub secrets_provider { - command_usage(1) if @_ > 1; # target is optional + command_usage(1) if @_ > 1; my %options = %{get_options(qw(interactive clear))}; $options{target} = shift if scalar(@_); @@ -459,7 +529,7 @@ sub secrets_provider { my %vault_info = $top->vault_status; if (%vault_info) { if ($vault_info{status} eq "unauthenticated") { - eval {$top->vault->authenticate}; # Try to auto-authenticate + eval {$top->vault->authenticate}; %vault_info = $top->vault_status; } info( diff --git a/t/unit-tests/genesis-cli.t b/t/unit-tests/genesis-cli.t index 976af7cf..52a2b135 100644 --- a/t/unit-tests/genesis-cli.t +++ b/t/unit-tests/genesis-cli.t @@ -390,57 +390,58 @@ subtest 'repo-init validation' => sub { require Genesis::Commands::Repo; delete $ENV{GENESIS_IGNORE_EVAL}; # Allow bail() to die instead of exit + $ENV{GIT_AUTHOR_NAME} = 'Test User'; + $ENV{GIT_AUTHOR_EMAIL} = 'test@example.com'; + $ENV{GIT_COMMITTER_NAME} = 'Test User'; + $ENV{GIT_COMMITTER_EMAIL} = 'test@example.com'; + + # Setup local kit fixtures for validation tests (no network access) + my $vt_devkit = workdir('validation-devkit'); + my $vt_tarball = workdir('validation-tarball'); + mkfile_or_fail("$vt_tarball/bosh-0.0.1.tar.gz", "fake kit tarball"); # Valid cases - prepare_command('repo-init', '-k', 'bosh', 'my-bosh'); + prepare_command('repo-init', '-k', "$vt_tarball/bosh-0.0.1.tar.gz", 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } - "valid: kit + name passes"; + "valid: local kit tarball + name passes"; - prepare_command('repo-init', '-k', 'bosh', '--skip-vault', 'my-bosh'); + prepare_command('repo-init', '-l', $vt_devkit, '--skip-vault', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } - "valid: skip-vault passes"; + "valid: link-dev-kit + skip-vault passes"; - prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'concourse', 'my-bosh'); + prepare_command('repo-init', '-k', "$vt_tarball/bosh-0.0.1.tar.gz", '--ci-provider', 'concourse', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } "valid: ci-provider concourse passes"; - prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'manual', 'my-bosh'); + prepare_command('repo-init', '-l', $vt_devkit, '--ci-provider', 'manual', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } "valid: ci-provider manual passes"; - prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'github-actions', 'my-bosh'); + prepare_command('repo-init', '-k', "$vt_tarball/bosh-0.0.1.tar.gz", '--ci-provider', 'github-actions', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } "valid: ci-provider github-actions passes"; - # Name derived from kit when not specified - prepare_command('repo-init', '-k', 'shield'); + # Name derived from link-dev-kit when not specified + prepare_command('repo-init', '-l', $vt_devkit); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); lives_ok { Genesis::Commands::Repo::_repo_init_validate() } - "valid: name derived from kit"; + "valid: name derived from link-dev-kit"; # Invalid: --vault and --skip-vault together - prepare_command('repo-init', '-k', 'bosh', '--vault', 'my-vault', '--skip-vault', 'my-bosh'); + prepare_command('repo-init', '-l', $vt_devkit, '--vault', 'my-vault', '--skip-vault', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); throws_ok { Genesis::Commands::Repo::_repo_init_validate() } qr/Cannot specify both --vault and --skip-vault/, "rejects --vault with --skip-vault"; # Invalid: --kit and --link-dev-kit together - prepare_command('repo-init', '-k', 'bosh', '-l', '/path/to/kit', 'my-bosh'); + prepare_command('repo-init', '-k', "$vt_tarball/bosh-0.0.1.tar.gz", '-l', $vt_devkit, 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); throws_ok { Genesis::Commands::Repo::_repo_init_validate() } qr/only specify one of kit.*or link/i, "rejects --kit with --link-dev-kit"; @@ -448,15 +449,13 @@ subtest 'repo-init validation' => sub { # Invalid: no name, no kit, no link-dev-kit prepare_command('repo-init'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); throws_ok { Genesis::Commands::Repo::_repo_init_validate() } qr/must specify a deployment name/i, "rejects empty invocation"; # Invalid: bad ci-provider value - prepare_command('repo-init', '-k', 'bosh', '--ci-provider', 'jenkins', 'my-bosh'); + prepare_command('repo-init', '-l', $vt_devkit, '--ci-provider', 'jenkins', 'my-bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); throws_ok { Genesis::Commands::Repo::_repo_init_validate() } qr/Invalid --ci-provider 'jenkins'/, "rejects invalid ci-provider value"; @@ -473,23 +472,26 @@ subtest 'repo-init execution (integration)' => sub { $ENV{GIT_COMMITTER_NAME} = 'Test User'; $ENV{GIT_COMMITTER_EMAIL} = 'test@example.com'; + # Local kit fixtures (no network access) + my $devkit = Cwd::abs_path('t/src/simple'); + my $kit_tarball = Cwd::abs_path('t/repos/compiled-kit-test/.genesis/kits/compiled-0.0.1.tar.gz'); + # Test 1: Basic --sub --skip-vault creates directory without .git my $basedir = workdir('repo-init-test-1'); pushd($basedir); # Init a git repo so --sub detection works run('git init 2>/dev/null'); - prepare_command('repo-init', '-k', 'bosh', '--sub', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--sub', '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); - Genesis::Commands::Repo::_repo_init_validate(); +Genesis::Commands::Repo::_repo_init_validate(); my $result = Genesis::Commands::Repo::_repo_init_execute(); ok(-d "$basedir/bosh", "bosh directory created"); ok(-d "$basedir/bosh/.genesis", ".genesis directory created"); ok(-f "$basedir/bosh/.genesis/config", ".genesis/config created"); ok(!-e "$basedir/bosh/.git", "no .git in subdirectory mode"); - ok(-d "$basedir/bosh/.genesis/kits", ".genesis/kits directory created"); + ok(-l "$basedir/bosh/dev", "dev symlink created for linked dev kit"); ok(!-d "$basedir/bosh/.genesis/bin", "no .genesis/bin without CI provider"); # Verify config has no secrets_provider @@ -504,14 +506,13 @@ subtest 'repo-init execution (integration)' => sub { ok($result->{submodule}, "result submodule is true"); popd; - # Test 2: --no-sub creates .git + # Test 2: --no-sub creates .git (using kit tarball) my $basedir2 = workdir('repo-init-test-2'); pushd($basedir2); run('git init 2>/dev/null'); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + prepare_command('repo-init', '-k', $kit_tarball, '--no-sub', '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result2 = Genesis::Commands::Repo::_repo_init_execute(); @@ -525,9 +526,8 @@ subtest 'repo-init execution (integration)' => sub { mkdir_or_fail("$basedir3/bosh"); mkfile_or_fail("$basedir3/bosh/sentinel.txt", "should be removed"); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '-F'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', '-F', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result3 = Genesis::Commands::Repo::_repo_init_execute(); @@ -535,18 +535,17 @@ subtest 'repo-init execution (integration)' => sub { ok(!-f "$basedir3/bosh/sentinel.txt", "old sentinel file removed by force"); popd; - # Test 4: Name derived from kit + # Test 4: Name derived from link-dev-kit basename my $basedir4 = workdir('repo-init-test-4'); pushd($basedir4); - prepare_command('repo-init', '-k', 'shield', '--no-sub', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result4 = Genesis::Commands::Repo::_repo_init_execute(); - ok(-d "$basedir4/shield/.genesis", "directory named from kit"); - is($result4->{name}, 'shield', "name derived from kit"); + ok(-d "$basedir4/simple/.genesis", "directory named from dev-kit basename"); + is($result4->{name}, 'simple', "name derived from dev-kit"); popd; # Test 5: Existing directory without --force bails (non-interactive) @@ -554,23 +553,20 @@ subtest 'repo-init execution (integration)' => sub { pushd($basedir5); mkdir_or_fail("$basedir5/bosh"); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); - Genesis::Commands::Repo::_repo_init_validate(); throws_ok { - Genesis::Commands::Repo::_repo_init_execute(); + Genesis::Commands::Repo::_repo_init_validate(); } qr/already exists.*-F/i, "bails on existing directory in non-interactive mode"; popd; - # Test 6: Custom --directory + # Test 6: Custom --directory (using kit tarball) my $basedir6 = workdir('repo-init-test-6'); pushd($basedir6); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '-d', 'my-custom-dir'); + prepare_command('repo-init', '-k', $kit_tarball, '--no-sub', '--skip-vault', '-d', 'my-custom-dir', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result6 = Genesis::Commands::Repo::_repo_init_execute(); @@ -584,14 +580,13 @@ subtest 'repo-init execution (integration)' => sub { my $basedir7 = workdir('repo-init-test-7'); pushd($basedir7); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', 'my-boshen'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', 'my-boshen'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result7 = Genesis::Commands::Repo::_repo_init_execute(); ok(-d "$basedir7/my-boshen/.genesis", "directory uses custom name, not kit name"); - ok(!-e "$basedir7/bosh", "default 'bosh' directory not created"); + ok(!-e "$basedir7/simple", "default dev-kit name directory not created"); is($result7->{name}, 'my-boshen', "result name matches custom name"); my $cfg7 = slurp("$basedir7/my-boshen/.genesis/config"); like($cfg7, qr/deployment_type: my-boshen/, "config deployment_type uses custom name"); @@ -606,8 +601,7 @@ subtest 'repo-init execution (integration)' => sub { prepare_command('repo-init', '-l', $devkit_dir, '--no-sub', '--skip-vault', 'my-devkit'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); - Genesis::Commands::Repo::_repo_init_validate(); +Genesis::Commands::Repo::_repo_init_validate(); my $result8 = Genesis::Commands::Repo::_repo_init_execute(); ok(-d "$basedir8/my-devkit/.genesis", ".genesis created for linked dev kit"); @@ -621,9 +615,8 @@ subtest 'repo-init execution (integration)' => sub { pushd($basedir9a); run('git init 2>/dev/null'); - prepare_command('repo-init', '-k', 'bosh', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result9a = Genesis::Commands::Repo::_repo_init_execute(); @@ -636,9 +629,8 @@ subtest 'repo-init execution (integration)' => sub { pushd($basedir10); # Do NOT run git init — this is not a git repo - prepare_command('repo-init', '-k', 'bosh', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result10 = Genesis::Commands::Repo::_repo_init_execute(); @@ -646,33 +638,31 @@ subtest 'repo-init execution (integration)' => sub { ok(!$result10->{submodule}, "outside git repo: result submodule is false"); popd; - # Test 11: Config values are correct across scenarios (detailed config validation) + # Test 11: Config values are correct (using kit tarball) my $basedir11 = workdir('repo-init-test-11'); pushd($basedir11); - prepare_command('repo-init', '-k', 'cf', '--no-sub', '--skip-vault'); + prepare_command('repo-init', '-k', $kit_tarball, '--no-sub', '--skip-vault', 'my-cf'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result11 = Genesis::Commands::Repo::_repo_init_execute(); - my $cfg9 = slurp("$basedir11/cf/.genesis/config"); - like($cfg9, qr/deployment_type: cf/, "cf config has deployment_type: cf"); - like($cfg9, qr/version: 2/, "config has version: 2"); - like($cfg9, qr/creator_version: 3\.2\.0-rc2/, "config has correct creator_version"); - like($cfg9, qr/minimum_version: 3\.2\.0-rc2/, "config has minimum_version"); - like($cfg9, qr/manifest_store: exodus/, "config has manifest_store: exodus"); - unlike($cfg9, qr/secrets_provider/, "config has no secrets_provider when skip-vault"); - unlike($cfg9, qr/kit_provider/, "config has no kit_provider (using default genesis-community)"); + my $cfg11 = slurp("$basedir11/my-cf/.genesis/config"); + like($cfg11, qr/deployment_type: my-cf/, "config has deployment_type: my-cf"); + like($cfg11, qr/version: 2/, "config has version: 2"); + like($cfg11, qr/creator_version: 3\.2\.0-rc2/, "config has correct creator_version"); + like($cfg11, qr/minimum_version: 3\.2\.0-rc2/, "config has minimum_version"); + like($cfg11, qr/manifest_store: exodus/, "config has manifest_store: exodus"); + unlike($cfg11, qr/secrets_provider/, "config has no secrets_provider when skip-vault"); + unlike($cfg11, qr/kit_provider/, "config has no kit_provider (using default genesis-community)"); popd; # Test 12: --ci-provider concourse writes config and creates ci dir my $basedir12 = workdir('repo-init-test-12'); pushd($basedir12); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '--ci-provider', 'concourse'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', '--ci-provider', 'concourse', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result12 = Genesis::Commands::Repo::_repo_init_execute(); @@ -690,15 +680,14 @@ subtest 'repo-init execution (integration)' => sub { my $basedir13 = workdir('repo-init-test-13'); pushd($basedir13); - prepare_command('repo-init', '-k', 'cf', '--no-sub', '--skip-vault', '--ci-provider', 'manual'); + prepare_command('repo-init', '-k', $kit_tarball, '--no-sub', '--skip-vault', '--ci-provider', 'manual', 'my-cf'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result13 = Genesis::Commands::Repo::_repo_init_execute(); - my $cfg13 = slurp("$basedir13/cf/.genesis/config"); + my $cfg13 = slurp("$basedir13/my-cf/.genesis/config"); like($cfg13, qr/type: manual/, "manual: ci.provider.type set"); - ok(-d "$basedir13/cf/.genesis/ci", "manual: .genesis/ci/ directory created"); + ok(-d "$basedir13/my-cf/.genesis/ci", "manual: .genesis/ci/ directory created"); is($result13->{ci_provider}, 'manual', "manual: result ci_provider correct"); popd; @@ -706,9 +695,8 @@ subtest 'repo-init execution (integration)' => sub { my $basedir14 = workdir('repo-init-test-14'); pushd($basedir14); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault', '--ci-provider', 'github-actions'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', '--ci-provider', 'github-actions', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result14 = Genesis::Commands::Repo::_repo_init_execute(); @@ -721,9 +709,8 @@ subtest 'repo-init execution (integration)' => sub { my $basedir15 = workdir('repo-init-test-15'); pushd($basedir15); - prepare_command('repo-init', '-k', 'bosh', '--no-sub', '--skip-vault'); + prepare_command('repo-init', '-l', $devkit, '--no-sub', '--skip-vault', 'bosh'); build_command_environment; - Genesis::Commands::Repo::_repo_init_parse(); Genesis::Commands::Repo::_repo_init_validate(); my $result15 = Genesis::Commands::Repo::_repo_init_execute(); From 0da5714c94236eee0de88daa8b1517793623a1ee Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 15 Apr 2026 12:29:50 -0700 Subject: [PATCH 012/103] Simplify repo-init success report --- lib/Genesis/Commands/Repo.pm | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 1f2ecd10..97bace52 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -306,26 +306,7 @@ sub _repo_init_execute { sub _repo_init_report { my ($result) = @_; - - my @details; - push @details, " - $result->{kit_desc}" if $result->{kit_desc}; - - if ($result->{vault}) { - push @details, " - using vault at #C{$result->{vault}}"; - } else { - push @details, " - #Y{vault not configured} (use #C{genesis secrets-provider} to set)"; - } - - if ($result->{ci_provider}) { - push @details, " - CI provider: #C{$result->{ci_provider}}"; - } - - if ($result->{submodule}) { - push @details, " - created as subdirectory (no separate git)"; - } - - info "\nInitialized Genesis repository in #C{%s}\n%s\n", - $result->{human_root}, join("\n", @details); + success "\nGenesis repository #C{%s} created successfully.\n", $result->{name}; } # -- Helpers ------------------------------------------------------------------- From a51dedaecfdb18e3b688187bddee1cccd8738293 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 15 Apr 2026 12:40:12 -0700 Subject: [PATCH 013/103] Clean up execute phase input handling --- lib/Genesis/Commands/Repo.pm | 42 +++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 97bace52..7f1e3500 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -203,16 +203,24 @@ sub _repo_init_validate { # on user input. Failures here are unexpected errors. # sub _repo_init_execute { - my %opts = %{get_options()}; - my $name = $opts{_name}; - my $dir = $opts{_dir}; - my $parent_dir = $opts{_parent_dir}; - my $target_path = $opts{_target_path}; - my $kit_file = $opts{_kit_file}; - my $use_subdir = $opts{_use_subdir}; - my $vault_target = $opts{_vault_target}; # undef = skip vault - my $replace_existing = $opts{_replace_existing}; - my $ci_provider = $opts{'ci-provider'}; + my ( + $name, # derived name of the deployment/repo + $dir, # target directory name (derived from name or specified by user) + $parent_dir, # parent directory where repo will be created (usually cwd) + $target_path, # full path to the target directory ($parent_dir/$dir) + $kit_file, # optional local kit file to copy into the repo + $kit_spec, # optional remote kit name[/version] to download + $use_subdir, # whether to create the repo as a subdirectory of an existing git repo + $vault_target, # vault target to configure, or undef to skip vault + $replace_existing, # whether to remove existing target directory if it exists + $linked_dev_kit, # optional path to a local dev kit to link into the repo + $ci_provider, # optional CI provider type + $directory, # optional custom directory name override + $kits_path, # optional custom kits path + ) = get_options()->@{qw/ + _name _dir _parent_dir _target_path _kit_file kit _use_subdir _vault_target _replace_existing + link-dev-kit ci-provider directory kits-path + /}; # Remove existing directory if validation approved it if ($replace_existing && -e $target_path) { @@ -221,8 +229,8 @@ sub _repo_init_execute { # Resolve link-dev-kit to absolute path before we potentially chdir my $abs_dev_target; - if ($opts{'link-dev-kit'}) { - $abs_dev_target = abs_path($opts{'link-dev-kit'}); + if ($linked_dev_kit) { + $abs_dev_target = abs_path($linked_dev_kit); } # Build create options for Top->create @@ -232,12 +240,12 @@ sub _repo_init_execute { } else { $create_opts{skip_vault} = 1; } - $create_opts{directory} = $opts{directory} if $opts{directory}; + $create_opts{directory} = $directory if $directory; # Kits path handling my $kit_path; - if (exists($opts{'kits-path'})) { - $kit_path = abs_path($opts{'kits-path'} // $ENV{HOME}.'/.genesis/kits'); + if (defined $kits_path) { + $kit_path = abs_path($kits_path // $ENV{HOME}.'/.genesis/kits'); mkdir_or_fail($kit_path) unless -d $kit_path; } @@ -261,8 +269,8 @@ sub _repo_init_execute { my $abs_src = $kit_file =~ m#^/# ? $kit_file : abs_path($ENV{GENESIS_CALLER_DIR}."/".$kit_file); copy_or_fail($abs_src, $target); $kit_desc = "using locally provided compiled kit #C{$kit_file}"; - } elsif ($opts{kit}) { - my ($kit_name, $kit_version) = $top->download_kit($opts{kit}); + } elsif ($kit_spec) { + my ($kit_name, $kit_version) = $top->download_kit($kit_spec); $kit_desc = "using the #C{$kit_name/$kit_version} kit"; } else { mkdir_or_fail("./dev"); From dd6bfdebc48f4fb30c7735d3fd89ca8ef987a4fe Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 15 Apr 2026 13:07:30 -0700 Subject: [PATCH 014/103] Improve repo-init validation ordering and kit caching Move directory existence check before kit provider network calls. Pass pre-built kit_provider through to Top->create to avoid re-fetching version list. Add "Removing existing directory" message in execute. Validate specific kit version against provider list. --- lib/Genesis/Commands/Repo.pm | 62 +++++++++++++++++++++--------------- lib/Genesis/Top.pm | 2 +- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 7f1e3500..da682567 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -86,31 +86,16 @@ sub _repo_init_validate { ) unless grep { $_ eq $opts{'ci-provider'} } @valid; } - # --- 3. Gather data: validate kit source, detect git repo --- + # --- 3. Gather data: validate local sources, detect git repo --- - # Validate kit source exists before any destructive actions - my ($resolved_kit_name, $resolved_kit_version); + # Validate local kit sources exist if ($kit_file) { bail("Local compiled kit file '%s' not found.", $kit_file) unless -f $kit_file; - } elsif ($opts{'link-dev-kit'}) { + } + if ($opts{'link-dev-kit'}) { bail("Dev kit link target '%s' not found.", $opts{'link-dev-kit'}) unless -e $opts{'link-dev-kit'}; - } elsif ($opts{kit}) { - # Remote kit — build provider and resolve name/version - ($resolved_kit_name, $resolved_kit_version) = split('/', $opts{kit}, 2); - - my %provider_opts; - Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); - my $provider = eval { Genesis::Kit::Provider->init(%provider_opts) }; - bail("Could not initialize kit provider: %s", $@) if $@; - - # Resolve version — also validates the kit exists on the provider - unless ($resolved_kit_version && $resolved_kit_version ne 'latest') { - $resolved_kit_version = eval { $provider->latest_version_of($resolved_kit_name) }; - bail("Kit '%s' not found or no versions available from %s.", - $resolved_kit_name, $provider->label) unless $resolved_kit_version; - } } # Check git author config @@ -134,7 +119,7 @@ sub _repo_init_validate { $use_subdir = 1; } - # --- 4. Check destructive prerequisites --- + # --- 4. Check destructive prerequisites (before expensive network calls) --- my $replace_existing = 0; if (-e $target_path) { @@ -149,7 +134,30 @@ sub _repo_init_validate { } } - # --- 5. Prompt for missing info --- + # --- 5. Validate remote kit availability (network, after directory check) --- + + my ($resolved_kit_name, $resolved_kit_version, $kit_provider); + if ($opts{kit} && !$kit_file) { + ($resolved_kit_name, $resolved_kit_version) = split('/', $opts{kit}, 2); + + my %provider_opts; + Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); + $kit_provider = eval { Genesis::Kit::Provider->init(%provider_opts) }; + bail("Could not initialize kit provider: %s", $@) if $@; + + if ($resolved_kit_version && $resolved_kit_version ne 'latest') { + my @versions = eval { $kit_provider->kit_versions($resolved_kit_name) }; + my $exists = grep { $_->{version} eq $resolved_kit_version } @versions; + bail("Kit '%s/%s' not found from %s.", + $resolved_kit_name, $resolved_kit_version, $kit_provider->label) unless $exists; + } else { + $resolved_kit_version = eval { $kit_provider->latest_version_of($resolved_kit_name) }; + bail("Kit '%s' not found or no versions available from %s.", + $resolved_kit_name, $kit_provider->label) unless $resolved_kit_version; + } + } + + # --- 6. Prompt for missing info --- my $vault_target; if ($opts{'skip-vault'}) { @@ -161,7 +169,7 @@ sub _repo_init_validate { $vault_target = $vault->{name} if $vault; } - # --- 6. Store derived values and summarize intent --- + # --- 8. Store derived values and summarize intent --- option_defaults( _name => $name, @@ -169,6 +177,7 @@ sub _repo_init_validate { _parent_dir => $parent_dir, _target_path => $target_path, _kit_file => $kit_file, + _kit_provider => $kit_provider, _use_subdir => $use_subdir, _vault_target => $vault_target, _replace_existing => $replace_existing, @@ -210,6 +219,7 @@ sub _repo_init_execute { $target_path, # full path to the target directory ($parent_dir/$dir) $kit_file, # optional local kit file to copy into the repo $kit_spec, # optional remote kit name[/version] to download + $kit_provider, # optional pre-built kit provider (from validation, with cached versions) $use_subdir, # whether to create the repo as a subdirectory of an existing git repo $vault_target, # vault target to configure, or undef to skip vault $replace_existing, # whether to remove existing target directory if it exists @@ -218,12 +228,13 @@ sub _repo_init_execute { $directory, # optional custom directory name override $kits_path, # optional custom kits path ) = get_options()->@{qw/ - _name _dir _parent_dir _target_path _kit_file kit _use_subdir _vault_target _replace_existing - link-dev-kit ci-provider directory kits-path + _name _dir _parent_dir _target_path _kit_file kit _kit_provider _use_subdir _vault_target + _replace_existing link-dev-kit ci-provider directory kits-path /}; # Remove existing directory if validation approved it if ($replace_existing && -e $target_path) { + info "Removing existing directory #C{%s}...", $dir; rmtree $target_path; } @@ -249,7 +260,8 @@ sub _repo_init_execute { mkdir_or_fail($kit_path) unless -d $kit_path; } - # Create the repo via Top->create + # Create the repo via Top->create (pass kit_provider if we already built one) + $create_opts{kit_provider} = $kit_provider if $kit_provider; my $top = Genesis::Top->create($parent_dir, $name, %create_opts, kits_path => $kit_path); $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider; diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 19930c70..04a8451b 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -138,7 +138,7 @@ sub create { my $self = $class->_build($path, %opts); $self->mkdir(".genesis"); - $self->{__kit_provider} = Genesis::Kit::Provider->init(%opts); + $self->{__kit_provider} = $opts{kit_provider} || Genesis::Kit::Provider->init(%opts); # Override vault if specified (will be saved to config later) $self->{__vault} = Service::Vault::Remote->target($opts{vault}) if $opts{vault}; my $kits_path = ''; From 1716dfb4b297b310477e7ed2352334b42af63a4e Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 16 Apr 2026 13:15:14 -0700 Subject: [PATCH 015/103] Add ci_control_branch accessor and constant Introduce DEFAULT_CONTROL_BRANCH ('control') and a ci_control_branch method on Genesis::Top that reads ci.control_branch from .genesis/config with the constant as a fallback. Not exposed as a user-facing option at this time; provides a single place to change the name later. --- lib/Genesis/Top.pm | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 04a8451b..5b9f599d 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -19,6 +19,15 @@ use Genesis::Config; use Cwd (); use File::Path qw/rmtree/; +# ---- Constants -------------------------------------------------------------- +# +# Default name of the CI "control" branch -- the branch from which +# environment branches are cut by 'genesis new' and against which +# 'genesis deploy' validates its working state. Captured as a constant +# (and a config key) so it can change without rippling through the +# codebase; not currently exposed to end users. +use constant DEFAULT_CONTROL_BRANCH => 'control'; + ### Config Section Delegation Registry {{{ # Modules may register themselves as handlers for specific top-level keys in # .genesis/config. Top.pm owns the core schema; registered handlers own their @@ -962,6 +971,19 @@ sub type { return $self->config->get("deployment_type"); } +# }}} +# ci_control_branch - return the configured CI control branch name {{{ +# +# Returns the branch name that Genesis pipeline tooling treats as the +# source of truth for this deployment repository. Reads from +# #C{ci.control_branch} in #C{.genesis/config}, defaulting to the +# value of the #C{DEFAULT_CONTROL_BRANCH} constant. Intentionally +# not exposed as a user-facing option at this time. +sub ci_control_branch { + my ($self) = @_; + return $self->config->get('ci.control_branch', DEFAULT_CONTROL_BRANCH); +} + # }}} # version - return the version of the cofiguration schema {{{ sub version { From e10ea48f615f782cec32e24af80c7e6011e8e492 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 16 Apr 2026 13:15:39 -0700 Subject: [PATCH 016/103] Refine repo-init options and error flow Drop --[no-]sub (subdir mode is now auto-detected) and --commit (auto-commit is the default). Add --no-commit to stage without committing, and --reason to override the commit message. --force also bypasses the "clean enclosing repo" preflight in subdir mode. Guardrail against nested Genesis repos via a walk-up helper that reuses Top->is_repo. In subdir mode, require the enclosing repo to be on the CI control branch (no bypass) and to have no tracked changes (bypassable with --force). Refactor the git flow: git init only in standalone mode; git add . in both; pathspec-scoped commit in subdir mode so unrelated parent index entries are not bundled. Standalone init forces the initial branch via git symbolic-ref so the first commit lands on 'control' regardless of init.defaultBranch. Always record ci.control_branch in .genesis/config. Move error detection, cleanup (rmtree, subdir index reset), and the final bail message from execute into report. Treat post-commit housekeeping failures (popd, vault URL resolution) as non-fatal: the repo is already created; warn instead of failing. --- bin/genesis | 22 ++-- lib/Genesis/Commands/Repo.pm | 227 +++++++++++++++++++++++++++++++---- 2 files changed, 217 insertions(+), 32 deletions(-) diff --git a/bin/genesis b/bin/genesis index ea65000a..6f16632f 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1237,21 +1237,21 @@ define_command("repo-init", { "backward compatibility by setting the #Y{legacy_repo_suffix} option in ". "the Genesis configuration file \$HOME/.genesis/config.yml to true.", - "commit" => - "Automatically commit the initial state without prompting.", - "no-commit" => - "Stage files and show the summary but skip the commit entirely.", + "Stage the new repository contents but skip the initial commit. ". + "By default, #C{repo-init} commits the new repository automatically.", - 'sub!' => - "Control whether the new repo is created as a subdirectory of the ". - "current git repository (no separate .git). If not specified and ". - "the current directory is inside a git repo, subdirectory mode is ". - "used automatically.", + "reason=s" => + "Commit message for the initial commit. Defaults to a message ". + "describing the new Genesis deployment. Ignored if #C{--no-commit} ". + "is set.", 'force|F' => - "If the target directory already exists, remove it and recreate. ". - "Without this flag, you will be prompted to confirm replacement.", + "If the target directory already exists, remove it and recreate ". + "(without this flag, you will be prompted to confirm ". + "replacement). Also bypasses the check that the enclosing git ". + "repository is clean when creating the new repo as a ". + "subdirectory; use with care.", 'skip-vault' => "Defer vault configuration. The repo will be created without a ". diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index da682567..498cc2d1 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -11,7 +11,7 @@ use Genesis::Kit::Provider; use Genesis::UI; use Cwd qw/getcwd abs_path/; -use File::Basename qw/basename/; +use File::Basename qw/basename dirname/; use File::Path qw/rmtree/; use JSON::PP qw/encode_json/; @@ -21,8 +21,7 @@ use JSON::PP qw/encode_json/; sub repo_init { _repo_init_validate(); - my $result = _repo_init_execute(); - _repo_init_report($result); + _repo_init_report(scalar _repo_init_execute()); exit 0; } @@ -112,11 +111,73 @@ sub _repo_init_validate { 'git config user.email'); } - # Detect git repo for subdirectory mode - my $in_git_repo = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null'); - my $use_subdir = $opts{sub}; - if ($in_git_repo && !defined($use_subdir)) { - $use_subdir = 1; + # Guardrail: forbid creating a new Genesis repo inside (or under) an + # existing one. Nested Genesis repos are never correct in this + # ecosystem -- pipeline tooling, kit discovery, and .genesis/config + # resolution all assume a single enclosing deployment root. + if (my $enclosing = _find_enclosing_genesis_repo()) { + bail( + "Cannot create a new Genesis deployment repository inside an ". + "existing one.\n Enclosing deployment repo: #C{%s}", + humanize_path($enclosing) + ); + } + + # Auto-detect whether we're inside an existing (non-Genesis) git + # worktree. If so, the new repo is created as a subdirectory with no + # separate .git -- it will share history with the surrounding repo. + # This is reported in the plan below so the user can notice an + # unexpected enclosure. + my $use_subdir = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null') ? 1 : 0; + + # In subdir mode, require the enclosing repo to have no staged or + # unstaged changes to tracked files. Untracked files are fine. + # This means that if execution fails and we need to roll back the + # index, we can do so without risking the user's in-progress work. + # #C{-F|--force} bypasses this check for users who understand the + # risk (e.g. scripted environments, known-safe index). + if ($use_subdir && !$opts{force}) { + my ($dirty) = run({}, 'git status --porcelain --untracked-files=no'); + if (defined($dirty) && $dirty =~ /\S/) { + bail( + "Cannot create a Genesis deployment repository inside an ". + "enclosing git repository that has uncommitted changes to ". + "tracked files.\n". + " Please commit, stash, or discard the following first, ". + "or rerun with #C{-F|--force} to bypass this check:\n\n%s\n", + $dirty + ); + } + } + + # In subdir mode, require the enclosing repo to be on the CI + # control branch. Genesis pipeline tooling treats this branch as + # the single source of truth from which environment branches are + # cut (#C{genesis new}) and against which deploys are validated + # (#C{genesis deploy}). We do not rename or create branches in + # the enclosing repo -- the user must set this up themselves. No + # #C{--force} bypass: this is a topology requirement, not a safety + # check. + my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH; + if ($use_subdir) { + my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); + chomp $branch if defined $branch; + if (!defined($branch) || $branch ne $control_branch) { + bail( + "Genesis requires the enclosing git repository to be on a ". + "branch named #C{%s}, but it is currently on #C{%s}.\n". + " Please switch to the #C{%s} branch before running ". + "#C{repo-init}:\n\n". + " git checkout %s\n\n". + " or, if it doesn't exist yet:\n\n". + " git checkout -b %s\n", + $control_branch, + $branch // '', + $control_branch, + $control_branch, + $control_branch + ); + } } # --- 4. Check destructive prerequisites (before expensive network calls) --- @@ -198,7 +259,8 @@ sub _repo_init_validate { push @plan, "vault: #C{$vault_target}" if $vault_target; push @plan, "vault: #Yi{deferred}" unless $vault_target; push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; - push @plan, "subdirectory: #C{yes} (no separate git)" if $use_subdir; + push @plan, "control branch: #C{$control_branch}"; + push @plan, "subdirectory of enclosing git repo: #C{yes} (no separate .git, auto-detected)" if $use_subdir; info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; info " %s", $_ for @plan; info ""; @@ -213,6 +275,7 @@ sub _repo_init_validate { # sub _repo_init_execute { my ( + # Derived and validated values from validation phase (prefixed with _) $name, # derived name of the deployment/repo $dir, # target directory name (derived from name or specified by user) $parent_dir, # parent directory where repo will be created (usually cwd) @@ -224,12 +287,16 @@ sub _repo_init_execute { $vault_target, # vault target to configure, or undef to skip vault $replace_existing, # whether to remove existing target directory if it exists $linked_dev_kit, # optional path to a local dev kit to link into the repo + + # User provided options (validated but not altered) $ci_provider, # optional CI provider type $directory, # optional custom directory name override $kits_path, # optional custom kits path + $no_commit, # skip the initial commit (stage only) + $reason, # optional commit message override ) = get_options()->@{qw/ _name _dir _parent_dir _target_path _kit_file kit _kit_provider _use_subdir _vault_target - _replace_existing link-dev-kit ci-provider directory kits-path + _replace_existing link-dev-kit ci-provider directory kits-path no-commit reason /}; # Remove existing directory if validation approved it @@ -294,43 +361,161 @@ sub _repo_init_execute { _create_ci_scaffold($top, $ci_provider); } - # Git init (skip if subdirectory of existing repo) + # Record the CI control branch name in .genesis/config so + # pipeline tooling (genesis new, genesis deploy, compiler) can + # read it via $top->ci_control_branch. Written AFTER + # _create_ci_scaffold because that helper replaces the ci: + # hash wholesale. + $top->config->set( + 'ci.control_branch', + Genesis::Top::DEFAULT_CONTROL_BRANCH, + 1, # save + ); + + # Only create a new .git when we're not already sitting inside + # an enclosing git worktree; in subdir mode we share the + # parent's .git and just stage into its index. In standalone + # mode, force the initial branch name to match the CI control + # branch (e.g. 'control') so the initial commit lands there + # instead of on whatever git's init.defaultBranch happens to + # be. #C{git symbolic-ref HEAD} works on all git versions, + # unlike #C{git init -b} which requires >= 2.28. unless ($use_subdir) { + my $branch = $top->ci_control_branch; run({ onfailure => "Failed to initialize git in $human_root/" }, - 'git init && git add .'); - run({ onfailure => "Failed to commit initial repository in $human_root/" }, - 'git commit -m "Initial Genesis Repo"'); + "git init && git symbolic-ref HEAD refs/heads/$branch"); + } + run({ onfailure => "Failed to stage repository in $human_root/" }, + 'git add .'); + + # Show a summary of what we just staged so the user can see + # exactly what the initial commit (or leftover stage) contains. + my ($stat) = run({}, 'git diff --cached --stat -- .'); + if ($stat && $stat =~ /\S/) { + info "\n#G{Files staged for initial commit:}"; + info " %s", $_ for split /\n/, $stat; + info ""; + } + + # Commit unless the user explicitly opted out. In subdir mode + # we scope the commit with a pathspec ('-- .') so any unrelated + # changes already staged in the enclosing repo are not bundled + # into this commit. + if ($no_commit) { + info "Skipping initial commit (#C{--no-commit} set); files remain staged."; + } else { + my $message = $reason || "Initial Genesis repo for $name"; + my @cmd = ('git', 'commit', '-m', $message); + push @cmd, '--', '.' if $use_subdir; + run({ onfailure => "Failed to commit initial repository in $human_root/" }, @cmd); } }; my $err = $@; - popd; + eval { popd }; # always restore cwd; swallow errors to not mask $err + + # On failure, return only what the report phase needs to clean up + # and bail -- the $top object may be in a partial state and its + # accessors (vault, etc.) are not safe to call. if ($err) { - debug("removing incomplete Genesis repository at #C{$root} due to failed creation"); - rmtree $root; - bail $err; + return { + error => $err, + root => $root, + human_root => $human_root, + target_path => $target_path, + parent_dir => $parent_dir, + use_subdir => $use_subdir, + }; } - return { + # Success: the repository is created and (if requested) committed. + # Resolving the vault URL for the report is not part of the + # command's core purpose -- if it trips a runtime error, warn the + # user but do NOT fail the command (the repo is already on disk). + my %result = ( + error => undef, root => $root, human_root => $human_root, name => $name, kit_desc => $kit_desc, ci_provider => $ci_provider, vault_skipped => $vault_target ? 0 : 1, - vault => $vault_target ? ($top->vault ? $top->vault->url : $vault_target) : undef, + vault => $vault_target, submodule => $use_subdir, + ); + eval { + $result{vault} = $vault_target + ? ($top->vault ? $top->vault->url : $vault_target) + : undef; }; + if (my $housekeeping_err = $@) { + warning( + "Repository was created successfully, but a post-commit ". + "housekeeping step failed (non-fatal):\n%s", + $housekeeping_err + ); + } + return \%result; } # -- Phase 3: Report ---------------------------------------------------------- - +# +# Handles both success and failure. On failure, cleans up any partial +# initialization (removing the new directory and, in subdir mode, +# unstaging from the enclosing repo's index) before bailing with an +# appropriate message. On success, prints the summary. +# sub _repo_init_report { my ($result) = @_; + + if ($result->{error}) { + debug( + "removing incomplete Genesis repository at #C{%s} due to failed creation", + $result->{root} + ); + # In subdir mode, unstage anything we added to the enclosing + # repo's index before deleting the directory so the parent + # repo's working state is left clean. This is safe because + # validation ensured the parent repo had no other tracked + # changes (or the user passed --force); the pathspec scopes + # the reset strictly to what we just staged. + if ($result->{use_subdir}) { + eval { + run({ dir => $result->{parent_dir} }, + 'git', 'reset', 'HEAD', '--', $result->{target_path}); + }; + } + rmtree $result->{root} if -e $result->{root}; + bail( + "Failed to create Genesis repository at #C{%s}:\n%s", + $result->{human_root}, $result->{error} + ); + } + success "\nGenesis repository #C{%s} created successfully.\n", $result->{name}; } # -- Helpers ------------------------------------------------------------------- +# Walk upward from the current working directory looking for an +# enclosing Genesis deployment repository. Returns the absolute path +# of the enclosing repo if found, otherwise undef. +# +# The global #C{in_repo_dir()} helper only checks cwd, and almost every +# consumer downstream uses #C{Genesis::Top->new('.')} which assumes +# cwd IS the repo root -- so teaching the global helper to walk up +# would ripple. This variant is scoped to #C{repo-init}'s guardrail +# against creating a nested deployment repo. +sub _find_enclosing_genesis_repo { + my $dir = abs_path(getcwd()); + while (defined($dir) && length($dir) > 1) { + return $dir if Genesis::Top->is_repo($dir); + my $parent = dirname($dir); + last if $parent eq $dir; + $dir = $parent; + } + return undef; +} + sub _select_vault_target { my $configure = Genesis::UI::prompt_for_boolean( "Would you like to configure a secrets vault now? [y|n] ", 1 From 37110153971a64f4d365fdaf8ed7f0281b7921d1 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 08:28:51 -0700 Subject: [PATCH 017/103] Fix repo-init post-merge issues Park Tristan's duplicate repo-init definitions: rename his handler to repo_configure_ci and comment out his define_command block so our phased version dispatches correctly. Move Kit::Provider->parse_opts to the top of validate so passthrough flags are stripped before reading $args[0] as the deployment name. Gate the control-branch check and git symbolic-ref on --ci-provider; without it no pipeline topology is established. Drop ci.control_branch config write and control-branch plan line. Report commit hash and message after the initial commit. --- bin/genesis | 77 +++++++++++++----------- lib/Genesis/Commands/Repo.pm | 112 +++++++++++++++++++++++------------ 2 files changed, 116 insertions(+), 73 deletions(-) diff --git a/bin/genesis b/bin/genesis index 6f16632f..2b19fb09 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1337,42 +1337,49 @@ define_command("kit-provider", { }); # }}} +# PARKED: Tristan's CI-only repo-init define_command block -- kept +# verbatim here for the upcoming merge meeting, but commented out so +# it does NOT override our phased repo-init definition earlier in this +# file. The handler sub has been renamed in lib/Genesis/Commands/Repo.pm +# to repo_configure_ci. If this block is ever re-enabled, the final +# dispatcher arg below must be updated to match that new name. +# # genesis repo-init - one-time CI provider setup {{{ -define_command("repo-init", { - summary => "Initialize CI configuration for this Genesis deployment repository.", - usage => "repo-init [--ci-provider PROVIDER] [--git-uri URI] [--git-branch BRANCH] [--vault-url URL] [--pipeline-name NAME]", - description => - "One-time setup that writes a #C{ci:} section to #C{.genesis/config} and ". - "generates the #C{.genesis/ci/} scaffold files required by the pipeline ". - "compiler.\n". - "\n". - "Errors if CI is already configured for this repository. Use ". - "#C{genesis repo-update} to modify an existing configuration.\n". - "\n". - "When required flags are omitted an interactive wizard collects them.", - function_group => Genesis::Commands::REPOSITORY, - scope => 'repo', - option_group => Genesis::Commands::REPO_OPTIONS, - options => [ - 'ci-provider|P=s' => - "CI provider to configure: #C{concourse} (default), ". - "#C{github-actions}, or #C{none}.", - - 'git-uri|g=s' => - "Git repository URI (e.g. #C{git\@github.com:org/repo.git}).", - - 'git-branch|b=s' => - "Default branch for the source-control integration. Defaults to #C{main}.", - - 'vault-url|V=s' => - "Vault URL for the secrets integration ". - "(e.g. #C{https://vault.example.com:8200}).", - - 'pipeline-name|n=s' => - "Name written into #C{pipeline.yml} metadata. Defaults to the ". - "deployment type.", - ], -}, 'Genesis::Commands::Repo::repo_init'); +# define_command("repo-init", { +# summary => "Initialize CI configuration for this Genesis deployment repository.", +# usage => "repo-init [--ci-provider PROVIDER] [--git-uri URI] [--git-branch BRANCH] [--vault-url URL] [--pipeline-name NAME]", +# description => +# "One-time setup that writes a #C{ci:} section to #C{.genesis/config} and ". +# "generates the #C{.genesis/ci/} scaffold files required by the pipeline ". +# "compiler.\n". +# "\n". +# "Errors if CI is already configured for this repository. Use ". +# "#C{genesis repo-update} to modify an existing configuration.\n". +# "\n". +# "When required flags are omitted an interactive wizard collects them.", +# function_group => Genesis::Commands::REPOSITORY, +# scope => 'repo', +# option_group => Genesis::Commands::REPO_OPTIONS, +# options => [ +# 'ci-provider|P=s' => +# "CI provider to configure: #C{concourse} (default), ". +# "#C{github-actions}, or #C{none}.", +# +# 'git-uri|g=s' => +# "Git repository URI (e.g. #C{git\@github.com:org/repo.git}).", +# +# 'git-branch|b=s' => +# "Default branch for the source-control integration. Defaults to #C{main}.", +# +# 'vault-url|V=s' => +# "Vault URL for the secrets integration ". +# "(e.g. #C{https://vault.example.com:8200}).", +# +# 'pipeline-name|n=s' => +# "Name written into #C{pipeline.yml} metadata. Defaults to the ". +# "deployment type.", +# ], +# }, 'Genesis::Commands::Repo::repo_configure_ci'); # }}} # genesis repo-update - idempotent CI config update {{{ define_command("repo-update", { diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 498cc2d1..d31c200c 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -30,10 +30,11 @@ sub repo_init { # Order within validation: # 1. Parse options and derive values (fast, no side effects) # 2. Check invalid option combinations (fast bail) -# 3. Gather data: resolve kit provider, check kit availability, detect git repo -# 4. Check for destructive prerequisites (existing directory) -# 5. Prompt for missing info (vault selection) -# 6. Summarize intent +# 3. Gather data: validate local sources, detect git repo (no network) +# 4. Check destructive prerequisites (existing directory, before network) +# 5. Validate remote kit availability (network calls) +# 6. Prompt for missing info (vault selection) +# 7. Store derived values and summarize intent # sub _repo_init_validate { my %opts = %{get_options()}; @@ -41,6 +42,25 @@ sub _repo_init_validate { # --- 1. Parse and derive --- + # The #C{repo-init} command uses #C{option_passthrough => 1}, so + # any option that isn't declared in bin/genesis (e.g. the + # #C{--kit-provider-*} flags provided by #C{Genesis::Kit::Provider} + # and, in the future, the CI-provider-specific flags) is left + # untouched in @args alongside true positional arguments. We must + # consume those passthrough options here, BEFORE reading + # #C{$args[0]} as the deployment name -- otherwise a flag value + # (or the flag itself) could be mistaken for the name. + # + # parse_opts mutates @args in place: it strips recognised flags + # into %provider_opts and leaves the remaining positional args + # behind. The provider object itself is built later, only when + # $opts{kit} is actually set. + # + # When the CI-provider parse_opts analogue lands, call it from + # here as well. + my %provider_opts; + Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); + my $name = $args[0]; my $kit_file; @@ -150,24 +170,29 @@ sub _repo_init_validate { } } - # In subdir mode, require the enclosing repo to be on the CI - # control branch. Genesis pipeline tooling treats this branch as - # the single source of truth from which environment branches are - # cut (#C{genesis new}) and against which deploys are validated + # In subdir mode AND when a CI provider is being configured, + # require the enclosing repo to be on the CI control branch. + # Genesis pipeline tooling treats this branch as the single source + # of truth from which environment branches are cut + # (#C{genesis new}) and against which deploys are validated # (#C{genesis deploy}). We do not rename or create branches in # the enclosing repo -- the user must set this up themselves. No # #C{--force} bypass: this is a topology requirement, not a safety # check. - my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH; - if ($use_subdir) { + # + # Without #C{--ci-provider}, no pipeline topology is being + # established, so the branch name is irrelevant at this point. + my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + if ($use_subdir && $opts{'ci-provider'}) { my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); chomp $branch if defined $branch; if (!defined($branch) || $branch ne $control_branch) { bail( - "Genesis requires the enclosing git repository to be on a ". - "branch named #C{%s}, but it is currently on #C{%s}.\n". + "Configuring a CI provider requires the enclosing git ". + "repository to be on a branch named #C{%s}, but it is ". + "currently on #C{%s}.\n". " Please switch to the #C{%s} branch before running ". - "#C{repo-init}:\n\n". + "#C{repo-init --ci-provider ...}:\n\n". " git checkout %s\n\n". " or, if it doesn't exist yet:\n\n". " git checkout -b %s\n", @@ -201,8 +226,10 @@ sub _repo_init_validate { if ($opts{kit} && !$kit_file) { ($resolved_kit_name, $resolved_kit_version) = split('/', $opts{kit}, 2); - my %provider_opts; - Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); + # %provider_opts was populated at the top of this sub so + # that positional-arg parsing wasn't fooled by passthrough + # flags. Build the provider object now that we know we + # need it. $kit_provider = eval { Genesis::Kit::Provider->init(%provider_opts) }; bail("Could not initialize kit provider: %s", $@) if $@; @@ -259,7 +286,6 @@ sub _repo_init_validate { push @plan, "vault: #C{$vault_target}" if $vault_target; push @plan, "vault: #Yi{deferred}" unless $vault_target; push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; - push @plan, "control branch: #C{$control_branch}"; push @plan, "subdirectory of enclosing git repo: #C{yes} (no separate .git, auto-detected)" if $use_subdir; info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; info " %s", $_ for @plan; @@ -356,34 +382,31 @@ sub _repo_init_execute { $kit_desc = "with an empty development kit in #C{$human_root/dev}"; } - # CI provider scaffold + # CI provider scaffold (only when --ci-provider is given) if ($ci_provider) { _create_ci_scaffold($top, $ci_provider); } - # Record the CI control branch name in .genesis/config so - # pipeline tooling (genesis new, genesis deploy, compiler) can - # read it via $top->ci_control_branch. Written AFTER - # _create_ci_scaffold because that helper replaces the ci: - # hash wholesale. - $top->config->set( - 'ci.control_branch', - Genesis::Top::DEFAULT_CONTROL_BRANCH, - 1, # save - ); - # Only create a new .git when we're not already sitting inside # an enclosing git worktree; in subdir mode we share the # parent's .git and just stage into its index. In standalone - # mode, force the initial branch name to match the CI control - # branch (e.g. 'control') so the initial commit lands there - # instead of on whatever git's init.defaultBranch happens to - # be. #C{git symbolic-ref HEAD} works on all git versions, - # unlike #C{git init -b} which requires >= 2.28. + # mode, if the user is establishing pipeline topology + # (--ci-provider given), force the initial branch name to the + # control branch so the initial commit lands there instead of + # on whatever git's init.defaultBranch happens to be. When + # no CI provider is being configured, let git choose its + # default branch -- pipeline topology is not relevant yet. + # #C{git symbolic-ref HEAD} works on all git versions, unlike + # #C{git init -b} which requires >= 2.28. unless ($use_subdir) { - my $branch = $top->ci_control_branch; - run({ onfailure => "Failed to initialize git in $human_root/" }, - "git init && git symbolic-ref HEAD refs/heads/$branch"); + if ($ci_provider) { + my $branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + run({ onfailure => "Failed to initialize git in $human_root/" }, + "git init && git symbolic-ref HEAD refs/heads/$branch"); + } else { + run({ onfailure => "Failed to initialize git in $human_root/" }, + 'git init'); + } } run({ onfailure => "Failed to stage repository in $human_root/" }, 'git add .'); @@ -408,6 +431,13 @@ sub _repo_init_execute { my @cmd = ('git', 'commit', '-m', $message); push @cmd, '--', '.' if $use_subdir; run({ onfailure => "Failed to commit initial repository in $human_root/" }, @cmd); + + # Report the commit that was just made so the user has a + # clear record of what was recorded (and, in subdir mode, + # where in the enclosing repo's history to find it). + my ($sha) = run({}, 'git rev-parse --short HEAD'); + chomp $sha if defined $sha; + info "#G{Committed} #C{%s} -- %s", $sha // '', $message; } }; my $err = $@; @@ -801,7 +831,13 @@ sub kit_provider { info(" Kits: %s\n\n", $kit_list) if $info{status} eq "ok"; } -sub repo_init { +# PARKED: this is Tristan's CI-only repo-init handler from the upstream +# merge. It conflicts with our phased repo-init at the top of this file +# (both registered under the same command name, causing Perl to redefine +# our sub). Renamed to repo_configure_ci as a parking slot until the +# folding meeting resolves how his CI-scaffold logic merges into our +# _create_ci_scaffold. Not currently dispatched by any command. +sub repo_configure_ci { my %options = %{get_options()}; command_usage(1) if @_; @@ -864,7 +900,7 @@ sub repo_update { ### Private helpers ########################################################### -# _empty_ci_defaults - blank defaults for repo_init wizard +# _empty_ci_defaults - blank defaults for repo_configure_ci wizard sub _empty_ci_defaults { my ($top) = @_; return { From 8bebc76ec113f57598d8320476e532749d120d2a Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:37:37 -0400 Subject: [PATCH 018/103] Add CI provider abstraction and options system Introduce a unified CI provider abstraction and provider options system. Adds a PipelineProvider registry and CLI parsing/helpers for provider-specific flags and help text; extends the Compiler to accept provider_opts and validate ci.provider config against provider schemas. Adds a Genesis::CI::Provider factory/base class and concrete Provider implementations for Concourse, GitHub Actions, and Manual (including config/interactive helpers and CLI specs). Concourse provider updated with option defaults, provider_option accessors, describe_provider, and deploy logic using three-tier option resolution (CLI > ci.provider config > defaults). Integrates provider parsing and deployment into Commands/Pipelines and repo-init flow, updates validation in Validator/Compiler, and adds tests for the provider option system. --- bin/genesis | 4 +- lib/Genesis/CI/Compiler.pm | 55 +++- lib/Genesis/CI/Compiler/Parser.pm | 1 + lib/Genesis/CI/Compiler/PipelineProvider.pm | 252 ++++++++++++++- .../CI/Compiler/Providers/Concourse.pm | 248 +++++++++++++-- lib/Genesis/CI/Compiler/Validator.pm | 68 ++++ lib/Genesis/CI/Provider.pm | 193 ++++++++++++ lib/Genesis/CI/Provider/Concourse.pm | 146 +++++++++ lib/Genesis/CI/Provider/GithubActions.pm | 136 ++++++++ lib/Genesis/CI/Provider/Manual.pm | 85 +++++ lib/Genesis/Commands/Pipelines.pm | 70 +++-- lib/Genesis/Commands/Repo.pm | 57 ++-- t/ci-compiler.t | 241 +++++++++++++++ t/ci-provider.t | 290 ++++++++++++++++++ 14 files changed, 1760 insertions(+), 86 deletions(-) create mode 100644 lib/Genesis/CI/Provider.pm create mode 100644 lib/Genesis/CI/Provider/Concourse.pm create mode 100644 lib/Genesis/CI/Provider/GithubActions.pm create mode 100644 lib/Genesis/CI/Provider/Manual.pm create mode 100644 t/ci-provider.t diff --git a/bin/genesis b/bin/genesis index 2b19fb09..8fe1ab6e 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1271,7 +1271,9 @@ define_command("repo-init", { ], extended_usage => sub { require Genesis::Kit::Provider; - Genesis::Kit::Provider->opts_help(); + require Genesis::CI::Provider; + Genesis::Kit::Provider->opts_help() . + Genesis::CI::Provider->opts_help(); } }); diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 4d33a08f..3038580d 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -98,7 +98,19 @@ sub compile { eval { require $provider_info->{file} } ## no critic or bail("Failed to load CI provider '%s': %s", $provider_type, $@); - my $provider = $provider_info->{class}->new(ast => $ast, top => $self->{top}); + # Extract provider options from parsed config (ci.provider: section) + # and merge with any caller-supplied opts. These are stored in the + # provider object and used by deploy() at deploy time. + my $provider_opts = { + %{ $parsed->{provider} || {} }, # from ci.provider: section + %{ $opts{provider_opts} || {} }, # caller-supplied overrides + }; + + my $provider = $provider_info->{class}->new( + ast => $ast, + top => $self->{top}, + provider_opts => $provider_opts, + ); my $raw_output = $provider->generate_from_ast($ast); # Wrap raw output into file map using provider's output_files manifest @@ -191,6 +203,47 @@ sub validate_config_section { bail("'ci.integrations.source_control' is required and must be a hash") unless ref($data->{integrations}{source_control}) eq 'HASH'; + + # Validate ci.provider: section against the provider's own schema + if (my $provider_data = $data->{provider}) { + bail("'ci.provider' must be a hash") + unless ref($provider_data) eq 'HASH'; + + my $type = $provider_data->{type}; + bail("'ci.provider.type' is required") unless $type; + + # Load provider class to get its schema + my $provider_info = eval { $class->_resolve_provider_class($type) }; + if ($@) { + bail("'ci.provider.type' is '%s', which is not a known CI provider type. ". + "Valid types: %s", $type, + join(', ', Genesis::CI::Compiler::PipelineProvider->known_providers())); + } + + eval { require $provider_info->{file} }; ## no critic + if ($@) { + bail("Failed to load CI provider '%s' for config validation: %s", $type, $@); + } + + my $schema = $provider_info->{class}->provider_options_schema(); + my $defaults = $provider_info->{class}->provider_options_defaults(); + + # Check required keys + for my $key (sort keys %$schema) { + my $spec = $schema->{$key}; + next unless $spec->{required}; + bail("'ci.provider.%s' is required for provider type '%s'", $key, $type) + unless defined $provider_data->{$key}; + } + + # Check unknown keys + for my $key (sort keys %$provider_data) { + next if exists $schema->{$key}; + bail("'ci.provider.%s' is not a recognized option for provider type '%s'. ". + "Valid options: %s", + $key, $type, join(', ', sort keys %$schema)); + } + } } # }}} diff --git a/lib/Genesis/CI/Compiler/Parser.pm b/lib/Genesis/CI/Compiler/Parser.pm index bba9cf89..160b1d89 100644 --- a/lib/Genesis/CI/Compiler/Parser.pm +++ b/lib/Genesis/CI/Compiler/Parser.pm @@ -132,6 +132,7 @@ sub _parse_genesis_config { $parsed{targets} = $data->{targets} || {}; $parsed{integrations} = $data->{integrations} || {}; $parsed{scripts} = $data->{scripts} || {}; + $parsed{provider} = $data->{provider} || {}; $parsed{provider_config} = $data->{provider_config} || {}; # When no pipeline section is provided, workflow topology is derived from diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 73ffb37f..efde61f6 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -4,7 +4,30 @@ use warnings; use Genesis; use JSON::PP; +use Getopt::Long qw/GetOptionsFromArray/; + +### Provider Registry {{{ +# Maps provider type strings to their class and file paths. +# Used by parse_cli_opts(), all_cli_opts_help(), and the compiler itself. + +my %_providers = ( + 'concourse' => { + class => 'Genesis::CI::Concourse', + file => 'Genesis/CI/Compiler/Providers/Concourse.pm', + }, + 'github-actions' => { + class => 'Genesis::CI::GithubActions', + file => 'Genesis/CI/Compiler/Providers/GithubActions.pm', + }, +); + +# known_providers - return list of known provider type strings {{{ +sub known_providers { + return sort keys %_providers; +} +# }}} +# }}} ### Constructor {{{ # new - create a new provider instance {{{ @@ -16,8 +39,9 @@ sub new { if $class eq __PACKAGE__; return bless({ - ast => $opts{ast}, - top => $opts{top}, + ast => $opts{ast}, + top => $opts{top}, + provider_opts => $opts{provider_opts} || {}, }, $class); } @@ -33,6 +57,13 @@ sub platform_name { bug("Subclass '%s' must implement platform_name()", ref($self)); } +# }}} +# provider_type - return canonical type string, e.g. 'concourse' {{{ +sub provider_type { + my ($self) = @_; + bug("Subclass '%s' must implement provider_type()", ref($self)); +} + # }}} # generate_from_ast - generate platform-specific output from AST {{{ sub generate_from_ast { @@ -49,6 +80,223 @@ sub output_files { # }}} # }}} +### Provider Options Contract {{{ +# These methods define the provider options system, modelled after +# Genesis::Kit::Provider. Subclasses override them to expose their +# platform-specific flags, help text, and config-section schemas. + +# cli_opts - Getopt::Long option specs for deploy-time command-line flags {{{ +# +# Returns a list of Getopt::Long spec strings, e.g.: +# qw( ci-target=s ci-team=s ci-pause ci-expose ) +# +# All CI-provider opts are prefixed with 'ci-' to avoid collisions with +# top-level genesis option names (--target, --dry-run, etc.). +# +# Subclasses override to declare their provider-specific options. +# Base implementation returns empty list (no provider-specific opts). +sub cli_opts { + return qw//; +} + +# }}} +# cli_opts_help - formatted help text for cli_opts() {{{ +# +# Returns a multi-line string (heredoc) documenting each option. +# Format mirrors Genesis::Kit::Provider::Github::opts_help(): +# +# --ci-target (required) +# The fly target to deploy to. +# +# --ci-team (optional, default: "main") +# The Concourse team name. +# +# %config may include: +# valid_types - arrayref of provider type strings to show help for +# +# Subclasses override to document their specific options. +sub cli_opts_help { + my ($class, %config) = @_; + return ''; +} + +# }}} +# provider_options_schema - schema for the ci.provider: config section {{{ +# +# Returns a hashref whose structure mirrors Top::_repo_config_schema(): +# +# { +# type => { type => 'string', required => 1, description => '...' }, +# target => { type => 'string', description => '...' }, +# team => { type => 'string', default => 'main', description => '...' }, +# expose => { type => 'boolean', default => 0, description => '...' }, +# ... +# } +# +# The base class always contributes the common 'type' key; subclasses add +# their own keys. Used by validate_config_section() in Compiler.pm and +# _validate_multi_file() in Validator.pm. +sub provider_options_schema { + return { + type => { + type => 'string', + required => 1, + description => 'CI provider type (concourse, github-actions)', + }, + }; +} + +# }}} +# provider_options_defaults - default values for provider options {{{ +# +# Returns a flat hashref of key => default_value. Keys match those in +# provider_options_schema(). Values here are NOT included in the config() +# output — only explicitly-set non-default values are saved. +sub provider_options_defaults { + return {}; +} + +# }}} +# provider_config - return stored provider options (non-defaults only) {{{ +# +# Returns a hashref suitable for round-tripping through the ci.provider: +# config section — i.e. the type key plus any explicitly-set values that +# differ from provider_options_defaults(). +sub provider_config { + my ($self) = @_; + my $defaults = $self->provider_options_defaults(); + my $opts = $self->{provider_opts} || {}; + my %out = ( type => $self->provider_type() ); + for my $k (keys %$opts) { + next if exists $defaults->{$k} && $defaults->{$k} eq ($opts->{$k} // ''); + $out{$k} = $opts->{$k}; + } + return \%out; +} + +# }}} +# provider_option - get a single provider option, applying defaults {{{ +sub provider_option { + my ($self, $key) = @_; + my $opts = $self->{provider_opts} || {}; + my $defaults = $self->provider_options_defaults(); + return exists $opts->{$key} ? $opts->{$key} + : exists $defaults->{$key} ? $defaults->{$key} + : undef; +} + +# }}} +# describe_provider - structured hash describing this provider instance {{{ +# +# Returns a hash suitable for human-readable display, analogous to +# Genesis::Kit::Provider::Github::status(). +# +# type => 'concourse' +# label => 'Concourse' # human platform name +# extras => [qw(Target Team Pipeline)] # keys to show in order +# Target => 'my-target' +# Team => 'main' +# Pipeline => 'cf' +# status => 'ok' # or error message +# +# Subclasses override to add platform-specific fields. +sub describe_provider { + my ($self) = @_; + return ( + type => $self->provider_type(), + label => $self->platform_name(), + extras => [], + status => 'ok', + ); +} + +# }}} +# }}} +### Class Methods — Provider Options Parsing {{{ + +# parse_cli_opts - two-pass CLI option parsing (mirrors Kit::Provider::parse_opts) {{{ +# +# Usage: +# Genesis::CI::Compiler::PipelineProvider->parse_cli_opts( +# \@ARGV, # args array (modified in place) +# \%opts, # options hash (populated in place) +# $provider_type, # optional: already-known provider type +# ); +# +# Pass 1: parse --ci-provider (if not already known) +# Pass 2: load provider class, get cli_opts(), parse provider-specific flags +# +# Returns 1. Remaining unparsed args are put back into $args. +sub parse_cli_opts { + my ($class, $args, $opts, $provider_type) = @_; + + Getopt::Long::Configure( + qw(pass_through permute no_auto_abbrev no_ignore_case bundling) + ); + + # Collect args up to '--' + my $opt_args = []; + while (scalar(@$args) && $args->[0] ne '--') { + push @$opt_args, shift @$args; + } + + # Pass 1: extract --ci-provider if not already known + unless ($provider_type) { + GetOptionsFromArray($opt_args, $opts, 'ci-provider=s'); + $provider_type = $opts->{'ci-provider'} // $opts->{platform}; + } + + # Pass 2: load provider and parse provider-specific flags + if ($provider_type && exists $_providers{$provider_type}) { + my $info = $_providers{$provider_type}; + eval { require $info->{file} } ## no critic + or bail("Failed to load CI provider '%s': %s", $provider_type, $@); + + my @extra_opts = $info->{class}->cli_opts(); + GetOptionsFromArray($opt_args, $opts, @extra_opts) if @extra_opts; + } + + # Return unparsed args to caller + while (scalar(@$opt_args)) { unshift @$args, pop @$opt_args } + + return 1; +} + +# }}} +# all_cli_opts_help - assembled help text for all known providers {{{ +# +# Prints shared CI flags, then delegates to each provider's cli_opts_help(). +# Mirrors Genesis::Kit::Provider::opts_help() in structure. +sub all_cli_opts_help { + my ($class, %config) = @_; + + $config{valid_types} ||= [sort keys %_providers]; + + # Load all provider classes for their help text + for my $type (sort keys %_providers) { + eval { require $_providers{$type}{file} }; ## no critic + } + + my $provider_help = join('', + map { $_providers{$_}{class}->cli_opts_help(%config) } + grep { eval { require $_providers{$_}{file}; 1 } } ## no critic + sort keys %_providers + ); + + return < (optional, defaults to "concourse") + The CI provider to use for pipeline generation and deployment. + Available types: ${\ join(', ', sort keys %_providers) } + +$provider_help +EOF +} + +# }}} +# }}} +### Shared Helper Methods {{{ ### Shared Helper Methods {{{ # ast - get stored AST {{{ diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 83b507a4..ee069641 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -10,6 +10,16 @@ use Genesis::CI::Legacy; use Genesis::CI::Compiler::PipelineDescriptor; use JSON::PP; +### Provider Constants {{{ + +use constant { + DEFAULT_TEAM => 'main', + DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top + DEFAULT_EXPOSE => 0, + DEFAULT_PAUSE_AFTER_SET => 0, +}; + +# }}} ### Class Methods {{{ # new - constructor for compiler pipeline path {{{ @@ -19,8 +29,9 @@ sub new { # Compiler construction path: receives AST and optionally top if ($opts{ast}) { return bless({ - ast => $opts{ast}, - top => $opts{top}, + ast => $opts{ast}, + top => $opts{top}, + provider_opts => $opts{provider_opts} || {}, }, $class); } @@ -30,23 +41,159 @@ sub new { } # }}} -# init - initialize Concourse provider {{{ +# init - initialize Concourse provider (trait interface path) {{{ sub init { my ($class, %opts) = @_; my $self = bless({ - file => $opts{file}, - top => $opts{top}, - layout => $opts{layout}, - _platform => $opts{platform} || '', - config => undef, - ast => undef, - errors => [], + file => $opts{file}, + top => $opts{top}, + layout => $opts{layout}, + _platform => $opts{platform} || '', + provider_opts => $opts{provider_opts} || {}, + config => undef, + ast => undef, + errors => [], }, $class); return $self; } +# }}} +# }}} +### Provider Options System {{{ +# Modelled after Genesis::Kit::Provider::Github — each method mirrors its +# kit-provider counterpart so the patterns are interchangeable. + +# provider_type - canonical type string {{{ +sub provider_type { 'concourse' } + +# }}} +# cli_opts - Getopt::Long specs for deploy-time command-line flags {{{ +# +# All CI provider options are prefixed with 'ci-' to avoid clashing with +# top-level genesis option names. +sub cli_opts { + qw/ + ci-target=s + ci-team=s + ci-pipeline-name=s + ci-pause + ci-expose + /; +} + +# }}} +# cli_opts_help - formatted help text for Concourse CLI flags {{{ +sub cli_opts_help { + my ($class, %config) = @_; + return '' unless grep { $_ eq 'concourse' } @{$config{valid_types} || ['concourse']}; + return <<'EOF'; + CI Provider `concourse`: + + Deploys Genesis pipelines to a Concourse CI server using the fly CLI. + Requires a configured fly target (see: fly login). + + --ci-target (required if not set in .genesis/config ci.provider.target) + The fly target alias that identifies the Concourse server. + Create a target with: fly login -t -c + + --ci-team (optional, default: "main") + The Concourse team to use when setting the pipeline. + Must match an existing team on the target Concourse server. + + --ci-pipeline-name (optional, default: deployment type from .genesis/config) + Override the pipeline name used in Concourse. + Defaults to the repository's deployment_type (e.g., "cf", "bosh"). + + --ci-pause (optional, default: false) + Leave the pipeline in a paused state after fly set-pipeline completes. + By default the pipeline is unpaused immediately after being set. + + --ci-expose (optional, default: false) + Run fly expose-pipeline after setting, making the pipeline publicly + viewable without authentication. Useful for open-source pipelines. + +EOF +} + +# }}} +# provider_options_schema - schema for ci.provider: when type=concourse {{{ +# +# Keys map directly to the ci.provider: sub-keys in .genesis/config. +# This schema is used by: +# - Genesis::CI::Compiler::validate_config_section() +# - Genesis::CI::Compiler::Validator::_validate_multi_file() +sub provider_options_schema { + return { + type => { + type => 'string', + required => 1, + description => 'Provider type (must be "concourse")', + }, + target => { + type => 'string', + description => 'Fly target alias (fly login -t )', + }, + team => { + type => 'string', + default => DEFAULT_TEAM, + description => 'Concourse team name', + }, + pipeline_name => { + type => 'string', + description => 'Pipeline name override (defaults to deployment_type)', + }, + expose => { + type => 'boolean', + default => DEFAULT_EXPOSE, + description => 'Make pipeline publicly viewable (fly expose-pipeline)', + }, + pause_after_set => { + type => 'boolean', + default => DEFAULT_PAUSE_AFTER_SET, + description => 'Leave pipeline paused after fly set-pipeline', + }, + }; +} + +# }}} +# provider_options_defaults - default values for all Concourse options {{{ +sub provider_options_defaults { + return { + team => DEFAULT_TEAM, + expose => DEFAULT_EXPOSE, + pause_after_set => DEFAULT_PAUSE_AFTER_SET, + }; +} + +# }}} +# describe_provider - structured self-description for display {{{ +# +# Mirrors Genesis::Kit::Provider::Github::status() — returns a hash with +# type, label, an ordered 'extras' list, and a per-key status structure. +sub describe_provider { + my ($self) = @_; + + my $target = $self->provider_option('ci-target') || $self->provider_option('target') || '(not set)'; + my $team = $self->provider_option('ci-team') || $self->provider_option('team') || DEFAULT_TEAM; + my $pipe_name = $self->provider_option('ci-pipeline-name') || $self->provider_option('pipeline_name') || '(deployment type)'; + my $expose = $self->provider_option('expose') ? 'yes' : 'no'; + my $paused = $self->provider_option('pause_after_set') ? 'yes' : 'no'; + + return ( + type => 'concourse', + label => 'Concourse', + extras => [qw(Target Team Pipeline Expose PauseAfterSet)], + Target => $target, + Team => $team, + Pipeline => $pipe_name, + Expose => $expose, + PauseAfterSet => $paused, + status => 'ok', + ); +} + # }}} # }}} ### Trait Interface Implementation {{{ @@ -127,67 +274,100 @@ sub generate { # }}} # deploy - deploy pipeline to Concourse via fly CLI {{{ +# +# Option resolution priority (highest to lowest): +# 1. Caller-supplied %opts (from command-line flags via parse_cli_opts) +# 2. provider_opts stored in $self (loaded from ci.provider: in .genesis/config) +# 3. Legacy $self->{layout} (backward compat) +# 4. Built-in defaults (team: main, etc.) sub deploy { my ($self, %opts) = @_; - - my $target = $opts{target} || $self->{layout}; - my $pipeline_name = $self->{config}{pipeline}{name}; - my $dry_run = $opts{'dry-run'}; - my $yes = $opts{yes}; - my $paused = $opts{paused}; - + bail("Must call parse() before deploy()") unless $self->{config}; - + + # --- Resolve options from three tiers --- + + # Target: CLI > provider_opts > legacy layout + my $target = $opts{'ci-target'} + || $self->provider_option('target') + || $self->{layout}; + bail("No Concourse target specified. Use --ci-target or set ci.provider.target in .genesis/config") + unless $target; + + # Team: CLI > provider_opts > default + my $team = $opts{'ci-team'} + || $self->provider_option('team') + || DEFAULT_TEAM; + + # Pipeline name: CLI > provider_opts > config name > deployment_type + my $pipeline_name = $opts{'ci-pipeline-name'} + || $self->provider_option('pipeline_name') + || $self->{config}{pipeline}{name} + || ($self->{top} ? $self->{top}->type : undef); + bail("Cannot determine pipeline name — set ci.provider.pipeline_name or ensure deployment_type is set") + unless $pipeline_name; + + # Pause/expose: CLI flags > provider_opts > defaults + my $dry_run = $opts{'dry-run'} || $opts{'ci-dry-run'}; + my $yes = $opts{yes} || $opts{'-y'}; + my $paused = $opts{'ci-pause'} + || $self->provider_option('pause_after_set') + || DEFAULT_PAUSE_AFTER_SET; + my $expose = $opts{'ci-expose'} + || $self->provider_option('expose') + || _yaml_bool(($self->{config}{pipeline} || {})->{public}, DEFAULT_EXPOSE); + my $yaml = $self->generate(); - + if ($dry_run) { output({raw => 1}, $yaml); return; } - - # Pause pipeline before updating + + # Pause pipeline before updating (safe to do even when not found yet) my ($out, $rc) = run( 'fly -t $1 pause-pipeline -p $2', $target, $pipeline_name ); bail("Could not pause pipeline '%s': %s", $pipeline_name, $out) unless $rc == 0 || $out =~ /pipeline '.*' not found/; - + # Write pipeline to temp file my $dir = workdir; mkfile_or_fail("${dir}/pipeline.yml", $yaml); - + # Upload pipeline my $yes_flag = $yes ? '-n' : ''; + my $team_flag = " --team=$team"; run({ interactive => 1, - onfailure => "Could not upload pipeline $pipeline_name" + onfailure => "Could not upload pipeline $pipeline_name", }, - 'fly -t $1 set-pipeline '.$yes_flag.' -p $2 -c $3/pipeline.yml', + "fly -t \$1 set-pipeline${yes_flag}${team_flag} -p \$2 -c \$3/pipeline.yml", $target, $pipeline_name, $dir ); - - # Unpause pipeline (unless --paused) + + # Unpause pipeline (unless --ci-pause / pause_after_set) unless ($paused) { run({ interactive => 1, - onfailure => "Could not unpause pipeline $pipeline_name" + onfailure => "Could not unpause pipeline $pipeline_name", }, 'fly -t $1 unpause-pipeline -p $2', $target, $pipeline_name ); } - - # Set visibility (public/private) - my $action = $self->{config}{pipeline}{public} ? 'expose' : 'hide'; + + # Set visibility (expose vs hide) + my $action = $expose ? 'expose' : 'hide'; run({ interactive => 1, - onfailure => "Could not $action pipeline $pipeline_name" + onfailure => "Could not $action pipeline $pipeline_name", }, - 'fly -t $1 '.$action.'-pipeline -p $2', + "fly -t \$1 ${action}-pipeline -p \$2", $target, $pipeline_name ); - + return; } diff --git a/lib/Genesis/CI/Compiler/Validator.pm b/lib/Genesis/CI/Compiler/Validator.pm index c3fe7568..63ddd40a 100644 --- a/lib/Genesis/CI/Compiler/Validator.pm +++ b/lib/Genesis/CI/Compiler/Validator.pm @@ -438,10 +438,78 @@ sub _validate_multi_file { # Validate integrations section $self->_validate_integrations_section($parsed->{integrations}); + # Validate provider options section (ci.provider: or provider-config/concourse.yml) + $self->_validate_provider_section($parsed->{provider}) + if $parsed->{provider}; + # Cross-reference validation $self->_validate_cross_references($parsed); } +# }}} +# _validate_provider_section - validate ci.provider: options against provider schema {{{ +# +# Called when a 'provider' key is present in the parsed config (inline in +# .genesis/config or as provider-config/concourse.yml). Loads the named +# provider class and validates subkeys against its provider_options_schema(). +sub _validate_provider_section { + my ($self, $provider) = @_; + + unless (ref($provider) eq 'HASH') { + $self->_error("'provider' section must be a hash"); + return; + } + + my $type = $provider->{type}; + unless ($type) { + $self->_error("'provider.type' is required"); + return; + } + + # Load provider class to get its schema + require Genesis::CI::Compiler::PipelineProvider; + my %known = map { $_ => 1 } Genesis::CI::Compiler::PipelineProvider->known_providers(); + unless ($known{$type}) { + $self->_error(sprintf( + "'provider.type' is '%s', which is not a known CI provider. Valid types: %s", + $type, join(', ', sort keys %known) + )); + return; + } + + # Dynamically load provider to get its schema + my $provider_info; + eval { + require Genesis::CI::Compiler; + $provider_info = Genesis::CI::Compiler->_resolve_provider_class($type); + require $provider_info->{file}; ## no critic + }; + if ($@) { + $self->_warn("Could not load provider '$type' for schema validation: $@"); + return; + } + + my $schema = $provider_info->{class}->provider_options_schema(); + + # Check required keys + for my $key (sort keys %$schema) { + my $spec = $schema->{$key}; + next unless $spec->{required}; + $self->_error(sprintf("'provider.%s' is required for provider type '%s'", $key, $type)) + unless defined $provider->{$key}; + } + + # Check unknown keys + for my $key (sort keys %$provider) { + unless (exists $schema->{$key}) { + $self->_error(sprintf( + "'provider.%s' is not a recognized option for provider type '%s'. Valid options: %s", + $key, $type, join(', ', sort keys %$schema) + )); + } + } +} + # }}} # _validate_pipeline_section - validate pipeline.yml structure {{{ sub _validate_pipeline_section { diff --git a/lib/Genesis/CI/Provider.pm b/lib/Genesis/CI/Provider.pm new file mode 100644 index 00000000..b7b1ac1e --- /dev/null +++ b/lib/Genesis/CI/Provider.pm @@ -0,0 +1,193 @@ +package Genesis::CI::Provider; +use strict; +use warnings; + +use Genesis; +use Getopt::Long qw/GetOptionsFromArray/; + +### Class Methods {{{ + +# new - builder for creating new instance of derived class based on config {{{ +sub new { + my ($class, %config) = @_; + bug("%s->new is calling %s->new illegally", $class, __PACKAGE__) + if $class ne __PACKAGE__; + + my $type = $config{type} || 'manual'; + + if ($type eq 'concourse') { + require Genesis::CI::Provider::Concourse; + return Genesis::CI::Provider::Concourse->new(%config); + } elsif ($type eq 'github-actions') { + require Genesis::CI::Provider::GithubActions; + return Genesis::CI::Provider::GithubActions->new(%config); + } elsif ($type eq 'manual') { + require Genesis::CI::Provider::Manual; + return Genesis::CI::Provider::Manual->new(%config); + } else { + bail("Unknown CI provider type '%s'. Valid types: concourse, github-actions, manual", $type); + } +} + +# }}} +# init - builder for creating new instance based on CLI options {{{ +sub init { + my ($class, %opts) = @_; + bug("%s->init is calling %s->init illegally", $class, __PACKAGE__) + if $class ne __PACKAGE__; + + my $type = $opts{'ci-provider'} || 'manual'; + + if ($type eq 'concourse') { + require Genesis::CI::Provider::Concourse; + return Genesis::CI::Provider::Concourse->init(%opts); + } elsif ($type eq 'github-actions') { + require Genesis::CI::Provider::GithubActions; + return Genesis::CI::Provider::GithubActions->init(%opts); + } elsif ($type eq 'manual') { + require Genesis::CI::Provider::Manual; + return Genesis::CI::Provider::Manual->init(%opts); + } else { + bail("Unknown CI provider type '%s'. Valid types: concourse, github-actions, manual", $type); + } +} + +# }}} +# parse_opts - two-pass extraction: first --ci-provider, then provider-specific flags {{{ +sub parse_opts { + my ($class, $args, $ci_opts) = @_; + Getopt::Long::Configure(qw(pass_through permute no_auto_abbrev no_ignore_case bundling)); + + # Stop at '--' + my $opt_args = []; + while (scalar(@$args) && $args->[0] ne '--') { + push @$opt_args, shift @$args; + } + + # First pass: extract --ci-provider + GetOptionsFromArray($opt_args, $ci_opts, qw/ci-provider=s/); + my $type = $ci_opts->{'ci-provider'}; + + # Second pass: extract provider-specific flags + my @extra_opts; + if (!$type || $type eq 'manual') { + require Genesis::CI::Provider::Manual; + @extra_opts = Genesis::CI::Provider::Manual->opts(); + } elsif ($type eq 'concourse') { + require Genesis::CI::Provider::Concourse; + @extra_opts = Genesis::CI::Provider::Concourse->opts(); + } elsif ($type eq 'github-actions') { + require Genesis::CI::Provider::GithubActions; + @extra_opts = Genesis::CI::Provider::GithubActions->opts(); + } else { + bail("Unknown CI provider type '%s'. Valid types: concourse, github-actions, manual", $type); + } + + GetOptionsFromArray($opt_args, $ci_opts, @extra_opts) if @extra_opts; + + # Return non-option args to $args + while (scalar(@$opt_args)) { unshift @$args, pop @$opt_args } + + return 1; +} + +# }}} +# opts - base class has no options of its own {{{ +sub opts { + qw//; +} + +# }}} +# opts_help - aggregate usage docs from all providers {{{ +sub opts_help { + my ($class, %config) = @_; + bug("%s->opts_help is calling %s->opts_help illegally", $class, __PACKAGE__) + if $class ne __PACKAGE__; + + require Genesis::CI::Provider::Concourse; + require Genesis::CI::Provider::GithubActions; + require Genesis::CI::Provider::Manual; + + $config{type_default_msg} ||= '(optional, defaults to "manual")'; + $config{valid_types} ||= [qw(concourse github-actions manual)]; + + < $config{type_default_msg} + The type of CI provider to configure. Valid types: + concourse, github-actions, manual. + +${\Genesis::CI::Provider::Concourse->opts_help(%config) +}${\Genesis::CI::Provider::GithubActions->opts_help(%config) +}${\Genesis::CI::Provider::Manual->opts_help(%config) +} +EOF +} + +# }}} +# }}} +### Instance Methods {{{ + +# label - human-readable name for this provider {{{ +sub label { + $_[0]->{label} // 'CI Provider'; +} + +# }}} +# config - returns hash for .genesis/config ci.provider section (abstract) {{{ +sub config { + my ($self) = @_; + bug("Abstract Method: %s class must define 'config'", ref($self)); +} + +# }}} +# interactive_wizard - prompt the user for provider config interactively (abstract) {{{ +sub interactive_wizard { + my ($self, $top) = @_; + bug("Abstract Method: %s class must define 'interactive_wizard'", ref($self)); +} + +# }}} +# }}} + +1; + +=head1 NAME + +Genesis::CI::Provider - CI provider factory and base class + +=head1 DESCRIPTION + +Genesis::CI::Provider is the factory and abstract base class for CI provider +configuration management. It follows the same pattern as Genesis::Kit::Provider. + +Concrete subclasses: Concourse, GithubActions, Manual. + +=head1 SYNOPSIS + + # Parse CLI opts (two-pass: --ci-provider first, then provider-specific) + my %ci_opts; + Genesis::CI::Provider->parse_opts(\@ARGV, \%ci_opts); + + # Build provider object from CLI opts + my $provider = Genesis::CI::Provider->init(%ci_opts); + + # Get config hash for .genesis/config ci.provider section + my %cfg = $provider->config(); + + # Reconstruct from stored config + my $provider = Genesis::CI::Provider->new(type => 'concourse', target => 'prod'); + +=head1 SEE ALSO + +Genesis::Kit::Provider, Genesis::CI::Compiler + +=cut + +# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm new file mode 100644 index 00000000..d9189bb3 --- /dev/null +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -0,0 +1,146 @@ +package Genesis::CI::Provider::Concourse; +use strict; +use warnings; + +use base 'Genesis::CI::Provider'; +use Genesis; +use Genesis::UI; + +use constant { + DEFAULT_TEAM => 'main', +}; + +### Class Methods {{{ + +# init - create a new Concourse provider from CLI options {{{ +sub init { + my ($class, %opts) = @_; + + bail("Concourse CI provider requires --ci-target") + unless $opts{'ci-target'}; + + $class->new( + type => 'concourse', + target => $opts{'ci-target'}, + team => $opts{'ci-team'} || DEFAULT_TEAM, + insecure => $opts{'ci-insecure'} ? 1 : 0, + ); +} + +# }}} +# new - create a Concourse provider from stored config {{{ +sub new { + my ($class, %config) = @_; + bless({ + label => 'Concourse', + target => $config{target}, + team => $config{team} || DEFAULT_TEAM, + insecure => $config{insecure} ? 1 : 0, + }, $class); +} + +# }}} +# opts - Getopt::Long spec for Concourse-specific CLI flags {{{ +sub opts { + qw/ + ci-target=s + ci-team=s + ci-insecure + /; +} + +# }}} +# opts_help - usage documentation for Concourse options {{{ +sub opts_help { + my ($class, %config) = @_; + return '' unless grep { $_ eq 'concourse' } @{$config{valid_types} || []}; + + <<'EOF'; + CI Provider `concourse`: + + --ci-target (required) + The Concourse target name (as configured in ~/.flyrc) to use when + setting the pipeline. This is the same target you would pass to + `fly -t `. + + --ci-team (optional, defaults to "main") + The Concourse team name to set the pipeline on. Defaults to the + "main" team if not specified. + + --ci-insecure (optional flag) + Skip TLS certificate verification when connecting to the Concourse + server. Equivalent to fly's --skip-ssl-validation flag. + Use with caution — only for self-signed or development endpoints. + +EOF +} + +# }}} +# }}} +### Instance Methods {{{ + +# label - human-readable name for this provider {{{ +sub label { 'Concourse' } + +# }}} +# config - returns hash for .genesis/config ci.provider section {{{ +sub config { + my ($self) = @_; + my %cfg = (type => 'concourse'); + $cfg{target} = $self->{target} if defined $self->{target}; + $cfg{team} = $self->{team} if defined $self->{team} && $self->{team} ne DEFAULT_TEAM; + $cfg{insecure} = 1 if $self->{insecure}; + return %cfg; +} + +# }}} +# interactive_wizard - prompt user for Concourse configuration {{{ +sub interactive_wizard { + my ($self, $top) = @_; + + my $target = prompt_for_line(undef, + "Concourse target name (from ~/.flyrc): ", ''); + bail("Concourse CI provider requires a target name") + unless $target && $target =~ /\S/; + + my $team = prompt_for_line(undef, + sprintf("Concourse team [%s]: ", DEFAULT_TEAM), DEFAULT_TEAM); + $team = DEFAULT_TEAM unless $team && $team =~ /\S/; + + my $insecure = prompt_for_boolean( + "Skip TLS certificate verification? [y|n] ", 0); + + return $self->new( + type => 'concourse', + target => $target, + team => $team, + insecure => $insecure ? 1 : 0, + ); +} + +# }}} +# }}} + +1; + +=head1 NAME + +Genesis::CI::Provider::Concourse - Concourse CI provider for Genesis repo-init + +=head1 DESCRIPTION + +Manages Concourse-specific CI configuration in .genesis/config ci.provider. + +=head1 SYNOPSIS + + my $p = Genesis::CI::Provider::Concourse->init( + 'ci-target' => 'prod', + 'ci-team' => 'platform', + 'ci-insecure' => 0, + ); + my %cfg = $p->config; + # { type => 'concourse', target => 'prod', team => 'platform' } + +=cut + +# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 diff --git a/lib/Genesis/CI/Provider/GithubActions.pm b/lib/Genesis/CI/Provider/GithubActions.pm new file mode 100644 index 00000000..2ff44142 --- /dev/null +++ b/lib/Genesis/CI/Provider/GithubActions.pm @@ -0,0 +1,136 @@ +package Genesis::CI::Provider::GithubActions; +use strict; +use warnings; + +use base 'Genesis::CI::Provider'; +use Genesis; +use Genesis::UI; + +use constant { + DEFAULT_BRANCH => 'main', +}; + +### Class Methods {{{ + +# init - create a new GithubActions provider from CLI options {{{ +sub init { + my ($class, %opts) = @_; + + bail("GitHub Actions CI provider requires --ci-github-repo (format: org/repo)") + unless $opts{'ci-github-repo'}; + + bail("--ci-github-repo must be in 'org/repo' format") + unless $opts{'ci-github-repo'} =~ m{^[^/]+/[^/]+$}; + + $class->new( + type => 'github-actions', + repo => $opts{'ci-github-repo'}, + branch => $opts{'ci-github-branch'} || DEFAULT_BRANCH, + ); +} + +# }}} +# new - create a GithubActions provider from stored config {{{ +sub new { + my ($class, %config) = @_; + bless({ + label => 'GitHub Actions', + repo => $config{repo}, + branch => $config{branch} || DEFAULT_BRANCH, + }, $class); +} + +# }}} +# opts - Getopt::Long spec for GitHub Actions-specific CLI flags {{{ +sub opts { + qw/ + ci-github-repo=s + ci-github-branch=s + /; +} + +# }}} +# opts_help - usage documentation for GitHub Actions options {{{ +sub opts_help { + my ($class, %config) = @_; + return '' unless grep { $_ eq 'github-actions' } @{$config{valid_types} || []}; + + <<'EOF'; + CI Provider `github-actions`: + + --ci-github-repo (required) + The GitHub repository to configure workflows for, in "org/repo" + format (e.g. "myorg/my-deployment-repo"). + + --ci-github-branch (optional, defaults to "main") + The branch that workflow dispatch events and pushes will trigger + pipeline runs on. Defaults to "main". + +EOF +} + +# }}} +# }}} +### Instance Methods {{{ + +# label - human-readable name for this provider {{{ +sub label { 'GitHub Actions' } + +# }}} +# config - returns hash for .genesis/config ci.provider section {{{ +sub config { + my ($self) = @_; + my %cfg = (type => 'github-actions'); + $cfg{repo} = $self->{repo} if defined $self->{repo}; + $cfg{branch} = $self->{branch} if defined $self->{branch} && $self->{branch} ne DEFAULT_BRANCH; + return %cfg; +} + +# }}} +# interactive_wizard - prompt user for GitHub Actions configuration {{{ +sub interactive_wizard { + my ($self, $top) = @_; + + my $repo = prompt_for_line(undef, + "GitHub repository (org/repo format): ", ''); + bail("GitHub Actions CI provider requires a repository") + unless $repo && $repo =~ /\S/; + bail("Repository must be in 'org/repo' format") + unless $repo =~ m{^[^/]+/[^/]+$}; + + my $branch = prompt_for_line(undef, + sprintf("Default branch [%s]: ", DEFAULT_BRANCH), DEFAULT_BRANCH); + $branch = DEFAULT_BRANCH unless $branch && $branch =~ /\S/; + + return $self->new( + type => 'github-actions', + repo => $repo, + branch => $branch, + ); +} + +# }}} +# }}} + +1; + +=head1 NAME + +Genesis::CI::Provider::GithubActions - GitHub Actions CI provider for Genesis repo-init + +=head1 DESCRIPTION + +Manages GitHub Actions-specific CI configuration in .genesis/config ci.provider. + +=head1 SYNOPSIS + + my $p = Genesis::CI::Provider::GithubActions->init( + 'ci-github-repo' => 'myorg/my-deployment', + 'ci-github-branch' => 'main', + ); + my %cfg = $p->config; + # { type => 'github-actions', repo => 'myorg/my-deployment' } + +=cut + +# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 diff --git a/lib/Genesis/CI/Provider/Manual.pm b/lib/Genesis/CI/Provider/Manual.pm new file mode 100644 index 00000000..3c0f9f9a --- /dev/null +++ b/lib/Genesis/CI/Provider/Manual.pm @@ -0,0 +1,85 @@ +package Genesis::CI::Provider::Manual; +use strict; +use warnings; + +use base 'Genesis::CI::Provider'; +use Genesis; + +### Class Methods {{{ + +# init - create a Manual provider (no options needed) {{{ +sub init { + my ($class, %opts) = @_; + $class->new(type => 'manual'); +} + +# }}} +# new - create a Manual provider {{{ +sub new { + my ($class, %config) = @_; + bless({ label => 'Manual' }, $class); +} + +# }}} +# opts - Manual provider takes no CLI flags {{{ +sub opts { + qw//; +} + +# }}} +# opts_help - usage documentation for Manual provider {{{ +sub opts_help { + my ($class, %config) = @_; + return '' unless grep { $_ eq 'manual' } @{$config{valid_types} || []}; + + <<'EOF'; + CI Provider `manual`: + + This provider type disables automated pipeline management. Genesis + will scaffold the repository structure but no CI system will be + automatically configured. Use this when you manage your pipeline + entirely outside of Genesis or through a separate process. + + No additional options are required. + +EOF +} + +# }}} +# }}} +### Instance Methods {{{ + +# label - human-readable name for this provider {{{ +sub label { 'Manual' } + +# }}} +# config - returns hash for .genesis/config ci.provider section {{{ +sub config { + my ($self) = @_; + return (type => 'manual'); +} + +# }}} +# interactive_wizard - no prompts needed for Manual {{{ +sub interactive_wizard { + my ($self, $top) = @_; + return $self->new(type => 'manual'); +} + +# }}} +# }}} + +1; + +=head1 NAME + +Genesis::CI::Provider::Manual - Manual (no-automation) CI provider for Genesis repo-init + +=head1 DESCRIPTION + +A no-op CI provider for repositories that manage their pipelines manually. +Takes no additional options and stores only C<< type => 'manual' >> in config. + +=cut + +# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 110d933c..739a376f 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -465,43 +465,30 @@ sub _repipe_compiled { exit 0; } - # For concourse, output is { 'pipeline.yml' => $yaml_string } + # For concourse: delegate deploy to the provider, which applies the + # three-tier option resolution (CLI > ci.provider: config > defaults). if ($platform eq 'concourse') { - my $yaml = $output->{'pipeline.yml'} - or bail("Concourse provider did not produce pipeline.yml"); + my $provider = $result->{provider}; + # --dry-run: print pipeline YAML and exit without deploying if (get_options->{'dry-run'}) { + my $yaml = $output->{'pipeline.yml'} + or bail("Concourse provider did not produce pipeline.yml"); output({raw => 1}, $yaml); exit 0; } - option_defaults(target => $layout || $name); - - my ($out,$rc) = run( - 'fly -t $1 pause-pipeline -p $2', - get_options->{target}, $name + # Pass all relevant CLI flags to deploy() for three-tier resolution. + # Provider reads: ci-target, ci-team, ci-pipeline-name, ci-pause, ci-expose, + # plus the stored provider_opts (from ci.provider: section in .genesis/config). + $provider->deploy( + 'ci-target' => get_options->{'ci-target'} || get_options->{target} || $layout, + 'ci-team' => get_options->{'ci-team'}, + 'ci-pipeline-name' => get_options->{'ci-pipeline-name'}, + 'ci-pause' => get_options->{'ci-pause'} || get_options->{paused}, + 'ci-expose' => get_options->{'ci-expose'}, + 'yes' => get_options->{yes}, ); - bail("Could not pause #C{%s} pipeline: $out", $name) - unless $rc == 0 || $out =~ /pipeline '.*' not found/; - - my $yes = get_options->{yes} ? ' -n ' : ''; - my $dir = workdir; - mkfile_or_fail("${dir}/pipeline.yml", $yaml); - run({ interactive => 1, onfailure => "Could not upload pipeline $name" }, - 'fly -t $1 set-pipeline '.$yes.' -p $2 -c $3/pipeline.yml', - get_options->{target}, $name, $dir); - - run( - { interactive => 1, onfailure => "Could not unpause pipeline $name" }, - 'fly -t $1 unpause-pipeline -p $2', - get_options->{target}, $name - ) unless (get_options->{paused}); - - my $public = $ast->configuration->{public} || 0; - my $action = ($public ? 'expose' : 'hide'); - run({ interactive => 1, onfailure => "Could not $action pipeline $name" }, - 'fly -t $1 '.$action.'-pipeline -p $2', - get_options->{target}, $name); } elsif ($platform eq 'github-actions') { # GitHub Actions outputs workflow YAML files to .github/workflows/ @@ -594,8 +581,31 @@ sub _compile_pipeline { info("Using legacy CI configuration from #C{%s}", $compiler_opts{file}); } + # Parse provider-specific CLI flags via the provider options system. + # This mirrors Kit::Provider::parse_opts() — first pass extracts the + # provider type, second pass loads that provider's cli_opts() and parses them. + my %provider_cli_opts; + { + # Build a synthetic argv from get_options so we can run GetOptionsFromArray. + # Provider flags are prefixed with 'ci-' and live alongside existing flags. + # We only need to pull the ones the provider declares. + require Genesis::CI::Compiler::PipelineProvider; + my @argv = (); # provider flags come from get_options, not ARGV at this point + Genesis::CI::Compiler::PipelineProvider->parse_cli_opts( + \@argv, \%provider_cli_opts, $platform + ); + # Merge any provider flags already captured by the outer option parser + for my $key (qw(ci-target ci-team ci-pipeline-name ci-pause ci-expose)) { + $provider_cli_opts{$key} = get_options->{$key} + if defined get_options->{$key}; + } + } + my $compiler = Genesis::CI::Compiler->new(%compiler_opts); - my $result = $compiler->compile(provider => $platform); + my $result = $compiler->compile( + provider => $platform, + provider_opts => \%provider_cli_opts, + ); # Dump debug artifacts if --debug-dir is specified if (my $debug_dir = get_options->{'debug-dir'}) { diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index d31c200c..dfb755cb 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -8,6 +8,7 @@ use Genesis::Commands; use Genesis::Term qw/in_controlling_terminal/; use Genesis::Top; use Genesis::Kit::Provider; +use Genesis::CI::Provider; use Genesis::UI; use Cwd qw/getcwd abs_path/; @@ -97,12 +98,30 @@ sub _repo_init_validate { "Cannot specify both --vault and --skip-vault." ) if $opts{vault} && $opts{'skip-vault'}; - if ($opts{'ci-provider'}) { - my @valid = qw(concourse github-actions manual); - bail( - "Invalid --ci-provider '%s'. Must be one of: %s", - $opts{'ci-provider'}, join(', ', @valid) - ) unless grep { $_ eq $opts{'ci-provider'} } @valid; + # --ci-provider is a declared option and is pre-parsed by get_options() into + # %opts; provider-specific flags (--ci-target, --ci-team, etc.) are NOT + # declared, so they stay in @args via option_passthrough. Seed ci-provider + # from %opts first so parse_opts can do the second pass for provider extras. + my %ci_provider_opts; + $ci_provider_opts{'ci-provider'} = delete $opts{'ci-provider'} + if $opts{'ci-provider'}; + Genesis::CI::Provider->parse_opts(\@args, \%ci_provider_opts); + + my $ci_provider_obj; + if ($ci_provider_opts{'ci-provider'}) { + $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; + if ($@) { + if (in_controlling_terminal) { + # Required flags omitted — run interactive wizard to collect them + $ci_provider_obj = eval { + Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) + ->interactive_wizard(undef); + }; + bail("CI provider wizard failed: %s", $@) if $@; + } else { + bail("Could not initialize CI provider: %s", $@); + } + } } # --- 3. Gather data: validate local sources, detect git repo --- @@ -266,6 +285,7 @@ sub _repo_init_validate { _target_path => $target_path, _kit_file => $kit_file, _kit_provider => $kit_provider, + _ci_provider_obj => $ci_provider_obj, _use_subdir => $use_subdir, _vault_target => $vault_target, _replace_existing => $replace_existing, @@ -285,7 +305,7 @@ sub _repo_init_validate { } push @plan, "vault: #C{$vault_target}" if $vault_target; push @plan, "vault: #Yi{deferred}" unless $vault_target; - push @plan, "ci provider: #C{$opts{'ci-provider'}}" if $opts{'ci-provider'}; + push @plan, "ci provider: #C{" . $ci_provider_obj->label . "}" if $ci_provider_obj; push @plan, "subdirectory of enclosing git repo: #C{yes} (no separate .git, auto-detected)" if $use_subdir; info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; info " %s", $_ for @plan; @@ -313,16 +333,16 @@ sub _repo_init_execute { $vault_target, # vault target to configure, or undef to skip vault $replace_existing, # whether to remove existing target directory if it exists $linked_dev_kit, # optional path to a local dev kit to link into the repo + $ci_provider_obj, # optional Genesis::CI::Provider object (from validation) # User provided options (validated but not altered) - $ci_provider, # optional CI provider type $directory, # optional custom directory name override $kits_path, # optional custom kits path $no_commit, # skip the initial commit (stage only) $reason, # optional commit message override ) = get_options()->@{qw/ _name _dir _parent_dir _target_path _kit_file kit _kit_provider _use_subdir _vault_target - _replace_existing link-dev-kit ci-provider directory kits-path no-commit reason + _replace_existing link-dev-kit _ci_provider_obj directory kits-path no-commit reason /}; # Remove existing directory if validation approved it @@ -356,7 +376,7 @@ sub _repo_init_execute { # Create the repo via Top->create (pass kit_provider if we already built one) $create_opts{kit_provider} = $kit_provider if $kit_provider; my $top = Genesis::Top->create($parent_dir, $name, %create_opts, kits_path => $kit_path); - $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider; + $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider_obj; my $root = $top->path; my $human_root = humanize_path($root); @@ -382,9 +402,9 @@ sub _repo_init_execute { $kit_desc = "with an empty development kit in #C{$human_root/dev}"; } - # CI provider scaffold (only when --ci-provider is given) - if ($ci_provider) { - _create_ci_scaffold($top, $ci_provider); + # CI provider scaffold + if ($ci_provider_obj) { + _create_ci_scaffold($top, $ci_provider_obj); } # Only create a new .git when we're not already sitting inside @@ -399,7 +419,7 @@ sub _repo_init_execute { # #C{git symbolic-ref HEAD} works on all git versions, unlike # #C{git init -b} which requires >= 2.28. unless ($use_subdir) { - if ($ci_provider) { + if ($ci_provider_obj) { my $branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); run({ onfailure => "Failed to initialize git in $human_root/" }, "git init && git symbolic-ref HEAD refs/heads/$branch"); @@ -467,7 +487,7 @@ sub _repo_init_execute { human_root => $human_root, name => $name, kit_desc => $kit_desc, - ci_provider => $ci_provider, + ci_provider => $ci_provider_obj ? $ci_provider_obj->label : undef, vault_skipped => $vault_target ? 0 : 1, vault => $vault_target, submodule => $use_subdir, @@ -560,11 +580,12 @@ sub _select_vault_target { sub _create_ci_scaffold { my ($top, $provider) = @_; + # $provider may be a Genesis::CI::Provider object or a plain string (legacy). + my %provider_cfg = ref($provider) ? $provider->config() : (type => $provider); + $top->config->set('ci', { enabled => Genesis::Config::TRUE, - provider => { - type => $provider, - }, + provider => \%provider_cfg, pipeline => { name => $top->config->get('deployment_type'), }, diff --git a/t/ci-compiler.t b/t/ci-compiler.t index dede486c..512142ff 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2215,6 +2215,28 @@ subtest 'Parser - genesis-config: produces correct normalized structure' => sub # env_dir must NOT be set when a pipeline section is provided ok(!$parsed->{env_dir}, "env_dir not set when pipeline section is provided"); + + # ci.provider: must be propagated into $parsed->{provider} so that + # Compiler->compile() can populate provider_opts from stored config. + ok ref($parsed->{provider}) eq 'HASH', "provider is a hash"; +}; + +subtest 'Parser - genesis-config: provider key propagated from ci.provider' => sub { + my $ci_with_provider = { + %$_ci_data, + provider => { type => 'concourse', target => 'prod', team => 'platform' }, + }; + my $top = MockTop->new( + config => MockConfig->new(ci => $ci_with_provider), + base => '/myrepo', + ); + my $parser = Genesis::CI::Compiler::Parser->new(top => $top); + my $parsed = eval { $parser->parse() }; + ok !$@, "parse() succeeds with provider in ci" or diag $@; + + is_deeply $parsed->{provider}, + { type => 'concourse', target => 'prod', team => 'platform' }, + "ci.provider data round-trips through parser into parsed->{provider}"; }; subtest 'Parser - genesis-config: sets env_dir when no pipeline section' => sub { @@ -2306,6 +2328,225 @@ subtest 'Top - register_config_section stores handler' => sub { ok 1, "re-registering same section does not error"; }; +### ============================================================ ### +### Phase E Provider Options System Tests +### ============================================================ ### + +subtest 'PipelineProvider - known_providers lists registered types' => sub { + my @providers = Genesis::CI::Compiler::PipelineProvider->known_providers(); + ok scalar(@providers) >= 1, "at least one provider registered"; + ok grep { $_ eq 'concourse' } @providers, "concourse is registered"; +}; + +subtest 'PipelineProvider - base class cli_opts returns empty list' => sub { + # We cannot call cli_opts on the abstract base directly (bug guard), + # so we test via the Concourse subclass and verify the pattern instead. + my @opts = Genesis::CI::Concourse->cli_opts(); + ok scalar(@opts) > 0, "Concourse declares at least one CLI opt"; + ok grep { $_ eq 'ci-target=s' } @opts, "ci-target=s declared"; + ok grep { $_ eq 'ci-team=s' } @opts, "ci-team=s declared"; + ok grep { $_ eq 'ci-pause' } @opts, "ci-pause declared (boolean flag)"; + ok grep { $_ eq 'ci-expose' } @opts, "ci-expose declared (boolean flag)"; +}; + +subtest 'Concourse - cli_opts_help contains required option documentation' => sub { + my $help = Genesis::CI::Concourse->cli_opts_help(valid_types => ['concourse']); + ok length($help) > 0, "help text is non-empty"; + like $help, qr/--ci-target/, "documents --ci-target"; + like $help, qr/--ci-team/, "documents --ci-team"; + like $help, qr/--ci-pipeline-name/, "documents --ci-pipeline-name"; + like $help, qr/--ci-pause/, "documents --ci-pause"; + like $help, qr/--ci-expose/, "documents --ci-expose"; + like $help, qr/required/i, "marks required options"; + like $help, qr/optional.*default|default.*optional/i, "marks optional options with defaults"; + like $help, qr/main/, "shows default team 'main'"; +}; + +subtest 'Concourse - cli_opts_help hidden when type not in valid_types' => sub { + my $help = Genesis::CI::Concourse->cli_opts_help(valid_types => ['github-actions']); + is $help, '', "help empty when concourse not in valid_types"; +}; + +subtest 'Concourse - provider_options_schema has correct structure' => sub { + my $schema = Genesis::CI::Concourse->provider_options_schema(); + ok ref($schema) eq 'HASH', "schema is a hash"; + ok exists $schema->{type}, "'type' key present"; + ok $schema->{type}{required}, "'type' is required"; + ok exists $schema->{target}, "'target' key present"; + ok exists $schema->{team}, "'team' key present"; + ok exists $schema->{expose}, "'expose' key present"; + ok exists $schema->{pause_after_set}, "'pause_after_set' key present"; + is $schema->{team}{default}, 'main', "team default is 'main'"; +}; + +subtest 'Concourse - provider_options_defaults returns expected defaults' => sub { + my $defaults = Genesis::CI::Concourse->provider_options_defaults(); + ok ref($defaults) eq 'HASH', "defaults is a hash"; + is $defaults->{team}, 'main', "team default is 'main'"; + is $defaults->{expose}, 0, "expose default is false"; + is $defaults->{pause_after_set}, 0, "pause_after_set default is false"; +}; + +subtest 'Concourse - provider_config omits default values' => sub { + # Only non-defaults should appear in config output + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { + type => 'concourse', + target => 'my-target', + team => 'main', # this IS the default — should be omitted + expose => 0, # this IS the default — should be omitted + }, + ); + my $config = $provider->provider_config(); + ok ref($config) eq 'HASH', "provider_config returns hash"; + is $config->{type}, 'concourse', "type always present"; + is $config->{target}, 'my-target', "non-default target included"; + ok !exists $config->{team}, "default team omitted"; + ok !exists $config->{expose}, "default expose omitted"; +}; + +subtest 'Concourse - provider_config includes non-default values' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { + type => 'concourse', + team => 'my-team', # non-default + pause_after_set => 1, # non-default + }, + ); + my $config = $provider->provider_config(); + is $config->{team}, 'my-team', "non-default team included"; + is $config->{pause_after_set}, 1, "non-default pause_after_set included"; +}; + +subtest 'Concourse - provider_option applies defaults when not set' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new(ast => $ast); + + is $provider->provider_option('team'), 'main', "team defaults to 'main'"; + is $provider->provider_option('expose'), 0, "expose defaults to 0"; + is $provider->provider_option('target'), undef, "target has no default"; +}; + +subtest 'Concourse - provider_option prefers stored opts over defaults' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { team => 'custom-team' }, + ); + is $provider->provider_option('team'), 'custom-team', + "stored team overrides default"; +}; + +subtest 'Concourse - describe_provider returns structured hash' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { + target => 'prod-concourse', + team => 'genesis', + }, + ); + my %info = $provider->describe_provider(); + + is $info{type}, 'concourse', "type is 'concourse'"; + is $info{label}, 'Concourse', "label is 'Concourse'"; + is $info{status}, 'ok', "status is 'ok'"; + ok ref($info{extras}) eq 'ARRAY', "extras is an arrayref"; + ok grep { $_ eq 'Target' } @{$info{extras}}, "Target in extras"; + ok grep { $_ eq 'Team' } @{$info{extras}}, "Team in extras"; + is $info{Target}, 'prod-concourse', "Target value correct"; + is $info{Team}, 'genesis', "Team value correct"; +}; + +subtest 'PipelineProvider - all_cli_opts_help covers all providers' => sub { + my $help = Genesis::CI::Compiler::PipelineProvider->all_cli_opts_help(); + like $help, qr/CI PROVIDER OPTIONS/, "contains header"; + like $help, qr/--ci-provider/, "documents --ci-provider"; + like $help, qr/concourse/, "mentions concourse"; + like $help, qr/--ci-target/, "includes Concourse-specific flag"; +}; + +subtest 'PipelineProvider - parse_cli_opts two-pass extraction' => sub { + # Simulate argv that includes a provider-specific flag + my @argv = ('--ci-target', 'my-fly-target', '--ci-team', 'ops', '--other-flag'); + my %opts; + + Genesis::CI::Compiler::PipelineProvider->parse_cli_opts( + \@argv, \%opts, 'concourse' + ); + + is $opts{'ci-target'}, 'my-fly-target', "ci-target extracted"; + is $opts{'ci-team'}, 'ops', "ci-team extracted"; + ok grep { $_ eq '--other-flag' } @argv, "unknown flag left in argv"; +}; + +subtest 'Compiler - validate_config_section validates ci.provider' => sub { + # Valid with a correct provider section + my $valid_data = { + %$_ci_data, + provider => { type => 'concourse', target => 'my-target' }, + }; + eval { Genesis::CI::Compiler->validate_config_section($valid_data, undef) }; + ok !$@, "valid ci.provider section passes" or diag $@; + + # Unknown provider type + my $bad_type = { + %$_ci_data, + provider => { type => 'kubernetes' }, + }; + eval { Genesis::CI::Compiler->validate_config_section($bad_type, undef) }; + like $@, qr/not a known CI provider/i, "unknown provider type fails"; + + # Unknown option key for concourse + my $bad_key = { + %$_ci_data, + provider => { type => 'concourse', bogus_option => 'foo' }, + }; + eval { Genesis::CI::Compiler->validate_config_section($bad_key, undef) }; + like $@, qr/not a recognized option/i, "unknown provider option key fails"; +}; + +subtest 'Validator - provider section validated in multi-file path' => sub { + my $v = Genesis::CI::Compiler::Validator->new(); + + # Valid provider section + $v->validate({ + _source_format => 'multi-file', + pipeline => {}, + targets => { sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } } }, + integrations => { + vault => { url => 'https://vault.example.com' }, + source_control => { provider => 'github', repository => 'org/repo' }, + }, + provider => { type => 'concourse', target => 'my-target', team => 'main' }, + scripts => {}, + provider_config => {}, + }); + ok !$v->has_errors, "valid provider section passes validator" + or diag join("\n", @{$v->errors}); + + # Unknown option + $v->validate({ + _source_format => 'multi-file', + pipeline => {}, + targets => { sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } } }, + integrations => { + vault => { url => 'https://vault.example.com' }, + source_control => { provider => 'github', repository => 'org/repo' }, + }, + provider => { type => 'concourse', unknown_key => 'bad' }, + scripts => {}, + provider_config => {}, + }); + ok $v->has_errors, "unknown provider key triggers validation error"; + like join(' ', @{$v->errors}), qr/not a recognized option/i, + "error message identifies the unknown key"; +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu diff --git a/t/ci-provider.t b/t/ci-provider.t new file mode 100644 index 00000000..d9cb28e2 --- /dev/null +++ b/t/ci-provider.t @@ -0,0 +1,290 @@ +#!perl +use strict; +use warnings; + +use Test::More; +use lib 'lib'; + +$ENV{GENESIS_TESTING} = "yes"; +$ENV{GENESIS_LIB} ||= 'lib'; + +use_ok 'Genesis::CI::Provider'; +use_ok 'Genesis::CI::Provider::Concourse'; +use_ok 'Genesis::CI::Provider::GithubActions'; +use_ok 'Genesis::CI::Provider::Manual'; + +### ============================================================ ### +### Factory: Genesis::CI::Provider->new +### ============================================================ ### + +subtest 'Provider->new dispatches on type' => sub { + my $c = Genesis::CI::Provider->new(type => 'concourse', target => 'myci'); + isa_ok $c, 'Genesis::CI::Provider::Concourse', 'type=concourse'; + + my $g = Genesis::CI::Provider->new(type => 'github-actions', repo => 'org/repo'); + isa_ok $g, 'Genesis::CI::Provider::GithubActions', 'type=github-actions'; + + my $m = Genesis::CI::Provider->new(type => 'manual'); + isa_ok $m, 'Genesis::CI::Provider::Manual', 'type=manual'; + + my $def = Genesis::CI::Provider->new(); # no type → manual + isa_ok $def, 'Genesis::CI::Provider::Manual', 'no type → manual'; +}; + +subtest 'Provider->new rejects unknown type' => sub { + eval { Genesis::CI::Provider->new(type => 'bogus') }; + like $@, qr/Unknown CI provider type/i, 'unknown type bails'; +}; + +### ============================================================ ### +### Factory: Genesis::CI::Provider->init +### ============================================================ ### + +subtest 'Provider->init dispatches on ci-provider opt' => sub { + my $c = Genesis::CI::Provider->init('ci-provider' => 'concourse', 'ci-target' => 'prod'); + isa_ok $c, 'Genesis::CI::Provider::Concourse', 'ci-provider=concourse'; + + my $g = Genesis::CI::Provider->init( + 'ci-provider' => 'github-actions', + 'ci-github-repo' => 'myorg/myrepo', + ); + isa_ok $g, 'Genesis::CI::Provider::GithubActions', 'ci-provider=github-actions'; + + my $m = Genesis::CI::Provider->init('ci-provider' => 'manual'); + isa_ok $m, 'Genesis::CI::Provider::Manual', 'ci-provider=manual'; + + my $def = Genesis::CI::Provider->init(); # no ci-provider → manual + isa_ok $def, 'Genesis::CI::Provider::Manual', 'no ci-provider → manual'; +}; + +subtest 'Provider->init rejects unknown type' => sub { + eval { Genesis::CI::Provider->init('ci-provider' => 'jenkins') }; + like $@, qr/Unknown CI provider type/i, 'unknown type bails'; +}; + +### ============================================================ ### +### parse_opts - two-pass extraction +### ============================================================ ### + +subtest 'parse_opts extracts --ci-provider and general args' => sub { + my @args = ('--ci-provider', 'concourse', '--ci-target', 'myci', 'other-arg'); + my %opts; + Genesis::CI::Provider->parse_opts(\@args, \%opts); + + is $opts{'ci-provider'}, 'concourse', 'ci-provider extracted'; + is $opts{'ci-target'}, 'myci', 'ci-target extracted'; + is_deeply \@args, ['other-arg'], 'non-option arg left in @args'; +}; + +subtest 'parse_opts stops at -- sentinel' => sub { + my @args = ('--ci-provider', 'concourse', '--', '--ci-target', 'x'); + my %opts; + Genesis::CI::Provider->parse_opts(\@args, \%opts); + + is $opts{'ci-provider'}, 'concourse', 'ci-provider extracted before --'; + ok !defined $opts{'ci-target'}, 'ci-target not extracted (after --)'; + is_deeply \@args, ['--', '--ci-target', 'x'], 'args after -- preserved'; +}; + +subtest 'parse_opts handles manual (no extra opts)' => sub { + my @args = ('--ci-provider', 'manual', 'leftover'); + my %opts; + Genesis::CI::Provider->parse_opts(\@args, \%opts); + + is $opts{'ci-provider'}, 'manual', 'ci-provider=manual'; + is_deeply \@args, ['leftover'], 'non-option preserved'; +}; + +subtest 'parse_opts handles github-actions flags' => sub { + my @args = ('--ci-provider', 'github-actions', + '--ci-github-repo', 'acme/deploy', + '--ci-github-branch', 'release'); + my %opts; + Genesis::CI::Provider->parse_opts(\@args, \%opts); + + is $opts{'ci-provider'}, 'github-actions', 'ci-provider'; + is $opts{'ci-github-repo'}, 'acme/deploy', 'ci-github-repo'; + is $opts{'ci-github-branch'}, 'release', 'ci-github-branch'; + is_deeply \@args, [], 'all args consumed'; +}; + +subtest 'parse_opts defaults to manual when no --ci-provider' => sub { + my @args = ('other-arg'); + my %opts; + Genesis::CI::Provider->parse_opts(\@args, \%opts); + ok !defined $opts{'ci-provider'}, 'ci-provider undef when not supplied'; + is_deeply \@args, ['other-arg'], 'non-option preserved'; +}; + +### ============================================================ ### +### opts_help +### ============================================================ ### + +subtest 'opts_help aggregates all providers' => sub { + my $help = Genesis::CI::Provider->opts_help(); + like $help, qr/CI PROVIDERS/, 'header present'; + like $help, qr/--ci-provider/, 'general flag documented'; + like $help, qr/concourse/, 'concourse section present'; + like $help, qr/github-actions/, 'github-actions section present'; + like $help, qr/manual/, 'manual section present'; + like $help, qr/--ci-target/, 'concourse flag in help'; + like $help, qr/--ci-github-repo/, 'gha flag in help'; +}; + +### ============================================================ ### +### Genesis::CI::Provider::Concourse +### ============================================================ ### + +subtest 'Concourse->opts returns Getopt spec' => sub { + my @opts = Genesis::CI::Provider::Concourse->opts(); + ok grep { $_ eq 'ci-target=s' } @opts, 'ci-target=s present'; + ok grep { $_ eq 'ci-team=s' } @opts, 'ci-team=s present'; + ok grep { $_ eq 'ci-insecure' } @opts, 'ci-insecure present'; +}; + +subtest 'Concourse->init requires --ci-target' => sub { + eval { Genesis::CI::Provider::Concourse->init() }; + like $@, qr/requires --ci-target/i, 'bails without ci-target'; +}; + +subtest 'Concourse->init defaults team to main' => sub { + my $p = Genesis::CI::Provider::Concourse->init('ci-target' => 'ci.example.com'); + is $p->{team}, 'main', 'team defaults to main'; + is $p->{insecure}, 0, 'insecure defaults to 0'; +}; + +subtest 'Concourse->init honours all opts' => sub { + my $p = Genesis::CI::Provider::Concourse->init( + 'ci-target' => 'ci.example.com', + 'ci-team' => 'platform', + 'ci-insecure' => 1, + ); + is $p->{target}, 'ci.example.com', 'target set'; + is $p->{team}, 'platform', 'team set'; + is $p->{insecure}, 1, 'insecure set'; +}; + +subtest 'Concourse->config omits default team' => sub { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'myci', + ); + my %cfg = $p->config; + is $cfg{type}, 'concourse', 'type present'; + is $cfg{target}, 'myci', 'target present'; + ok !exists $cfg{team}, 'default team omitted'; + ok !exists $cfg{insecure}, 'insecure omitted when false'; +}; + +subtest 'Concourse->config includes non-default team and insecure' => sub { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'ci', + team => 'ops', + insecure => 1, + ); + my %cfg = $p->config; + is $cfg{team}, 'ops', 'non-default team in config'; + is $cfg{insecure}, 1, 'insecure in config'; +}; + +subtest 'Concourse->label' => sub { + my $p = Genesis::CI::Provider::Concourse->new(type => 'concourse', target => 'x'); + is $p->label, 'Concourse', 'label is Concourse'; +}; + +subtest 'Concourse->opts_help contains required flag docs' => sub { + my $help = Genesis::CI::Provider::Concourse->opts_help( + valid_types => [qw(concourse)] + ); + like $help, qr/--ci-target/, 'ci-target documented'; + like $help, qr/--ci-team/, 'ci-team documented'; + like $help, qr/--ci-insecure/, 'ci-insecure documented'; +}; + +subtest 'Concourse->opts_help empty when not in valid_types' => sub { + my $help = Genesis::CI::Provider::Concourse->opts_help( + valid_types => [qw(github-actions)] + ); + is $help, '', 'empty when concourse not in valid_types'; +}; + +### ============================================================ ### +### Genesis::CI::Provider::GithubActions +### ============================================================ ### + +subtest 'GithubActions->opts returns Getopt spec' => sub { + my @opts = Genesis::CI::Provider::GithubActions->opts(); + ok grep { $_ eq 'ci-github-repo=s' } @opts, 'ci-github-repo=s present'; + ok grep { $_ eq 'ci-github-branch=s' } @opts, 'ci-github-branch=s present'; +}; + +subtest 'GithubActions->init requires --ci-github-repo' => sub { + eval { Genesis::CI::Provider::GithubActions->init() }; + like $@, qr/requires --ci-github-repo/i, 'bails without ci-github-repo'; +}; + +subtest 'GithubActions->init validates org/repo format' => sub { + eval { Genesis::CI::Provider::GithubActions->init('ci-github-repo' => 'invalid-no-slash') }; + like $@, qr/org\/repo/i, 'bails on bad format'; +}; + +subtest 'GithubActions->init defaults branch to main' => sub { + my $p = Genesis::CI::Provider::GithubActions->init('ci-github-repo' => 'acme/deploy'); + is $p->{branch}, 'main', 'branch defaults to main'; +}; + +subtest 'GithubActions->config omits default branch' => sub { + my $p = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', + repo => 'acme/deploy', + ); + my %cfg = $p->config; + is $cfg{type}, 'github-actions', 'type present'; + is $cfg{repo}, 'acme/deploy', 'repo present'; + ok !exists $cfg{branch}, 'default branch omitted'; +}; + +subtest 'GithubActions->config includes non-default branch' => sub { + my $p = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', + repo => 'acme/deploy', + branch => 'release', + ); + my %cfg = $p->config; + is $cfg{branch}, 'release', 'non-default branch in config'; +}; + +subtest 'GithubActions->label' => sub { + my $p = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', repo => 'acme/x' + ); + is $p->label, 'GitHub Actions', 'label is GitHub Actions'; +}; + +### ============================================================ ### +### Genesis::CI::Provider::Manual +### ============================================================ ### + +subtest 'Manual->opts returns empty list' => sub { + my @opts = Genesis::CI::Provider::Manual->opts(); + is scalar @opts, 0, 'Manual has no opts'; +}; + +subtest 'Manual->init returns Manual object' => sub { + my $m = Genesis::CI::Provider::Manual->init(); + isa_ok $m, 'Genesis::CI::Provider::Manual', 'Manual->init returns Manual'; +}; + +subtest 'Manual->config returns only type' => sub { + my $m = Genesis::CI::Provider::Manual->new(type => 'manual'); + my %cfg = $m->config; + is_deeply \%cfg, { type => 'manual' }, 'config is {type => manual}'; +}; + +subtest 'Manual->label' => sub { + my $m = Genesis::CI::Provider::Manual->new(type => 'manual'); + is $m->label, 'Manual', 'label is Manual'; +}; + +done_testing; From 014ec171e2468b546fdbb58d70183212ac46f79d Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:06:58 -0400 Subject: [PATCH 019/103] Add pipeline options and metadata injection Introduce --prior-env, --require-pr and --manual CLI options and interactive prompts for pipeline configuration when a CI provider is present. Validate --prior-env against existing environments, provide a numbered choice menu in interactive mode, and support non-interactive scripted use. Only write a pipeline: section if there is data to record; avoid overwriting an existing pipeline section and inject metadata after the env: line using flexible indentation, with warnings if injection fails. Also update bin/genesis help text and display the configured CI provider type. --- bin/genesis | 16 ++++- lib/Genesis/Commands/Env.pm | 121 ++++++++++++++++++++++++++++-------- 2 files changed, 109 insertions(+), 28 deletions(-) diff --git a/bin/genesis b/bin/genesis index 8fe1ab6e..054af923 100755 --- a/bin/genesis +++ b/bin/genesis @@ -254,7 +254,21 @@ define_command("create", { "repository for the environment.", "force|f" => - "Create a new environment file even if it already exists." + "Create a new environment file even if it already exists.", + + "prior-env=s" => + "Name of the prior environment in the pipeline chain (the environment ". + "that must succeed before this one deploys). Only used when a CI ". + "provider is configured in #C{.genesis/config}. Omit or leave blank ". + "to designate this environment as a pipeline entrypoint.", + + "require-pr" => + "Require a pull-request gate before this environment deploys. ". + "Only used when a CI provider is configured in #C{.genesis/config}.", + + "manual" => + "Require a manual CI trigger before this environment deploys. ". + "Only used when a CI provider is configured in #C{.genesis/config}." ], deprecated_options => [ "^prefix=s" => diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 2f525aba..73c34b60 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -76,45 +76,112 @@ sub create { my $env = $top->create_env($name, $kit, %{get_options()}); bail "Failed to create environment $name" unless $env; - # Phase C: prompt for pipeline metadata when CI provider is configured - if ($top->config->has('ci.provider')) { + # Phase C: write pipeline metadata when CI provider is configured. + # Runs interactively when in a controlling terminal; honours --prior-env, + # --require-pr, and --manual flags for non-interactive (scripted) use. + if ($top->config->has('ci.provider.type')) { + my $ci_type = $top->config->get('ci.provider.type') // 'unknown'; info( - "\n#G{Pipeline configuration} (ci.provider: #C{%s})\n", - $top->config->get('ci.provider') + "\n#G{Pipeline configuration} (ci provider: #C{%s})\n", + $ci_type ); - my $prior_env = prompt_for_line( - "Prior environment (leave blank if this is the pipeline entrypoint):", - "prior env", - "", - ); + my %cli_opts = %{get_options()}; + my $interactive = in_controlling_terminal; - my $require_pr = prompt_for_boolean( - "Require a PR gate before this environment deploys? [y|n]", - "n", - ); + # --- prior_env (Issue 1: use choice menu, not freeform) --- + my $prior_env; + if (exists $cli_opts{'prior-env'}) { + # Non-interactive path: flag supplied; validate it refers to a real env. + $prior_env = $cli_opts{'prior-env'} // ''; + if (length($prior_env)) { + my %known = map { $_->name => 1 } $top->envs(); + bail( + "--prior-env '%s' does not match any environment in this repository.", + $prior_env + ) unless $known{$prior_env}; + } + } elsif ($interactive) { + # Interactive path: present numbered menu of existing envs. + my @existing = grep { $_->name ne $name } $top->envs(); + if (@existing) { + my @env_names = map { $_->name } @existing; + my @choices = ('', @env_names); + my @labels = ('(none — pipeline entrypoint)', @env_names); + $prior_env = prompt_for_choice( + "Select prior environment (the environment that must succeed before this one):", + \@choices, + '', + \@labels, + "Please select a number from the list", + "environment", + ); + } else { + # No other envs yet — must be the entrypoint. + $prior_env = ''; + info("No other environments found — #C{%s} will be the pipeline entrypoint.", $name); + } + } - my $manual = prompt_for_boolean( - "Require a manual CI trigger before this environment deploys? [y|n]", - "n", - ); + # --- require_pr --- + my $require_pr; + if (exists $cli_opts{'require-pr'}) { + $require_pr = $cli_opts{'require-pr'} ? 1 : 0; + } elsif ($interactive) { + $require_pr = prompt_for_boolean( + "Require a PR gate before this environment deploys? [y|n]", + "n", + ); + } - if (length($prior_env)) { + # --- manual --- + my $manual; + if (exists $cli_opts{manual}) { + $manual = $cli_opts{manual} ? 1 : 0; + } elsif ($interactive) { + $manual = prompt_for_boolean( + "Require a manual CI trigger before this environment deploys? [y|n]", + "n", + ); + } + + # Write pipeline: section when there is something to record. + # Entrypoints (no prior_env) can still carry manual: true. + if (length($prior_env // '') || $require_pr || $manual) { my $pipeline_yaml = " pipeline:\n"; - $pipeline_yaml .= " prior_env: $prior_env\n"; - $pipeline_yaml .= " require_pr: true\n" if $require_pr; - $pipeline_yaml .= " manual: true\n" if $manual; + $pipeline_yaml .= " prior_env: $prior_env\n" if length($prior_env // ''); + $pipeline_yaml .= " require_pr: true\n" if $require_pr; + $pipeline_yaml .= " manual: true\n" if $manual; my $file = $env->path($env->file); my $contents = slurp($file); - unless ($contents =~ /^ pipeline:/m) { - $contents =~ s/^( env:\s+\S[^\n]*\n)/$1$pipeline_yaml/m; - mkfile_or_fail($file, $contents); - } - info("#G{Pipeline metadata written to} #C{%s}", $env->file); + if ($contents =~ /^\s+pipeline:/m) { + # pipeline: section already present (kit wrote one) — don't overwrite. + info( + "#Y{Note}: pipeline section already present in #C{%s}, skipping injection.", + $env->file + ); + } else { + # Inject after the env: line; flexible indentation (Issue 4). + my $injected = ($contents =~ s/^((\s+)env:\s+\S[^\n]*\n)/$1$pipeline_yaml/m); + if ($injected) { + mkfile_or_fail($file, $contents); + info("#G{Pipeline metadata written to} #C{%s}", $env->file); + } else { + warning( + "Could not inject pipeline metadata into #C{%s}: ". + "'env:' key not found at expected indentation. ". + "Add the pipeline section manually:\n%s", + $env->file, $pipeline_yaml + ); + } + } } else { - info("No prior environment — #C{%s} is a pipeline entrypoint, no pipeline section written.", $name); + info( + "#C{%s} is a pipeline entrypoint with no gate flags — no pipeline section written.", + $name + ); } } From d9de0c0da29858a5d6b6bebeb3a46b6b4f7e09a8 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:13:51 -0400 Subject: [PATCH 020/103] Add CI provider prerequisite checks Introduce a check_prereqs() hook on Provider and PipelineProvider to allow providers to verify required external tooling before performing operations. Concourse implementations now check for the fly CLI in PATH and optionally enforce a minimum fly version (min_fly_version), emitting user-friendly error messages when unmet. Commands that invoke provider actions (repo init and pipeline deploy) now call check_prereqs() and bail on failure to avoid runtime errors. Tests updated/added to cover presence/absence of fly and version checks. --- lib/Genesis/CI/Compiler/PipelineProvider.pm | 16 ++++ .../CI/Compiler/Providers/Concourse.pm | 21 ++++++ lib/Genesis/CI/Provider.pm | 16 ++++ lib/Genesis/CI/Provider/Concourse.pm | 39 ++++++++++ lib/Genesis/Commands/Pipelines.pm | 4 + lib/Genesis/Commands/Repo.pm | 4 + t/ci-compiler.t | 57 +++++++++++++++ t/ci-provider.t | 73 +++++++++++++++++++ 8 files changed, 230 insertions(+) diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index efde61f6..134bc483 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -78,6 +78,22 @@ sub output_files { bug("Subclass '%s' must implement output_files()", ref($self)); } +# }}} +# }}} +### Prerequisite Checking {{{ + +# check_prereqs - verify all required external tools are available {{{ +# +# Called before deploy() to confirm the provider's toolchain is present. +# Returns 1 when all prereqs are satisfied; returns 0 and emits error() +# messages for each unmet prereq (caller decides whether to bail). +# +# Subclasses override to check their specific requirements. +# Base implementation has no prereqs and always returns 1. +sub check_prereqs { + return 1; +} + # }}} # }}} ### Provider Options Contract {{{ diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index ee069641..bb935473 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -68,6 +68,27 @@ sub init { # provider_type - canonical type string {{{ sub provider_type { 'concourse' } +# }}} +# check_prereqs - verify fly CLI is installed before attempting deploy {{{ +sub check_prereqs { + my ($self) = @_; + my $ok = 1; + + chomp(my $fly_path = `which fly 2>/dev/null`); + unless ($fly_path) { + error( + "Cannot deploy Concourse pipeline: the #C{fly} CLI was not found in ". + "your PATH.\n". + " Install it from your Concourse server:\n". + " #C{/api/v1/cli?arch=amd64&platform=}\n". + " Or log in via the Concourse UI and download fly from the bottom-right icon.", + ); + return 0; + } + + return $ok; +} + # }}} # cli_opts - Getopt::Long specs for deploy-time command-line flags {{{ # diff --git a/lib/Genesis/CI/Provider.pm b/lib/Genesis/CI/Provider.pm index b7b1ac1e..956af2a2 100644 --- a/lib/Genesis/CI/Provider.pm +++ b/lib/Genesis/CI/Provider.pm @@ -146,6 +146,22 @@ sub config { bug("Abstract Method: %s class must define 'config'", ref($self)); } +# }}} +# check_prereqs - verify all required external tools are available {{{ +# +# Called before any provider-dependent operation (repo-init, repipe, etc.) +# to confirm the provider's toolchain is present and meets any minimum +# version requirements. +# +# Returns 1 when all prereqs are satisfied; returns 0 and emits error() +# for each unmet prereq. Caller decides whether to bail. +# +# Subclasses override to add provider-specific checks. +# Base implementation has no prereqs and always returns 1. +sub check_prereqs { + return 1; +} + # }}} # interactive_wizard - prompt the user for provider config interactively (abstract) {{{ sub interactive_wizard { diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index d9189bb3..9b7178d5 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -82,6 +82,45 @@ EOF # label - human-readable name for this provider {{{ sub label { 'Concourse' } +# }}} +# check_prereqs - verify fly CLI is installed and meets min version {{{ +sub check_prereqs { + my ($self) = @_; + my $ok = 1; + + # Require fly in PATH + chomp(my $fly_path = `which fly 2>/dev/null`); + unless ($fly_path) { + error( + "The Concourse CI provider requires the #C{fly} CLI but it was not ". + "found in your PATH.\n". + " Install it from your Concourse server:\n". + " #C{/api/v1/cli?arch=amd64&platform=}\n". + " Or log in via the Concourse UI and download fly from the bottom-right icon.", + ); + return 0; + } + + # Optional minimum version enforcement + if (defined $self->{min_fly_version}) { + my ($ver_out) = run({ stderr => 0 }, 'fly', '--version'); + chomp(my $fly_version = $ver_out // ''); + $fly_version =~ s/\s.*$//; # strip trailing build info if any + unless (new_enough($fly_version, $self->{min_fly_version})) { + error( + "Concourse CI provider requires fly version #C{%s} or later ". + "(found: #Y{%s}).\n". + " Sync fly with your Concourse server: #C{fly -t sync}", + $self->{min_fly_version}, + $fly_version || 'unknown', + ); + $ok = 0; + } + } + + return $ok; +} + # }}} # config - returns hash for .genesis/config ci.provider section {{{ sub config { diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 739a376f..544597bf 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -478,6 +478,10 @@ sub _repipe_compiled { exit 0; } + # Verify the provider's toolchain is available before attempting deploy. + bail("CI provider prerequisite check failed") + unless $provider->check_prereqs(); + # Pass all relevant CLI flags to deploy() for three-tier resolution. # Provider reads: ci-target, ci-team, ci-pipeline-name, ci-pause, ci-expose, # plus the stored provider_opts (from ci.provider: section in .genesis/config). diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index dfb755cb..af1eb3bf 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -122,6 +122,10 @@ sub _repo_init_validate { bail("Could not initialize CI provider: %s", $@); } } + + # Verify the provider's toolchain is available before we do any work + bail("CI provider prerequisite check failed") + unless $ci_provider_obj->check_prereqs(); } # --- 3. Gather data: validate local sources, detect git repo --- diff --git a/t/ci-compiler.t b/t/ci-compiler.t index 512142ff..996dc3ad 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2547,6 +2547,63 @@ subtest 'Validator - provider section validated in multi-file path' => sub { "error message identifies the unknown key"; }; +### ============================================================ ### +### PipelineProvider - check_prereqs +### ============================================================ ### + +subtest 'PipelineProvider - base class check_prereqs returns 1' => sub { + # Base class has no prereqs; GHA provider inherits this no-op default. + eval { require 'Genesis/CI/Compiler/Providers/GithubActions.pm' }; + if ($@) { + pass 'skipped: GithubActions provider not available'; + return; + } + my $gha = Genesis::CI::GithubActions->new( + ast => Genesis::CI::Compiler::AST->new( + metadata => { name => 'test', version => '2.0', source => 'modern' }, + branches => { live => 'main', target_prefix => 'target/' }, + integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, + targets => {}, + workflows => {}, + ), + top => undef, + provider_opts => {}, + ); + ok $gha->check_prereqs(), 'GithubActions PipelineProvider check_prereqs returns 1'; +}; + +subtest 'PipelineProvider::Concourse - check_prereqs returns 1 when fly present' => sub { + my $fly = `which fly 2>/dev/null`; + chomp $fly; + unless ($fly) { + pass 'skipped: fly not installed in this environment'; + return; + } + my $ast = Genesis::CI::Compiler::AST->new( + metadata => { name => 'test', version => '2.0', source => 'modern' }, + branches => { live => 'main', target_prefix => 'target/' }, + integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, + targets => {}, + workflows => {}, + ); + my $p = Genesis::CI::Concourse->new(ast => $ast, top => undef, provider_opts => {}); + ok $p->check_prereqs(), 'check_prereqs returns 1 when fly is present'; +}; + +subtest 'PipelineProvider::Concourse - check_prereqs returns 0 when fly absent' => sub { + local $ENV{PATH} = '/nonexistent'; + my $ast = Genesis::CI::Compiler::AST->new( + metadata => { name => 'test', version => '2.0', source => 'modern' }, + branches => { live => 'main', target_prefix => 'target/' }, + integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, + targets => {}, + workflows => {}, + ); + my $p = Genesis::CI::Concourse->new(ast => $ast, top => undef, provider_opts => {}); + my $result = $p->check_prereqs(); + ok !$result, 'check_prereqs returns 0 when fly is not in PATH'; +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu diff --git a/t/ci-provider.t b/t/ci-provider.t index d9cb28e2..8c90d8de 100644 --- a/t/ci-provider.t +++ b/t/ci-provider.t @@ -287,4 +287,77 @@ subtest 'Manual->label' => sub { is $m->label, 'Manual', 'label is Manual'; }; +### ============================================================ ### +### check_prereqs +### ============================================================ ### + +subtest 'check_prereqs: base Provider always returns 1' => sub { + # Base class has no prereqs; all three concrete providers inherit this + # as a no-op default except where they override it. + my $m = Genesis::CI::Provider::Manual->new(type => 'manual'); + ok $m->check_prereqs(), 'Manual->check_prereqs returns 1'; + + my $g = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', repo => 'acme/x' + ); + ok $g->check_prereqs(), 'GithubActions->check_prereqs returns 1'; +}; + +subtest 'check_prereqs: Concourse returns 1 when fly is in PATH' => sub { + # Only run if fly is actually installed in this environment. + my $fly = `which fly 2>/dev/null`; + chomp $fly; + if ($fly) { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', target => 'test' + ); + ok $p->check_prereqs(), 'check_prereqs returns 1 when fly is present'; + } else { + pass 'skipped: fly not installed in this environment'; + } +}; + +subtest 'check_prereqs: Concourse returns 0 when fly is absent' => sub { + # Temporarily shadow PATH so fly cannot be found. + local $ENV{PATH} = '/nonexistent'; + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', target => 'test' + ); + my $result = $p->check_prereqs(); + ok !$result, 'check_prereqs returns 0 when fly is not in PATH'; +}; + +subtest 'check_prereqs: Concourse min_fly_version satisfied' => sub { + my $fly = `which fly 2>/dev/null`; + chomp $fly; + unless ($fly) { + pass 'skipped: fly not installed in this environment'; + return; + } + # Require a minimum of 0.0.1 — any real fly version will satisfy this. + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'test', + min_fly_version => '0.0.1', + ); + ok $p->check_prereqs(), 'check_prereqs passes with trivially low min version'; +}; + +subtest 'check_prereqs: Concourse min_fly_version not satisfied' => sub { + my $fly = `which fly 2>/dev/null`; + chomp $fly; + unless ($fly) { + pass 'skipped: fly not installed in this environment'; + return; + } + # Require an impossibly high minimum — should fail. + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'test', + min_fly_version => '9999.0.0', + ); + my $result = $p->check_prereqs(); + ok !$result, 'check_prereqs returns 0 when fly version too old'; +}; + done_testing; From 2c62d3a909c9efe3611d56501687d9835bc486c6 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:19:20 -0400 Subject: [PATCH 021/103] Exit with code 86 on CI prereq failure Replace bail("CI provider prerequisite check failed") unless check_prereqs() with check_prereqs() or exit 86 in Pipelines.pm and Repo.pm. This makes the process terminate with status 86 when a CI provider prerequisite check fails instead of calling bail(). --- lib/Genesis/Commands/Pipelines.pm | 3 +-- lib/Genesis/Commands/Repo.pm | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 544597bf..f02b3ad2 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -479,8 +479,7 @@ sub _repipe_compiled { } # Verify the provider's toolchain is available before attempting deploy. - bail("CI provider prerequisite check failed") - unless $provider->check_prereqs(); + $provider->check_prereqs() or exit 86; # Pass all relevant CLI flags to deploy() for three-tier resolution. # Provider reads: ci-target, ci-team, ci-pipeline-name, ci-pause, ci-expose, diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index af1eb3bf..1ab4cbe7 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -124,8 +124,7 @@ sub _repo_init_validate { } # Verify the provider's toolchain is available before we do any work - bail("CI provider prerequisite check failed") - unless $ci_provider_obj->check_prereqs(); + $ci_provider_obj->check_prereqs() or exit 86; } # --- 3. Gather data: validate local sources, detect git repo --- From 21f08dad50721c9261160d284c4552d45684ff78 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:04:57 -0400 Subject: [PATCH 022/103] Prefill CI provider interactive wizards Allow CI provider interactive_wizard methods to accept preset CLI flags so wizards can be pre-filled when invoked non-interactively. Concourse and GitHub Actions wizards now take %opts (supporting ci-target, ci-team, ci-insecure, ci-github-repo, ci-github-branch) and fall back to prompts when options are absent. Repo command now forwards %ci_provider_opts into the wizard when running interactively. Also: simplify the check_prereqs comment, switch Concourse's fly lookup to use run('type -p fly') (with safe chomp), remove the unused Manual::opts stub, and preserve boolean handling for insecure. These changes improve scripting and automation of CI setup. --- lib/Genesis/CI/Provider.pm | 12 +------- lib/Genesis/CI/Provider/Concourse.pm | 36 +++++++++++++++--------- lib/Genesis/CI/Provider/GithubActions.pm | 32 +++++++++++++-------- lib/Genesis/CI/Provider/Manual.pm | 6 ---- lib/Genesis/Commands/Repo.pm | 4 +-- 5 files changed, 46 insertions(+), 44 deletions(-) diff --git a/lib/Genesis/CI/Provider.pm b/lib/Genesis/CI/Provider.pm index 956af2a2..40c89793 100644 --- a/lib/Genesis/CI/Provider.pm +++ b/lib/Genesis/CI/Provider.pm @@ -147,17 +147,7 @@ sub config { } # }}} -# check_prereqs - verify all required external tools are available {{{ -# -# Called before any provider-dependent operation (repo-init, repipe, etc.) -# to confirm the provider's toolchain is present and meets any minimum -# version requirements. -# -# Returns 1 when all prereqs are satisfied; returns 0 and emits error() -# for each unmet prereq. Caller decides whether to bail. -# -# Subclasses override to add provider-specific checks. -# Base implementation has no prereqs and always returns 1. +# check_prereqs - returns 1 if toolchain is present, 0 + error() if not {{{ sub check_prereqs { return 1; } diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index 9b7178d5..6c6158cc 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -89,7 +89,8 @@ sub check_prereqs { my $ok = 1; # Require fly in PATH - chomp(my $fly_path = `which fly 2>/dev/null`); + my ($fly_path) = run({ stderr => 0 }, 'type -p fly'); + chomp($fly_path //= ''); unless ($fly_path) { error( "The Concourse CI provider requires the #C{fly} CLI but it was not ". @@ -135,25 +136,34 @@ sub config { # }}} # interactive_wizard - prompt user for Concourse configuration {{{ sub interactive_wizard { - my ($self, $top) = @_; - - my $target = prompt_for_line(undef, - "Concourse target name (from ~/.flyrc): ", ''); - bail("Concourse CI provider requires a target name") - unless $target && $target =~ /\S/; + my ($self, $top, %opts) = @_; + + my $target = $opts{'ci-target'}; + unless ($target && $target =~ /\S/) { + $target = prompt_for_line(undef, + "Concourse target name (from ~/.flyrc): ", ''); + bail("Concourse CI provider requires a target name") + unless $target && $target =~ /\S/; + } - my $team = prompt_for_line(undef, - sprintf("Concourse team [%s]: ", DEFAULT_TEAM), DEFAULT_TEAM); - $team = DEFAULT_TEAM unless $team && $team =~ /\S/; + my $team; + if (exists $opts{'ci-team'}) { + $team = $opts{'ci-team'} || DEFAULT_TEAM; + } else { + $team = prompt_for_line(undef, + sprintf("Concourse team [%s]: ", DEFAULT_TEAM), DEFAULT_TEAM); + $team = DEFAULT_TEAM unless $team && $team =~ /\S/; + } - my $insecure = prompt_for_boolean( - "Skip TLS certificate verification? [y|n] ", 0); + my $insecure = exists $opts{'ci-insecure'} + ? ($opts{'ci-insecure'} ? 1 : 0) + : prompt_for_boolean("Skip TLS certificate verification? [y|n] ", 0); return $self->new( type => 'concourse', target => $target, team => $team, - insecure => $insecure ? 1 : 0, + insecure => $insecure, ); } diff --git a/lib/Genesis/CI/Provider/GithubActions.pm b/lib/Genesis/CI/Provider/GithubActions.pm index 2ff44142..11f22f51 100644 --- a/lib/Genesis/CI/Provider/GithubActions.pm +++ b/lib/Genesis/CI/Provider/GithubActions.pm @@ -89,18 +89,26 @@ sub config { # }}} # interactive_wizard - prompt user for GitHub Actions configuration {{{ sub interactive_wizard { - my ($self, $top) = @_; - - my $repo = prompt_for_line(undef, - "GitHub repository (org/repo format): ", ''); - bail("GitHub Actions CI provider requires a repository") - unless $repo && $repo =~ /\S/; - bail("Repository must be in 'org/repo' format") - unless $repo =~ m{^[^/]+/[^/]+$}; - - my $branch = prompt_for_line(undef, - sprintf("Default branch [%s]: ", DEFAULT_BRANCH), DEFAULT_BRANCH); - $branch = DEFAULT_BRANCH unless $branch && $branch =~ /\S/; + my ($self, $top, %opts) = @_; + + my $repo = $opts{'ci-github-repo'}; + unless ($repo && $repo =~ m{^[^/]+/[^/]+$}) { + $repo = prompt_for_line(undef, + "GitHub repository (org/repo format): ", ''); + bail("GitHub Actions CI provider requires a repository") + unless $repo && $repo =~ /\S/; + bail("Repository must be in 'org/repo' format") + unless $repo =~ m{^[^/]+/[^/]+$}; + } + + my $branch; + if (exists $opts{'ci-github-branch'}) { + $branch = $opts{'ci-github-branch'} || DEFAULT_BRANCH; + } else { + $branch = prompt_for_line(undef, + sprintf("Default branch [%s]: ", DEFAULT_BRANCH), DEFAULT_BRANCH); + $branch = DEFAULT_BRANCH unless $branch && $branch =~ /\S/; + } return $self->new( type => 'github-actions', diff --git a/lib/Genesis/CI/Provider/Manual.pm b/lib/Genesis/CI/Provider/Manual.pm index 3c0f9f9a..a829e2b9 100644 --- a/lib/Genesis/CI/Provider/Manual.pm +++ b/lib/Genesis/CI/Provider/Manual.pm @@ -20,12 +20,6 @@ sub new { bless({ label => 'Manual' }, $class); } -# }}} -# opts - Manual provider takes no CLI flags {{{ -sub opts { - qw//; -} - # }}} # opts_help - usage documentation for Manual provider {{{ sub opts_help { diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 1ab4cbe7..58c85db3 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -112,10 +112,10 @@ sub _repo_init_validate { $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; if ($@) { if (in_controlling_terminal) { - # Required flags omitted — run interactive wizard to collect them + # Required flags omitted — run interactive wizard, pre-filling any flags supplied $ci_provider_obj = eval { Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) - ->interactive_wizard(undef); + ->interactive_wizard(undef, %ci_provider_opts); }; bail("CI provider wizard failed: %s", $@) if $@; } else { From 2897d7496975db43078eb5239f1b7f2d9dc42265 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:10:22 -0400 Subject: [PATCH 023/103] Use 'type -p' to locate fly; tidy comments Replace shell backtick `which fly` with run({ stderr => 0 }, 'type -p fly') in Concourse provider to more reliably detect the fly CLI and avoid relying on external which; ensure $fly_path is defined and remove an unused $ok variable. Also simplify and reword prerequisite comment in PipelineProvider, and remove several redundant/comment-only lines in Env command (no functional behavior changes). These edits improve portability and clean up documentation in the codebase. --- lib/Genesis/CI/Compiler/PipelineProvider.pm | 9 +-------- lib/Genesis/CI/Compiler/Providers/Concourse.pm | 8 ++++---- lib/Genesis/Commands/Env.pm | 9 +-------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 134bc483..535e7155 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -82,14 +82,7 @@ sub output_files { # }}} ### Prerequisite Checking {{{ -# check_prereqs - verify all required external tools are available {{{ -# -# Called before deploy() to confirm the provider's toolchain is present. -# Returns 1 when all prereqs are satisfied; returns 0 and emits error() -# messages for each unmet prereq (caller decides whether to bail). -# -# Subclasses override to check their specific requirements. -# Base implementation has no prereqs and always returns 1. +# check_prereqs - returns 1 if toolchain is present, 0 + error() if not {{{ sub check_prereqs { return 1; } diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index bb935473..2d72e75d 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -69,12 +69,12 @@ sub init { sub provider_type { 'concourse' } # }}} -# check_prereqs - verify fly CLI is installed before attempting deploy {{{ +# check_prereqs - returns 1 if fly is in PATH, 0 + error() if not {{{ sub check_prereqs { my ($self) = @_; - my $ok = 1; - chomp(my $fly_path = `which fly 2>/dev/null`); + my ($fly_path) = run({ stderr => 0 }, 'type -p fly'); + chomp($fly_path //= ''); unless ($fly_path) { error( "Cannot deploy Concourse pipeline: the #C{fly} CLI was not found in ". @@ -86,7 +86,7 @@ sub check_prereqs { return 0; } - return $ok; + return 1; } # }}} diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 73c34b60..c4ece0ce 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -89,10 +89,8 @@ sub create { my %cli_opts = %{get_options()}; my $interactive = in_controlling_terminal; - # --- prior_env (Issue 1: use choice menu, not freeform) --- my $prior_env; if (exists $cli_opts{'prior-env'}) { - # Non-interactive path: flag supplied; validate it refers to a real env. $prior_env = $cli_opts{'prior-env'} // ''; if (length($prior_env)) { my %known = map { $_->name => 1 } $top->envs(); @@ -102,7 +100,6 @@ sub create { ) unless $known{$prior_env}; } } elsif ($interactive) { - # Interactive path: present numbered menu of existing envs. my @existing = grep { $_->name ne $name } $top->envs(); if (@existing) { my @env_names = map { $_->name } @existing; @@ -117,13 +114,12 @@ sub create { "environment", ); } else { - # No other envs yet — must be the entrypoint. $prior_env = ''; info("No other environments found — #C{%s} will be the pipeline entrypoint.", $name); } } - # --- require_pr --- + # --- require_pr / manual --- my $require_pr; if (exists $cli_opts{'require-pr'}) { $require_pr = $cli_opts{'require-pr'} ? 1 : 0; @@ -134,7 +130,6 @@ sub create { ); } - # --- manual --- my $manual; if (exists $cli_opts{manual}) { $manual = $cli_opts{manual} ? 1 : 0; @@ -157,13 +152,11 @@ sub create { my $contents = slurp($file); if ($contents =~ /^\s+pipeline:/m) { - # pipeline: section already present (kit wrote one) — don't overwrite. info( "#Y{Note}: pipeline section already present in #C{%s}, skipping injection.", $env->file ); } else { - # Inject after the env: line; flexible indentation (Issue 4). my $injected = ($contents =~ s/^((\s+)env:\s+\S[^\n]*\n)/$1$pipeline_yaml/m); if ($injected) { mkfile_or_fail($file, $contents); From 91763f269b26b391528ade72423217e25d34db1b Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 09:14:03 -0700 Subject: [PATCH 024/103] Add extended_handlers framework to Commands Replace the ad-hoc extended_usage closure pattern with a structured extended_handlers registration in define_command. Each handler class implements parse_opts, opts_help, and opts_slot; the framework validates and loads handlers in parse_options, calls each in declared order to consume passthrough flags, merges results into COMMAND_OPTIONS under the handler's declared slot name, and rejects any unclaimed option-like tokens left in COMMAND_ARGS. extended_handlers implies option_passthrough so commands no longer need to declare both. Legacy extended_usage is preserved as a fallback for unmigrated commands. Migrate repo-init and kit-provider to extended_handlers. Simplify _repo_init_validate to read kit provider opts from get_options() instead of calling parse_opts directly. --- bin/genesis | 18 +-------- lib/Genesis/CI/Provider.pm | 4 ++ lib/Genesis/Commands.pm | 78 +++++++++++++++++++++++++++++++++++- lib/Genesis/Commands/Repo.pm | 48 ++++++---------------- lib/Genesis/Kit/Provider.pm | 4 ++ 5 files changed, 99 insertions(+), 53 deletions(-) diff --git a/bin/genesis b/bin/genesis index 054af923..524ffcf3 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1271,24 +1271,13 @@ define_command("repo-init", { "Defer vault configuration. The repo will be created without a ". "secrets provider. You must configure one via #C{genesis secrets-provider} ". "before creating environments.", - - 'ci-provider=s' => - "CI provider for pipeline automation: 'concourse', 'github-actions', ". - "or 'manual'. When specified, writes the ci: section to .genesis/config ". - "and generates the .genesis/ci/ scaffold.", ], - option_passthrough => 1, + extended_handlers => ['Genesis::Kit::Provider', 'Genesis::CI::Provider'], arguments => [ "name?" => "If the name argument is not specified, it will default to the same ". "name as the kit. You must specify either name or kit." ], - extended_usage => sub { - require Genesis::Kit::Provider; - require Genesis::CI::Provider; - Genesis::Kit::Provider->opts_help() . - Genesis::CI::Provider->opts_help(); - } }); # }}} @@ -1346,10 +1335,7 @@ define_command("kit-provider", { "export-config" => "Export the current kit provider information." ], - extended_usage => sub { - require Genesis::Kit::Provider; - Genesis::Kit::Provider->opts_help(); - } + extended_handlers => ['Genesis::Kit::Provider'], }); # }}} diff --git a/lib/Genesis/CI/Provider.pm b/lib/Genesis/CI/Provider.pm index 40c89793..d341d88b 100644 --- a/lib/Genesis/CI/Provider.pm +++ b/lib/Genesis/CI/Provider.pm @@ -91,6 +91,10 @@ sub parse_opts { return 1; } +# }}} +# opts_slot - key name for this handler's parsed options in $COMMAND_OPTIONS {{{ +sub opts_slot { 'ci_provider' } + # }}} # opts - base class has no options of its own {{{ sub opts { diff --git a/lib/Genesis/Commands.pm b/lib/Genesis/Commands.pm index 4b245751..c38807af 100644 --- a/lib/Genesis/Commands.pm +++ b/lib/Genesis/Commands.pm @@ -177,6 +177,12 @@ sub define_command { # {{{ }; $PROPS{$name} = {%$default_props, %$props}; + + # extended_handlers implies option_passthrough: the main parser + # must leave unrecognised flags in @args for the handlers to claim. + if ($PROPS{$name}{extended_handlers}) { + $PROPS{$name}{option_passthrough} = 1; + } my $fn_require = ''; if (ref($fn) ne "CODE") { if (defined($fn)) { @@ -282,6 +288,31 @@ sub parse_options { # {{{ my $args = shift; my $args_copy = [@$args]; + + # Validate extended handlers once, up front, before any option + # parsing. This is the single gateway for both command execution + # and help rendering (prepare_command always calls parse_options + # first), so we validate here rather than duplicating in + # command_help. Each handler class must be loadable and must + # implement the required contract methods. + if (my $handlers = $PROPS{$COMMAND}{extended_handlers}) { + for my $class (@$handlers) { + (my $file = $class) =~ s|::|/|g; + eval { require "$file.pm" }; + bail( + "Extended handler #C{%s} for command #C{%s} could not be loaded:\n%s", + $class, $COMMAND, $@ + ) if $@; + for my $method (qw(parse_opts opts_help opts_slot)) { + bail( + "Extended handler #C{%s} for command #C{%s} does not implement ". + "the required #C{%s} method.", + $class, $COMMAND, $method + ) unless $class->can($method); + } + } + } + my @base_spec = keys %{({map {@$_} @global_options[0..$PROPS{$COMMAND}{option_group}]})}; my @opts_spec = ( @@ -328,6 +359,31 @@ sub parse_options { # {{{ shift @$args if ($args->[0]||'') eq '--'; @COMMAND_ARGS = (@$args); + # Extended handlers: each registered handler class gets a chance to + # consume its flags from @COMMAND_ARGS and populate a slot in + # $COMMAND_OPTIONS. Handlers run in declared order so each sees + # only what prior handlers left behind. The classes were already + # required and validated at the top of this sub. + if (my $handlers = $PROPS{$COMMAND}{extended_handlers}) { + for my $class (@$handlers) { + my %slot; + $class->parse_opts(\@COMMAND_ARGS, \%slot); + $COMMAND_OPTIONS->{$class->opts_slot()} = \%slot; + } + + # After all handlers have run, anything still looking like an + # option is unclaimed -- reject it the same way the main parser + # would without option_passthrough. + my @unknown = grep { /^-/ } @COMMAND_ARGS; + if (@unknown) { + command_usage(1, sprintf( + "Unknown option%s: %s", + @unknown > 1 ? 's' : '', + join(', ', @unknown) + )); + } + } + # Extract Core options $ENV{NOCOLOR} = 'y' if defined($COMMAND_OPTIONS->{color}) && !delete($COMMAND_OPTIONS->{color}); $ENV{QUIET} = 'y' if delete($COMMAND_OPTIONS->{quiet}); @@ -650,8 +706,26 @@ sub command_usage { # {{{ $out .= "#i{To see only global options, use }#g{${\(humanize_bin)}} #y{--globals}\n"; } - # TODO: Integrate extended usage better than just dumping it at the end - if (ref($PROPS{$command}{extended_usage}) eq "CODE") { + # Extended usage: render help from registered handlers (preferred), + # or fall back to legacy extended_usage closure if no handlers are + # registered. + # Extended handlers were already required and validated in + # parse_options (which always runs before command_help via + # prepare_command). + if (my $handlers = $PROPS{$command}{extended_handlers}) { + my $extended_usage = ''; + for my $class (@$handlers) { + my $help = $class->opts_help(); + if ($help) { + $help =~ s/\s*$//s; + $extended_usage .= $help . "\n"; + } + } + if ($extended_usage =~ /\S/) { + $out .= "\n#Wku{Extended Usage Information}\n"; + $out .= "\n$extended_usage\n"; + } + } elsif (ref($PROPS{$command}{extended_usage}) eq "CODE") { my $extended_usage = $PROPS{$command}{extended_usage}->(); if ($extended_usage) { $extended_usage =~ s/\s*$//s; diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 58c85db3..7d39efd4 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -7,8 +7,6 @@ use Genesis; use Genesis::Commands; use Genesis::Term qw/in_controlling_terminal/; use Genesis::Top; -use Genesis::Kit::Provider; -use Genesis::CI::Provider; use Genesis::UI; use Cwd qw/getcwd abs_path/; @@ -38,30 +36,15 @@ sub repo_init { # 7. Store derived values and summarize intent # sub _repo_init_validate { + # Passthrough options (e.g. --kit-provider-*) have already been + # parsed by the framework's extended_handlers and are available in + # get_options()->{kit_provider}. @args from get_args() + # contains only true positional arguments. my %opts = %{get_options()}; my @args = get_args(); # --- 1. Parse and derive --- - # The #C{repo-init} command uses #C{option_passthrough => 1}, so - # any option that isn't declared in bin/genesis (e.g. the - # #C{--kit-provider-*} flags provided by #C{Genesis::Kit::Provider} - # and, in the future, the CI-provider-specific flags) is left - # untouched in @args alongside true positional arguments. We must - # consume those passthrough options here, BEFORE reading - # #C{$args[0]} as the deployment name -- otherwise a flag value - # (or the flag itself) could be mistaken for the name. - # - # parse_opts mutates @args in place: it strips recognised flags - # into %provider_opts and leaves the remaining positional args - # behind. The provider object itself is built later, only when - # $opts{kit} is actually set. - # - # When the CI-provider parse_opts analogue lands, call it from - # here as well. - my %provider_opts; - Genesis::Kit::Provider->parse_opts(\@args, \%provider_opts); - my $name = $args[0]; my $kit_file; @@ -98,15 +81,11 @@ sub _repo_init_validate { "Cannot specify both --vault and --skip-vault." ) if $opts{vault} && $opts{'skip-vault'}; - # --ci-provider is a declared option and is pre-parsed by get_options() into - # %opts; provider-specific flags (--ci-target, --ci-team, etc.) are NOT - # declared, so they stay in @args via option_passthrough. Seed ci-provider - # from %opts first so parse_opts can do the second pass for provider extras. - my %ci_provider_opts; - $ci_provider_opts{'ci-provider'} = delete $opts{'ci-provider'} - if $opts{'ci-provider'}; - Genesis::CI::Provider->parse_opts(\@args, \%ci_provider_opts); - + # CI provider options (--ci-provider, --ci-target, etc.) have been + # parsed by the framework's extended_handlers into the ci_provider + # slot of get_options(). Build the provider object if one was + # requested. + my %ci_provider_opts = %{$opts{ci_provider} // {}}; my $ci_provider_obj; if ($ci_provider_opts{'ci-provider'}) { $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; @@ -205,7 +184,7 @@ sub _repo_init_validate { # Without #C{--ci-provider}, no pipeline topology is being # established, so the branch name is irrelevant at this point. my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - if ($use_subdir && $opts{'ci-provider'}) { + if ($use_subdir && $ci_provider_obj) { my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); chomp $branch if defined $branch; if (!defined($branch) || $branch ne $control_branch) { @@ -248,10 +227,9 @@ sub _repo_init_validate { if ($opts{kit} && !$kit_file) { ($resolved_kit_name, $resolved_kit_version) = split('/', $opts{kit}, 2); - # %provider_opts was populated at the top of this sub so - # that positional-arg parsing wasn't fooled by passthrough - # flags. Build the provider object now that we know we - # need it. + # Kit provider options were parsed by the framework's + # extended_handlers into the kit_provider slot. + my %provider_opts = %{$opts{kit_provider} // {}}; $kit_provider = eval { Genesis::Kit::Provider->init(%provider_opts) }; bail("Could not initialize kit provider: %s", $@) if $@; diff --git a/lib/Genesis/Kit/Provider.pm b/lib/Genesis/Kit/Provider.pm index 2ba61ee3..b70f7a7f 100644 --- a/lib/Genesis/Kit/Provider.pm +++ b/lib/Genesis/Kit/Provider.pm @@ -65,6 +65,10 @@ sub default_provider { return Genesis::Kit::Provider::GenesisCommunity->new(); } +# }}} +# opts_slot - key name for this handler's parsed options in $COMMAND_OPTIONS {{{ +sub opts_slot { 'kit_provider' } + # }}} # opts - list of options supported by init method {{{ sub opts { From 548691c3b9c9620bf3769cbd7860b9c9c8e2b029 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 11:13:46 -0700 Subject: [PATCH 025/103] Improve Concourse wizard and fix UI bugs Concourse interactive_wizard reads fly targets output via parse_fixed_width_table for name/url/team/expiry, merges with flyrc for the insecure flag. Columnar display with aligned secure/insecure and EXPIRED! flags. Adds "Create a new target" option with separator. Runs fly login inline for new targets. Fix bless-into-reference error (use ref($self) in wizard). Reorder _repo_init_validate so the CI provider wizard runs after directory and kit validation, not before. Fix new_prompt_for_choice selecting first item when no default specified (undef == 0 in numeric context). --- lib/Genesis/CI/Provider/Concourse.pm | 144 ++++++++++++++++++++++++--- lib/Genesis/Commands/Repo.pm | 62 +++++++----- lib/Genesis/UI.pm | 2 +- 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index 6c6158cc..c0bca836 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -6,6 +6,8 @@ use base 'Genesis::CI::Provider'; use Genesis; use Genesis::UI; +use POSIX qw(mktime); + use constant { DEFAULT_TEAM => 'main', }; @@ -138,28 +140,113 @@ sub config { sub interactive_wizard { my ($self, $top, %opts) = @_; - my $target = $opts{'ci-target'}; - unless ($target && $target =~ /\S/) { - $target = prompt_for_line(undef, - "Concourse target name (from ~/.flyrc): ", ''); - bail("Concourse CI provider requires a target name") - unless $target && $target =~ /\S/; + my ($target, $team, $insecure); + + my @fly_targets; + my $flyrc = "$ENV{HOME}/.flyrc"; + my %flyrc_data; + if (-f $flyrc) { + my $data = load_yaml_file($flyrc); + if (ref($data) eq 'HASH' && ref($data->{targets}) eq 'HASH') { + %flyrc_data = %{$data->{targets}}; + } + } + + my ($fly_out) = run({ stderr => '/dev/null' }, + 'fly targets --print-table-headers'); + if ($fly_out && $fly_out =~ /\S/) { + my @lines = split /\n/, $fly_out; + my @rows = parse_fixed_width_table(@lines); + for my $row (@rows) { + my $name = $row->{name} // next; + my $flyrc_entry = $flyrc_data{$name} // {}; + push @fly_targets, { + name => $name, + api => $row->{url} // '(unknown)', + team => $row->{team} // DEFAULT_TEAM, + insecure => $flyrc_entry->{insecure} ? 1 : 0, + expired => _token_expired($row->{expiry}), + }; + } + } + + if (@fly_targets) { + # Compute column widths for aligned display + my $w_name = 0; + my $w_api = 0; + my $w_team = 0; + for (@fly_targets) { + $w_name = length($_->{name}) if length($_->{name}) > $w_name; + $w_api = length($_->{api}) if length($_->{api}) > $w_api; + $w_team = length($_->{team}) if length($_->{team}) > $w_team; + } + + my @choices = map {{ + value => $_->{name}, + label => sprintf("#C{%-${w_name}s} %-${w_api}s %-${w_team}s %s%s", + $_->{name}, $_->{api}, $_->{team}, + $_->{insecure} ? "#Yi{insecure}" : "#G{secure} ", + $_->{expired} ? " #R{EXPIRED!}" : ""), + summary => $_->{name}, + }} @fly_targets; + + # Separator + "create new" option at the bottom + push @choices, { separator => 1 }; + push @choices, { + value => '__new__', + label => '#Yi{Create a new Concourse target}', + summary => '(new target)', + }; + + $target = new_prompt_for_choice( + header => "Select a Concourse target:", + choices => \@choices, + default => $self->{target}, + description => "target", + ); + + if ($target ne '__new__') { + # Pre-fill team and insecure from the selected flyrc entry + my ($selected) = grep { $_->{name} eq $target } @fly_targets; + if ($selected) { + $team = $selected->{team}; + $insecure = $selected->{insecure}; + } + } } - my $team; - if (exists $opts{'ci-team'}) { - $team = $opts{'ci-team'} || DEFAULT_TEAM; - } else { + # Create a new target: prompt for details and run fly login + if (!$target || $target eq '__new__') { + my $name = prompt_for_line(undef, + "Target name (short alias for this Concourse): ", ''); + bail("A target name is required.") unless $name && $name =~ /\S/; + + my $url = prompt_for_line(undef, + "Concourse URL (e.g. https://ci.example.com): ", ''); + bail("A Concourse URL is required.") unless $url && $url =~ m{^https?://}; + $team = prompt_for_line(undef, - sprintf("Concourse team [%s]: ", DEFAULT_TEAM), DEFAULT_TEAM); + sprintf("Team name [%s]: ", DEFAULT_TEAM), DEFAULT_TEAM); $team = DEFAULT_TEAM unless $team && $team =~ /\S/; - } - my $insecure = exists $opts{'ci-insecure'} - ? ($opts{'ci-insecure'} ? 1 : 0) - : prompt_for_boolean("Skip TLS certificate verification? [y|n] ", 0); + $insecure = prompt_for_boolean( + "Skip TLS certificate verification? [y|n] ", 0); + + # Run fly login to create the target in ~/.flyrc and + # authenticate the user. This is interactive — fly will + # prompt for credentials or open a browser. + info "\nLogging in to Concourse as #C{%s} on #C{%s}...\n", $team, $url; + my @fly_cmd = ('fly', '-t', $name, 'login', + '-c', $url, '-n', $team); + push @fly_cmd, '-k' if $insecure; + run({ interactive => 1, + onfailure => "fly login failed for target '$name'" }, + join(' ', map { /\s/ ? "\"$_\"" : $_ } @fly_cmd)); + + $target = $name; + } - return $self->new( + return (ref($self) || $self)->new( type => 'concourse', target => $target, team => $team, @@ -167,6 +254,31 @@ sub interactive_wizard { ); } +# }}} + +# _token_expired - check if a fly targets expiry string is in the past {{{ +my %_months = ( + Jan => 0, Feb => 1, Mar => 2, Apr => 3, May => 4, Jun => 5, + Jul => 6, Aug => 7, Sep => 8, Oct => 9, Nov => 10, Dec => 11, +); +sub _token_expired { + my ($expiry_str) = @_; + return 0 unless $expiry_str && $expiry_str =~ /\S/; + + # fly targets format: "Sat, 08 Feb 2025 18:53:16 UTC" + if ($expiry_str =~ /(\d{2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+UTC/) { + my ($day, $mon, $year, $hour, $min, $sec) = ($1, $2, $3, $4, $5, $6); + return 0 unless defined $_months{$mon}; + my $exp = eval { + local $ENV{TZ} = 'UTC'; + POSIX::mktime($sec, $min, $hour, $day, $_months{$mon}, $year - 1900); + }; + return 0 unless $exp; + return $exp < time() ? 1 : 0; + } + return 0; +} + # }}} # }}} diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 7d39efd4..e5685be5 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -30,10 +30,11 @@ sub repo_init { # 1. Parse options and derive values (fast, no side effects) # 2. Check invalid option combinations (fast bail) # 3. Gather data: validate local sources, detect git repo (no network) -# 4. Check destructive prerequisites (existing directory, before network) -# 5. Validate remote kit availability (network calls) -# 6. Prompt for missing info (vault selection) -# 7. Store derived values and summarize intent +# 4. Subdir preflight: dirty check, control-branch check +# 5. Check destructive prerequisites (existing directory, before network) +# 6. Validate remote kit availability (network calls) +# 7. Prompt for missing info (vault, CI provider wizard) +# 8. Store derived values and summarize intent # sub _repo_init_validate { # Passthrough options (e.g. --kit-provider-*) have already been @@ -83,28 +84,11 @@ sub _repo_init_validate { # CI provider options (--ci-provider, --ci-target, etc.) have been # parsed by the framework's extended_handlers into the ci_provider - # slot of get_options(). Build the provider object if one was - # requested. + # slot. We extract the opts hash here for quick flag checks (e.g. + # branch validation), but defer building the actual provider object + # until step 6 (after directory/kit validation passes) so the + # interactive wizard doesn't run if we're going to bail anyway. my %ci_provider_opts = %{$opts{ci_provider} // {}}; - my $ci_provider_obj; - if ($ci_provider_opts{'ci-provider'}) { - $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; - if ($@) { - if (in_controlling_terminal) { - # Required flags omitted — run interactive wizard, pre-filling any flags supplied - $ci_provider_obj = eval { - Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) - ->interactive_wizard(undef, %ci_provider_opts); - }; - bail("CI provider wizard failed: %s", $@) if $@; - } else { - bail("Could not initialize CI provider: %s", $@); - } - } - - # Verify the provider's toolchain is available before we do any work - $ci_provider_obj->check_prereqs() or exit 86; - } # --- 3. Gather data: validate local sources, detect git repo --- @@ -184,7 +168,7 @@ sub _repo_init_validate { # Without #C{--ci-provider}, no pipeline topology is being # established, so the branch name is irrelevant at this point. my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - if ($use_subdir && $ci_provider_obj) { + if ($use_subdir && $ci_provider_opts{'ci-provider'}) { my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); chomp $branch if defined $branch; if (!defined($branch) || $branch ne $control_branch) { @@ -246,6 +230,10 @@ sub _repo_init_validate { } # --- 6. Prompt for missing info --- + # + # Interactive steps are deferred until here so we don't waste the + # user's time if validation is going to bail (directory exists, + # kit not found, wrong branch, etc.). my $vault_target; if ($opts{'skip-vault'}) { @@ -257,7 +245,27 @@ sub _repo_init_validate { $vault_target = $vault->{name} if $vault; } - # --- 8. Store derived values and summarize intent --- + # Build the CI provider object now that all preflight checks have + # passed. If required flags (--ci-target, etc.) were omitted and + # we have a controlling terminal, fall back to the interactive + # wizard to collect them. + my $ci_provider_obj; + if ($ci_provider_opts{'ci-provider'}) { + $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; + if ($@) { + if (in_controlling_terminal) { + $ci_provider_obj = eval { + Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) + ->interactive_wizard(undef); + }; + bail("CI provider wizard failed: %s", $@) if $@; + } else { + bail("Could not initialize CI provider: %s", $@); + } + } + } + + # --- 7. Store derived values and summarize intent --- option_defaults( _name => $name, diff --git a/lib/Genesis/UI.pm b/lib/Genesis/UI.pm index e99024cf..9434752c 100644 --- a/lib/Genesis/UI.pm +++ b/lib/Genesis/UI.pm @@ -556,7 +556,7 @@ sub new_prompt_for_choice { my $choice = $i+1-$section_offset; $selection_map{$choice} = $choices->[$i]; $form .= csprintf("\n %*s) %s", $iw, $choice, $choices->[$i]{label}); - $default_choice = $choice if $i == $default_idx; + $default_choice = $choice if defined($default_idx) && $i == $default_idx; } } From ddcae861ad91fe220d4eaceb6090e8ae5eb27377 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 12:19:45 -0700 Subject: [PATCH 026/103] Fix new_prompt_for_choice and add strict to UI Add use v5.20, use warnings, use strict to Genesis::UI. Fix all violations: undeclared $section_offset (was a leaking package global that corrupted numbering across calls), $in predeclaration, $sections hash vs hashref, $terminal_width missing parens, $default_choice/$selection_map scoping. Fix undef == 0 bug where no default silently selected the first item (add defined() guard on $default_idx comparison). Add three regression tests for new_prompt_for_choice: no-default re-prompt, separator numbering, and sequential call independence. --- lib/Genesis/UI.pm | 17 ++++-- t/unit-tests/genesis_ui-choice.t | 91 ++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/lib/Genesis/UI.pm b/lib/Genesis/UI.pm index 9434752c..7489a046 100644 --- a/lib/Genesis/UI.pm +++ b/lib/Genesis/UI.pm @@ -1,4 +1,7 @@ package Genesis::UI; +use v5.20; +use warnings; +use strict; use base 'Exporter'; our @EXPORT = qw/ @@ -130,7 +133,8 @@ sub __prompt_for_block { $prompt = "$prompt (Enter to end)"; (my $line = $prompt) =~ s/./-/g; print csprintf("%s","\n$prompt\n$line\n"); - open(my $in, '<&', fileno(STDIN)) or $in = \*STDIN; + my $in; + open($in, '<&', fileno(STDIN)) or $in = \*STDIN; my @data = <$in>; return join("", @data); } @@ -427,7 +431,7 @@ sub new_prompt_for_choice { } my $label = $choice->{label} // $choice->{value}; $max_label_len = max($max_label_len, length($label)+ $max_number_width + 2); # 2 for column separator - push @{$sections{$section_headers[-1] //= []}}, $choice; + push @{$sections->{$section_headers[-1] //= []}}, $choice; } $columns = int($max_width / $max_label_len); $col_width = int($max_width / $columns); @@ -454,8 +458,13 @@ sub new_prompt_for_choice { # Display header my $form = $options{header}."\n"; + my %selection_map; + my $default_choice; # Handle user input my $display_choices = sub { + my $section_offset = 0; + %selection_map = (); + $default_choice = undef; for my $section_header (@section_headers) { if ($section_header) { @@ -467,7 +476,7 @@ sub new_prompt_for_choice { $form .= csprintf("\n #Wku{%s}\n", $section_header); } } - my $section_choices = $sections{$section_header}; + my $section_choices = $sections->{$section_header}; }; # Calculate item ranges for pagination @@ -487,7 +496,7 @@ sub new_prompt_for_choice { } # Calculate number of columns that fit - my $cols = max(1, int($terminal_width / ($max_label_len + 2))); + my $cols = max(1, int(terminal_width() / ($max_label_len + 2))); # Display items in columns my $col = 0; diff --git a/t/unit-tests/genesis_ui-choice.t b/t/unit-tests/genesis_ui-choice.t index a276a621..19d7f440 100644 --- a/t/unit-tests/genesis_ui-choice.t +++ b/t/unit-tests/genesis_ui-choice.t @@ -525,4 +525,95 @@ subtest 'new_prompt_for_choice header auto-generated when omitted' => sub { like($out, qr/Select one of the following/, 'auto-generated header shown'); }; +# ── Regression tests for bugs found during FWT-915/919 ────────────────────── + +subtest 'new_prompt_for_choice no default does not auto-select first item' => sub { + plan tests => 3; + + # Without a default, pressing Enter should re-prompt (not select + # item 1). Feed blank + valid selection to verify re-prompting + # occurred. Bug: undef == 0 in numeric context made the first + # item the implicit default. + set_stdin("\n2\n"); + my $result; + my $out = combined_from { + $result = new_prompt_for_choice( + header => "Pick:", + choices => [qw(alpha beta gamma)], + # no default + ); + }; + reset_stdin(); + + is($result, 'beta', 'blank enter without default re-prompts; second input accepted'); + my @prompts = ($out =~ /Select choice >/g); + cmp_ok(scalar @prompts, '>=', 2, 'prompt appeared at least twice (re-prompted after blank enter)'); + like($out, qr/No default/, 'error message shown on blank enter without default'); +}; + +subtest 'new_prompt_for_choice separator does not corrupt numbering' => sub { + plan tests => 2; + + # Choices with a separator: the separator should not get a number, + # and items after it should be numbered contiguously. + # Bug: $section_offset was a package global that leaked between + # calls, causing items to start at 0 instead of 1. + set_stdin("3\n"); + my $result; + my $out = combined_from { + $result = new_prompt_for_choice( + header => "Pick:", + choices => [ + {value => 'a', label => 'Alpha'}, + {value => 'b', label => 'Beta'}, + {separator => 1}, + {value => 'c', label => 'Gamma'}, + ], + ); + }; + reset_stdin(); + + is($result, 'c', 'item after separator selected by correct number'); + like($out, qr/1\) Alpha.*2\) Beta.*3\) Gamma/s, + 'items numbered 1-3 with separator consuming no number'); +}; + +subtest 'new_prompt_for_choice numbering correct across sequential calls' => sub { + plan tests => 2; + + # Two calls in a row: the second call should start numbering at 1, + # not carry over $section_offset from the first call. + # Bug: $section_offset was a package global. + + # First call — has a separator + set_stdin("1\n"); + my $r1; + combined_from { + $r1 = new_prompt_for_choice( + header => "First:", + choices => [ + {value => 'x'}, + {separator => 1}, + {value => 'y'}, + ], + ); + }; + reset_stdin(); + + is($r1, 'x', 'first call: item 1 selected correctly'); + + # Second call — should not be affected by the first + set_stdin("1\n"); + my $r2; + my $out = combined_from { + $r2 = new_prompt_for_choice( + header => "Second:", + choices => [qw(a b c)], + ); + }; + reset_stdin(); + + is($r2, 'a', 'second call: numbering starts fresh at 1'); +}; + done_testing; From b70c6b13db8e7ba8b4397d314ef1c52130e52130 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 12:20:22 -0700 Subject: [PATCH 027/103] Implement Concourse CI provider wizard and opts Wizard reads fly targets + flyrc for columnar display with name/url/team/secure/expired flags. User picks an existing target (re-authenticates if expired) or creates a new one by entering url/team/insecure and running fly login inline. Update init() to support two modes: existing target by name (resolved from flyrc, rejects conflicting flags) or new target from --ci-url + --ci-team + optional --ci-target override. Add --ci-url to opts(). Store url in new() and config(). Derive target names from url subdomain + team. --- lib/Genesis/CI/Provider/Concourse.pm | 269 ++++++++++++++++----------- 1 file changed, 157 insertions(+), 112 deletions(-) diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index c0bca836..d94d1fdf 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -15,16 +15,64 @@ use constant { ### Class Methods {{{ # init - create a new Concourse provider from CLI options {{{ +# +# Two modes: +# 1. Existing target: --ci-target (looks up url/team/insecure +# from ~/.flyrc) +# 2. New target: --ci-url --ci-team [--ci-insecure] +# [--ci-target ] (derives target name if not specified) +# sub init { my ($class, %opts) = @_; - bail("Concourse CI provider requires --ci-target") - unless $opts{'ci-target'}; + my $has_target = $opts{'ci-target'}; + my $has_new = $opts{'ci-url'} || $opts{'ci-team'} || $opts{'ci-insecure'}; + + if ($has_target) { + # Check if the target already exists in flyrc + my $flyrc = "$ENV{HOME}/.flyrc"; + my $entry; + if (-f $flyrc) { + my $data = load_yaml_file($flyrc); + $entry = $data->{targets}{$opts{'ci-target'}} + if ref($data) eq 'HASH' && ref($data->{targets}) eq 'HASH'; + } + + if ($entry) { + # Existing target — reject conflicting flags + bail( + "Target '%s' already exists in ~/.flyrc.\n". + " Cannot combine --ci-target with --ci-url, --ci-team, ". + "or --ci-insecure when the target already exists.", + $opts{'ci-target'} + ) if $has_new; + + return $class->new( + type => 'concourse', + target => $opts{'ci-target'}, + url => $entry->{api}, + team => $entry->{team} || DEFAULT_TEAM, + insecure => $entry->{insecure} ? 1 : 0, + ); + } + + # Target name provided but doesn't exist yet — fall through + # to new-target creation below (--ci-url and --ci-team required) + } - $class->new( + # New target: --ci-url and --ci-team are required; --ci-target is + # an optional name override (derived from url/team if omitted). + bail("Concourse CI provider requires --ci-url and --ci-team for a new target") + unless $opts{'ci-url'} && $opts{'ci-team'}; + + my $target = $opts{'ci-target'} + // _derive_target_name($opts{'ci-url'}, $opts{'ci-team'}); + + return $class->new( type => 'concourse', - target => $opts{'ci-target'}, - team => $opts{'ci-team'} || DEFAULT_TEAM, + target => $target, + url => $opts{'ci-url'}, + team => $opts{'ci-team'}, insecure => $opts{'ci-insecure'} ? 1 : 0, ); } @@ -33,9 +81,11 @@ sub init { # new - create a Concourse provider from stored config {{{ sub new { my ($class, %config) = @_; + $class = ref($class) || $class; bless({ label => 'Concourse', target => $config{target}, + url => $config{url}, team => $config{team} || DEFAULT_TEAM, insecure => $config{insecure} ? 1 : 0, }, $class); @@ -46,6 +96,7 @@ sub new { sub opts { qw/ ci-target=s + ci-url=s ci-team=s ci-insecure /; @@ -60,19 +111,31 @@ sub opts_help { <<'EOF'; CI Provider `concourse`: - --ci-target (required) - The Concourse target name (as configured in ~/.flyrc) to use when - setting the pipeline. This is the same target you would pass to - `fly -t `. + Use an existing fly target: + + --ci-target + An existing Concourse target name from ~/.flyrc. URL, team, and + TLS settings are read from the target. Cannot be combined with + --ci-url, --ci-team, or --ci-insecure when the target exists. + + Or specify a new target: - --ci-team (optional, defaults to "main") - The Concourse team name to set the pipeline on. Defaults to the - "main" team if not specified. + --ci-url (required for new targets) + The Concourse server URL (e.g. https://ci.example.com). + + --ci-team (required for new targets) + The Concourse team name. + + --ci-target (optional for new targets) + Override the fly target name. Defaults to a name derived from + the URL and team (e.g. ci/platform). --ci-insecure (optional flag) Skip TLS certificate verification when connecting to the Concourse server. Equivalent to fly's --skip-ssl-validation flag. - Use with caution — only for self-signed or development endpoints. + + When neither --ci-target nor --ci-url is provided, an interactive + wizard guides target selection or creation. EOF } @@ -84,97 +147,31 @@ EOF # label - human-readable name for this provider {{{ sub label { 'Concourse' } -# }}} -# check_prereqs - verify fly CLI is installed and meets min version {{{ -sub check_prereqs { - my ($self) = @_; - my $ok = 1; - - # Require fly in PATH - my ($fly_path) = run({ stderr => 0 }, 'type -p fly'); - chomp($fly_path //= ''); - unless ($fly_path) { - error( - "The Concourse CI provider requires the #C{fly} CLI but it was not ". - "found in your PATH.\n". - " Install it from your Concourse server:\n". - " #C{/api/v1/cli?arch=amd64&platform=}\n". - " Or log in via the Concourse UI and download fly from the bottom-right icon.", - ); - return 0; - } - - # Optional minimum version enforcement - if (defined $self->{min_fly_version}) { - my ($ver_out) = run({ stderr => 0 }, 'fly', '--version'); - chomp(my $fly_version = $ver_out // ''); - $fly_version =~ s/\s.*$//; # strip trailing build info if any - unless (new_enough($fly_version, $self->{min_fly_version})) { - error( - "Concourse CI provider requires fly version #C{%s} or later ". - "(found: #Y{%s}).\n". - " Sync fly with your Concourse server: #C{fly -t sync}", - $self->{min_fly_version}, - $fly_version || 'unknown', - ); - $ok = 0; - } - } - - return $ok; -} - # }}} # config - returns hash for .genesis/config ci.provider section {{{ sub config { my ($self) = @_; my %cfg = (type => 'concourse'); $cfg{target} = $self->{target} if defined $self->{target}; + $cfg{url} = $self->{url} if defined $self->{url}; $cfg{team} = $self->{team} if defined $self->{team} && $self->{team} ne DEFAULT_TEAM; $cfg{insecure} = 1 if $self->{insecure}; return %cfg; } # }}} -# interactive_wizard - prompt user for Concourse configuration {{{ +# interactive_wizard - select existing target or create a new one {{{ sub interactive_wizard { - my ($self, $top, %opts) = @_; + my ($self, $top) = @_; - my ($target, $team, $insecure); + my ($target, $url, $team, $insecure); - my @fly_targets; - my $flyrc = "$ENV{HOME}/.flyrc"; - my %flyrc_data; - if (-f $flyrc) { - my $data = load_yaml_file($flyrc); - if (ref($data) eq 'HASH' && ref($data->{targets}) eq 'HASH') { - %flyrc_data = %{$data->{targets}}; - } - } - - my ($fly_out) = run({ stderr => '/dev/null' }, - 'fly targets --print-table-headers'); - if ($fly_out && $fly_out =~ /\S/) { - my @lines = split /\n/, $fly_out; - my @rows = parse_fixed_width_table(@lines); - for my $row (@rows) { - my $name = $row->{name} // next; - my $flyrc_entry = $flyrc_data{$name} // {}; - push @fly_targets, { - name => $name, - api => $row->{url} // '(unknown)', - team => $row->{team} // DEFAULT_TEAM, - insecure => $flyrc_entry->{insecure} ? 1 : 0, - expired => _token_expired($row->{expiry}), - }; - } - } + # --- Gather existing targets from fly targets + flyrc --- + my @fly_targets = _load_fly_targets(); if (@fly_targets) { # Compute column widths for aligned display - my $w_name = 0; - my $w_api = 0; - my $w_team = 0; + my ($w_name, $w_api, $w_team) = (0, 0, 0); for (@fly_targets) { $w_name = length($_->{name}) if length($_->{name}) > $w_name; $w_api = length($_->{api}) if length($_->{api}) > $w_api; @@ -190,7 +187,6 @@ sub interactive_wizard { summary => $_->{name}, }} @fly_targets; - # Separator + "create new" option at the bottom push @choices, { separator => 1 }; push @choices, { value => '__new__', @@ -198,30 +194,35 @@ sub interactive_wizard { summary => '(new target)', }; - $target = new_prompt_for_choice( + my $selection = new_prompt_for_choice( header => "Select a Concourse target:", choices => \@choices, default => $self->{target}, description => "target", ); - if ($target ne '__new__') { - # Pre-fill team and insecure from the selected flyrc entry - my ($selected) = grep { $_->{name} eq $target } @fly_targets; - if ($selected) { - $team = $selected->{team}; - $insecure = $selected->{insecure}; + if ($selection ne '__new__') { + my ($selected) = grep { $_->{name} eq $selection } @fly_targets; + $target = $selected->{name}; + $url = $selected->{api}; + $team = $selected->{team}; + $insecure = $selected->{insecure}; + + # Re-authenticate if the token has expired + if ($selected->{expired}) { + info "\nRe-authenticating expired target #C{%s}...\n", $target; + my @cmd = ('fly', '-t', $target, 'login', '-n', $team); + push @cmd, '-k' if $insecure; + run({ interactive => 1, + onfailure => "fly login failed for target '$target'" }, + @cmd); } } } # Create a new target: prompt for details and run fly login - if (!$target || $target eq '__new__') { - my $name = prompt_for_line(undef, - "Target name (short alias for this Concourse): ", ''); - bail("A target name is required.") unless $name && $name =~ /\S/; - - my $url = prompt_for_line(undef, + unless ($target) { + $url = prompt_for_line(undef, "Concourse URL (e.g. https://ci.example.com): ", ''); bail("A Concourse URL is required.") unless $url && $url =~ m{^https?://}; @@ -232,30 +233,74 @@ sub interactive_wizard { $insecure = prompt_for_boolean( "Skip TLS certificate verification? [y|n] ", 0); - # Run fly login to create the target in ~/.flyrc and - # authenticate the user. This is interactive — fly will - # prompt for credentials or open a browser. - info "\nLogging in to Concourse as #C{%s} on #C{%s}...\n", $team, $url; - my @fly_cmd = ('fly', '-t', $name, 'login', + $target = _derive_target_name($url, $team); + + info "\nLogging in to #C{%s} as team #C{%s} (target: #C{%s})...\n", + $url, $team, $target; + my @cmd = ('fly', '-t', $target, 'login', '-c', $url, '-n', $team); - push @fly_cmd, '-k' if $insecure; + push @cmd, '-k' if $insecure; run({ interactive => 1, - onfailure => "fly login failed for target '$name'" }, - join(' ', map { /\s/ ? "\"$_\"" : $_ } @fly_cmd)); - - $target = $name; + onfailure => "fly login failed for target '$target'" }, + @cmd); } - return (ref($self) || $self)->new( + return $self->new( type => 'concourse', target => $target, + url => $url, team => $team, - insecure => $insecure, + insecure => $insecure ? 1 : 0, ); } # }}} +# _load_fly_targets - merge fly targets output with flyrc insecure flags {{{ +sub _load_fly_targets { + my @targets; + + my $flyrc = "$ENV{HOME}/.flyrc"; + my %flyrc_data; + if (-f $flyrc) { + my $data = load_yaml_file($flyrc); + if (ref($data) eq 'HASH' && ref($data->{targets}) eq 'HASH') { + %flyrc_data = %{$data->{targets}}; + } + } + + my ($fly_out) = run({ stderr => '/dev/null' }, + 'fly targets --print-table-headers'); + if ($fly_out && $fly_out =~ /\S/) { + my @lines = split /\n/, $fly_out; + my @rows = parse_fixed_width_table(@lines); + for my $row (@rows) { + my $name = $row->{name} // next; + my $flyrc_entry = $flyrc_data{$name} // {}; + push @targets, { + name => $name, + api => $row->{url} // '(unknown)', + team => $row->{team} // DEFAULT_TEAM, + insecure => $flyrc_entry->{insecure} ? 1 : 0, + expired => _token_expired($row->{expiry}), + }; + } + } + return @targets; +} + +# }}} +# _derive_target_name - build a target name from url and team {{{ +sub _derive_target_name { + my ($url, $team) = @_; + # https://ci.example.com → ci + # https://pipes.scalecf.net → pipes + (my $host = $url) =~ s{^https?://}{}; $host =~ s{[:/].*}{}; + my $subdomain = (split /\./, $host)[0] // $host; + return $team eq 'main' ? $subdomain : "$subdomain/$team"; +} + +# }}} # _token_expired - check if a fly targets expiry string is in the past {{{ my %_months = ( Jan => 0, Feb => 1, Mar => 2, Apr => 3, May => 4, Jun => 5, From f2ecbf77023af63b29fc6c52254b7e2bea00005e Mon Sep 17 00:00:00 2001 From: Tristan James Poland Date: Fri, 17 Apr 2026 12:29:10 -0700 Subject: [PATCH 028/103] Restore check_prereqs and wizard pre-fill Re-add check_prereqs call and %ci_provider_opts pre-fill to the CI provider init block at step 7 of _repo_init_validate. These were in Tristan's commits but lost when the block was moved from step 2 to step 7 during the validation reorder. --- lib/Genesis/Commands/Repo.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index e5685be5..20209b8b 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -256,13 +256,16 @@ sub _repo_init_validate { if (in_controlling_terminal) { $ci_provider_obj = eval { Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) - ->interactive_wizard(undef); + ->interactive_wizard(undef, %ci_provider_opts); }; bail("CI provider wizard failed: %s", $@) if $@; } else { bail("Could not initialize CI provider: %s", $@); } } + + # Verify the provider's toolchain is available before we do any work + $ci_provider_obj->check_prereqs() or exit 86; } # --- 7. Store derived values and summarize intent --- From 0fb3c314854ab3f6e392df4c2ef7452e8a46d1f5 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 17 Apr 2026 13:34:59 -0700 Subject: [PATCH 029/103] Skip commit when only metadata changed After staging, check if the only diff is the "Last updated" comment and/or updater/creator_version in .genesis/config. If so, roll back the config file and report no meaningful changes instead of creating an empty-content commit. --- lib/Genesis/Commands/Repo.pm | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 20209b8b..9df12236 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -423,6 +423,27 @@ sub _repo_init_execute { run({ onfailure => "Failed to stage repository in $human_root/" }, 'git add .'); + # Check if the only staged changes are metadata-only (the + # "Last updated" comment and/or updater/creator_version in + # .genesis/config). If so, roll them back — re-running + # repo-init with --force on the same kit version shouldn't + # produce a commit with no meaningful content. + my ($diff_names) = run({}, 'git diff --cached --name-only -- .'); + if ($diff_names && $diff_names =~ /\S/) { + my @changed = grep { /\S/ } split /\n/, $diff_names; + if (@changed == 1 && $changed[0] =~ m{\.genesis/config$}) { + my ($diff_content) = run({}, 'git diff --cached -- .genesis/config'); + my @meaningful = grep { + /^[-+]/ && !/^[-+]{3}\s/ && + $_ !~ /^[-+]\s*#\s*Last updated by\b/ && + $_ !~ /^[-+]\s*(updater_version|creator_version):/ + } split /\n/, $diff_content; + unless (@meaningful) { + run({}, 'git checkout -- .genesis/config'); + } + } + } + # Show a summary of what we just staged so the user can see # exactly what the initial commit (or leftover stage) contains. my ($stat) = run({}, 'git diff --cached --stat -- .'); @@ -430,13 +451,16 @@ sub _repo_init_execute { info "\n#G{Files staged for initial commit:}"; info " %s", $_ for split /\n/, $stat; info ""; + } else { + info "\nNo changes to commit — repository contents are unchanged."; + $kit_desc = "unchanged (already up to date)"; } - # Commit unless the user explicitly opted out. In subdir mode - # we scope the commit with a pathspec ('-- .') so any unrelated - # changes already staged in the enclosing repo are not bundled - # into this commit. - if ($no_commit) { + # Commit unless the user explicitly opted out or there is + # nothing staged. + if (!$stat || $stat !~ /\S/) { + # nothing to commit — already reported above + } elsif ($no_commit) { info "Skipping initial commit (#C{--no-commit} set); files remain staged."; } else { my $message = $reason || "Initial Genesis repo for $name"; From 78da531948b0ba29af1e55b8af62dc035362db3d Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Sun, 19 Apr 2026 13:26:50 -0700 Subject: [PATCH 030/103] Make genesis new pipeline-aware Add control-branch validation, git commit, and environment branch creation to genesis new when CI is configured. Add --no-commit and --reason flags. Migrate prior_env prompt to new_prompt_for_choice. Standardize --force to -f everywhere. Fix iaas() for non-OCFP kits: derive IaaS from kit.features when kit.iaas is absent, using lookup to avoid is_ocfp/features hook recursion. Fix create-env default for BOSH director kits with use_create_env: allow. Fix remove_secrets purge mode (all => 'purge') to skip secrets_plan when env file doesn't exist. Fix --force to move existing env file to .old before running the new hook. Fix credhub_connection_env to skip empty bosh_env for create-env environments. --- bin/genesis | 14 +++- lib/Genesis/Commands/Env.pm | 92 ++++++++++++++++---- lib/Genesis/Env.pm | 163 ++++++++++++++++++++++++++++++------ 3 files changed, 228 insertions(+), 41 deletions(-) diff --git a/bin/genesis b/bin/genesis index 524ffcf3..450ffa69 100755 --- a/bin/genesis +++ b/bin/genesis @@ -268,7 +268,17 @@ define_command("create", { "manual" => "Require a manual CI trigger before this environment deploys. ". - "Only used when a CI provider is configured in #C{.genesis/config}." + "Only used when a CI provider is configured in #C{.genesis/config}.", + + "no-commit" => + "Stage the new environment file but skip the git commit and ". + "environment branch creation. Only applies when a CI ". + "provider is configured.", + + "reason=s" => + "Commit message for the environment commit. Defaults to ". + "#C{Add environment }. Only applies when a CI ". + "provider is configured and #C{--no-commit} is not set.", ], deprecated_options => [ "^prefix=s" => @@ -1260,7 +1270,7 @@ define_command("repo-init", { "describing the new Genesis deployment. Ignored if #C{--no-commit} ". "is set.", - 'force|F' => + 'force|f' => "If the target directory already exists, remove it and recreate ". "(without this flag, you will be prompted to confirm ". "replacement). Also bypasses the check that the enclosing git ". diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index c4ece0ce..44ebd3d3 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -71,6 +71,26 @@ sub create { # check version prereqs $kit->check_prereqs() or exit 86; + # Pipeline-aware repos require new environments to be created on the + # control branch so the topology is visible to pipeline tooling and + # the environment branch can be cut from the right point. + my $ci_configured = $top->config->has('ci.provider.type'); + if ($ci_configured) { + my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); + chomp $branch if defined $branch; + if (!defined($branch) || $branch ne $control) { + bail( + "Creating environments requires being on the #C{%s} branch, ". + "but you are currently on #C{%s}.\n\n". + " git checkout %s\n", + $control, + $branch // '', + $control + ); + } + } + # create the environment info("\nSetting up new environment #C{$name} based on kit %s ...", $kit->id); my $env = $top->create_env($name, $kit, %{get_options()}); @@ -103,15 +123,17 @@ sub create { my @existing = grep { $_->name ne $name } $top->envs(); if (@existing) { my @env_names = map { $_->name } @existing; - my @choices = ('', @env_names); - my @labels = ('(none — pipeline entrypoint)', @env_names); - $prior_env = prompt_for_choice( - "Select prior environment (the environment that must succeed before this one):", - \@choices, - '', - \@labels, - "Please select a number from the list", - "environment", + my @choices = map {{ value => $_, label => $_ }} @env_names; + push @choices, { separator => 1 }; + push @choices, { + value => '', + label => '#Yi{(none — pipeline entrypoint)}', + summary => '(entrypoint)', + }; + $prior_env = new_prompt_for_choice( + header => "Select prior environment (must succeed before this one):", + choices => \@choices, + description => "environment", ); } else { $prior_env = ''; @@ -178,13 +200,53 @@ sub create { } } + # Git operations: stage, commit, and create an environment branch. + # Only when CI is configured and --no-commit is not set. + my %cli_opts_git = %{get_options()}; + if ($ci_configured) { + my $env_file = $env->file; + run({ onfailure => "Failed to stage $env_file" }, + 'git', 'add', $env_file); + + if ($cli_opts_git{'no-commit'}) { + info "Skipping commit (#C{--no-commit} set); #C{%s} remains staged.", $env_file; + } else { + my $message = $cli_opts_git{reason} + || "Add environment $name"; + run({ onfailure => "Failed to commit $env_file" }, + 'git', 'commit', '-m', $message); + + my ($sha) = run({}, 'git rev-parse --short HEAD'); + chomp $sha if defined $sha; + info "#G{Committed} #C{%s} -- %s", $sha // '', $message; + + # Create the environment branch at the current commit. + # This is the branch where future config changes and + # deploys for this environment will happen. We stay on + # the control branch. + run({ onfailure => "Failed to create branch '$name'" }, + 'git', 'branch', $name); + info "Environment branch #C{%s} created.", $name; + } + } + # let the user know - info( - "New environment $env->{name} provisioned!\n\n". - "To deploy, run this:\n\n". - " #C{genesis deploy '%s'}\n", - $env->{name} - ); + if ($ci_configured && !$cli_opts_git{'no-commit'}) { + info( + "\nNew environment #C{%s} provisioned!\n\n". + "To deploy, switch to the environment branch and run:\n\n". + " #C{git checkout '%s'}\n". + " #C{genesis deploy '%s'}\n", + $env->{name}, $name, $env->{name} + ); + } else { + info( + "\nNew environment #C{%s} provisioned!\n\n". + "To deploy, run this:\n\n". + " #C{genesis deploy '%s'}\n", + $env->{name}, $env->{name} + ); + } } sub edit { diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 86020a5e..de74f51d 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -340,9 +340,19 @@ sub create { my $env = $class->new(get_opts(\%opts, qw(name top kit))); my $create_env = $opts{'create-env'}; - # environment must not already exist... - die "Environment file $env->{file} already exists.\n" - if -f $env->path($env->{file}); + # environment must not already exist (unless --force) + my $env_path = $env->path($env->{file}); + if (-f $env_path) { + die "Environment file $env->{file} already exists.\n" + unless $opts{force}; + # Move the existing file out of the way so the new hook + # starts fresh and $self->exists returns false. The .old + # file is kept for reference (manual diff / re-create). + my $old_path = "${env_path}.old"; + rename($env_path, $old_path) + or bail("Could not move existing %s to %s: %s", $env->{file}, "$env->{file}.old", $!); + info("Moved existing #C{%s} to #C{%s.old}", $env->{file}, $env->{file}); + } # Sanitize the vault descriptor, if present if ($opts{vault}) { @@ -389,15 +399,30 @@ sub create { $env->{__params}{genesis}{use_create_env} = 0; $env->{__params}{genesis}{bosh_env} = $bosh_env//$opts{name}; } else { - # Complicated state: the kit allows but does not require create-env. - warning( - "\nKit #M{%s} supports both bosh and create-env deployment. No --create-env ". - "option specified, so using bosh deployment method.", - $env->kit->id - ) unless defined($create_env) || $bosh_env; + # Kit allows but does not require create-env. When neither + # --create-env nor --bosh-env is given, default based on + # whether the kit is a BOSH director: director kits default + # to create-env (they create the director itself), everything + # else defaults to bosh deployment. This matches the + # pre-2.8.0 heuristic and the use_create_env accessor logic. bail( "Cannot specify a bosh environment for environments that use create-env deployment method." ) if $create_env && $bosh_env; + unless (defined($create_env) || $bosh_env) { + if ($env->is_bosh_director) { + $create_env = 1; + warning( + "\nNo #C{--bosh-env} specified — defaulting to #C{create-env} deployment ". + "for this BOSH director kit." + ); + } else { + warning( + "\nKit #M{%s} supports both bosh and create-env deployment. No --create-env ". + "option specified, so using bosh deployment method.", + $env->kit->id + ); + } + } $env->{__params}{genesis}{use_create_env} = $create_env//0; $env->{__params}{genesis}{bosh_env} = $create_env ? '' : $bosh_env || $opts{name}; } @@ -419,7 +444,7 @@ sub create { bail("No vault specified or configured.") unless $env->vault; - my ($results) = $env->remove_secrets(all => 1, no_populate => 1); + my ($results) = $env->remove_secrets(all => 'purge'); bail "Cannot continue with existing secrets for this environment" if ($results->{abort} || $results->{error}); @@ -1458,28 +1483,61 @@ sub scale { # }}} # iaas - returns the iaas for the environment {{{ +# +# Resolution order: +# 1. kit.iaas (explicit — works for both OCFP and non-OCFP) +# 2. kit.features (non-OCFP only: silently upconvert IaaS feature) +# 3. Director exodus data (inherited from parent BOSH director) +# 4. Bail with context-appropriate message +# +my @_known_iaas = qw(vsphere aws azure google openstack warden); sub iaas { my ($self) = @_; - my $iaas = $self->lookup('kit.iaas'); + # 1. Explicit kit.iaas (OCFP sets this; non-OCFP may also use it) + my $iaas = $self->lookup('kit.iaas'); return lc($iaas) if $iaas; - bail( - "No IaaS type set for %s environment, which uses a create-env deployment. ". - "Please set the `kit.iaas` in the enviroment file -- you can use #G{%s ". - "%s edit} to do this.", - $self->name, - humanize_bin(), - humanize_path($self->path($self->name)) - ) if $self->use_create_env; + # 2. Non-OCFP: silently derive from kit.features. + # Check kit.features directly via lookup (returns from __params + # cache during create) — do NOT call is_ocfp()/has_feature() + # which triggers features() → features hook → get_environment_variables + # → iaas() recursion. + my @features = @{$self->lookup('kit.features', [])}; + my $is_ocfp = grep { $_ eq 'ocfp' } @features; + unless ($is_ocfp) { + for my $f (@features) { + return lc($f) if grep { $f eq $_ } @_known_iaas; + } + } - eval {$iaas = $self->director_exodus_lookup('iaas') }; # FIXME: How to handle multiple CPIs? + # 3. Inherit from the BOSH director's exodus data + unless ($self->use_create_env) { + eval { $iaas = $self->director_exodus_lookup('iaas') }; # FIXME: How to handle multiple CPIs? + return lc($iaas) if $iaas; + } + # 4. Nothing found — bail with appropriate guidance + if ($is_ocfp) { + bail( + "No IaaS type set for OCFP environment %s. ". + "Set #C{kit.iaas} in the environment file or ensure it is ". + "inherited from the parent BOSH director.", + $self->name, + ); + } elsif ($self->use_create_env) { + bail( + "No IaaS type found for %s environment (create-env deployment). ". + "Set #C{kit.iaas} in the environment file or include the IaaS ". + "as a kit feature (e.g. #C{features: [vsphere, ...]}).", + $self->name, + ); + } bail( "No IaaS type set for %s environment, and no default IaaS type set for ". "deployments under %s bosh director.", $self->name, $self->bosh->alias - ) if ! $iaas && $self->kit->requires_iaas($self); + ) if $self->kit->requires_iaas($self); return lc($iaas//''); } @@ -1876,11 +1934,21 @@ sub credhub_connection_env { my ($credhub_src,$credhub_src_key) = $self->lookup( ['genesis.credhub_env','genesis.bosh_env','params.bosh','genesis.env','params.env'] ); + + # create-env environments have genesis.bosh_env = '' (empty string) + # which lookup finds as "defined". Fall through to the next keys + # so we don't try to parse an empty string as a BOSH env descriptor. + if ($credhub_src_key && $credhub_src_key eq 'genesis.bosh_env' && !length($credhub_src // '')) { + ($credhub_src,$credhub_src_key) = $self->lookup( + ['genesis.env','params.env'] + ); + } + my %env=(); my $credhub_info = {}; $env{GENESIS_CREDHUB_EXODUS_SOURCE_OVERRIDE} = ""; - if ($credhub_src_key eq 'genesis.bosh_env') { + if ($credhub_src_key && $credhub_src_key eq 'genesis.bosh_env') { my ($bosh_alias,$bosh_dep_type,$bosh_exodus_vault,$bosh_exodus_mount) = $self->_parse_bosh_env($credhub_src); $bosh_alias //= $self->name; $bosh_dep_type //= 'bosh'; @@ -4456,14 +4524,61 @@ sub rotate_secrets { sub remove_secrets { my ($self, %opts) = @_; + # Modes: + # all => 'purge' — pre-create: wipe vault paths without a + # secrets plan (env file may not exist yet) + # all => 1 — interactive: wipe with plan-based labeling + # (neither) — targeted removal by filter + # + # Legacy: all => 1, no_populate => 1 is treated as all => 'purge' + if ($opts{all} && $opts{no_populate}) { + $opts{all} = 'purge'; + delete $opts{no_populate}; + } + my $store = $self->secrets_store(%opts); - # Determine secrets_store from kit - assume vault for now (credhub ignored) if ($opts{all}) { - # TODO: extract this to a method in Genesis::Env::Secrets::Store my @paths = $store->store_paths(); return ({empty => 1}) unless scalar(@paths); + # Purge mode: called from create() before the env file exists. + # We can't build a secrets_plan (the blueprint hook needs the + # env file), so skip plan-based labeling and just wipe the + # vault paths directly after prompting with raw paths. + if ($opts{all} eq 'purge') { + unless ($opts{'no-prompt'}) { + die_unless_controlling_terminal( + "\nCannot prompt for confirmation to remove all secrets outside a ". + "controlling terminal. Use #C{-y|--no-prompt} option to provide ". + "confirmation to bypass this limitation." + ); + warning( + "\nExisting secrets found under '#C{%s}' from a previous run.\n". + "The following %d path(s) will be removed:\n", + $self->secrets_base, scalar(@paths) + ); + my $prefix = $store->base =~ s/^\///r; + for my $full_path (sort @paths) { + info(bullet("#C{%s}", $full_path =~ s/^$prefix//r)); + } + my $response = prompt_for_line(undef, "Type 'yes' to remove these secrets; anything else will abort",""); + if ($response ne 'yes') { + return ({abort => 1}, sprintf( + "Keeping all existing secrets under '#C{%s}'.", + $store->base + )); + } + } + output {pending => 1}, "Deleting existing secrets under '#C{%s}'...", $store->base; + my ($out,$rc) = $store->service->query('rm', '-rf', $store->base); + if ($rc) { + my $msg = "Failed to remove secrets under '#C{%s}':\n%s"; + return ({error => 1}, sprintf($msg, $store->base, $out)); + } + return ({success => 1}, "#G{All applicable secrets removed.}"); + } + my $plan = $self->secrets_plan(%opts, silent => 1); unless ($opts{'no-prompt'}) { die_unless_controlling_terminal( From f4e8a932ccdee291a5616dd69aadec64dec1f293 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:50:59 -0400 Subject: [PATCH 031/103] Add config validation to CI providers Call per-provider validation from Provider->new and fail fast with a clear, aggregated error message. Introduce a no-op validate_config in the base class and implement field checks for Concourse (requires target, optional url must be http(s) prefixed) and GithubActions (requires repo in org/repo format). Update Concourse init to accept either an existing target or a url+team for new targets and adjust tests accordingly; add comprehensive tests for validate_config behavior and factory bailing on invalid configs. This enforces local, fast configuration checks without network calls and improves error reporting when constructing CI provider objects. --- lib/Genesis/CI/Provider.pm | 26 ++++- lib/Genesis/CI/Provider/Concourse.pm | 18 +++- lib/Genesis/CI/Provider/GithubActions.pm | 12 +++ t/ci-provider.t | 115 +++++++++++++++++++++-- 4 files changed, 157 insertions(+), 14 deletions(-) diff --git a/lib/Genesis/CI/Provider.pm b/lib/Genesis/CI/Provider.pm index d341d88b..2d487854 100644 --- a/lib/Genesis/CI/Provider.pm +++ b/lib/Genesis/CI/Provider.pm @@ -15,18 +15,27 @@ sub new { my $type = $config{type} || 'manual'; + my $obj; if ($type eq 'concourse') { require Genesis::CI::Provider::Concourse; - return Genesis::CI::Provider::Concourse->new(%config); + $obj = Genesis::CI::Provider::Concourse->new(%config); } elsif ($type eq 'github-actions') { require Genesis::CI::Provider::GithubActions; - return Genesis::CI::Provider::GithubActions->new(%config); + $obj = Genesis::CI::Provider::GithubActions->new(%config); } elsif ($type eq 'manual') { require Genesis::CI::Provider::Manual; - return Genesis::CI::Provider::Manual->new(%config); + $obj = Genesis::CI::Provider::Manual->new(%config); } else { bail("Unknown CI provider type '%s'. Valid types: concourse, github-actions, manual", $type); } + + my @errors = $obj->validate_config; + bail( + "Invalid CI provider configuration for type '%s':\n%s", + $type, join("\n", map { " - $_" } @errors) + ) if @errors; + + return $obj; } # }}} @@ -156,6 +165,17 @@ sub check_prereqs { return 1; } +# }}} +# validate_config - check stored config fields; returns list of error strings {{{ +# +# Called by Provider->new after the subclass object is constructed. +# Subclasses override this to assert that all required fields are present and +# well-formed. Returning an empty list means the config is valid. +# No network calls should be made here — this is a fast, local check only. +sub validate_config { + return (); +} + # }}} # interactive_wizard - prompt the user for provider config interactively (abstract) {{{ sub interactive_wizard { diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index d94d1fdf..a986fce9 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -62,8 +62,10 @@ sub init { # New target: --ci-url and --ci-team are required; --ci-target is # an optional name override (derived from url/team if omitted). - bail("Concourse CI provider requires --ci-url and --ci-team for a new target") - unless $opts{'ci-url'} && $opts{'ci-team'}; + bail( + "Concourse CI provider requires --ci-target (existing fly target) ". + "or --ci-url and --ci-team (new target)" + ) unless $opts{'ci-url'} && $opts{'ci-team'}; my $target = $opts{'ci-target'} // _derive_target_name($opts{'ci-url'}, $opts{'ci-team'}); @@ -147,6 +149,18 @@ EOF # label - human-readable name for this provider {{{ sub label { 'Concourse' } +# }}} +# validate_config - assert required fields are present in stored config {{{ +sub validate_config { + my ($self) = @_; + my @errors; + push @errors, "'target' is required for the Concourse provider" + unless $self->{target}; + push @errors, "'url' must begin with http:// or https://" + if $self->{url} && $self->{url} !~ m{^https?://}; + return @errors; +} + # }}} # config - returns hash for .genesis/config ci.provider section {{{ sub config { diff --git a/lib/Genesis/CI/Provider/GithubActions.pm b/lib/Genesis/CI/Provider/GithubActions.pm index 11f22f51..540849af 100644 --- a/lib/Genesis/CI/Provider/GithubActions.pm +++ b/lib/Genesis/CI/Provider/GithubActions.pm @@ -76,6 +76,18 @@ EOF # label - human-readable name for this provider {{{ sub label { 'GitHub Actions' } +# }}} +# validate_config - assert required fields are present in stored config {{{ +sub validate_config { + my ($self) = @_; + my @errors; + push @errors, "'repo' is required for the GitHub Actions provider" + unless $self->{repo}; + push @errors, "'repo' must be in 'org/repo' format" + if $self->{repo} && $self->{repo} !~ m{^[^/]+/[^/]+$}; + return @errors; +} + # }}} # config - returns hash for .genesis/config ci.provider section {{{ sub config { diff --git a/t/ci-provider.t b/t/ci-provider.t index 8c90d8de..99ae4b8c 100644 --- a/t/ci-provider.t +++ b/t/ci-provider.t @@ -41,7 +41,11 @@ subtest 'Provider->new rejects unknown type' => sub { ### ============================================================ ### subtest 'Provider->init dispatches on ci-provider opt' => sub { - my $c = Genesis::CI::Provider->init('ci-provider' => 'concourse', 'ci-target' => 'prod'); + my $c = Genesis::CI::Provider->init( + 'ci-provider' => 'concourse', + 'ci-url' => 'https://ci.example.com', + 'ci-team' => 'main', + ); isa_ok $c, 'Genesis::CI::Provider::Concourse', 'ci-provider=concourse'; my $g = Genesis::CI::Provider->init( @@ -142,21 +146,22 @@ subtest 'Concourse->opts returns Getopt spec' => sub { ok grep { $_ eq 'ci-insecure' } @opts, 'ci-insecure present'; }; -subtest 'Concourse->init requires --ci-target' => sub { +subtest 'Concourse->init requires --ci-target or --ci-url+--ci-team' => sub { eval { Genesis::CI::Provider::Concourse->init() }; - like $@, qr/requires --ci-target/i, 'bails without ci-target'; + like $@, qr/requires --ci-target/i, 'bails without ci-target or url+team'; }; -subtest 'Concourse->init defaults team to main' => sub { - my $p = Genesis::CI::Provider::Concourse->init('ci-target' => 'ci.example.com'); - is $p->{team}, 'main', 'team defaults to main'; - is $p->{insecure}, 0, 'insecure defaults to 0'; +subtest 'Concourse defaults team to main and insecure to 0' => sub { + my $p = Genesis::CI::Provider::Concourse->new(type => 'concourse', target => 'ci.example.com'); + is $p->{team}, 'main', 'team defaults to main'; + is $p->{insecure}, 0, 'insecure defaults to 0'; }; -subtest 'Concourse->init honours all opts' => sub { +subtest 'Concourse->init honours all opts (new-target path)' => sub { my $p = Genesis::CI::Provider::Concourse->init( - 'ci-target' => 'ci.example.com', + 'ci-url' => 'https://ci.example.com', 'ci-team' => 'platform', + 'ci-target' => 'ci.example.com', 'ci-insecure' => 1, ); is $p->{target}, 'ci.example.com', 'target set'; @@ -360,4 +365,96 @@ subtest 'check_prereqs: Concourse min_fly_version not satisfied' => sub { ok !$result, 'check_prereqs returns 0 when fly version too old'; }; +### ============================================================ ### +### validate_config — per-provider field validation +### ============================================================ ### + +subtest 'validate_config: Manual always passes' => sub { + my $m = Genesis::CI::Provider::Manual->new(type => 'manual'); + my @errors = $m->validate_config; + is scalar @errors, 0, 'Manual has no required fields'; +}; + +subtest 'validate_config: Concourse passes with target' => sub { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', target => 'my-ci', + ); + my @errors = $p->validate_config; + is scalar @errors, 0, 'no errors when target is present'; +}; + +subtest 'validate_config: Concourse fails without target' => sub { + my $p = Genesis::CI::Provider::Concourse->new(type => 'concourse'); + my @errors = $p->validate_config; + ok scalar @errors > 0, 'errors returned when target missing'; + like $errors[0], qr/target.*required/i, 'error mentions target'; +}; + +subtest 'validate_config: Concourse passes with target and valid url' => sub { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'my-ci', + url => 'https://ci.example.com', + ); + my @errors = $p->validate_config; + is scalar @errors, 0, 'no errors with target and valid https url'; +}; + +subtest 'validate_config: Concourse rejects malformed url' => sub { + my $p = Genesis::CI::Provider::Concourse->new( + type => 'concourse', + target => 'my-ci', + url => 'not-a-url', + ); + my @errors = $p->validate_config; + ok scalar @errors > 0, 'error returned for malformed url'; + like $errors[0], qr/url.*http/i, 'error mentions url format'; +}; + +subtest 'validate_config: GithubActions passes with valid repo' => sub { + my $p = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', repo => 'acme/deploy', + ); + my @errors = $p->validate_config; + is scalar @errors, 0, 'no errors when repo is valid'; +}; + +subtest 'validate_config: GithubActions fails without repo' => sub { + my $p = Genesis::CI::Provider::GithubActions->new(type => 'github-actions'); + my @errors = $p->validate_config; + ok scalar @errors > 0, 'errors returned when repo missing'; + like $errors[0], qr/repo.*required/i, 'error mentions repo'; +}; + +subtest 'validate_config: GithubActions fails on bad repo format' => sub { + my $p = Genesis::CI::Provider::GithubActions->new( + type => 'github-actions', repo => 'noslash', + ); + my @errors = $p->validate_config; + ok scalar @errors > 0, 'errors returned for bad repo format'; + like $errors[0], qr/org.repo/i, 'error mentions org/repo format'; +}; + +subtest 'Provider->new bails when validate_config returns errors' => sub { + # Concourse with no target should be rejected by the factory + eval { Genesis::CI::Provider->new(type => 'concourse') }; + like $@, qr/Invalid CI provider configuration/i, 'factory bails on invalid config'; + like $@, qr/target.*required/i, 'bail message includes field-level error'; +}; + +subtest 'Provider->new accepts valid Concourse config' => sub { + my $p = Genesis::CI::Provider->new(type => 'concourse', target => 'prod'); + isa_ok $p, 'Genesis::CI::Provider::Concourse', 'valid Concourse config accepted'; +}; + +subtest 'Provider->new bails on invalid GithubActions config' => sub { + eval { Genesis::CI::Provider->new(type => 'github-actions') }; + like $@, qr/Invalid CI provider configuration/i, 'factory bails on missing repo'; +}; + +subtest 'Provider->new accepts valid GithubActions config' => sub { + my $p = Genesis::CI::Provider->new(type => 'github-actions', repo => 'org/repo'); + isa_ok $p, 'Genesis::CI::Provider::GithubActions', 'valid GHA config accepted'; +}; + done_testing; From d0ac344bd92b017e0f62053ab9ceaf3c16589aec Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:03:04 -0400 Subject: [PATCH 032/103] Concourse: add fly prereq checks; minor fixes Add runtime prerequisite checks for the Concourse provider: new min_fly_version attribute and check_prereqs method that verifies the fly CLI is in PATH and meets a minimum semver, with user-friendly error messages. Improve robustness when writing temporary override files in Compiler.pm by checking print/close results and bailing on failure. Rename AST graphviz reference to mermaid in AST.pm. Remove a duplicate header line in PipelineProvider.pm. These changes improve error handling and CI provider validation. --- lib/Genesis/CI/Compiler.pm | 6 ++- lib/Genesis/CI/Compiler/AST.pm | 2 +- lib/Genesis/CI/Compiler/PipelineProvider.pm | 1 - lib/Genesis/CI/Provider/Concourse.pm | 51 +++++++++++++++++++-- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 3038580d..2dec6384 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -281,8 +281,10 @@ sub _apply_provider_overrides { my $base_path = "$dir/override-base-${filename}"; open(my $fh, '>', $base_path) or bail("Cannot write temporary override base %s: %s", $base_path, $!); - print $fh $content; - close $fh; + print $fh $content + or bail("Cannot write to temporary override base %s: %s", $base_path, $!); + close $fh + or bail("Cannot flush temporary override base %s: %s", $base_path, $!); my ($merged_yaml, $rc) = run( 'spruce', 'merge', $base_path, $override_file diff --git a/lib/Genesis/CI/Compiler/AST.pm b/lib/Genesis/CI/Compiler/AST.pm index 0a2b5570..703a6b47 100644 --- a/lib/Genesis/CI/Compiler/AST.pm +++ b/lib/Genesis/CI/Compiler/AST.pm @@ -350,7 +350,7 @@ the generic pipeline and should not be accessed by providers directly. my $resources = $ast->pipeline_resources; my $jobs = $ast->jobs; my $groups = $ast->groups; - my $dot = $ast->graphviz; + my $dot = $ast->mermaid; my $text = $ast->description; =head1 SEE ALSO diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 535e7155..862db531 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -306,7 +306,6 @@ EOF # }}} # }}} ### Shared Helper Methods {{{ -### Shared Helper Methods {{{ # ast - get stored AST {{{ sub ast { diff --git a/lib/Genesis/CI/Provider/Concourse.pm b/lib/Genesis/CI/Provider/Concourse.pm index a986fce9..2a186587 100644 --- a/lib/Genesis/CI/Provider/Concourse.pm +++ b/lib/Genesis/CI/Provider/Concourse.pm @@ -85,14 +85,55 @@ sub new { my ($class, %config) = @_; $class = ref($class) || $class; bless({ - label => 'Concourse', - target => $config{target}, - url => $config{url}, - team => $config{team} || DEFAULT_TEAM, - insecure => $config{insecure} ? 1 : 0, + label => 'Concourse', + target => $config{target}, + url => $config{url}, + team => $config{team} || DEFAULT_TEAM, + insecure => $config{insecure} ? 1 : 0, + min_fly_version => $config{min_fly_version} || undef, }, $class); } +# }}} +# check_prereqs - verify fly CLI is present and meets min version {{{ +sub check_prereqs { + my ($self) = @_; + + my ($fly_path) = run({ stderr => 0 }, 'type -p fly'); + chomp($fly_path //= ''); + unless ($fly_path) { + error( + "Concourse CI provider requires the #C{fly} CLI but it was not found in PATH.\n". + " Download it from your Concourse server's home page or:\n". + " #C{/api/v1/cli?arch=amd64&platform=}" + ); + return 0; + } + + if ($self->{min_fly_version}) { + my ($ver_out) = run({ stderr => 0 }, 'fly --version'); + chomp($ver_out //= ''); + if ($ver_out && $ver_out =~ /^(\d+)\.(\d+)\.(\d+)/) { + my @got = ($1+0, $2+0, $3+0); + my @min = map { $_ + 0 } split(/\./, $self->{min_fly_version}, 3); + push @min, 0 while @min < 3; + for my $i (0..2) { + if ($got[$i] < ($min[$i]//0)) { + error( + "Concourse CI provider requires fly >= %s but found %s.\n". + " Upgrade fly from your Concourse server.", + $self->{min_fly_version}, $ver_out + ); + return 0; + } + last if $got[$i] > ($min[$i]//0); + } + } + } + + return 1; +} + # }}} # opts - Getopt::Long spec for Concourse-specific CLI flags {{{ sub opts { From babc5a6800d1dde8c0ff7cff5da2a5cff6602353 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:43:29 -0400 Subject: [PATCH 033/103] Normalize CI provider CLI options and Concourse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centralize and normalize CI provider CLI options so callers and providers use consistent config-style keys. - Added cli_key_to_config_key, normalize_provider_opts and cli_opt_keys to PipelineProvider to convert ci-* hyphenated CLI keys into unprefixed underscored config keys and to expose provider-specific CLI keys. - Compiler now normalizes caller-supplied provider opts before merging them into provider options. - Commands (Pipeline/Pipelines) now pass normalized/plain keys to provider->deploy and delegate Concourse fly operations to the provider. Pipelines captures provider CLI opts and normalizes them before calling deploy(). - Concourse provider API updated: renamed pause_after_set -> pause (and DEFAULT_PAUSE constant), updated schema/defaults/describe/deploy to use config-style keys and three-tier resolution (call-site > provider_opts > defaults). This reduces duplication between the commands and provider layers, supports both legacy and new option forms, and centralizes mapping logic (ci-* → config) in one place. --- lib/Genesis/CI/Compiler.pm | 12 ++- lib/Genesis/CI/Compiler/PipelineProvider.pm | 44 +++++++++++ .../CI/Compiler/Providers/Concourse.pm | 79 ++++++++++--------- lib/Genesis/Commands/Pipeline.pm | 37 +++------ lib/Genesis/Commands/Pipelines.pm | 26 +++--- 5 files changed, 118 insertions(+), 80 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 2dec6384..d240fdaa 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -99,11 +99,15 @@ sub compile { or bail("Failed to load CI provider '%s': %s", $provider_type, $@); # Extract provider options from parsed config (ci.provider: section) - # and merge with any caller-supplied opts. These are stored in the - # provider object and used by deploy() at deploy time. + # and merge with any caller-supplied opts. Normalize caller opts from their + # CLI form (ci-* prefixed, hyphenated) to config/schema form (unprefixed, underscored) + # so that provider_option() and provider_config() always see consistent keys. + require Genesis::CI::Compiler::PipelineProvider; my $provider_opts = { - %{ $parsed->{provider} || {} }, # from ci.provider: section - %{ $opts{provider_opts} || {} }, # caller-supplied overrides + %{ $parsed->{provider} || {} }, + %{ Genesis::CI::Compiler::PipelineProvider->normalize_provider_opts( + $opts{provider_opts} || {} + ) }, }; my $provider = $provider_info->{class}->new( diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 862db531..48ea5523 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -271,6 +271,50 @@ sub parse_cli_opts { return 1; } +# }}} +# cli_key_to_config_key - convert a ci-* CLI key to its config/schema key {{{ +# +# The convention is: strip the 'ci-' prefix, convert hyphens to underscores. +# Examples: +# ci-target => target +# ci-team => team +# ci-pipeline-name => pipeline_name +# ci-pause => pause +# ci-expose => expose +# +# Non-ci-prefixed keys are returned unchanged (already in config form). +sub cli_key_to_config_key { + my ($class, $key) = @_; + $key =~ s/^ci-//; + $key =~ s/-/_/g; + return $key; +} + +# }}} +# normalize_provider_opts - convert a hash of CLI-keyed opts to config keys {{{ +sub normalize_provider_opts { + my ($class, $opts) = @_; + my %out; + for my $k (keys %{ $opts || {} }) { + $out{ $class->cli_key_to_config_key($k) } = $opts->{$k}; + } + return \%out; +} + +# }}} +# cli_opt_keys - return parsed option key names for a given provider type {{{ +# +# Strips Getopt::Long type suffixes (=s, !, +, etc.) leaving bare key names. +# Used by the commands layer to copy matching options from get_options without +# hardcoding provider-specific names. +sub cli_opt_keys { + my ($class, $provider_type) = @_; + my $info = $_providers{$provider_type} or return (); + eval { require $info->{file} } ## no critic + or bail("Failed to load CI provider '%s': %s", $provider_type, $@); + return map { (split /[=!+:]/, $_)[0] } $info->{class}->cli_opts(); +} + # }}} # all_cli_opts_help - assembled help text for all known providers {{{ # diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 2d72e75d..e733aa46 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -13,10 +13,10 @@ use JSON::PP; ### Provider Constants {{{ use constant { - DEFAULT_TEAM => 'main', - DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top - DEFAULT_EXPOSE => 0, - DEFAULT_PAUSE_AFTER_SET => 0, + DEFAULT_TEAM => 'main', + DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top + DEFAULT_EXPOSE => 0, + DEFAULT_PAUSE => 0, }; # }}} @@ -170,9 +170,9 @@ sub provider_options_schema { default => DEFAULT_EXPOSE, description => 'Make pipeline publicly viewable (fly expose-pipeline)', }, - pause_after_set => { + pause => { type => 'boolean', - default => DEFAULT_PAUSE_AFTER_SET, + default => DEFAULT_PAUSE, description => 'Leave pipeline paused after fly set-pipeline', }, }; @@ -182,9 +182,9 @@ sub provider_options_schema { # provider_options_defaults - default values for all Concourse options {{{ sub provider_options_defaults { return { - team => DEFAULT_TEAM, - expose => DEFAULT_EXPOSE, - pause_after_set => DEFAULT_PAUSE_AFTER_SET, + team => DEFAULT_TEAM, + expose => DEFAULT_EXPOSE, + pause => DEFAULT_PAUSE, }; } @@ -196,11 +196,11 @@ sub provider_options_defaults { sub describe_provider { my ($self) = @_; - my $target = $self->provider_option('ci-target') || $self->provider_option('target') || '(not set)'; - my $team = $self->provider_option('ci-team') || $self->provider_option('team') || DEFAULT_TEAM; - my $pipe_name = $self->provider_option('ci-pipeline-name') || $self->provider_option('pipeline_name') || '(deployment type)'; - my $expose = $self->provider_option('expose') ? 'yes' : 'no'; - my $paused = $self->provider_option('pause_after_set') ? 'yes' : 'no'; + my $target = $self->provider_option('target') || '(not set)'; + my $team = $self->provider_option('team') || DEFAULT_TEAM; + my $pipe_name = $self->provider_option('pipeline_name') || '(deployment type)'; + my $expose = $self->provider_option('expose') ? 'yes' : 'no'; + my $paused = $self->provider_option('pause') ? 'yes' : 'no'; return ( type => 'concourse', @@ -306,37 +306,40 @@ sub deploy { bail("Must call parse() before deploy()") unless $self->{config}; + # All keys use config-file names (target, team, pipeline_name, pause, expose). + # Callers are responsible for normalizing cli-prefixed keys before calling deploy(). + # --- Resolve options from three tiers --- - # Target: CLI > provider_opts > legacy layout - my $target = $opts{'ci-target'} - || $self->provider_option('target') - || $self->{layout}; + # Target: call-site override > provider_opts > legacy layout + my $target = $opts{target} + // $self->provider_option('target') + // $self->{layout}; bail("No Concourse target specified. Use --ci-target or set ci.provider.target in .genesis/config") unless $target; - # Team: CLI > provider_opts > default - my $team = $opts{'ci-team'} - || $self->provider_option('team') - || DEFAULT_TEAM; + # Team: call-site override > provider_opts > default + my $team = $opts{team} + // $self->provider_option('team') + // DEFAULT_TEAM; - # Pipeline name: CLI > provider_opts > config name > deployment_type - my $pipeline_name = $opts{'ci-pipeline-name'} - || $self->provider_option('pipeline_name') - || $self->{config}{pipeline}{name} - || ($self->{top} ? $self->{top}->type : undef); + # Pipeline name: call-site override > provider_opts > config name > deployment_type + my $pipeline_name = $opts{pipeline_name} + // $self->provider_option('pipeline_name') + // $self->{config}{pipeline}{name} + // ($self->{top} ? $self->{top}->type : undef); bail("Cannot determine pipeline name — set ci.provider.pipeline_name or ensure deployment_type is set") unless $pipeline_name; - # Pause/expose: CLI flags > provider_opts > defaults - my $dry_run = $opts{'dry-run'} || $opts{'ci-dry-run'}; - my $yes = $opts{yes} || $opts{'-y'}; - my $paused = $opts{'ci-pause'} - || $self->provider_option('pause_after_set') - || DEFAULT_PAUSE_AFTER_SET; - my $expose = $opts{'ci-expose'} - || $self->provider_option('expose') - || _yaml_bool(($self->{config}{pipeline} || {})->{public}, DEFAULT_EXPOSE); + # Pause/expose/dry-run: call-site override > provider_opts > defaults + my $dry_run = $opts{'dry-run'}; + my $yes = $opts{yes}; + my $pause = $opts{pause} + // $self->provider_option('pause') + // DEFAULT_PAUSE; + my $expose = $opts{expose} + // $self->provider_option('expose') + // _yaml_bool(($self->{config}{pipeline} || {})->{public}, DEFAULT_EXPOSE); my $yaml = $self->generate(); @@ -368,8 +371,8 @@ sub deploy { $target, $pipeline_name, $dir ); - # Unpause pipeline (unless --ci-pause / pause_after_set) - unless ($paused) { + # Unpause pipeline (unless ci.provider.pause or --ci-pause override) + unless ($pause) { run({ interactive => 1, onfailure => "Could not unpause pipeline $pipeline_name", diff --git a/lib/Genesis/Commands/Pipeline.pm b/lib/Genesis/Commands/Pipeline.pm index c4415d09..89cbe2e6 100644 --- a/lib/Genesis/Commands/Pipeline.pm +++ b/lib/Genesis/Commands/Pipeline.pm @@ -50,38 +50,25 @@ sub apply { } if ($platform eq 'concourse') { - my $yaml = $output->{'pipeline.yml'} - or bail("Concourse provider did not produce pipeline.yml"); + my $provider = $result->{provider}; if ($opts->{'dry-run'}) { + my $yaml = $output->{'pipeline.yml'} + or bail("Concourse provider did not produce pipeline.yml"); output({raw => 1}, $yaml); exit 0; } - my $target = $opts->{target} || $layout || $name; + $provider->check_prereqs() or exit 86; - my ($out, $rc) = run('fly -t $1 pause-pipeline -p $2', $target, $name); - bail("Could not pause #C{%s} pipeline: %s", $name, $out) - unless $rc == 0 || $out =~ /pipeline '.*' not found/; - - my $yes = $opts->{yes} ? ' -n ' : ''; - my $dir = workdir; - mkfile_or_fail("$dir/pipeline.yml", $yaml); - run({ interactive => 1, onfailure => "Could not upload pipeline $name" }, - 'fly -t $1 set-pipeline '.$yes.' -p $2 -c $3/pipeline.yml', - $target, $name, $dir); - - unless ($opts->{paused}) { - run({ interactive => 1, onfailure => "Could not unpause pipeline $name" }, - 'fly -t $1 unpause-pipeline -p $2', - $target, $name); - } - - my $public = $ast->configuration->{public} || 0; - my $action = $public ? 'expose' : 'hide'; - run({ interactive => 1, onfailure => "Could not $action pipeline $name" }, - 'fly -t $1 '.$action.'-pipeline -p $2', - $target, $name); + # Delegate all fly operations to the Concourse provider. Pass generic + # command-level flags using plain keys; deploy() accepts both plain and + # ci-* prefixed forms so neither layer has to know the other's naming. + $provider->deploy( + target => $opts->{target} || $layout || $name, + pause => $opts->{paused}, + yes => $opts->{yes}, + ); } elsif ($platform eq 'github-actions') { if ($opts->{'dry-run'}) { diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index f02b3ad2..030de973 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -481,17 +481,15 @@ sub _repipe_compiled { # Verify the provider's toolchain is available before attempting deploy. $provider->check_prereqs() or exit 86; - # Pass all relevant CLI flags to deploy() for three-tier resolution. - # Provider reads: ci-target, ci-team, ci-pipeline-name, ci-pause, ci-expose, - # plus the stored provider_opts (from ci.provider: section in .genesis/config). - $provider->deploy( - 'ci-target' => get_options->{'ci-target'} || get_options->{target} || $layout, - 'ci-team' => get_options->{'ci-team'}, - 'ci-pipeline-name' => get_options->{'ci-pipeline-name'}, - 'ci-pause' => get_options->{'ci-pause'} || get_options->{paused}, - 'ci-expose' => get_options->{'ci-expose'}, - 'yes' => get_options->{yes}, - ); + # Normalize provider CLI opts (ci-* → config keys) then merge legacy + # repipe aliases (--target → target, --paused → pause) before calling deploy(). + require Genesis::CI::Compiler::PipelineProvider; + my %deploy_opts = %{ Genesis::CI::Compiler::PipelineProvider->normalize_provider_opts( + $result->{provider_cli_opts} || {} + ) }; + $deploy_opts{target} //= get_options->{target} // $layout; + $deploy_opts{pause} //= get_options->{paused}; + $provider->deploy(%deploy_opts, yes => get_options->{yes}); } elsif ($platform eq 'github-actions') { # GitHub Actions outputs workflow YAML files to .github/workflows/ @@ -597,8 +595,9 @@ sub _compile_pipeline { Genesis::CI::Compiler::PipelineProvider->parse_cli_opts( \@argv, \%provider_cli_opts, $platform ); - # Merge any provider flags already captured by the outer option parser - for my $key (qw(ci-target ci-team ci-pipeline-name ci-pause ci-expose)) { + # Merge any provider-specific flags already captured by the outer option parser. + # Ask the provider what keys it owns — don't hardcode them here. + for my $key (Genesis::CI::Compiler::PipelineProvider->cli_opt_keys($platform)) { $provider_cli_opts{$key} = get_options->{$key} if defined get_options->{$key}; } @@ -615,6 +614,7 @@ sub _compile_pipeline { _dump_debug_artifacts($debug_dir, $result, $platform); } + $result->{provider_cli_opts} = \%provider_cli_opts; return $result; } From 3cb0171624acc26b75f4727c463b4f54d3f7e395 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 21 Apr 2026 12:09:34 -0700 Subject: [PATCH 034/103] Add config v3 schema with CI section Bump LATEST_CONFIG_VERSION to 3. New repos start at v3; v2 configs augmented in-memory with ci.enabled:false defaults (not persisted). v3 adds structured ci section with enabled, provider, and pipeline subsections. Includes polymorphic conditional required for schema validation, legacy ci.yml detection, validation re-entry guard, ci_enabled/ci_configured accessors, and 29 tests. --- lib/Genesis/Config.pm | 33 +++- lib/Genesis/Top.pm | 119 ++++++++++++- t/unit-tests/genesis_commands_repo_ci-core.t | 168 +++++++++++++++++++ t/unit-tests/genesis_config-core.t | 50 ++++++ t/unit-tests/genesis_top-core.t | 4 +- 5 files changed, 364 insertions(+), 10 deletions(-) diff --git a/lib/Genesis/Config.pm b/lib/Genesis/Config.pm index e767234f..276a9574 100644 --- a/lib/Genesis/Config.pm +++ b/lib/Genesis/Config.pm @@ -265,8 +265,10 @@ sub validate { # Set default only if not loaded or explicitly set $self->_update_source('default', $key, $schema->{$key}{default}); } elsif ($schema->{$key}{required} and ! struct_has($self->{loaded_values}, $key) and ! struct_has($self->{set_values}, $key)) { - push @errors, "#R{$key}: missing required key"; - next; + if (_is_required($schema->{$key}{required}, $self->_contents)) { + push @errors, "#R{$key}: missing required key"; + next; + } } } @@ -366,6 +368,29 @@ sub _signature { my $explicit = priority_merge($self->{set_values}, $self->{loaded_values}); return sha1_hex(JSON::PP->new->canonical->encode($explicit)); } +# }}} +# _is_required - evaluate whether a 'required' constraint is active {{{ +# 1 → always required +# 'sibling' → required when sibling is truthy +# { sibling => v } → required when sibling eq v +# +# Multiple hash keys are OR'd together. Future: support arrayref values +# for multi-match on a single sibling, and negation (e.g. { '!key' => v }). +sub _is_required { + my ($req, $siblings) = @_; + return 0 unless $req; + return 1 unless ref($req) || $req =~ /\D/; + if (ref($req) eq 'HASH') { + for my $dep (keys %$req) { + return 1 if exists($siblings->{$dep}) && defined($siblings->{$dep}) + && $siblings->{$dep} eq $req->{$dep}; + } + return 0; + } + # String (non-numeric): truthy check on named sibling + return $siblings->{$req} ? 1 : 0; +} + # }}} # _validate_key - validate a value against a schema {{{ sub _validate_key { @@ -419,7 +444,9 @@ sub _validate_key { my $source = $loaded_is_empty_hash ? 'loaded' : 'default'; $self->_update_source($source, "$key.$subkey", $subschema->{default}); } elsif ($subschema->{required} and ! exists($value->{$subkey})) { - push @errors, "#R{$key}: missing required key #ri{$subkey}"; + if (_is_required($subschema->{required}, $value)) { + push @errors, "#R{$key}: missing required key #ri{$subkey}"; + } } } # Refetch value to include newly-added defaults for recursive validation diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 5b9f599d..0bc3c3d0 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -27,6 +27,7 @@ use File::Path qw/rmtree/; # (and a config key) so it can change without rippling through the # codebase; not currently exposed to end users. use constant DEFAULT_CONTROL_BRANCH => 'control'; +use constant LATEST_CONFIG_VERSION => 3; ### Config Section Delegation Registry {{{ # Modules may register themselves as handlers for specific top-level keys in @@ -169,7 +170,7 @@ sub create { # Write new configuration - Set defaults $self->config->set('deployment_type',$name); - $self->config->set('version',2); + $self->config->set('version', LATEST_CONFIG_VERSION); $self->config->set('creator_version', $Genesis::VERSION); $self->config->set('minimum_version', $Genesis::VERSION) unless $Genesis::VERSION eq '(development)'; $self->config->set('manifest_store', 'exodus'); @@ -984,6 +985,20 @@ sub ci_control_branch { return $self->config->get('ci.control_branch', DEFAULT_CONTROL_BRANCH); } +# }}} +# ci_enabled - return whether CI pipeline is enabled {{{ +sub ci_enabled { + my ($self) = @_; + return $self->config->get('ci.enabled'); +} + +# }}} +# ci_configured - return whether CI is enabled AND has a provider configured {{{ +sub ci_configured { + my ($self) = @_; + return $self->config->get('ci.enabled') && $self->config->has('ci.provider.type'); +} + # }}} # version - return the version of the cofiguration schema {{{ sub version { @@ -1186,6 +1201,8 @@ sub download_kit { # _validate_config - validate the configuration of the repo {{{ sub _validate_config { my ($self) = @_; + return 1 if $self->{__config_validated}; + $self->{__config_validated} = 1; my $config_version = $self->config->get(version => 1); if ($config_version == 1 || $config_version =~ /^\d+\.\d+\.\d+(-[A-Za-z0-9_-]\.?\d+)?$/) { @@ -1198,7 +1215,50 @@ sub _validate_config { $self->_upgrade_config_to_v2($config_version, $upgrade_automatically); } elsif ($config_version == 2){ + $self->config->validate($self->_repo_config_schema_v2()); + $self->{__config_disk_version} = 2; + + # Check for legacy ci.yml — bail until post-MVP migration is implemented. + # Only flag it if it looks like a pipeline config (has 'pipeline:' key), + # not an environment file (which would have 'kit:' or 'genesis:'). + my $ci_yml = $self->path('ci.yml'); + if (-f $ci_yml && _is_legacy_ci_file($ci_yml)) { + bail( + "Legacy CI configuration detected at #C{ci.yml}.\n". + "Pipeline support requires a v3 config. Please run:\n\n". + " genesis repo-init --upgrade\n\n". + "to migrate your CI configuration into the v3 config format." + ); + } + + # Augment in-memory with v3 defaults so downstream code sees + # a uniform v3 shape. These go into the 'default' layer and + # will NOT be persisted to disk on save. + $self->config->_update_source('default', 'ci', { + enabled => Genesis::Config::FALSE, + }); + + } elsif ($config_version == 3){ $self->config->validate($self->_repo_config_schema()); + $self->{__config_disk_version} = 3; + + # Detect legacy ci.yml alongside v3 config + my $ci_yml = $self->path('ci.yml'); + if (-f $ci_yml && _is_legacy_ci_file($ci_yml)) { + if ($self->config->get('ci.enabled') && $self->config->has('ci.provider.type')) { + bail( + "Legacy #C{ci.yml} conflicts with the v3 CI configuration.\n". + "Remove #C{ci.yml} — CI is already configured in #C{.genesis/config}." + ); + } else { + bail( + "Legacy CI configuration detected at #C{ci.yml}.\n". + "Pipeline support requires migrating this into the v3 config. Please run:\n\n". + " genesis repo-init --upgrade\n\n". + "to migrate your CI configuration into the v3 config format." + ); + } + } # Delegate validation of registered sections to their owning modules for my $section (sort keys %_config_section_handlers) { @@ -1279,8 +1339,8 @@ sub _upgrade_config_to_v2 { } # }}} -# _repo_config_schema - return the repository configuration validation schema {{{ -sub _repo_config_schema { +# _repo_config_schema_v2 - v2 configuration validation schema {{{ +sub _repo_config_schema_v2 { my ($self) = @_; return { deployment_type => { @@ -1360,13 +1420,62 @@ sub _repo_config_schema { envvar => 'GENESIS_CONFIRM_RELEASE_OVERRIDES', description => 'Confirm release overrides' }, + }; +} + +# }}} +# _repo_config_schema - v3 configuration validation schema (superset of v2) {{{ +sub _repo_config_schema { + my ($self) = @_; + return { + %{$self->_repo_config_schema_v2()}, + version => { + type => '"3"', + required => 1, + description => 'Configuration schema version' + }, ci => { - type => 'opaque', - description => 'CI pipeline configuration (owned by Genesis::CI::Compiler)', + type => 'hash', + description => 'CI pipeline configuration', + schema => { + enabled => {type => 'boolean', default => Genesis::Config::FALSE, description => 'Whether CI pipeline is active'}, + provider => { + type => 'hash', + required => 'enabled', + description => 'CI provider connection details', + schema => { + type => {type => 'enum', values => ['concourse', 'gha', 'manual'], description => 'CI system type'}, + target => {type => 'string', description => 'Provider target name (e.g., fly target)'}, + url => {type => 'string', description => 'Provider API URL'}, + team => {type => 'string', description => 'Provider team/org'}, + insecure => {type => 'boolean', default => Genesis::Config::FALSE, description => 'Skip TLS verification'}, + } + }, + pipeline => { + type => 'hash', + description => 'Pipeline generation settings', + schema => { + name => {type => 'string', description => 'Pipeline name (defaults to deployment_type)'}, + } + }, + } }, }; } +# }}} +# _is_legacy_ci_file - detect whether ci.yml is a pipeline config (not an env file) {{{ +sub _is_legacy_ci_file { + my ($path) = @_; + open my $fh, '<', $path or return 0; + while (<$fh>) { + return 1 if /^pipeline:/; + return 0 if /^kit:/ || /^genesis:/; + } + close $fh; + return 0; +} + # }}} 1; diff --git a/t/unit-tests/genesis_commands_repo_ci-core.t b/t/unit-tests/genesis_commands_repo_ci-core.t index 0bec1ef8..b9ecd9b7 100644 --- a/t/unit-tests/genesis_commands_repo_ci-core.t +++ b/t/unit-tests/genesis_commands_repo_ci-core.t @@ -8,6 +8,7 @@ use helper; use Genesis; use Genesis::Config; use_ok 'Genesis::Commands::Repo'; +use_ok 'Genesis::Top'; my $tmp = workdir(); @@ -230,4 +231,171 @@ YAML is $git_branch, 'trunk', "branch extracted"; }; +### Config v3 validation tests ################################################ + +$Genesis::RC = Genesis::Config->new("$ENV{HOME}/.genesis/config"); + +sub make_v3_repo { + my ($dir, %opts) = @_; + my $genesis_dir = "$dir/.genesis"; + mkdir_or_fail($genesis_dir); + + my $version = $opts{version} // 3; + my $content = "---\ncreator_version: 3.2.0\ndeployment_type: test-kit\nversion: $version\n"; + if ($opts{ci}) { + $content .= "ci:\n"; + for my $key (sort keys %{$opts{ci}}) { + my $val = $opts{ci}{$key}; + if (ref($val) eq 'HASH') { + $content .= " $key:\n"; + for my $subkey (sort keys %$val) { + $content .= " $subkey: $val->{$subkey}\n"; + } + } else { + $content .= " $key: $val\n"; + } + } + } + mkfile_or_fail("$genesis_dir/config", $content); + return $dir; +} + +subtest 'v2 config loads and augments ci.enabled default' => sub { + my $dir = make_v3_repo(workdir("v2-augment"), version => 2); + + my $top = Genesis::Top->new($dir, no_vault => 1); + ok !$top->ci_enabled, "ci_enabled is false for v2 config"; + ok !$top->ci_configured, "ci_configured is false for v2 config"; + is $top->config->get('ci.enabled'), 0, "ci.enabled defaults to false"; + is $top->config->get_source('ci'), 'default', "ci section comes from default layer"; +}; + +subtest 'v2 config with ci.yml bails' => sub { + my $dir = make_v3_repo(workdir("v2-ci-yml"), version => 2); + mkfile_or_fail("$dir/ci.yml", "---\npipeline:\n layouts:\n - sandbox\n"); + + my $top = Genesis::Top->new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/Legacy CI configuration/, "v2 + ci.yml bails with migration message"; + like $@, qr/repo-init --upgrade/, "bail message mentions upgrade command"; +}; + +subtest 'v3 config validates with CI disabled' => sub { + my $dir = make_v3_repo(workdir("v3-disabled"), ci => { enabled => 'false' }); + + my $top = Genesis::Top->new($dir, no_vault => 1); + ok !$top->ci_enabled, "ci_enabled is false"; + ok !$top->ci_configured, "ci_configured is false"; +}; + +subtest 'v3 config validates with CI enabled and provider' => sub { + my $dir = make_v3_repo(workdir("v3-enabled"), ci => { + enabled => 'true', + provider => { type => 'concourse', target => 'pipes/lmelt', url => 'https://pipes.example.com', team => 'lmelt' }, + pipeline => { name => 'bosh' }, + }); + + my $top = Genesis::Top->new($dir, no_vault => 1); + ok $top->ci_enabled, "ci_enabled is true"; + ok $top->ci_configured, "ci_configured is true"; + is $top->config->get('ci.provider.type'), 'concourse', "provider type is concourse"; + is $top->config->get('ci.provider.target'), 'pipes/lmelt', "provider target correct"; + is $top->config->get('ci.pipeline.name'), 'bosh', "pipeline name correct"; +}; + +subtest 'v3 config bails when enabled but no provider' => sub { + my $dir = make_v3_repo(workdir("v3-no-provider"), ci => { enabled => 'true' }); + + my $top = Genesis::Top->new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/missing required key/, "bails when enabled without provider"; + like $@, qr/provider/, "bail message references provider"; +}; + +subtest 'v3 config with ci.yml and CI configured bails with conflict' => sub { + my $dir = make_v3_repo(workdir("v3-conflict"), ci => { + enabled => 'true', + provider => { type => 'concourse', target => 'pipes/test', url => 'https://ci.example.com', team => 'test' }, + }); + mkfile_or_fail("$dir/ci.yml", "---\npipeline:\n layouts:\n - sandbox\n"); + + my $top = Genesis::Top->new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/conflicts/, "v3 + ci.yml + configured CI bails with conflict message"; +}; + +subtest 'v3 config with ci.yml and CI not configured bails with migrate' => sub { + my $dir = make_v3_repo(workdir("v3-migrate"), ci => { enabled => 'false' }); + mkfile_or_fail("$dir/ci.yml", "---\npipeline:\n layouts:\n - sandbox\n"); + + my $top = Genesis::Top->new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/Legacy CI configuration/, "v3 + ci.yml + disabled CI bails with migrate message"; + like $@, qr/repo-init --upgrade/, "mentions upgrade command"; +}; + +subtest 'v3 config rejects unknown ci keys' => sub { + my $dir = workdir("v3-unknown-key"); + mkdir_or_fail("$dir/.genesis"); + mkfile_or_fail("$dir/.genesis/config", <new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/unknown configuration key/, "rejects unknown key in ci section"; + like $@, qr/bogus_key/, "error mentions the offending key"; +}; + +subtest 'v3 config rejects invalid provider type' => sub { + my $dir = workdir("v3-bad-provider"); + mkdir_or_fail("$dir/.genesis"); + mkfile_or_fail("$dir/.genesis/config", <new($dir, no_vault => 1); + eval { $top->config }; + like $@, qr/jenkins|expected/, "rejects invalid provider type enum value"; +}; + +subtest 'v2 config write-back does not persist ci defaults' => sub { + my $dir = make_v3_repo(workdir("v2-writeback"), version => 2); + + my $top = Genesis::Top->new($dir, no_vault => 1); + # ci.enabled should be accessible + is $top->config->get('ci.enabled'), 0, "ci.enabled is available via get"; + + # But _explicit_contents (what gets saved) should NOT have ci + my $explicit = $top->config->_explicit_contents; + ok !exists $explicit->{ci}, "ci section not in explicit contents (won't persist)"; +}; + +subtest 'new repos created with LATEST_CONFIG_VERSION' => sub { + is Genesis::Top::LATEST_CONFIG_VERSION(), 3, "LATEST_CONFIG_VERSION is 3"; +}; + +subtest 'ci_control_branch returns constant for MVP' => sub { + my $dir = make_v3_repo(workdir("v3-control"), ci => { + enabled => 'true', + provider => { type => 'concourse', target => 'pipes/test', url => 'https://ci.example.com', team => 'test' }, + }); + + my $top = Genesis::Top->new($dir, no_vault => 1); + is $top->ci_control_branch, 'control', "ci_control_branch returns 'control'"; +}; + done_testing; diff --git a/t/unit-tests/genesis_config-core.t b/t/unit-tests/genesis_config-core.t index e744a0da..a8ac3861 100644 --- a/t/unit-tests/genesis_config-core.t +++ b/t/unit-tests/genesis_config-core.t @@ -894,6 +894,56 @@ subtest 'validate() integer type accepts zero and rejects leading zeros' => sub ); }; +subtest 'conditional required (polymorphic)' => sub { + local $ENV{NOCOLOR} = 1; + my $schema = { + mode => {type => 'string', required => 1}, + target => {type => 'string', required => 'mode'}, + url => {type => 'string', required => {mode => 'remote'}}, + port => {type => 'number', required => {mode => 'remote'}}, + }; + + # mode=local, no target → fails (required => 'mode' means required when mode is truthy) + my $c1 = Genesis::Config->new(undef, 0, {mode => 'local'}); + throws_ok( + sub { $c1->validate($schema) }, + qr/target: missing required key/, + "required => 'sibling': fails when sibling is truthy and key missing" + ); + + # mode=local, target present, no url → ok (url only required when mode eq 'remote') + my $c2 = Genesis::Config->new(undef, 0, {mode => 'local', target => '/tmp'}); + lives_ok( + sub { $c2->validate($schema) }, + "required => {mode => 'remote'}: passes when mode is 'local'" + ); + + # mode=remote, target present, no url → fails + my $c3 = Genesis::Config->new(undef, 0, {mode => 'remote', target => 'srv1'}); + throws_ok( + sub { $c3->validate($schema) }, + qr/url: missing required key/, + "required => {mode => 'remote'}: fails when mode is 'remote' and key missing" + ); + + # mode=remote, all present → passes + my $c4 = Genesis::Config->new(undef, 0, {mode => 'remote', target => 'srv1', url => 'http://x', port => 8080}); + lives_ok( + sub { $c4->validate($schema) }, + "required => {mode => 'remote'}: passes when all keys present" + ); + + # required => 0 means not required + my $schema2 = { + name => {type => 'string', required => 0}, + }; + my $c5 = Genesis::Config->new(undef, 0, {}); + lives_ok( + sub { $c5->validate($schema2) }, + "required => 0 means not required" + ); +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu diff --git a/t/unit-tests/genesis_top-core.t b/t/unit-tests/genesis_top-core.t index e5c259e2..0241d112 100644 --- a/t/unit-tests/genesis_top-core.t +++ b/t/unit-tests/genesis_top-core.t @@ -199,7 +199,7 @@ subtest 'repo creation with no_vault' => sub { # Verify config contents my $config = $top->config->_explicit_contents; - is($config->{version}, 2, "config has version 2"); + is($config->{version}, 3, "config has version 3 (LATEST_CONFIG_VERSION)"); is($config->{deployment_type}, "testkit", "config has correct deployment_type"); is($config->{creator_version}, "3.1.0", "config has creator_version"); @@ -216,7 +216,7 @@ subtest 'repo creation with no_vault' => sub { # Verify the actual .genesis/config file matches expected structure yaml_is get_file("$tmp/.genesis/config"), < Date: Tue, 21 Apr 2026 12:09:45 -0700 Subject: [PATCH 035/103] Move add_secrets out of Env::create Secrets generation now runs in Commands::Env::create after pipeline metadata and git operations, so a failure does not prevent env file commit or branch creation. Failure is non-fatal with guidance to retry or defer. Also aligns pipeline YAML values at column 16 and uses notice logger for force-move messaging. --- lib/Genesis/Commands/Env.pm | 27 +++++++++++++++++++----- lib/Genesis/Env.pm | 42 +++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 44ebd3d3..3e6757d5 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -74,7 +74,7 @@ sub create { # Pipeline-aware repos require new environments to be created on the # control branch so the topology is visible to pipeline tooling and # the environment branch can be cut from the right point. - my $ci_configured = $top->config->has('ci.provider.type'); + my $ci_configured = $top->ci_configured; if ($ci_configured) { my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); @@ -99,7 +99,7 @@ sub create { # Phase C: write pipeline metadata when CI provider is configured. # Runs interactively when in a controlling terminal; honours --prior-env, # --require-pr, and --manual flags for non-interactive (scripted) use. - if ($top->config->has('ci.provider.type')) { + if ($ci_configured) { my $ci_type = $top->config->get('ci.provider.type') // 'unknown'; info( "\n#G{Pipeline configuration} (ci provider: #C{%s})\n", @@ -166,9 +166,9 @@ sub create { # Entrypoints (no prior_env) can still carry manual: true. if (length($prior_env // '') || $require_pr || $manual) { my $pipeline_yaml = " pipeline:\n"; - $pipeline_yaml .= " prior_env: $prior_env\n" if length($prior_env // ''); - $pipeline_yaml .= " require_pr: true\n" if $require_pr; - $pipeline_yaml .= " manual: true\n" if $manual; + $pipeline_yaml .= " prior_env: $prior_env\n" if length($prior_env // ''); + $pipeline_yaml .= " require_pr: true\n" if $require_pr; + $pipeline_yaml .= " manual: true\n" if $manual; my $file = $env->path($env->file); my $contents = slurp($file); @@ -230,6 +230,23 @@ sub create { } } + # Generate secrets. Non-fatal — the env file, pipeline metadata, and + # git branch are already in place; secrets can be retried later with + # `genesis add-secrets` or will be generated at deploy time. + my $secrets_ok = eval { $env->add_secrets(verbose => 1, import => 1) }; + if (!$secrets_ok) { + my $err = $@ || ''; + $err =~ s/\s+$//; + warning( + "Secret generation incomplete for #C{%s}.%s\n". + "Run #C{genesis add-secrets '%s'} to retry, or secrets will be\n". + "generated at deploy time.", + $name, + $err ? "\n$err" : '', + $name + ); + } + # let the user know if ($ci_configured && !$cli_opts_git{'no-commit'}) { info( diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index de74f51d..200015aa 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -351,7 +351,7 @@ sub create { my $old_path = "${env_path}.old"; rename($env_path, $old_path) or bail("Could not move existing %s to %s: %s", $env->{file}, "$env->{file}.old", $!); - info("Moved existing #C{%s} to #C{%s.old}", $env->{file}, $env->{file}); + notice("\nMoved existing #C{%s} to #C{%s.old}", $env->{file}, $env->{file}); } # Sanitize the vault descriptor, if present @@ -480,11 +480,6 @@ sub create { } } - if (! $env->add_secrets(verbose=>1, import => 1)) { - $env->remove_secrets(all => 1, 'no-prompt' => 1); - unlink $env->file; - return undef; - } return $env; } @@ -5660,8 +5655,17 @@ sub _cap_yaml_file { my $cap_file = $self->workpath("fin.yml"); my $now = strftime(EXODUS_TIME_FORMAT, gmtime()); - my $bosh_target = $self->use_create_env ? "~" : $self->bosh_env->{description}; - my $bosh_exodus_path = $self->use_create_env ? "~" : $self->bosh->exodus_path; + my ($bosh_target, $bosh_exodus_path); + if ($self->use_create_env) { + $bosh_target = "~"; + $bosh_exodus_path = "~"; + } else { + $bosh_target = $self->bosh_env->{description}; + # Prefer live director; fall back to bosh_env-derived path when + # the parent director hasn't been deployed yet. + my $bosh = eval { $self->bosh }; + $bosh_exodus_path = $bosh ? $bosh->exodus_path : $self->bosh_env->{exodus_path}; + } mkfile_or_fail($cap_file, 0644, <secrets_mount) =~ s#/?$#/#; + $mount = "${sm}exodus/"; + } + $mount =~ s#/?$#/#; + my $bosh_exodus_path = sprintf("%s%s/%s", $mount, $name, $dep_type || 'bosh'); + return wantarray ? ($name, $dep_type, $vault_url, $exodus_mount) : { - name => $name, - dep_type => $dep_type, - vault_url => $vault_url, - exodus_mount => $exodus_mount, - description => $bosh_env_description, + name => $name, + dep_type => $dep_type, + vault_url => $vault_url, + exodus_mount => $exodus_mount, + exodus_path => $bosh_exodus_path, + description => $bosh_env_description, }; } From 9d81500db19f53cfc598587b85c856589fa57e7e Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 21 Apr 2026 12:09:53 -0700 Subject: [PATCH 036/103] Use env vars for vault in ui_prompt_for Replace Top/Env object loading with GENESIS_SECRETS_BASE env var and vault rebind. Eliminates unnecessary BOSH director resolution during kit hook secret prompts. --- lib/Genesis/Commands/Utility.pm | 44 +++++++++++---------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/lib/Genesis/Commands/Utility.pm b/lib/Genesis/Commands/Utility.pm index ef9e5b2a..794c95d6 100644 --- a/lib/Genesis/Commands/Utility.pm +++ b/lib/Genesis/Commands/Utility.pm @@ -101,20 +101,15 @@ sub multi_select_prompt_handler { sub secret_line_prompt_handler { my ($prompt,%opts) = @_; my $secret = delete $opts{secret}; - my $env = delete $opts{env}; + my $secrets_base = delete $opts{secrets_base}; validate_prompt_opts("secret-line", \%opts, qw(echo)); - my $vault; - if ($env && $env->kit->feature_compatibility('2.7.0-rc4')) { - $secret = $env->secrets_base.$secret unless $secret =~ /^\//; - $vault = $env->vault; - } else { - $secret = "secret/$secret"; - require Service::Vault::Remote; - $vault = Service::Vault::Remote->current || Service::Vault::Remote->rebind(); - } - my ($path, $key) = split /:/, $secret; + require Service::Vault::Remote; + my $vault = Service::Vault::Remote->current || Service::Vault::Remote->rebind(); bail("No vault selected!") unless $vault; + + $secret = $secrets_base.$secret unless $secret =~ /^\//; + my ($path, $key) = split /:/, $secret; print "\n"; $vault->query( { interactive => 1, onfailure => "Failed to save data to #C{$secret} in vault" }, @@ -123,21 +118,16 @@ sub secret_line_prompt_handler { sub secret_block_prompt_handler { my ($prompt,%opts) = @_; my $secret = delete $opts{secret}; - my $env = delete $opts{env}; + my $secrets_base = delete $opts{secrets_base}; validate_prompt_opts("secret-block", \%opts, ()); my $file = mkfile_or_fail(workdir()."/param", prompt_for_block($prompt)); - my $vault; - if ($env && $env->kit->feature_compatibility('2.7.0-rc4')) { - $secret = $env->secrets_base.$secret unless $secret =~ /^\//; - $vault = $env->vault; - } else { - $secret = "secret/$secret"; - require Service::Vault::Remote; - $vault = Service::Vault::Remote->current || Service::Vault::Remote->rebind(); - } - my ($path, $key) = split /:/, $secret; + require Service::Vault::Remote; + my $vault = Service::Vault::Remote->current || Service::Vault::Remote->rebind(); bail("No vault selected!") unless $vault; + + $secret = $secrets_base.$secret unless $secret =~ /^\//; + my ($path, $key) = split /:/, $secret; print "\n"; $vault->query( { onfailure => "Failed to save data to #C{$secret} in vault" }, @@ -173,14 +163,8 @@ sub ui_prompt_for { my $use_vault = ($type =~ /^secret-/); if ($use_vault) { get_options->{secret} = $path; - eval { - require Genesis::Top; - get_options->{env} = Genesis::Top->new($ENV{GENESIS_ROOT})->load_env($ENV{GENESIS_ENVIRONMENT}); - }; - if ($@) { - debug "Failed in ui-prompt-for attempting to load environment:\n$@"; - bail "Cannot prompt for secrets outside a kit hook"; - } + get_options->{secrets_base} = $ENV{GENESIS_SECRETS_BASE} + || bail("Cannot prompt for secrets: GENESIS_SECRETS_BASE not set (not in a kit hook?)"); } bail( From 7e30ba92d66a7418156aa37a5b6ea25936128ce1 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 21 Apr 2026 12:10:04 -0700 Subject: [PATCH 037/103] Graceful fallback when parent BOSH undeployed Synthesize bosh exodus path from bosh_env metadata in _cap_yaml_file when the live director is unavailable. Suppress vault export warning for new environments. --- lib/Genesis/Env/Secrets/Store/Vault.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Genesis/Env/Secrets/Store/Vault.pm b/lib/Genesis/Env/Secrets/Store/Vault.pm index 8d34183b..7bd5857a 100644 --- a/lib/Genesis/Env/Secrets/Store/Vault.pm +++ b/lib/Genesis/Env/Secrets/Store/Vault.pm @@ -116,7 +116,8 @@ sub store_data { if (defined $data) { $self->{__data} = $data; } else { - warning("Vault export returned no data for %s", $self->base); + warning("Vault export returned no data for %s", $self->base) + if $self->env->exists; } } return $self->{__data} // {}; From 42c274b652c97956790634774e9a744962a1fc76 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 21 Apr 2026 12:10:23 -0700 Subject: [PATCH 038/103] Fix from_environment missing env arg Pass explicit undef for $env positional parameter in from_alias call, and skip deployment opt when undef. Add debug logging to from_alias and from_environment. --- lib/Service/BOSH/Director.pm | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/Service/BOSH/Director.pm b/lib/Service/BOSH/Director.pm index cb55352d..96f400ea 100644 --- a/lib/Service/BOSH/Director.pm +++ b/lib/Service/BOSH/Director.pm @@ -143,6 +143,7 @@ sub from_exodus { # }}} # from_alias - create a BOSH director object that uses a local config alias {{{ sub from_alias { + debug("from_alias called with %d args: [%s]", scalar(@_)-1, join(', ', map {defined($_) ? "'$_'" : 'undef'} @_[1..$#_])); my ($class, $alias, $env, %opts) = @_; my $config_home = $opts{config_home} || "$ENV{HOME}/.bosh/config"; @@ -177,6 +178,9 @@ sub from_environment { # # 2. We need to indicate if we want the self or parent BOSH director + debug("from_environment: BOSH_ALIAS=%s BOSH_ENVIRONMENT=%s BOSH_CLIENT=%s BOSH_DEPLOYMENT=%s", + $ENV{BOSH_ALIAS}//'(undef)', $ENV{BOSH_ENVIRONMENT}//'(undef)', + $ENV{BOSH_CLIENT}//'(undef)', $ENV{BOSH_DEPLOYMENT}//'(undef)'); if (is_valid_uri($ENV{BOSH_ENVIRONMENT}) && $ENV{BOSH_CLIENT}) { return $class->new( $ENV{BOSH_ALIAS}, @@ -188,7 +192,8 @@ sub from_environment { deployment => $ENV{BOSH_DEPLOYMENT} ); } else { - return $class->from_alias($ENV{BOSH_ALIAS} || $ENV{BOSH_ENVIRONMENT}, deployment => $ENV{BOSH_DEPLOYMENT}); + return $class->from_alias($ENV{BOSH_ALIAS} || $ENV{BOSH_ENVIRONMENT}, undef, + $ENV{BOSH_DEPLOYMENT} ? (deployment => $ENV{BOSH_DEPLOYMENT}) : ()); } } From 1c32f18d73b1ba4a4274c4a1c8192d47db92d9ae Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Tue, 21 Apr 2026 12:23:48 -0700 Subject: [PATCH 039/103] Drop legacy scaffold checks from CI validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ci: section is now validated by the v3 schema in Top.pm. Remove targets/integrations checks from Compiler::validate_config_section — only provider- specific delegation remains. --- lib/Genesis/CI/Compiler.pm | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index d240fdaa..d85ecabd 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -195,20 +195,7 @@ sub validate_config_section { bail("'ci' configuration in .genesis/config must be a hash") unless ref($data) eq 'HASH'; - bail("'ci.targets' is required and must be a non-empty hash") - unless ref($data->{targets}) eq 'HASH' && %{$data->{targets}}; - - bail("'ci.integrations' is required and must be a hash") - unless ref($data->{integrations}) eq 'HASH'; - - bail("'ci.integrations.vault' must be a hash if present") - if defined $data->{integrations}{vault} - && ref($data->{integrations}{vault}) ne 'HASH'; - - bail("'ci.integrations.source_control' is required and must be a hash") - unless ref($data->{integrations}{source_control}) eq 'HASH'; - - # Validate ci.provider: section against the provider's own schema + # Validate ci.provider section against the provider's own schema if (my $provider_data = $data->{provider}) { bail("'ci.provider' must be a hash") unless ref($provider_data) eq 'HASH'; From 052b25c85b84140652499709d3fe7ad9ad182c4f Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:35:13 -0400 Subject: [PATCH 040/103] Update Concourse.pm --- .../CI/Compiler/Providers/Concourse.pm | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index e733aa46..61b2151b 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -17,6 +17,7 @@ use constant { DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top DEFAULT_EXPOSE => 0, DEFAULT_PAUSE => 0, + DEFAULT_INSECURE => 0, }; # }}} @@ -101,6 +102,7 @@ sub cli_opts { ci-pipeline-name=s ci-pause ci-expose + ci-insecure /; } @@ -135,6 +137,11 @@ sub cli_opts_help { Run fly expose-pipeline after setting, making the pipeline publicly viewable without authentication. Useful for open-source pipelines. + --ci-insecure (optional, default: false) + Skip TLS certificate verification when communicating with Concourse. + Passes --skip-ssl-validation (-k) to all fly commands. Use when the + Concourse server uses a self-signed or otherwise untrusted certificate. + EOF } @@ -175,6 +182,11 @@ sub provider_options_schema { default => DEFAULT_PAUSE, description => 'Leave pipeline paused after fly set-pipeline', }, + insecure => { + type => 'boolean', + default => DEFAULT_INSECURE, + description => 'Skip TLS certificate verification (fly --skip-ssl-validation)', + }, }; } @@ -182,9 +194,10 @@ sub provider_options_schema { # provider_options_defaults - default values for all Concourse options {{{ sub provider_options_defaults { return { - team => DEFAULT_TEAM, - expose => DEFAULT_EXPOSE, - pause => DEFAULT_PAUSE, + team => DEFAULT_TEAM, + expose => DEFAULT_EXPOSE, + pause => DEFAULT_PAUSE, + insecure => DEFAULT_INSECURE, }; } @@ -199,18 +212,20 @@ sub describe_provider { my $target = $self->provider_option('target') || '(not set)'; my $team = $self->provider_option('team') || DEFAULT_TEAM; my $pipe_name = $self->provider_option('pipeline_name') || '(deployment type)'; - my $expose = $self->provider_option('expose') ? 'yes' : 'no'; - my $paused = $self->provider_option('pause') ? 'yes' : 'no'; + my $expose = $self->provider_option('expose') ? 'yes' : 'no'; + my $paused = $self->provider_option('pause') ? 'yes' : 'no'; + my $insecure = $self->provider_option('insecure') ? 'yes' : 'no'; return ( type => 'concourse', label => 'Concourse', - extras => [qw(Target Team Pipeline Expose PauseAfterSet)], + extras => [qw(Target Team Pipeline Expose PauseAfterSet Insecure)], Target => $target, Team => $team, Pipeline => $pipe_name, Expose => $expose, PauseAfterSet => $paused, + Insecure => $insecure, status => 'ok', ); } From fb04caee30ee80bd877704c2e281f91bae631d9c Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:35:29 -0400 Subject: [PATCH 041/103] Update Concourse.pm --- .../CI/Compiler/Providers/Concourse.pm | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 61b2151b..07cdd775 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -346,15 +346,18 @@ sub deploy { bail("Cannot determine pipeline name — set ci.provider.pipeline_name or ensure deployment_type is set") unless $pipeline_name; - # Pause/expose/dry-run: call-site override > provider_opts > defaults - my $dry_run = $opts{'dry-run'}; - my $yes = $opts{yes}; - my $pause = $opts{pause} + # Pause/expose/dry-run/insecure: call-site override > provider_opts > defaults + my $dry_run = $opts{'dry-run'}; + my $yes = $opts{yes}; + my $pause = $opts{pause} // $self->provider_option('pause') // DEFAULT_PAUSE; - my $expose = $opts{expose} + my $expose = $opts{expose} // $self->provider_option('expose') // _yaml_bool(($self->{config}{pipeline} || {})->{public}, DEFAULT_EXPOSE); + my $insecure = $opts{insecure} + // $self->provider_option('insecure') + // DEFAULT_INSECURE; my $yaml = $self->generate(); @@ -363,9 +366,11 @@ sub deploy { return; } + my $k_flag = $insecure ? ' -k' : ''; + # Pause pipeline before updating (safe to do even when not found yet) my ($out, $rc) = run( - 'fly -t $1 pause-pipeline -p $2', + "fly${k_flag} -t \$1 pause-pipeline -p \$2", $target, $pipeline_name ); bail("Could not pause pipeline '%s': %s", $pipeline_name, $out) @@ -376,13 +381,13 @@ sub deploy { mkfile_or_fail("${dir}/pipeline.yml", $yaml); # Upload pipeline - my $yes_flag = $yes ? '-n' : ''; + my $yes_flag = $yes ? ' -n' : ''; my $team_flag = " --team=$team"; run({ interactive => 1, onfailure => "Could not upload pipeline $pipeline_name", }, - "fly -t \$1 set-pipeline${yes_flag}${team_flag} -p \$2 -c \$3/pipeline.yml", + "fly${k_flag} -t \$1 set-pipeline${yes_flag}${team_flag} -p \$2 -c \$3/pipeline.yml", $target, $pipeline_name, $dir ); @@ -392,7 +397,7 @@ sub deploy { interactive => 1, onfailure => "Could not unpause pipeline $pipeline_name", }, - 'fly -t $1 unpause-pipeline -p $2', + "fly${k_flag} -t \$1 unpause-pipeline -p \$2", $target, $pipeline_name ); } @@ -403,7 +408,7 @@ sub deploy { interactive => 1, onfailure => "Could not $action pipeline $pipeline_name", }, - "fly -t \$1 ${action}-pipeline -p \$2", + "fly${k_flag} -t \$1 ${action}-pipeline -p \$2", $target, $pipeline_name ); From 896d8f02e09e8cc585eabc724ee378fd2415b475 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 09:41:08 -0700 Subject: [PATCH 042/103] Add url option to Concourse provider schema [Improvements] - Add `url` field to Concourse provider options schema for documenting the Concourse API URL used by `fly login` --- lib/Genesis/CI/Compiler/Providers/Concourse.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 07cdd775..0c6913fd 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -163,6 +163,10 @@ sub provider_options_schema { type => 'string', description => 'Fly target alias (fly login -t )', }, + url => { + type => 'string', + description => 'Concourse API URL (informational; used by fly login)', + }, team => { type => 'string', default => DEFAULT_TEAM, From 08f898c59ff2b7796eb07626ea5f57d58b057542 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:29:53 -0400 Subject: [PATCH 043/103] Move/refactor pipeline commands into Pipelines module Remove the legacy Genesis::Commands::Pipeline module and consolidate/refactor pipeline-related commands into Genesis::Commands::Pipelines. Update bin/genesis to register commands against the new Pipelines handlers. Introduce a unified _compile_pipeline helper and modernized implementations for apply, pipeline_graph, pipeline_describe, diff, status, pause, resume (including provider-driven deploy flow, provider CLI opts parsing, and debug-artifact dumping). Keep deprecated aliases (repipe, graph, describe) delegating to the new implementations. Small adjustments: update a Concourse comment reference, tweak CI task code paths and some minor output formatting/cleanup. --- bin/genesis | 14 +- .../CI/Compiler/Providers/Concourse.pm | 2 +- lib/Genesis/Commands/Pipeline.pm | 597 -------------- lib/Genesis/Commands/Pipelines.pm | 764 +++++++++++------- t/ci-pipeline.t | 44 +- 5 files changed, 481 insertions(+), 940 deletions(-) delete mode 100644 lib/Genesis/Commands/Pipeline.pm diff --git a/bin/genesis b/bin/genesis index 450ffa69..3840a8bb 100755 --- a/bin/genesis +++ b/bin/genesis @@ -2021,7 +2021,7 @@ define_command("pipeline-apply", { 'debug-dir=s' => "Write intermediate compiler artifacts to this directory.", ], -}, 'Genesis::Commands::Pipeline::apply'); +}, 'Genesis::Commands::Pipelines::apply'); # }}} # genesis pipeline-graph - write Mermaid pipeline.md {{{ define_command("pipeline-graph", { @@ -2040,7 +2040,7 @@ define_command("pipeline-graph", { 'platform|provider|p=s' => "CI provider: 'concourse' (default) or 'github-actions'.", ], -}, 'Genesis::Commands::Pipeline::graph'); +}, 'Genesis::Commands::Pipelines::pipeline_graph'); # }}} # genesis pipeline-describe - human-readable pipeline description {{{ define_command("pipeline-describe", { @@ -2059,7 +2059,7 @@ define_command("pipeline-describe", { 'platform|provider|p=s' => "CI provider: 'concourse' (default) or 'github-actions'.", ], -}, 'Genesis::Commands::Pipeline::describe'); +}, 'Genesis::Commands::Pipelines::pipeline_describe'); # }}} # genesis pipeline-diff - show compiled vs live pipeline delta {{{ define_command("pipeline-diff", { @@ -2085,7 +2085,7 @@ define_command("pipeline-diff", { 'skip-vault' => "Skip vault connectivity when compiling.", ], -}, 'Genesis::Commands::Pipeline::diff'); +}, 'Genesis::Commands::Pipelines::diff'); # }}} # genesis pipeline-status - show per-environment job health {{{ define_command("pipeline-status", { @@ -2113,7 +2113,7 @@ define_command("pipeline-status", { 'skip-vault' => "Skip vault connectivity when compiling.", ], -}, 'Genesis::Commands::Pipeline::status'); +}, 'Genesis::Commands::Pipelines::status'); # }}} # genesis pipeline-pause - pause an environment job or entire pipeline {{{ define_command("pipeline-pause", { @@ -2141,7 +2141,7 @@ define_command("pipeline-pause", { 'skip-vault' => "Skip vault connectivity when compiling.", ], -}, 'Genesis::Commands::Pipeline::pause'); +}, 'Genesis::Commands::Pipelines::pause'); # }}} # genesis pipeline-resume - resume an environment job or entire pipeline {{{ define_command("pipeline-resume", { @@ -2169,7 +2169,7 @@ define_command("pipeline-resume", { 'skip-vault' => "Skip vault connectivity when compiling.", ], -}, 'Genesis::Commands::Pipeline::resume'); +}, 'Genesis::Commands::Pipelines::resume'); # }}} # }}} diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index 0c6913fd..b73a29ac 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -546,7 +546,7 @@ sub graph_md { } # }}} -# generate_description - alias for describe(); called by Genesis::Commands::Pipeline {{{ +# generate_description - alias for describe(); called by Genesis::Commands::Pipelines {{{ sub generate_description { $_[0]->describe() } # }}} diff --git a/lib/Genesis/Commands/Pipeline.pm b/lib/Genesis/Commands/Pipeline.pm deleted file mode 100644 index 89cbe2e6..00000000 --- a/lib/Genesis/Commands/Pipeline.pm +++ /dev/null @@ -1,597 +0,0 @@ -package Genesis::Commands::Pipeline; -use strict; -use warnings; - -use Genesis; -use Genesis::Commands; -use Genesis::Top; -use Genesis::CI::Compiler; -use JSON::PP; -use File::Basename qw/dirname/; - -### Public Command Functions {{{ - -# apply - compile and deploy pipeline (replaces genesis repipe) {{{ -sub apply { - my ($layout) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - - bail("--output-dir requires --platform") - if $opts->{'output-dir'} && !$opts->{platform}; - bail("--debug-dir requires --platform") - if $opts->{'debug-dir'} && !$opts->{platform}; - - my $top = _get_top($opts); - my $result = _compile($top, $platform, $opts); - - _dump_debug_artifacts($opts->{'debug-dir'}, $result, $platform) - if $opts->{'debug-dir'}; - - my $ast = $result->{ast}; - my $output = $result->{output}; - - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - - # --output-dir: write artifacts to directory and exit - if (my $out_dir = $opts->{'output-dir'}) { - mkdir_or_fail($out_dir); - for my $file (sort keys %$output) { - mkfile_or_fail("$out_dir/$file", $output->{$file}); - info("Wrote #C{%s/%s}", $out_dir, $file); - } - mkfile_or_fail("$out_dir/ast.json", - JSON::PP->new->pretty->canonical->encode({%$ast})); - info("Wrote #C{%s/ast.json}", $out_dir); - exit 0; - } - - if ($platform eq 'concourse') { - my $provider = $result->{provider}; - - if ($opts->{'dry-run'}) { - my $yaml = $output->{'pipeline.yml'} - or bail("Concourse provider did not produce pipeline.yml"); - output({raw => 1}, $yaml); - exit 0; - } - - $provider->check_prereqs() or exit 86; - - # Delegate all fly operations to the Concourse provider. Pass generic - # command-level flags using plain keys; deploy() accepts both plain and - # ci-* prefixed forms so neither layer has to know the other's naming. - $provider->deploy( - target => $opts->{target} || $layout || $name, - pause => $opts->{paused}, - yes => $opts->{yes}, - ); - - } elsif ($platform eq 'github-actions') { - if ($opts->{'dry-run'}) { - for my $file (sort keys %$output) { - output "#G{--- %s ---}", $file; - output({raw => 1}, $output->{$file}); - } - exit 0; - } - for my $file (sort keys %$output) { - my $path = ".github/workflows/$file"; - mkdir_or_fail(dirname($path)); - mkfile_or_fail($path, $output->{$file}); - info("Wrote #C{%s}", $path); - } - info("GitHub Actions workflows written. Commit and push to activate."); - - } else { - bail("Unsupported platform '%s' for pipeline apply", $platform); - } - - exit 0; -} - -# }}} -# graph - write pipeline.md with Mermaid flowchart (replaces genesis graph) {{{ -sub graph { - my ($layout) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - my $top = Genesis::Top->new('.'); - - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $provider = $result->{provider}; - - my $md; - if ($provider->can('graph_md')) { - $md = $provider->graph_md(); - } else { - $md = _mermaid_md($ast); - } - - mkfile_or_fail('pipeline.md', $md); - info("Wrote #C{pipeline.md}"); - exit 0; -} - -# }}} -# describe - human-readable pipeline progression (replaces genesis describe) {{{ -sub describe { - my ($layout) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - my $top = Genesis::Top->new('.'); - - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $provider = $result->{provider}; - - if ($provider->can('generate_description')) { - $provider->generate_description($ast); - } else { - _print_description($ast, $platform); - } - exit 0; -} - -# }}} -# diff - show compiled vs live pipeline delta {{{ -sub diff { - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - - bail("diff is only supported for the 'concourse' platform") - unless $platform eq 'concourse'; - - my $top = _get_top($opts, skip_vault => 1); - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $output = $result->{output}; - - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; - - my $compiled = $output->{'pipeline.yml'} - or bail("Concourse provider did not produce pipeline.yml"); - - my $dir = workdir; - mkfile_or_fail("$dir/compiled.yml", $compiled); - - my ($live, $rc) = run('fly -t $1 get-pipeline -p $2', $target, $name); - if ($rc != 0) { - info("#Y{Pipeline '%s' does not exist on target '%s' — nothing to diff against.}", - $name, $target); - info("Run #C{genesis pipeline-apply} to deploy it first."); - exit 0; - } - mkfile_or_fail("$dir/live.yml", $live); - - my ($diff_out, $diff_rc) = run( - 'diff -u --label live --label compiled $1 $2', - "$dir/live.yml", "$dir/compiled.yml" - ); - - if ($diff_rc == 0) { - info("#G{No differences} — compiled pipeline matches live pipeline."); - } else { - output({raw => 1}, $diff_out); - } - exit 0; -} - -# }}} -# status - show per-env job health {{{ -sub status { - my ($filter_env) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - - bail("status is only supported for the 'concourse' platform") - unless $platform eq 'concourse'; - - my $top = _get_top($opts, skip_vault => 1); - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; - - my ($json_out, $rc) = run('fly -t $1 jobs -p $2 --json', $target, $name); - bail("Could not get jobs for pipeline '%s' on target '%s': %s", $name, $target, $json_out) - unless $rc == 0; - - my $jobs; - eval { $jobs = JSON::PP->new->decode($json_out) }; - bail("Failed to parse fly jobs output: %s", $@) if $@; - - output "#G{Pipeline}: #C{%s} (#Yi{target}: %s)", $name, $target; - output ""; - - # Index jobs by name for ordered lookup - my %job_by_name = map { $_->{name} => $_ } @$jobs; - - # Use AST pipeline order when available; fall back to alphabetical - my @ordered_names; - for my $wf_name ($ast->workflow_names) { - my @stage_order = eval { $ast->workflow_stage_order($wf_name) }; - if (@stage_order) { - my $nodes = ($ast->workflows->{$wf_name} || {})->{graph}{nodes} || {}; - push @ordered_names, map { $nodes->{$_}{alias} || $_ } @stage_order; - } - } - # Append any jobs from fly that didn't appear in the AST (e.g. update-pipeline) - my %seen = map { $_ => 1 } @ordered_names; - push @ordered_names, sort grep { !$seen{$_} } keys %job_by_name; - - my $col_w = 40; - output " %-${col_w}s %-10s %s", "Environment", "Status", "Notes"; - output " %s %s %s", '-' x $col_w, '-' x 10, '-' x 20; - - for my $job_name (@ordered_names) { - next if $filter_env && $job_name ne $filter_env; - my $job = $job_by_name{$job_name} or next; - - my $status = _job_status_label($job); - my @notes; - push @notes, 'paused' if $job->{paused}; - push @notes, 'errored' if ($job->{finished_build} || {})->{status} eq 'errored'; - - output " %-${col_w}s %-10s %s", - $job_name, - $status, - join(', ', @notes) || ''; - } - output ""; - exit 0; -} - -# }}} -# pause - pause env job or entire pipeline {{{ -sub pause { - my ($env) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - - bail("pause is only supported for the 'concourse' platform") - unless $platform eq 'concourse'; - - my $top = _get_top($opts, skip_vault => 1); - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; - - if ($env) { - run({ interactive => 1, - onfailure => "Could not pause job '$env' in pipeline '$name'" }, - 'fly -t $1 pause-job -p $2 -j $3', - $target, $name, $env); - info("Paused job #C{%s} in pipeline #C{%s}", $env, $name); - } else { - run({ interactive => 1, - onfailure => "Could not pause pipeline '$name'" }, - 'fly -t $1 pause-pipeline -p $2', - $target, $name); - info("Paused pipeline #C{%s}", $name); - } - exit 0; -} - -# }}} -# resume - resume env job or entire pipeline {{{ -sub resume { - my ($env) = @_; - option_defaults(config => 'ci.yml'); - - my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; - - bail("resume is only supported for the 'concourse' platform") - unless $platform eq 'concourse'; - - my $top = _get_top($opts, skip_vault => 1); - my $result = _compile($top, $platform, $opts); - my $ast = $result->{ast}; - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; - - if ($env) { - run({ interactive => 1, - onfailure => "Could not resume job '$env' in pipeline '$name'" }, - 'fly -t $1 unpause-job -p $2 -j $3', - $target, $name, $env); - info("Resumed job #C{%s} in pipeline #C{%s}", $env, $name); - } else { - run({ interactive => 1, - onfailure => "Could not resume pipeline '$name'" }, - 'fly -t $1 unpause-pipeline -p $2', - $target, $name); - info("Resumed pipeline #C{%s}", $name); - } - exit 0; -} - -# }}} -# }}} -### Internal Helpers {{{ - -# _compile - detect config source and run the compiler {{{ -sub _compile { - my ($top, $platform, $opts) = @_; - - my %compiler_opts = (top => $top); - - my $ci_dir = '.genesis/ci'; - if (-d $ci_dir && -f "$ci_dir/pipeline.yml") { - $compiler_opts{ci_dir} = $ci_dir; - info("Using multi-file configuration from #C{%s/}", $ci_dir); - } elsif (Genesis::CI::Compiler->can_compile_from_env_files($ci_dir)) { - $compiler_opts{ci_dir} = $ci_dir; - $compiler_opts{env_dir} = '.'; - info("Using env-file topology with config from #C{%s/}", $ci_dir); - } else { - $compiler_opts{file} = $opts->{config} || 'ci.yml'; - info("Using legacy configuration from #C{%s}", $compiler_opts{file}); - } - - my $compiler = Genesis::CI::Compiler->new(%compiler_opts); - return $compiler->compile(provider => $platform); -} - -# }}} -# _get_top - create a Genesis::Top object, with or without vault {{{ -sub _get_top { - my ($opts, %defaults) = @_; - - my $skip = $opts->{'skip-vault'} || $defaults{skip_vault}; - if ($skip) { - return Genesis::Top->new('.'); - } - - my $top = Genesis::Top->new('.', vault => $opts->{vault}); - bail( - "No vault specified or configured.\n". - "Use --skip-vault to compile without vault access." - ) unless $top->vault; - return $top; -} - -# }}} -# _mermaid_md - generate Mermaid pipeline.md from a bare AST {{{ -sub _mermaid_md { - my ($ast) = @_; - - my $name = $ast->metadata->{name} || 'genesis-pipeline'; - my @lines = ('flowchart LR'); - - for my $wf_name ($ast->workflow_names) { - my $wf = $ast->workflows->{$wf_name}; - next unless $wf->{graph}; - - my $nodes = $wf->{graph}{nodes} || {}; - my $edges = $wf->{graph}{edges} || []; - - my %in_any_edge; - for my $edge (@$edges) { - $in_any_edge{$edge->{from}} = 1; - $in_any_edge{$edge->{to}} = 1; - } - - for my $edge (@$edges) { - my $from = $nodes->{$edge->{from}}{alias} || $edge->{from}; - my $to = $nodes->{$edge->{to}}{alias} || $edge->{to}; - ($from) =~ s/[^a-zA-Z0-9_]/_/g; - ($to) =~ s/[^a-zA-Z0-9_]/_/g; - push @lines, " $from --> $to"; - } - - for my $n (sort keys %$nodes) { - next if $in_any_edge{$n}; - my $alias = $nodes->{$n}{alias} || $n; - $alias =~ s/[^a-zA-Z0-9_]/_/g; - push @lines, " $alias"; - } - } - - my $mermaid = join("\n", @lines) . "\n"; - return "# Pipeline: $name\n\n\`\`\`mermaid\n${mermaid}\`\`\`\n"; -} - -# }}} -# _print_description - human-readable AST description {{{ -sub _print_description { - my ($ast, $platform) = @_; - - output "#G{Pipeline}: #C{%s}", $ast->metadata->{name} || '(unnamed)'; - output " #Yi{Platform}: %s", $platform; - output " #Yi{Source}: %s", $ast->metadata->{source} || 'unknown'; - output ""; - - my $integrations = $ast->integrations || {}; - if (my $sc = $integrations->{source_control}) { - output "#G{Source Control}:"; - output " Provider: %s", $sc->{provider} || 'unknown'; - output " Repository: %s", $sc->{repository} || 'unknown'; - } - - my @targets = $ast->target_names; - if (@targets) { - output ""; - output "#G{Targets}: (%d)", scalar @targets; - output " - #C{%s}", $_ for sort @targets; - } - - my @workflows = $ast->workflow_names; - if (@workflows) { - output ""; - output "#G{Workflows}: (%d)", scalar @workflows; - for my $wf_name (sort @workflows) { - my $wf = $ast->workflows->{$wf_name}; - output " #Yi{%s} (%s)", $wf_name, $wf->{type} || 'deployment'; - - if ($wf->{graph} && $wf->{graph}{nodes}) { - my $nodes = $wf->{graph}{nodes}; - my @order = eval { $ast->workflow_stage_order($wf_name) }; - my @stages = @order - ? map { $nodes->{$_}{alias} || $_ } @order - : sort keys %$nodes; - output " Progression: %s", join(' -> ', @stages); - } - } - } - - output ""; -} - -# }}} -# _dump_debug_artifacts - write compiler intermediates to a directory {{{ -sub _dump_debug_artifacts { - my ($debug_dir, $result, $platform) = @_; - - mkdir_or_fail($debug_dir); - - my $json = JSON::PP->new->pretty->canonical; - - if ($result->{parsed}) { - mkfile_or_fail("$debug_dir/01-parsed.json", $json->encode($result->{parsed})); - info("Debug: wrote #C{%s/01-parsed.json}", $debug_dir); - } - - if (my $ast = $result->{ast}) { - my %source; - for my $key (qw(branches integrations targets workflows configuration - provider_config triggers resources)) { - my $accessor = $ast->can($key); - $source{$key} = $accessor->($ast) if $accessor; - } - $source{metadata} = $ast->metadata; - $source{scripts} = $ast->scripts; - - mkfile_or_fail("$debug_dir/02-ast-source.json", $json->encode(\%source)); - info("Debug: wrote #C{%s/02-ast-source.json}", $debug_dir); - - if ($ast->pipeline && %{$ast->pipeline}) { - my %pipeline = %{$ast->pipeline}; - my $pipeline_md = delete $pipeline{pipeline_md}; - my $description = delete $pipeline{description}; - delete $pipeline{mermaid}; - - mkfile_or_fail("$debug_dir/03-pipeline.json", $json->encode(\%pipeline)); - info("Debug: wrote #C{%s/03-pipeline.json}", $debug_dir); - - if ($pipeline_md) { - mkfile_or_fail("$debug_dir/04-pipeline.md", $pipeline_md); - info("Debug: wrote #C{%s/04-pipeline.md}", $debug_dir); - } - if ($description) { - mkfile_or_fail("$debug_dir/05-description.txt", $description); - info("Debug: wrote #C{%s/05-description.txt}", $debug_dir); - } - } - } - - if ($result->{output}) { - if (ref($result->{output}) eq 'HASH') { - for my $file (sort keys %{$result->{output}}) { - mkfile_or_fail("$debug_dir/06-output-$file", $result->{output}{$file}); - info("Debug: wrote #C{%s/06-output-%s}", $debug_dir, $file); - } - } else { - mkfile_or_fail("$debug_dir/06-output.yml", $result->{output}); - info("Debug: wrote #C{%s/06-output.yml}", $debug_dir); - } - } - - info("Debug artifacts written to #C{%s/}", $debug_dir); -} - -# }}} -# _job_status_label - derive a display status from a fly jobs JSON entry {{{ -sub _job_status_label { - my ($job) = @_; - return 'paused' if $job->{paused}; - my $fb = $job->{finished_build} || {}; - return $fb->{status} || 'pending'; -} - -# }}} -# }}} - -1; - -=head1 NAME - -Genesis::Commands::Pipeline - Pipeline management command suite - -=head1 DESCRIPTION - -Implements the C command family, replacing the legacy -C, C, and C commands with a unified interface. - -Commands are registered in C as flat names for 3.0.x -compatibility (C, C, etc.) -and the legacy C/C/C remain as aliases. - -=head1 COMMANDS - -=over 4 - -=item B [--platform PROVIDER] [--dry-run] [--paused] - -Compile and deploy the pipeline. Replaces C. -Defaults to Concourse. Supports C<--dry-run> (print YAML only) and -C<--output-dir> (write artifacts to a directory). - -=item B [--platform PROVIDER] - -Compile pipeline and write C containing a Mermaid flowchart. -Replaces C. - -=item B [--platform PROVIDER] - -Compile pipeline and print a human-readable ordered progression. -Replaces C. - -=item B [--target TARGET] - -Compare compiled pipeline YAML against the live pipeline via -C. Shows unified diff or reports no differences. - -=item B [] [--target TARGET] - -Query C for per-environment job status. Optionally filter to a -single environment. - -=item B [] [--target TARGET] - -Pause a specific environment's job, or the entire pipeline if no env given. - -=item B [] [--target TARGET] - -Resume a specific environment's job, or the entire pipeline if no env given. - -=back - -=head1 SEE ALSO - -Genesis::Commands::Pipelines (legacy), Genesis::CI::Compiler - -=cut - -# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 030de973..700a6bbc 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -16,95 +16,340 @@ use File::Basename qw/dirname/; use File::Path qw/rmtree/; use JSON::PP; +### Public Commands {{{ + +# embed - embed Genesis binary in the repository {{{ sub embed { command_usage(1) if @_; - - # FIXME: update .genesis/config with new version info Genesis::Top->new('.')->embed($ENV{GENESIS_CALLBACK_BIN} || $0); } -sub repipe { - warning("'genesis repipe' is deprecated and will be removed in a future version. Use 'genesis pipeline-apply' instead."); +# }}} +# apply - compile and deploy pipeline (replaces genesis repipe) {{{ +sub apply { + my ($layout) = @_; option_defaults(config => 'ci.yml'); - my $layout = $_[0]; - # Resolve --provider/--platform into a canonical platform value - my $platform = get_options->{platform}; + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; - bail("--output-dir requires --provider") - if get_options->{'output-dir'} && !$platform; + bail("--output-dir requires --platform") + if $opts->{'output-dir'} && !$opts->{platform}; + bail("--debug-dir requires --platform") + if $opts->{'debug-dir'} && !$opts->{platform}; - bail("--skip-vault requires --provider") - if get_options->{'skip-vault'} && !$platform; + my $top = _get_top($opts); + my $result = _compile_pipeline($top, $platform); - bail("--debug-dir requires --provider") - if get_options->{'debug-dir'} && !$platform; + _dump_debug_artifacts($opts->{'debug-dir'}, $result, $platform) + if $opts->{'debug-dir'}; - # New compiler pipeline when --provider/--platform is specified - if ($platform) { - my $top; - if (get_options->{'skip-vault'}) { - $top = Genesis::Top->new('.'); - } else { - $top = Genesis::Top->new('.', vault=>get_options->{vault}); - bail("No vault specified or configured.\n". - "Use --skip-vault with --platform to compile without vault access." - ) unless $top->vault; + my $ast = $result->{ast}; + my $output = $result->{output}; + my $name = $ast->metadata->{name} + or bail("Pipeline AST has no name defined"); + + if (my $out_dir = $opts->{'output-dir'}) { + mkdir_or_fail($out_dir); + for my $file (sort keys %$output) { + mkfile_or_fail("$out_dir/$file", $output->{$file}); + info("Wrote #C{%s/%s}", $out_dir, $file); } - return _repipe_compiled($top, $layout); + mkfile_or_fail("$out_dir/ast.json", + JSON::PP->new->pretty->canonical->encode({%$ast})); + info("Wrote #C{%s/ast.json}", $out_dir); + exit 0; } - my $top = Genesis::Top->new('.', vault=>get_options->{vault}); - bail( - "No vault specified or configured." - ) unless $top->vault; + if ($platform eq 'concourse') { + my $provider = $result->{provider}; - (my $pipeline, $layout) = Genesis::CI::Legacy::parse(get_options->{config}, $top, $layout); + if ($opts->{'dry-run'}) { + my $yaml = $output->{'pipeline.yml'} + or bail("Concourse provider did not produce pipeline.yml"); + output({raw => 1}, $yaml); + exit 0; + } + + $provider->check_prereqs() or exit 86; + + require Genesis::CI::Compiler::PipelineProvider; + my %deploy_opts = %{ Genesis::CI::Compiler::PipelineProvider->normalize_provider_opts( + $result->{provider_cli_opts} || {} + ) }; + $deploy_opts{target} //= $opts->{target} // $layout // $name; + $deploy_opts{pause} //= $opts->{paused}; + $provider->deploy(%deploy_opts, yes => $opts->{yes}); + + } elsif ($platform eq 'github-actions') { + if ($opts->{'dry-run'}) { + for my $file (sort keys %$output) { + output "#G{--- %s ---}", $file; + output({raw => 1}, $output->{$file}); + } + exit 0; + } + for my $file (sort keys %$output) { + my $path = ".github/workflows/$file"; + mkdir_or_fail(dirname($path)); + mkfile_or_fail($path, $output->{$file}); + info("Wrote #C{%s}", $path); + } + info("GitHub Actions workflows written. Commit and push to activate."); + + } else { + bail("Unsupported platform '%s' for pipeline apply", $platform); + } + + exit 0; +} + +# }}} +# pipeline_graph - write pipeline.md with Mermaid flowchart {{{ +sub pipeline_graph { + my ($layout) = @_; + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + my $top = Genesis::Top->new('.'); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $provider = $result->{provider}; + + my $md = $provider->can('graph_md') + ? $provider->graph_md() + : _ast_to_mermaid_md($ast); + + mkfile_or_fail('pipeline.md', $md); + info("Wrote #C{pipeline.md}"); + exit 0; +} + +# }}} +# pipeline_describe - human-readable pipeline progression {{{ +sub pipeline_describe { + my ($layout) = @_; + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + my $top = Genesis::Top->new('.'); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $provider = $result->{provider}; - option_defaults(target => $layout); - my $yaml = Genesis::CI::Legacy::generate_pipeline_concourse_yaml($pipeline, $top); - if (get_options->{'dry-run'}) { - output({raw => 1}, $yaml); + if ($provider->can('generate_description')) { + $provider->generate_description($ast); + } else { + _describe_ast($ast, $platform); + } + exit 0; +} + +# }}} +# diff - show compiled vs live pipeline delta {{{ +sub diff { + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + + bail("diff is only supported for the 'concourse' platform") + unless $platform eq 'concourse'; + + my $top = _get_top($opts, skip_vault => 1); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $output = $result->{output}; + + my $name = $ast->metadata->{name} + or bail("Pipeline AST has no name defined"); + my $target = $opts->{target} || $name; + + my $compiled = $output->{'pipeline.yml'} + or bail("Concourse provider did not produce pipeline.yml"); + + my $dir = workdir; + mkfile_or_fail("$dir/compiled.yml", $compiled); + + my ($live, $rc) = run('fly -t $1 get-pipeline -p $2', $target, $name); + if ($rc != 0) { + info("#Y{Pipeline '%s' does not exist on target '%s' — nothing to diff against.}", + $name, $target); + info("Run #C{genesis pipeline-apply} to deploy it first."); exit 0; } + mkfile_or_fail("$dir/live.yml", $live); - my ($out,$rc) = run( - 'fly -t $1 pause-pipeline -p $2', - get_options->{target}, $pipeline->{pipeline}{name} + my ($diff_out, $diff_rc) = run( + 'diff -u --label live --label compiled $1 $2', + "$dir/live.yml", "$dir/compiled.yml" ); - bail("Could not pause #C{%s} pipeline: $out", $pipeline->{pipeline}{name}) - unless $rc == 0 || $out =~ /pipeline '.*' not found/; - my $yes = get_options->{yes} ? ' -n ' : ''; - my $dir = workdir; - mkfile_or_fail("${dir}/pipeline.yml", $yaml); - run({ interactive => 1, onfailure => "Could not upload pipeline $pipeline->{pipeline}{name}" }, - 'fly -t $1 set-pipeline '.$yes.' -p $2 -c $3/pipeline.yml', - get_options->{target}, $pipeline->{pipeline}{name}, $dir); + if ($diff_rc == 0) { + info("#G{No differences} — compiled pipeline matches live pipeline."); + } else { + output({raw => 1}, $diff_out); + } + exit 0; +} - run( - { interactive => 1, onfailure => "Could not unpause pipeline $pipeline->{pipeline}{name}" }, - 'fly -t $1 unpause-pipeline -p $2', - get_options->{target}, $pipeline->{pipeline}{name} - ) unless (get_options->{paused}); +# }}} +# status - show per-env job health {{{ +sub status { + my ($filter_env) = @_; + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + + bail("status is only supported for the 'concourse' platform") + unless $platform eq 'concourse'; + + my $top = _get_top($opts, skip_vault => 1); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $name = $ast->metadata->{name} + or bail("Pipeline AST has no name defined"); + my $target = $opts->{target} || $name; + + my ($json_out, $rc) = run('fly -t $1 jobs -p $2 --json', $target, $name); + bail("Could not get jobs for pipeline '%s' on target '%s': %s", $name, $target, $json_out) + unless $rc == 0; + + my $jobs; + eval { $jobs = JSON::PP->new->decode($json_out) }; + bail("Failed to parse fly jobs output: %s", $@) if $@; + + output "#G{Pipeline}: #C{%s} (#Yi{target}: %s)", $name, $target; + output ""; - my $action = ($pipeline->{pipeline}{public} ? 'expose' : 'hide'); - run({ interactive => 1, onfailure => "Could not $action pipeline $pipeline->{pipeline}{name}" }, - 'fly -t $1 '.$action.'-pipeline -p $2', - get_options->{target}, $pipeline->{pipeline}{name}); + my %job_by_name = map { $_->{name} => $_ } @$jobs; + my @ordered_names; + for my $wf_name ($ast->workflow_names) { + my @stage_order = eval { $ast->workflow_stage_order($wf_name) }; + if (@stage_order) { + my $nodes = ($ast->workflows->{$wf_name} || {})->{graph}{nodes} || {}; + push @ordered_names, map { $nodes->{$_}{alias} || $_ } @stage_order; + } + } + my %seen = map { $_ => 1 } @ordered_names; + push @ordered_names, sort grep { !$seen{$_} } keys %job_by_name; + + my $col_w = 40; + output " %-${col_w}s %-10s %s", "Environment", "Status", "Notes"; + output " %s %s %s", '-' x $col_w, '-' x 10, '-' x 20; + + for my $job_name (@ordered_names) { + next if $filter_env && $job_name ne $filter_env; + my $job = $job_by_name{$job_name} or next; + + my $status = _job_status_label($job); + my @notes; + push @notes, 'paused' if $job->{paused}; + push @notes, 'errored' if ($job->{finished_build} || {})->{status} eq 'errored'; + + output " %-${col_w}s %-10s %s", + $job_name, + $status, + join(', ', @notes) || ''; + } + output ""; + exit 0; +} + +# }}} +# pause - pause env job or entire pipeline {{{ +sub pause { + my ($env) = @_; + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + + bail("pause is only supported for the 'concourse' platform") + unless $platform eq 'concourse'; + + my $top = _get_top($opts, skip_vault => 1); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $name = $ast->metadata->{name} + or bail("Pipeline AST has no name defined"); + my $target = $opts->{target} || $name; + + if ($env) { + run({ interactive => 1, + onfailure => "Could not pause job '$env' in pipeline '$name'" }, + 'fly -t $1 pause-job -p $2 -j $3', + $target, $name, $env); + info("Paused job #C{%s} in pipeline #C{%s}", $env, $name); + } else { + run({ interactive => 1, + onfailure => "Could not pause pipeline '$name'" }, + 'fly -t $1 pause-pipeline -p $2', + $target, $name); + info("Paused pipeline #C{%s}", $name); + } + exit 0; +} + +# }}} +# resume - resume env job or entire pipeline {{{ +sub resume { + my ($env) = @_; + option_defaults(config => 'ci.yml'); + + my $opts = get_options; + my $platform = $opts->{platform} || 'concourse'; + + bail("resume is only supported for the 'concourse' platform") + unless $platform eq 'concourse'; + + my $top = _get_top($opts, skip_vault => 1); + my $result = _compile_pipeline($top, $platform); + my $ast = $result->{ast}; + my $name = $ast->metadata->{name} + or bail("Pipeline AST has no name defined"); + my $target = $opts->{target} || $name; + + if ($env) { + run({ interactive => 1, + onfailure => "Could not resume job '$env' in pipeline '$name'" }, + 'fly -t $1 unpause-job -p $2 -j $3', + $target, $name, $env); + info("Resumed job #C{%s} in pipeline #C{%s}", $env, $name); + } else { + run({ interactive => 1, + onfailure => "Could not resume pipeline '$name'" }, + 'fly -t $1 unpause-pipeline -p $2', + $target, $name); + info("Resumed pipeline #C{%s}", $name); + } exit 0; } +# }}} +# }}} +### Deprecated Commands {{{ + +# repipe - deprecated; delegates to apply {{{ +sub repipe { + warning("'genesis repipe' is deprecated and will be removed in a future version. Use 'genesis pipeline-apply' instead."); + apply(@_); +} + +# }}} +# graph - deprecated; legacy graphviz without --platform, modern pipeline.md with {{{ sub graph { warning("'genesis graph' is deprecated and will be removed in a future version. Use 'genesis pipeline-graph' instead."); option_defaults(config => 'ci.yml'); my $layout = $_[0]; - my $top = Genesis::Top->new('.'); + my $top = Genesis::Top->new('.'); - # New compiler pipeline when --platform is specified if (get_options->{platform}) { - return _graph_compiled($top, $layout); + return pipeline_graph($layout); } (my $pipeline, $layout) = Genesis::CI::Legacy::parse(get_options->{config}, $top, $layout); @@ -113,29 +358,22 @@ sub graph { exit 0; } +# }}} +# describe - deprecated; delegates to pipeline_describe {{{ sub describe { warning("'genesis describe' is deprecated and will be removed in a future version. Use 'genesis pipeline-describe' instead."); - option_defaults(config => 'ci.yml'); - my $layout = $_[0]; - my $top = Genesis::Top->new('.'); - - # New compiler pipeline when --platform is specified - if (get_options->{platform}) { - return _describe_compiled($top, $layout); - } - - (my $pipeline, $layout) = Genesis::CI::Legacy::parse(get_options->{config}, $top, $layout); - Genesis::CI::Legacy::generate_pipeline_human_description($pipeline); - exit 0; + pipeline_describe(@_); } +# }}} +# }}} +### CI Task Commands {{{ + sub ci_pipeline_deploy { command_usage(1) if @_; info("[#G{genesis} ci-pipeline-deploy] v#G{$Genesis::VERSION}\n"); - # TODO: support detection of required vars in the prepare_command step. (maybe - # show optional variables with a ? after them, or ?1a ?1b to show either/or my @undefined = grep { !$ENV{$_} } qw/CURRENT_ENV GIT_BRANCH OUT_DIR WORKING_DIR VAULT_ROLE_ID VAULT_SECRET_ID VAULT_ADDR/; push @undefined, "CACHE_DIR" if ($ENV{PREVIOUS_ENV} && ! $ENV{CACHE_DIR}); @@ -145,21 +383,18 @@ sub ci_pipeline_deploy { "The pipeline must specify either GIT_PRIVATE_KEY, or GIT_USERNAME and ". "GIT_PASSWORD" ) unless $ENV{GIT_PRIVATE_KEY} || ($ENV{GIT_USERNAME} && $ENV{GIT_PASSWORD}); - # FIXME: Support Bearer Token _vault_auth(); _propagate_previous_passed_files(); - # Load the environment in order to check other required variables my $workdir = $ENV{WORKING_DIR}; $workdir .= "/$ENV{GIT_GENESIS_ROOT}" if (defined($ENV{GIT_GENESIS_ROOT}) && $ENV{GIT_GENESIS_ROOT} ne ""); pushd $workdir; my $env = Genesis::Top->new('.')->load_env($ENV{CURRENT_ENV})->with_vault(); if ($env->use_create_env) { - # Make sure that state is up to date. Keep environment changes local to this scope. - my $tmp = workdir; + my $tmp = workdir; my $git_env = _get_git_env($tmp); run({ onfailure => "Could not reset to the latest state file from origin. State file may not exist, which occurs if the proto bosh has not been deployed once manually.", interactive => 1, @@ -168,7 +403,7 @@ sub ci_pipeline_deploy { $ENV{GIT_BRANCH}, $ENV{CURRENT_ENV}); } - _bail_on_missing_pipeline_environment_variables(@undefined); # FIXME -- is this needed? + _bail_on_missing_pipeline_environment_variables(@undefined); info "Preparing to deploy #C{%s}:\n - based on kit #c{%s}", $env->name, $env->kit->id; if ($env->use_create_env) { @@ -180,11 +415,11 @@ sub ci_pipeline_deploy { my $result; my %deploy_opts = ( - redact => !envset('CI_NO_REDACT'), - 'disable-reactions' => 0, - 'yes' => 1, + redact => !envset('CI_NO_REDACT'), + 'disable-reactions' => 0, + 'yes' => 1, ); - $ENV{BOSH_NON_INTERACTIVE} = 'true'; # Doesn't work without this... + $ENV{BOSH_NON_INTERACTIVE} = 'true'; eval { $result = $env->with_bosh ->download_required_configs('deploy') @@ -193,7 +428,6 @@ sub ci_pipeline_deploy { if ($@ || !$result) { error "#R{Deployment failed!}\n%s", $@ || ""; - # Make sure to commit the state file in the case of failure if ($env->use_create_env) { popd; _commit_changes( @@ -206,13 +440,10 @@ sub ci_pipeline_deploy { } if ($ENV{PREVIOUS_ENV}) { - ## rm cache dir - ## copy previous env cache dir - # leaving as system calls for Concourse so output shows up in log system("rm -rf .genesis/config .genesis/kits .genesis/cached") == 0 or exit 1; - system("git checkout .genesis/config"); # ignore failure for git checkout so that - system("git checkout .genesis/kits"); # we don't cause problems if these files dont - system("git checkout .genesis/cached"); # yet exist in the working tree (but did in the cache tree) + system("git checkout .genesis/config"); + system("git checkout .genesis/kits"); + system("git checkout .genesis/cached"); } popd; _commit_changes($ENV{WORKING_DIR}, $ENV{OUT_DIR}, $ENV{GIT_BRANCH}, @@ -238,7 +469,6 @@ sub ci_show_changes { my $mismatches = _propagate_previous_passed_files(); - # Load the environment in order to check other required variables my $workdir = $ENV{WORKING_DIR}; $workdir .= "/$ENV{GIT_GENESIS_ROOT}" if (defined($ENV{GIT_GENESIS_ROOT}) && $ENV{GIT_GENESIS_ROOT} ne ""); pushd $workdir; @@ -297,7 +527,7 @@ current_variables="$(bosh int <(bosh curl "/deployments/${deployment}/variables" bosh diff-config --json \ --from-content <(bosh int <(spruce merge --fallback-append <(echo "${current_configs}") <(echo "${current_manifest}") <(echo "${current_variables}"))) \ --to-content <(bosh int <(spruce merge --fallback-append <(echo "${new_configs}") <(echo "${new_manifest}") <(echo "${new_variables}")) -l ${vars_file}) \ - | jq -r '.Tables[0].Rows[0] | if (.diff == "" ) then "No differences found." else .diff end' + | jq -r '.Tables[0].Rows[0] | if (.diff == "" ) then "[32;1mNo differences found.[0m" else .diff end' EOF my %envvars = $env->get_environment_variables(); @@ -320,10 +550,9 @@ EOF $differences += 1; last; } my $cache_file = slurp("$ENV{CACHE_DIR}/$_"); - my $work_file = slurp("$ENV{WORKING_DIR}/$_"); - unless ($cache_file eq $work_file) { $differences += 1; last; }; + my $work_file = slurp("$ENV{WORKING_DIR}/$_"); + unless ($cache_file eq $work_file) { $differences += 1; last; } } - @extra = () unless $differences; } @@ -356,23 +585,8 @@ sub ci_generate_cache { command_usage(1) if @_; - # environment variables we should have - # CURRENT_ENV - Name of the current environment - # GIT_BRANCH - Name of the git branch to push commits to. post-deploy - # GIT_PRIVATE_KEY - Private Key to use for pushing commits, post-deploy, ssh - # GIT_USERNAME - Username to use for pushing commits, post-deploy, https - # GIT_PASSWORD - Password to use for pushing commits, post-deploy, https - # PREVIOUS_ENV - Name of the previous env, or null if none - # CACHE_DIR - Path to the directory of the previous environment's cache - # WORKING_DIR - Path to the directory to deploy/work from - # OUT_DIR - Path to the directory to output to - # - # TODO: Detect if we need to run genesis from a different directory, based - # on the min genesis version of the environment for cached, changes, or git - # repo. my @undefined = grep { !$ENV{$_} } - qw/CURRENT_ENV GIT_BRANCH - WORKING_DIR OUT_DIR/; + qw/CURRENT_ENV GIT_BRANCH WORKING_DIR OUT_DIR/; push(@undefined, 'CACHE_DIR') if $ENV{PREVIOUS_ENV} && ! $ENV{CACHE_DIR}; _bail_on_missing_pipeline_environment_variables(@undefined); bail("The pipeline must specify either GIT_PRIVATE_KEY, or GIT_USERNAME and GIT_PASSWORD") @@ -414,18 +628,6 @@ sub ci_pipeline_run_errand { command_usage(1) if @_; - # environment variables we should have - # CURRENT_ENV - Name of the current environment - # ERRAND_NAME - Name of the Smoke Test errand to run - # - # VAULT_ROLE_ID - Vault RoleID to authenticate to Vault with - # VAULT_SECRET_ID - Vault SecretID to authenticate to Vault with - # VAULT_ADDR - URL of the Vault to use for credentials retrieval - # VAULT_SKIP_VERIFY - Whether or not to enforce SSL/TLS validation - # VAULT_NAMESPACE - Set for enterprise vaults that require namespaces - # VAULT_NO_STRONGBOX - Set true for non Genesis vault deployments - # VAULT_SECRETS_MOUNT - Set if vault secrets are not found under /secret - my @undefined = grep { !$ENV{$_} } qw/CURRENT_ENV ERRAND_NAME VAULT_ROLE_ID VAULT_SECRET_ID VAULT_ADDR/; _bail_on_missing_pipeline_environment_variables(@undefined); @@ -437,166 +639,39 @@ sub ci_pipeline_run_errand { exit 0; } -### Compiler Pipeline Functions {{{ - -# _repipe_compiled - deploy pipeline using the new compiler system {{{ -sub _repipe_compiled { - my ($top, $layout) = @_; - my $platform = get_options->{platform}; - - my $result = _compile_pipeline($top, $platform); - my $ast = $result->{ast}; - my $output = $result->{output}; - - my $name = $ast->metadata->{name} - or bail("Pipeline AST has no name defined"); - - # --output-dir: write artifacts to directory and exit - if (my $out_dir = get_options->{'output-dir'}) { - mkdir_or_fail($out_dir); - for my $file (sort keys %$output) { - mkfile_or_fail("$out_dir/$file", $output->{$file}); - info("Wrote #C{%s/%s}", $out_dir, $file); - } - mkfile_or_fail("$out_dir/ast.json", - JSON::PP->new->pretty->canonical->encode({%$ast})); - info("Wrote #C{%s/ast.json}", $out_dir); - info("Pipeline artifacts written to #C{%s/}", $out_dir); - exit 0; - } - - # For concourse: delegate deploy to the provider, which applies the - # three-tier option resolution (CLI > ci.provider: config > defaults). - if ($platform eq 'concourse') { - my $provider = $result->{provider}; - - # --dry-run: print pipeline YAML and exit without deploying - if (get_options->{'dry-run'}) { - my $yaml = $output->{'pipeline.yml'} - or bail("Concourse provider did not produce pipeline.yml"); - output({raw => 1}, $yaml); - exit 0; - } - - # Verify the provider's toolchain is available before attempting deploy. - $provider->check_prereqs() or exit 86; - - # Normalize provider CLI opts (ci-* → config keys) then merge legacy - # repipe aliases (--target → target, --paused → pause) before calling deploy(). - require Genesis::CI::Compiler::PipelineProvider; - my %deploy_opts = %{ Genesis::CI::Compiler::PipelineProvider->normalize_provider_opts( - $result->{provider_cli_opts} || {} - ) }; - $deploy_opts{target} //= get_options->{target} // $layout; - $deploy_opts{pause} //= get_options->{paused}; - $provider->deploy(%deploy_opts, yes => get_options->{yes}); - - } elsif ($platform eq 'github-actions') { - # GitHub Actions outputs workflow YAML files to .github/workflows/ - if (get_options->{'dry-run'}) { - for my $file (sort keys %$output) { - output "#G{--- %s ---}", $file; - output({raw => 1}, $output->{$file}); - } - exit 0; - } - - for my $file (sort keys %$output) { - my $path = ".github/workflows/$file"; - mkdir_or_fail(dirname($path)); - mkfile_or_fail($path, $output->{$file}); - info("Wrote #C{%s}", $path); - } - info("GitHub Actions workflows written. Commit and push to activate."); - } else { - bail("Unsupported platform '%s' for repipe", $platform); - } - - exit 0; -} - # }}} -# _graph_compiled - write pipeline.md with Mermaid flowchart {{{ -sub _graph_compiled { - my ($top, $layout) = @_; - my $platform = get_options->{platform}; - - my $result = _compile_pipeline($top, $platform); - my $ast = $result->{ast}; - my $provider = $result->{provider}; - - # Use provider's graph_md method if available - my $md; - if ($provider->can('graph_md')) { - $md = $provider->graph_md(); - } else { - $md = _ast_to_mermaid_md($ast); - } - - mkfile_or_fail('pipeline.md', $md); - info("Wrote #C{pipeline.md}"); - exit 0; -} - # }}} -# _describe_compiled - describe compiled pipeline in human-readable form {{{ -sub _describe_compiled { - my ($top, $layout) = @_; - my $platform = get_options->{platform}; - - my $result = _compile_pipeline($top, $platform); - my $ast = $result->{ast}; - my $provider = $result->{provider}; - - # Use provider's describe method if available - if ($provider->can('generate_description')) { - $provider->generate_description($ast); - exit 0; - } - - # Fall back to generic AST-based description - _describe_ast($ast, $platform); - exit 0; -} +### Internal Compiler Helpers {{{ -# }}} -# _compile_pipeline - run the compiler pipeline and return results {{{ +# _compile_pipeline - detect config source and compile; returns result hash {{{ sub _compile_pipeline { my ($top, $platform) = @_; my %compiler_opts = (top => $top); - # Detect configuration source — priority order: - # 1. .genesis/ci/ directory with targets.yml or pipeline.yml (multi-file) - # 2. ci: section in .genesis/config (genesis-config, Phase E) - # 3. Legacy ci.yml / --config file (backward compat) + # Priority order: + # 1. .genesis/ci/pipeline.yml or targets.yml (multi-file) + # 2. ci: section in .genesis/config (genesis-config) + # 3. Legacy ci.yml / --config file (backward compat) my $ci_dir = $top->path('.genesis/ci'); if (-d $ci_dir && (-f "$ci_dir/pipeline.yml" || -f "$ci_dir/targets.yml")) { $compiler_opts{ci_dir} = $ci_dir; info("Using multi-file CI configuration from #C{.genesis/ci/}"); } elsif (Genesis::CI::Compiler->can_compile_from_genesis_config($top)) { - # Parser reads ci: from $top->config — no ci_dir or file needed info("Using inline CI configuration from #C{.genesis/config}"); } else { $compiler_opts{file} = get_options->{config} || $top->path('ci.yml'); info("Using legacy CI configuration from #C{%s}", $compiler_opts{file}); } - # Parse provider-specific CLI flags via the provider options system. - # This mirrors Kit::Provider::parse_opts() — first pass extracts the - # provider type, second pass loads that provider's cli_opts() and parses them. + # Parse provider-specific CLI flags my %provider_cli_opts; { - # Build a synthetic argv from get_options so we can run GetOptionsFromArray. - # Provider flags are prefixed with 'ci-' and live alongside existing flags. - # We only need to pull the ones the provider declares. require Genesis::CI::Compiler::PipelineProvider; - my @argv = (); # provider flags come from get_options, not ARGV at this point + my @argv = (); Genesis::CI::Compiler::PipelineProvider->parse_cli_opts( \@argv, \%provider_cli_opts, $platform ); - # Merge any provider-specific flags already captured by the outer option parser. - # Ask the provider what keys it owns — don't hardcode them here. for my $key (Genesis::CI::Compiler::PipelineProvider->cli_opt_keys($platform)) { $provider_cli_opts{$key} = get_options->{$key} if defined get_options->{$key}; @@ -604,12 +679,11 @@ sub _compile_pipeline { } my $compiler = Genesis::CI::Compiler->new(%compiler_opts); - my $result = $compiler->compile( + my $result = $compiler->compile( provider => $platform, provider_opts => \%provider_cli_opts, ); - # Dump debug artifacts if --debug-dir is specified if (my $debug_dir = get_options->{'debug-dir'}) { _dump_debug_artifacts($debug_dir, $result, $platform); } @@ -619,7 +693,25 @@ sub _compile_pipeline { } # }}} -# _dump_debug_artifacts - write compiler intermediates to debug directory {{{ +# _get_top - create Genesis::Top, optionally skipping vault {{{ +sub _get_top { + my ($opts, %defaults) = @_; + + my $skip = $opts->{'skip-vault'} || $defaults{skip_vault}; + if ($skip) { + return Genesis::Top->new('.'); + } + + my $top = Genesis::Top->new('.', vault => $opts->{vault}); + bail( + "No vault specified or configured.\n". + "Use --skip-vault to compile without vault access." + ) unless $top->vault; + return $top; +} + +# }}} +# _dump_debug_artifacts - write compiler intermediates to a directory {{{ sub _dump_debug_artifacts { my ($debug_dir, $result, $platform) = @_; @@ -627,18 +719,16 @@ sub _dump_debug_artifacts { my $json = JSON::PP->new->pretty->canonical; - # 1. Parsed config (what the parser produced) if ($result->{parsed}) { mkfile_or_fail("$debug_dir/01-parsed.json", $json->encode($result->{parsed})); info("Debug: wrote #C{%s/01-parsed.json}", $debug_dir); } - # 2. AST source representation (internal Genesis concepts) if (my $ast = $result->{ast}) { my %source; for my $key (qw(branches integrations targets workflows configuration - provider_config triggers resources)) { + provider_config triggers resources)) { my $accessor = $ast->can($key); $source{$key} = $accessor->($ast) if $accessor; } @@ -649,10 +739,8 @@ sub _dump_debug_artifacts { $json->encode(\%source)); info("Debug: wrote #C{%s/02-ast-source.json}", $debug_dir); - # 3. Resolved generic pipeline (what PipelineDescriptor produced) if ($ast->pipeline && %{$ast->pipeline}) { - # Write pipeline structure (minus visualization/description for readability) - my %pipeline = %{$ast->pipeline}; + my %pipeline = %{$ast->pipeline}; my $mermaid = delete $pipeline{mermaid}; my $pipeline_md = delete $pipeline{pipeline_md}; my $description = delete $pipeline{description}; @@ -661,13 +749,11 @@ sub _dump_debug_artifacts { $json->encode(\%pipeline)); info("Debug: wrote #C{%s/03-pipeline.json}", $debug_dir); - # 4. Mermaid pipeline.md if ($pipeline_md) { mkfile_or_fail("$debug_dir/04-pipeline.md", $pipeline_md); info("Debug: wrote #C{%s/04-pipeline.md}", $debug_dir); } - # 5. Human description if ($description) { mkfile_or_fail("$debug_dir/05-description.txt", $description); info("Debug: wrote #C{%s/05-description.txt}", $debug_dir); @@ -675,7 +761,6 @@ sub _dump_debug_artifacts { } } - # 6. Provider output (final platform-specific YAML) if ($result->{output}) { if (ref($result->{output}) eq 'HASH') { for my $file (sort keys %{$result->{output}}) { @@ -693,7 +778,7 @@ sub _dump_debug_artifacts { } # }}} -# _ast_to_mermaid_md - fallback Mermaid pipeline.md from a bare AST {{{ +# _ast_to_mermaid_md - generate pipeline.md Mermaid content from a bare AST {{{ sub _ast_to_mermaid_md { my ($ast) = @_; @@ -734,7 +819,7 @@ sub _ast_to_mermaid_md { } # }}} -# _describe_ast - describe AST contents in human-readable form {{{ +# _describe_ast - human-readable AST description {{{ sub _describe_ast { my ($ast, $platform) = @_; @@ -743,25 +828,20 @@ sub _describe_ast { output " #Yi{Source}: %s", $ast->metadata->{source} || 'unknown'; output ""; - # Integrations my $integrations = $ast->integrations || {}; if (my $sc = $integrations->{source_control}) { output "#G{Source Control}:"; - output " Provider: %s", $sc->{provider} || 'unknown'; + output " Provider: %s", $sc->{provider} || 'unknown'; output " Repository: %s", $sc->{repository} || 'unknown'; } - # Targets my @targets = $ast->target_names; if (@targets) { output ""; output "#G{Targets}: (%d)", scalar @targets; - for my $name (sort @targets) { - output " - #C{%s}", $name; - } + output " - #C{%s}", $_ for sort @targets; } - # Workflows my @workflows = $ast->workflow_names; if (@workflows) { output ""; @@ -787,10 +867,19 @@ sub _describe_ast { } # }}} -# }}} +# _job_status_label - derive a display status from a fly jobs JSON entry {{{ +sub _job_status_label { + my ($job) = @_; + return 'paused' if $job->{paused}; + my $fb = $job->{finished_build} || {}; + return $fb->{status} || 'pending'; +} -### Support functions +# }}} +# }}} +### CI Task Helpers {{{ +# _vault_auth - authenticate to vault using AppRole credentials from env {{{ sub _vault_auth { my @missing_variables = grep { !exists($ENV{$_}) || !defined($ENV{$_}) @@ -800,18 +889,18 @@ sub _vault_auth { join(", ", @missing_variables) ) if @missing_variables; - # TODO: This should be handled by repo or env vault yaml entries (#BETTERVAULTTARGET) Service::Vault::Remote->create( $ENV{VAULT_ADDR}, 'deployments-vault', - skip_verify => envset("VAULT_SKIP_VERIFY"), - namespace => $ENV{VAULT_NAMESPACE}, + skip_verify => envset("VAULT_SKIP_VERIFY"), + namespace => $ENV{VAULT_NAMESPACE}, no_strongbox => envset("VAULT_NO_STRONGBOX"), - mount => $ENV{VAULT_SECRETS_MOUNT} + mount => $ENV{VAULT_SECRETS_MOUNT} )->connect_and_validate(); } -# _bail_on_missing_pipeline_environment_variables - provide consistent error message when missing pipeline environment variables {{{ +# }}} +# _bail_on_missing_pipeline_environment_variables - consistent error for missing CI env vars {{{ sub _bail_on_missing_pipeline_environment_variables { if (@_) { error("The following #R{required} environment variables have not been defined:"); @@ -823,12 +912,12 @@ sub _bail_on_missing_pipeline_environment_variables { } # }}} -# _propagate_previous_passed_files - copy cached files from previous pipeline environment {{{ +# _propagate_previous_passed_files - copy cached files from previous pipeline env {{{ sub _propagate_previous_passed_files { return unless $ENV{PREVIOUS_ENV}; - bail "No CACHE_DIR set - cannot propagate passed values" unless $ENV{CACHE_DIR}; + bail "No CACHE_DIR set - cannot propagate passed values" unless $ENV{CACHE_DIR}; bail "No WORKING_DIR set - cannot propagate passed values" unless $ENV{WORKING_DIR}; my $workdir = $ENV{WORKING_DIR}; @@ -838,7 +927,7 @@ sub _propagate_previous_passed_files { $cachedir .= "/$ENV{GIT_GENESIS_ROOT}"; } - my @cachables=( + my @cachables = ( ".genesis/cached", ".genesis/config", ".genesis/kits" @@ -851,26 +940,24 @@ sub _propagate_previous_passed_files { ) for (@cachables); info "\n#C{Copying over cached files from $ENV{PREVIOUS_ENV} environment:}"; - run( { onfailure => "#R{[ERROR]} Failed to copy '$cachedir/$_' to '$workdir/$_'", interactive => 1 }, 'cp', '-Rv', "$cachedir/$_", "$workdir/$_" ) for (@cachables); - my $env = Genesis::Top->new($workdir)->load_env($ENV{CURRENT_ENV}); + my $env = Genesis::Top->new($workdir)->load_env($ENV{CURRENT_ENV}); my @files = map {(my $s = $_) =~ s/^\.\///; $s} grep {$_ !~ /^not-shared/} $env->relate($ENV{PREVIOUS_ENV},'','not-shared'); return { missing => [grep {-f "$workdir/$_" && ! -f "$cachedir/$_"} @files], extra => [grep {-f "$cachedir/$_" && ! -f "$workdir/$_"} @files], - } + }; } # }}} -# _get_git_env - setup a git configuration in the 'home' directory under the given dir, and return env {{{ +# _get_git_env - set up git credentials in a temp home dir, return env hash {{{ sub _get_git_env { - my ($env_dir) = @_; my %env; $env{HOME} = "$env_dir/home"; @@ -890,8 +977,8 @@ Host * LogLevel QUIET IdentityFile $env_dir/home/.ssh/key EOF - $env{GIT_SSH_COMMAND} = "ssh -F $env_dir/home/.ssh/config"; - }; + $env{GIT_SSH_COMMAND} = "ssh -F $env_dir/home/.ssh/config"; + } if ($ENV{GIT_USERNAME}) { mkdir_or_fail( "$env_dir/home", 0700); mkfile_or_fail("$env_dir/home/credential-helper.sh", 0755, <<'EOF'); @@ -903,39 +990,25 @@ EOF } return wantarray ? %env : \%env; - } # }}} -# _commit_changes - commit changes back to genesis that incorporate the upstream values {{{ +# _commit_changes - commit post-deploy changes back to the git repo {{{ sub _commit_changes { my ($indir, $outdir, $branch, $message, $filter) = @_; - # the below copying of files into new repos from older repos is all - # done in the name of avoiding merge conflicts, or weird errors when - # rebasing, and git discovers that there are no changes after you rebase - - # remove any old directories if they are left over (only really happens when - # debugging a failed run) run( {onfailure => "#R{[ERROR]} Failed to remove remnant of previous changes commit"}, 'rm -rf "$1"', $outdir ) if -d $outdir; - # create an output git repo based off of latest origin/$branch run({ interactive => 1, passfail => 1}, 'cp -R "$1" "$2"', $indir, $outdir) or exit 1; pushd $outdir; - my $tmp = workdir; + my $tmp = workdir; my $git_env = _get_git_env($tmp); - # What's Going On? - # reset --hard : reset the repo to the current commit on branch - # clean -df: remove any untracked files (ie the new cached files) - # checkout: ensure we're on the named branch, to correctly push later - # pull origin: ensure we're up-to-date with any changes that may have - # happened during pipeline run({ onfailure => "Could not reset to the newest applicable ref in git", interactive => 1, env => $git_env }, @@ -943,13 +1016,12 @@ sub _commit_changes { $branch); popd; - # find and copy (or remove if appropriate) all potential changes to the outdir pushd $indir; my @output = lines(run({env => $git_env}, 'git status --porcelain')); popd; info "Detected the following changes in repo:" if @output; for my $change (@output) { - my ($action,$file)= $change =~ /^(..).(.*)$/; + my ($action,$file) = $change =~ /^(..).(.*)$/; next if $filter && ref($filter) && ref($filter) =~ /^regexp$/i && $file !~ $filter; if ($action =~ /.D/) { info " - #R{removed:} $file"; @@ -960,7 +1032,6 @@ sub _commit_changes { } else { mkdir_or_fail(dirname("$outdir/$file")); info " - ".($action eq "??" ? "#G{added:} " : "#Y{changed:} ").$file; - run( { onfailure => "Could not copy changed files to output directory" }, 'cp -Rv "$1" "$2"', "$indir/$file", "$outdir/$file" @@ -968,12 +1039,10 @@ sub _commit_changes { } } - # check if any changes actually exist in the outdir (potential changes may have alread - # been tracked after $indir's commit, so they could disappear here), then commit them pushd $outdir; my ($output, undef) = run({env => $git_env}, 'git status --porcelain'); if ($output) { - run({ interactive => 1, # print output to Concourse log + run({ interactive => 1, env => $git_env }, 'git add -A && '. 'git status && '. @@ -981,6 +1050,73 @@ sub _commit_changes { 'git commit -m "$1"', "CI commit: $message"); } } + +# }}} # }}} 1; + +=head1 NAME + +Genesis::Commands::Pipelines - Pipeline management command suite + +=head1 DESCRIPTION + +Implements the C command family and the legacy +C, C, C, C, and C commands. + +=head1 COMMANDS + +=over 4 + +=item B [--platform PROVIDER] [--dry-run] [--paused] + +Compile and deploy the pipeline. Defaults to Concourse. Supports +C<--dry-run> (print YAML only) and C<--output-dir> (write artifacts). + +=item B [--platform PROVIDER] + +Compile pipeline and write C containing a Mermaid flowchart. + +=item B [--platform PROVIDER] + +Compile pipeline and print a human-readable ordered progression. + +=item B [--target TARGET] + +Compare compiled pipeline YAML against the live pipeline via +C. + +=item B [] [--target TARGET] + +Query C for per-environment job status. + +=item B [] [--target TARGET] + +Pause a specific environment's job, or the entire pipeline. + +=item B [] [--target TARGET] + +Resume a specific environment's job, or the entire pipeline. + +=item B (deprecated) + +Alias for C. + +=item B (deprecated) + +Legacy graphviz output without C<--platform>; C with it. + +=item B (deprecated) + +Alias for C. + +=back + +=head1 SEE ALSO + +Genesis::CI::Compiler, Genesis::CI::Legacy + +=cut + +# vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu diff --git a/t/ci-pipeline.t b/t/ci-pipeline.t index 49a6519c..59cb6c57 100644 --- a/t/ci-pipeline.t +++ b/t/ci-pipeline.t @@ -14,7 +14,7 @@ $ENV{GENESIS_LIB} ||= 'lib'; use_ok 'Genesis::CI::Compiler::Parser'; use_ok 'Genesis::CI::Compiler::ASTBuilder'; use_ok 'Genesis::CI::Compiler'; -use_ok 'Genesis::Commands::Pipeline'; +use_ok 'Genesis::Commands::Pipelines'; ### ============================================================ ### ### Phase A — env-files topology without pipeline.yml @@ -159,17 +159,19 @@ YAML ### Phase B — Genesis::Commands::Pipeline module ### ============================================================ ### -subtest 'Commands::Pipeline module loads correctly' => sub { - can_ok 'Genesis::Commands::Pipeline', 'apply'; - can_ok 'Genesis::Commands::Pipeline', 'graph'; - can_ok 'Genesis::Commands::Pipeline', 'describe'; - can_ok 'Genesis::Commands::Pipeline', 'diff'; - can_ok 'Genesis::Commands::Pipeline', 'status'; - can_ok 'Genesis::Commands::Pipeline', 'pause'; - can_ok 'Genesis::Commands::Pipeline', 'resume'; +subtest 'Commands::Pipelines module loads correctly' => sub { + can_ok 'Genesis::Commands::Pipelines', 'apply'; + can_ok 'Genesis::Commands::Pipelines', 'pipeline_graph'; + can_ok 'Genesis::Commands::Pipelines', 'pipeline_describe'; + can_ok 'Genesis::Commands::Pipelines', 'diff'; + can_ok 'Genesis::Commands::Pipelines', 'status'; + can_ok 'Genesis::Commands::Pipelines', 'pause'; + can_ok 'Genesis::Commands::Pipelines', 'resume'; + can_ok 'Genesis::Commands::Pipelines', 'graph'; + can_ok 'Genesis::Commands::Pipelines', 'describe'; }; -subtest 'Commands::Pipeline - _compile helper: detects multi-file config' => sub { +subtest 'Commands::Pipelines - _compile_pipeline helper: detects multi-file config' => sub { my $tmp = tempdir(CLEANUP => 1); mkpath("$tmp/.genesis/ci"); @@ -190,7 +192,7 @@ YAML my $top = bless({}, 'Genesis::Top'); # stub my $result = eval { - Genesis::Commands::Pipeline::_compile($top, 'concourse', {}) + Genesis::Commands::Pipelines::_compile_pipeline($top, 'concourse') }; # This will bail due to no vault/spruce in test environment, but we just # check the compiler path was resolved correctly @@ -203,7 +205,7 @@ YAML chdir($orig); }; -subtest 'Commands::Pipeline - _mermaid_md generates valid Mermaid markdown' => sub { +subtest 'Commands::Pipelines - _ast_to_mermaid_md generates valid Mermaid markdown' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test-pipeline' }, workflows => { @@ -225,7 +227,7 @@ subtest 'Commands::Pipeline - _mermaid_md generates valid Mermaid markdown' => s }, ); - my $md = Genesis::Commands::Pipeline::_mermaid_md($ast); + my $md = Genesis::Commands::Pipelines::_ast_to_mermaid_md($ast); like $md, qr/^# Pipeline: test-pipeline/m, "H1 heading present"; like $md, qr/```mermaid/, "mermaid fence open"; @@ -235,7 +237,7 @@ subtest 'Commands::Pipeline - _mermaid_md generates valid Mermaid markdown' => s like $md, qr/```/, "mermaid fence close"; }; -subtest 'Commands::Pipeline - _print_description does not die' => sub { +subtest 'Commands::Pipelines - _describe_ast does not die' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test-pipeline', source => 'multi-file' }, integrations => { @@ -259,25 +261,25 @@ subtest 'Commands::Pipeline - _print_description does not die' => sub { }, ); - eval { Genesis::Commands::Pipeline::_print_description($ast, 'concourse') }; - ok !$@, "_print_description does not die: $@"; + eval { Genesis::Commands::Pipelines::_describe_ast($ast, 'concourse') }; + ok !$@, "_describe_ast does not die: $@"; }; -subtest 'Commands::Pipeline - _job_status_label returns expected labels' => sub { - is Genesis::Commands::Pipeline::_job_status_label({ paused => 1 }), +subtest 'Commands::Pipelines - _job_status_label returns expected labels' => sub { + is Genesis::Commands::Pipelines::_job_status_label({ paused => 1 }), 'paused', "paused job returns 'paused'"; - is Genesis::Commands::Pipeline::_job_status_label({ + is Genesis::Commands::Pipelines::_job_status_label({ paused => 0, finished_build => { status => 'succeeded' }, }), 'succeeded', "succeeded job"; - is Genesis::Commands::Pipeline::_job_status_label({ + is Genesis::Commands::Pipelines::_job_status_label({ paused => 0, finished_build => { status => 'failed' }, }), 'failed', "failed job"; - is Genesis::Commands::Pipeline::_job_status_label({ + is Genesis::Commands::Pipelines::_job_status_label({ paused => 0, }), 'pending', "job with no build is pending"; }; From 5107bed7420859e1323c93e5c52ebd15c18f8497 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:40:28 -0400 Subject: [PATCH 044/103] Concourse: rename pause to pause_after_set Validate ci config and rename Concourse pause option. Add validation in Genesis::CI::Compiler to require ci.targets be a non-empty hash and to require ci.integrations.source_control (ensuring ci.integrations is a hash). In the Concourse provider, rename DEFAULT_PAUSE -> DEFAULT_PAUSE_AFTER_SET and replace the provider schema key pause with pause_after_set; update provider defaults, describe() and deploy() to use the new key. Add normalize_provider_opts() to map the legacy pause key to pause_after_set for backward compatibility. --- lib/Genesis/CI/Compiler.pm | 15 +++++++ .../CI/Compiler/Providers/Concourse.pm | 43 ++++++++++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index d85ecabd..8a203f5f 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -195,6 +195,21 @@ sub validate_config_section { bail("'ci' configuration in .genesis/config must be a hash") unless ref($data) eq 'HASH'; + # ci.targets must be a non-empty hash + my $targets = $data->{targets}; + bail("'ci.targets' is required and must define at least one target") + unless defined($targets) && ref($targets) eq 'HASH' && scalar(keys %$targets) > 0; + + # ci.integrations.source_control is required + my $integrations = $data->{integrations}; + if (defined $integrations) { + bail("'ci.integrations' must be a hash") unless ref($integrations) eq 'HASH'; + bail("'ci.integrations.source_control' is required") + unless defined $integrations->{source_control}; + } else { + bail("'ci.integrations.source_control' is required"); + } + # Validate ci.provider section against the provider's own schema if (my $provider_data = $data->{provider}) { bail("'ci.provider' must be a hash") diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index b73a29ac..af250c37 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -13,11 +13,11 @@ use JSON::PP; ### Provider Constants {{{ use constant { - DEFAULT_TEAM => 'main', - DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top - DEFAULT_EXPOSE => 0, - DEFAULT_PAUSE => 0, - DEFAULT_INSECURE => 0, + DEFAULT_TEAM => 'main', + DEFAULT_PIPELINE_NAME => undef, # falls back to deployment_type from Top + DEFAULT_EXPOSE => 0, + DEFAULT_PAUSE_AFTER_SET => 0, + DEFAULT_INSECURE => 0, }; # }}} @@ -181,9 +181,9 @@ sub provider_options_schema { default => DEFAULT_EXPOSE, description => 'Make pipeline publicly viewable (fly expose-pipeline)', }, - pause => { + pause_after_set => { type => 'boolean', - default => DEFAULT_PAUSE, + default => DEFAULT_PAUSE_AFTER_SET, description => 'Leave pipeline paused after fly set-pipeline', }, insecure => { @@ -198,13 +198,24 @@ sub provider_options_schema { # provider_options_defaults - default values for all Concourse options {{{ sub provider_options_defaults { return { - team => DEFAULT_TEAM, - expose => DEFAULT_EXPOSE, - pause => DEFAULT_PAUSE, - insecure => DEFAULT_INSECURE, + team => DEFAULT_TEAM, + expose => DEFAULT_EXPOSE, + pause_after_set => DEFAULT_PAUSE_AFTER_SET, + insecure => DEFAULT_INSECURE, }; } +# }}} +# normalize_provider_opts - remap ci-pause (CLI) to pause_after_set (schema key) {{{ +sub normalize_provider_opts { + my ($class, $opts) = @_; + my $normalized = $class->SUPER::normalize_provider_opts($opts); + if (exists $normalized->{pause} && !exists $normalized->{pause_after_set}) { + $normalized->{pause_after_set} = delete $normalized->{pause}; + } + return $normalized; +} + # }}} # describe_provider - structured self-description for display {{{ # @@ -217,7 +228,7 @@ sub describe_provider { my $team = $self->provider_option('team') || DEFAULT_TEAM; my $pipe_name = $self->provider_option('pipeline_name') || '(deployment type)'; my $expose = $self->provider_option('expose') ? 'yes' : 'no'; - my $paused = $self->provider_option('pause') ? 'yes' : 'no'; + my $paused = $self->provider_option('pause_after_set') ? 'yes' : 'no'; my $insecure = $self->provider_option('insecure') ? 'yes' : 'no'; return ( @@ -325,7 +336,7 @@ sub deploy { bail("Must call parse() before deploy()") unless $self->{config}; - # All keys use config-file names (target, team, pipeline_name, pause, expose). + # All keys use config-file names (target, team, pipeline_name, pause_after_set, expose). # Callers are responsible for normalizing cli-prefixed keys before calling deploy(). # --- Resolve options from three tiers --- @@ -353,9 +364,9 @@ sub deploy { # Pause/expose/dry-run/insecure: call-site override > provider_opts > defaults my $dry_run = $opts{'dry-run'}; my $yes = $opts{yes}; - my $pause = $opts{pause} - // $self->provider_option('pause') - // DEFAULT_PAUSE; + my $pause = $opts{pause_after_set} + // $self->provider_option('pause_after_set') + // DEFAULT_PAUSE_AFTER_SET; my $expose = $opts{expose} // $self->provider_option('expose') // _yaml_bool(($self->{config}{pipeline} || {})->{public}, DEFAULT_EXPOSE); From cb2a166a6ccc61a6e0bdcf1c74938478107d332b Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:38:49 -0400 Subject: [PATCH 045/103] Concourse provider: handle insecure and normalize opts Add support for the Concourse 'insecure' option and improve provider option handling and validation. Key changes: - Validate that ci.integrations.source_control is a hash in Compiler.pm. - PipelineProvider::provider_config now skips undefined keys and does a safer comparison against defaults (handles definedness and stringified comparison) so boolean false and undef are treated correctly. - Commands/Pipelines: stop requiring PipelineProvider class directly, use provider->normalize_provider_opts(), remap pause to pause_after_set, and derive Concourse fly target and '-k' insecure flag via a new _concourse_fly_flags() helper; update all fly invocations to include the computed k_flag. - Add extensive tests in t/ci-compiler.t covering the insecure option schema/default/CLI/description, normalize_provider_opts behavior (including ci-pause -> pause_after_set), provider_config boolean/undef handling, and validate_config_section source_control validation. These changes ensure proper handling of provider defaults, explicit false values, and support for insecure Concourse connections. --- lib/Genesis/CI/Compiler.pm | 7 +- lib/Genesis/CI/Compiler/PipelineProvider.pm | 5 +- lib/Genesis/Commands/Pipelines.pm | 44 ++++-- t/ci-compiler.t | 151 ++++++++++++++++++++ 4 files changed, 190 insertions(+), 17 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 8a203f5f..7785fa33 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -200,12 +200,15 @@ sub validate_config_section { bail("'ci.targets' is required and must define at least one target") unless defined($targets) && ref($targets) eq 'HASH' && scalar(keys %$targets) > 0; - # ci.integrations.source_control is required + # ci.integrations.source_control is required and must be a hash my $integrations = $data->{integrations}; if (defined $integrations) { bail("'ci.integrations' must be a hash") unless ref($integrations) eq 'HASH'; + my $sc = $integrations->{source_control}; bail("'ci.integrations.source_control' is required") - unless defined $integrations->{source_control}; + unless defined $sc; + bail("'ci.integrations.source_control' must be a hash") + unless ref($sc) eq 'HASH'; } else { bail("'ci.integrations.source_control' is required"); } diff --git a/lib/Genesis/CI/Compiler/PipelineProvider.pm b/lib/Genesis/CI/Compiler/PipelineProvider.pm index 48ea5523..73410ac6 100644 --- a/lib/Genesis/CI/Compiler/PipelineProvider.pm +++ b/lib/Genesis/CI/Compiler/PipelineProvider.pm @@ -177,7 +177,10 @@ sub provider_config { my $opts = $self->{provider_opts} || {}; my %out = ( type => $self->provider_type() ); for my $k (keys %$opts) { - next if exists $defaults->{$k} && $defaults->{$k} eq ($opts->{$k} // ''); + next unless defined $opts->{$k}; + next if exists $defaults->{$k} + && defined $defaults->{$k} + && "$defaults->{$k}" eq "$opts->{$k}"; $out{$k} = $opts->{$k}; } return \%out; diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 700a6bbc..abc274f3 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -73,12 +73,11 @@ sub apply { $provider->check_prereqs() or exit 86; - require Genesis::CI::Compiler::PipelineProvider; - my %deploy_opts = %{ Genesis::CI::Compiler::PipelineProvider->normalize_provider_opts( + my %deploy_opts = %{ $provider->normalize_provider_opts( $result->{provider_cli_opts} || {} ) }; - $deploy_opts{target} //= $opts->{target} // $layout // $name; - $deploy_opts{pause} //= $opts->{paused}; + $deploy_opts{target} //= $opts->{target} // $layout // $name; + $deploy_opts{pause_after_set} //= $opts->{paused}; $provider->deploy(%deploy_opts, yes => $opts->{yes}); } elsif ($platform eq 'github-actions') { @@ -165,7 +164,7 @@ sub diff { my $name = $ast->metadata->{name} or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; + my ($target, $k_flag) = _concourse_fly_flags($result, $opts, $name); my $compiled = $output->{'pipeline.yml'} or bail("Concourse provider did not produce pipeline.yml"); @@ -173,7 +172,7 @@ sub diff { my $dir = workdir; mkfile_or_fail("$dir/compiled.yml", $compiled); - my ($live, $rc) = run('fly -t $1 get-pipeline -p $2', $target, $name); + my ($live, $rc) = run("fly${k_flag} -t \$1 get-pipeline -p \$2", $target, $name); if ($rc != 0) { info("#Y{Pipeline '%s' does not exist on target '%s' — nothing to diff against.}", $name, $target); @@ -212,9 +211,9 @@ sub status { my $ast = $result->{ast}; my $name = $ast->metadata->{name} or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; + my ($target, $k_flag) = _concourse_fly_flags($result, $opts, $name); - my ($json_out, $rc) = run('fly -t $1 jobs -p $2 --json', $target, $name); + my ($json_out, $rc) = run("fly${k_flag} -t \$1 jobs -p \$2 --json", $target, $name); bail("Could not get jobs for pipeline '%s' on target '%s': %s", $name, $target, $json_out) unless $rc == 0; @@ -277,18 +276,18 @@ sub pause { my $ast = $result->{ast}; my $name = $ast->metadata->{name} or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; + my ($target, $k_flag) = _concourse_fly_flags($result, $opts, $name); if ($env) { run({ interactive => 1, onfailure => "Could not pause job '$env' in pipeline '$name'" }, - 'fly -t $1 pause-job -p $2 -j $3', + "fly${k_flag} -t \$1 pause-job -p \$2 -j \$3", $target, $name, $env); info("Paused job #C{%s} in pipeline #C{%s}", $env, $name); } else { run({ interactive => 1, onfailure => "Could not pause pipeline '$name'" }, - 'fly -t $1 pause-pipeline -p $2', + "fly${k_flag} -t \$1 pause-pipeline -p \$2", $target, $name); info("Paused pipeline #C{%s}", $name); } @@ -312,18 +311,18 @@ sub resume { my $ast = $result->{ast}; my $name = $ast->metadata->{name} or bail("Pipeline AST has no name defined"); - my $target = $opts->{target} || $name; + my ($target, $k_flag) = _concourse_fly_flags($result, $opts, $name); if ($env) { run({ interactive => 1, onfailure => "Could not resume job '$env' in pipeline '$name'" }, - 'fly -t $1 unpause-job -p $2 -j $3', + "fly${k_flag} -t \$1 unpause-job -p \$2 -j \$3", $target, $name, $env); info("Resumed job #C{%s} in pipeline #C{%s}", $env, $name); } else { run({ interactive => 1, onfailure => "Could not resume pipeline '$name'" }, - 'fly -t $1 unpause-pipeline -p $2', + "fly${k_flag} -t \$1 unpause-pipeline -p \$2", $target, $name); info("Resumed pipeline #C{%s}", $name); } @@ -866,6 +865,23 @@ sub _describe_ast { output ""; } +# }}} +# _concourse_fly_flags - derive (target, k_flag) from compiled result + CLI opts {{{ +# +# Target resolution: explicit --target CLI opt > ci.provider.target config > pipeline name. +# k_flag is ' -k' when insecure is set, '' otherwise. +sub _concourse_fly_flags { + my ($result, $opts, $name) = @_; + my $provider = $result->{provider}; + my $target = $opts->{target} + // ($provider->can('provider_option') ? $provider->provider_option('target') : undef) + // $name; + my $insecure = $provider->can('provider_option') + ? ($provider->provider_option('insecure') // 0) : 0; + my $k_flag = $insecure ? ' -k' : ''; + return ($target, $k_flag); +} + # }}} # _job_status_label - derive a display status from a fly jobs JSON entry {{{ sub _job_status_label { diff --git a/t/ci-compiler.t b/t/ci-compiler.t index 996dc3ad..c11fb1ad 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2604,6 +2604,157 @@ subtest 'PipelineProvider::Concourse - check_prereqs returns 0 when fly absent' ok !$result, 'check_prereqs returns 0 when fly is not in PATH'; }; +### ============================================================ ### +### Concourse insecure option +### ============================================================ ### + +subtest 'Concourse - insecure in provider_options_schema' => sub { + my $schema = Genesis::CI::Concourse->provider_options_schema(); + ok exists $schema->{insecure}, "'insecure' key present in schema"; + is $schema->{insecure}{type}, 'boolean', "insecure type is boolean"; + ok !$schema->{insecure}{required}, "insecure is not required"; + is $schema->{insecure}{default}, 0, "insecure default is 0"; +}; + +subtest 'Concourse - insecure in provider_options_defaults' => sub { + my $defaults = Genesis::CI::Concourse->provider_options_defaults(); + ok exists $defaults->{insecure}, "insecure present in defaults"; + is $defaults->{insecure}, 0, "insecure default is 0 (false)"; +}; + +subtest 'Concourse - ci-insecure declared in cli_opts' => sub { + my @opts = Genesis::CI::Concourse->cli_opts(); + ok grep { $_ eq 'ci-insecure' } @opts, "ci-insecure declared (boolean flag)"; +}; + +subtest 'Concourse - insecure omitted from provider_config when default (false)' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { type => 'concourse', target => 't', insecure => 0 }, + ); + my $config = $provider->provider_config(); + ok !exists $config->{insecure}, "insecure omitted when false (matches default)"; +}; + +subtest 'Concourse - insecure included in provider_config when true' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { type => 'concourse', target => 't', insecure => 1 }, + ); + my $config = $provider->provider_config(); + is $config->{insecure}, 1, "insecure=1 included in provider_config"; +}; + +subtest 'Concourse - provider_option insecure defaults to 0' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new(ast => $ast); + is $provider->provider_option('insecure'), 0, "insecure defaults to 0"; +}; + +subtest 'Concourse - describe_provider includes Insecure field' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { target => 'myci', insecure => 1 }, + ); + my %info = $provider->describe_provider(); + ok grep { $_ eq 'Insecure' } @{$info{extras}}, "Insecure in extras list"; + is $info{Insecure}, 'yes', "Insecure field is 'yes' when insecure=1"; +}; + +### ============================================================ ### +### Concourse normalize_provider_opts override +### ============================================================ ### + +subtest 'Concourse - normalize_provider_opts remaps ci-pause to pause_after_set' => sub { + my $normalized = Genesis::CI::Concourse->normalize_provider_opts({ + 'ci-pause' => 1, + }); + ok !exists $normalized->{pause}, "raw 'pause' key not present after remap"; + is $normalized->{pause_after_set}, 1, "pause_after_set=1 after remapping ci-pause"; +}; + +subtest 'Concourse - normalize_provider_opts does not remap if pause_after_set already set' => sub { + my $normalized = Genesis::CI::Concourse->normalize_provider_opts({ + 'ci-pause' => 0, + 'pause_after_set' => 1, + }); + is $normalized->{pause_after_set}, 1, + "explicit pause_after_set wins over ci-pause when both present"; +}; + +subtest 'Concourse - normalize_provider_opts handles full CLI key set' => sub { + my $normalized = Genesis::CI::Concourse->normalize_provider_opts({ + 'ci-target' => 'myci', + 'ci-team' => 'platform', + 'ci-pipeline-name' => 'cf-deploy', + 'ci-pause' => 1, + 'ci-expose' => 0, + 'ci-insecure' => 1, + }); + is $normalized->{target}, 'myci', "target normalized"; + is $normalized->{team}, 'platform', "team normalized"; + is $normalized->{pipeline_name}, 'cf-deploy', "pipeline_name normalized"; + is $normalized->{pause_after_set}, 1, "pause_after_set normalized from ci-pause"; + is $normalized->{expose}, 0, "expose normalized"; + is $normalized->{insecure}, 1, "insecure normalized"; + ok !exists $normalized->{pause}, "no stale 'pause' key present"; +}; + +### ============================================================ ### +### provider_config boolean comparison (PipelineProvider) +### ============================================================ ### + +subtest 'PipelineProvider - provider_config skips undef opts' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { type => 'concourse', team => undef }, + ); + my $config = $provider->provider_config(); + ok !exists $config->{team}, "undef opt not included in provider_config"; +}; + +subtest 'PipelineProvider - provider_config keeps boolean false when non-default' => sub { + my $ast = Genesis::CI::Compiler::AST->new(); + # expose default is 0; setting expose=>0 explicitly should still omit it + # insecure default is 0; setting insecure=>1 should include it + my $provider = Genesis::CI::Concourse->new( + ast => $ast, + provider_opts => { type => 'concourse', expose => 0, insecure => 1 }, + ); + my $config = $provider->provider_config(); + ok !exists $config->{expose}, "expose=0 (matches default) omitted"; + is $config->{insecure}, 1, "insecure=1 (non-default) included"; +}; + +### ============================================================ ### +### validate_config_section: source_control must be a hash +### ============================================================ ### + +subtest 'Compiler - validate_config_section: rejects scalar source_control' => sub { + my $bad = { + %$_ci_data, + integrations => { source_control => 1 }, + }; + eval { Genesis::CI::Compiler->validate_config_section($bad, undef) }; + like $@, qr/source_control.*must be a hash/i, + "scalar source_control triggers error"; +}; + +subtest 'Compiler - validate_config_section: accepts hash source_control' => sub { + my $good = { + %$_ci_data, + integrations => { + source_control => { provider => 'github', repository => 'org/repo' }, + }, + }; + eval { Genesis::CI::Compiler->validate_config_section($good, undef) }; + ok !$@, "hash source_control passes validation" or diag $@; +}; + done_testing; # vim: ts=2 sw=2 sts=2 noet fdm=marker foldlevel=1 nu From 7da14e21b49760279c2823eaa705b7254bee2873 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 10:04:20 -0700 Subject: [PATCH 046/103] Add ci-provider option to repo-init [New Features] - Add `--ci-provider=s` option to `genesis repo-init` accepting 'concourse', 'github-actions', or 'manual' to configure CI automation at repo creation time - When specified, writes the ci: section to .genesis/config and generates the .genesis/ci/ scaffold --- - Minor whitespace fix (double-space after periods) in the skip-vault option help text --- bin/genesis | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/genesis b/bin/genesis index 3840a8bb..d9f9b38f 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1278,9 +1278,14 @@ define_command("repo-init", { "subdirectory; use with care.", 'skip-vault' => - "Defer vault configuration. The repo will be created without a ". - "secrets provider. You must configure one via #C{genesis secrets-provider} ". + "Defer vault configuration. The repo will be created without a ". + "secrets provider. You must configure one via #C{genesis secrets-provider} ". "before creating environments.", + + 'ci-provider=s' => + "CI provider for pipeline automation: 'concourse', 'github-actions', ". + "or 'manual'. When specified, writes the ci: section to .genesis/config ". + "and generates the .genesis/ci/ scaffold.", ], extended_handlers => ['Genesis::Kit::Provider', 'Genesis::CI::Provider'], arguments => [ From 6214fc59ac52ab31678405e9228f81d175951ee0 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:25:14 -0400 Subject: [PATCH 047/103] Accept flat ci.* integrations; refine validation Allow integrations to be specified either nested under ci.integrations or as top-level flat keys (e.g. ci.vault, ci.source_control). Parser merges flat keys into the integrations hash while preserving explicit nested values. Compiler validation now accepts ci.source_control as an alternative to ci.integrations.source_control and emits clearer error messages. Validator message texts were clarified (prefix pipeline.* keys, make workflows optional when topology is derived from env files, and provide guidance for missing integrations.vault/source_control). These changes improve compatibility with alternative config layouts and make errors more actionable. --- lib/Genesis/CI/Compiler.pm | 15 +++-- lib/Genesis/CI/Compiler/Parser.pm | 8 ++- lib/Genesis/CI/Compiler/Validator.pm | 16 +++-- t/ci-compiler.t | 87 ++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 18 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 7785fa33..727ff86e 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -200,18 +200,17 @@ sub validate_config_section { bail("'ci.targets' is required and must define at least one target") unless defined($targets) && ref($targets) eq 'HASH' && scalar(keys %$targets) > 0; - # ci.integrations.source_control is required and must be a hash + # source_control is required — accepted at ci.integrations.source_control (nested) + # or ci.source_control (flat, per architecture config reference). my $integrations = $data->{integrations}; if (defined $integrations) { bail("'ci.integrations' must be a hash") unless ref($integrations) eq 'HASH'; - my $sc = $integrations->{source_control}; - bail("'ci.integrations.source_control' is required") - unless defined $sc; - bail("'ci.integrations.source_control' must be a hash") - unless ref($sc) eq 'HASH'; - } else { - bail("'ci.integrations.source_control' is required"); } + my $sc = ($integrations || {})->{source_control} // $data->{source_control}; + bail("'ci.source_control' (or 'ci.integrations.source_control') is required") + unless defined $sc; + bail("'ci.source_control' must be a hash") + unless ref($sc) eq 'HASH'; # Validate ci.provider section against the provider's own schema if (my $provider_data = $data->{provider}) { diff --git a/lib/Genesis/CI/Compiler/Parser.pm b/lib/Genesis/CI/Compiler/Parser.pm index 160b1d89..e38584c1 100644 --- a/lib/Genesis/CI/Compiler/Parser.pm +++ b/lib/Genesis/CI/Compiler/Parser.pm @@ -130,11 +130,17 @@ sub _parse_genesis_config { $parsed{pipeline} = $data->{pipeline} || {}; $parsed{targets} = $data->{targets} || {}; - $parsed{integrations} = $data->{integrations} || {}; $parsed{scripts} = $data->{scripts} || {}; $parsed{provider} = $data->{provider} || {}; $parsed{provider_config} = $data->{provider_config} || {}; + # Accept both nested (ci.integrations.*) and flat (ci.vault:, ci.source_control:) formats. + # Flat keys win only if the nested key is absent so explicit ci.integrations: always takes precedence. + my %integ = %{ $data->{integrations} || {} }; + $integ{vault} //= $data->{vault} if $data->{vault} && !$integ{vault}; + $integ{source_control} //= $data->{source_control} if $data->{source_control} && !$integ{source_control}; + $parsed{integrations} = \%integ; + # When no pipeline section is provided, workflow topology is derived from # genesis.pipeline.* keys in environment YAML files (same as multi-file # with no pipeline.yml). Point ASTBuilder at the repo root. diff --git a/lib/Genesis/CI/Compiler/Validator.pm b/lib/Genesis/CI/Compiler/Validator.pm index 63ddd40a..fa1ca901 100644 --- a/lib/Genesis/CI/Compiler/Validator.pm +++ b/lib/Genesis/CI/Compiler/Validator.pm @@ -526,23 +526,21 @@ sub _validate_pipeline_section { # Metadata if ($pipeline->{metadata}) { - $self->_error("'metadata.name' is required") + $self->_error("'pipeline.metadata.name' is required") unless $pipeline->{metadata}{name}; } # Branches if ($pipeline->{branches}) { - $self->_error("'branches.live' is required") + $self->_error("'pipeline.branches.live' is required") unless $pipeline->{branches}{live}; } - # Workflows + # Workflows are optional — absent means topology is derived from env files if ($pipeline->{workflows}) { for my $wf_name (keys %{$pipeline->{workflows}}) { $self->_validate_workflow($wf_name, $pipeline->{workflows}{$wf_name}); } - } else { - $self->_error("'workflows' section is required in pipeline.yml"); } } @@ -648,20 +646,20 @@ sub _validate_integrations_section { my ($self, $integrations) = @_; unless ($integrations && ref($integrations) eq 'HASH') { - $self->_error("Invalid or empty integrations.yml"); + $self->_error("'integrations' section is missing or invalid"); return; } # Vault is required unless ($integrations->{vault}) { - $self->_error("'vault' section is required in integrations.yml"); + $self->_error("'integrations.vault' is required (set ci.integrations.vault or ci.vault)"); } elsif (!$integrations->{vault}{url}) { - $self->_error("'vault.url' is required in integrations.yml"); + $self->_error("'integrations.vault.url' is required"); } # Source control is required unless ($integrations->{source_control}) { - $self->_error("'source_control' section is required in integrations.yml"); + $self->_error("'integrations.source_control' is required (set ci.integrations.source_control or ci.source_control)"); } } diff --git a/t/ci-compiler.t b/t/ci-compiler.t index c11fb1ad..5be18a52 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2734,6 +2734,93 @@ subtest 'PipelineProvider - provider_config keeps boolean false when non-default ### validate_config_section: source_control must be a hash ### ============================================================ ### +### ============================================================ ### +### Parser: flat-format vault/source_control normalization +### ============================================================ ### + +subtest 'Parser - genesis-config: flat vault/source_control lifted into integrations' => sub { + my $flat_ci = { + targets => { + sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } }, + }, + vault => { + url => 'https://vault.example.com', + auth => { role_id => { secret_ref => 'secret/ci:role_id' } }, + }, + source_control => { + provider => 'github', + repository => 'org/repo', + }, + pipeline => {}, + }; + + my $top = MockTop->new(config => MockConfig->new(ci => $flat_ci), base => '/repo'); + my $parser = Genesis::CI::Compiler::Parser->new(top => $top); + my $parsed = eval { $parser->parse() }; + ok !$@, "flat-format config parses without error" or diag $@; + + ok ref($parsed->{integrations}) eq 'HASH', "integrations is a hash"; + ok $parsed->{integrations}{vault}, + "ci.vault lifted into integrations.vault"; + is $parsed->{integrations}{vault}{url}, 'https://vault.example.com', + "vault url preserved"; + ok $parsed->{integrations}{source_control}, + "ci.source_control lifted into integrations.source_control"; + is $parsed->{integrations}{source_control}{provider}, 'github', + "source_control provider preserved"; +}; + +subtest 'Parser - genesis-config: nested integrations.* takes precedence over flat' => sub { + my $mixed_ci = { + targets => { sandbox => { type => 'bosh-director', connection => { url => 'u' } } }, + vault => { url => 'https://flat-vault.example.com' }, # flat — should be ignored + integrations => { + vault => { url => 'https://nested-vault.example.com' }, # nested wins + source_control => { provider => 'github', repository => 'org/repo' }, + }, + pipeline => {}, + }; + + my $top = MockTop->new(config => MockConfig->new(ci => $mixed_ci), base => '/repo'); + my $parser = Genesis::CI::Compiler::Parser->new(top => $top); + my $parsed = eval { $parser->parse() }; + ok !$@, "mixed flat+nested parses without error" or diag $@; + is $parsed->{integrations}{vault}{url}, 'https://nested-vault.example.com', + "nested integrations.vault takes precedence over flat ci.vault"; +}; + +### ============================================================ ### +### Validator: workflows optional in pipeline section +### ============================================================ ### + +subtest 'Validator - pipeline section with metadata but no workflows is valid' => sub { + my $v = Genesis::CI::Compiler::Validator->new(); + $v->validate({ + _source_format => 'genesis-config', + pipeline => { name => 'cf', branches => { propagation => 'push' } }, + integrations => { + vault => { url => 'https://vault.example.com' }, + source_control => { provider => 'github', repository => 'org/repo' }, + }, + targets => { + sandbox => { type => 'bosh-director', connection => { url => 'https://bosh' } }, + }, + }); + ok !$v->has_errors, + "pipeline with name/branches but no workflows passes validation (env-file topology)" + or diag join("\n", @{$v->errors}); +}; + +subtest 'Validator - validate_config_section: accepts flat-format source_control' => sub { + my $flat = { + targets => { sandbox => { type => 'bosh-director', connection => { url => 'u' } } }, + source_control => { provider => 'github', repository => 'org/repo' }, + # no integrations: key at all + }; + eval { Genesis::CI::Compiler->validate_config_section($flat, undef) }; + ok !$@, "flat ci.source_control accepted by validate_config_section" or diag $@; +}; + subtest 'Compiler - validate_config_section: rejects scalar source_control' => sub { my $bad = { %$_ci_data, From 7ce7146bf6617d142e498338faecabd9b4f416e3 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:26:09 -0400 Subject: [PATCH 048/103] Update ci-compiler.t --- t/ci-compiler.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/ci-compiler.t b/t/ci-compiler.t index 5be18a52..fd2b455d 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -2797,7 +2797,7 @@ subtest 'Validator - pipeline section with metadata but no workflows is valid' = my $v = Genesis::CI::Compiler::Validator->new(); $v->validate({ _source_format => 'genesis-config', - pipeline => { name => 'cf', branches => { propagation => 'push' } }, + pipeline => { name => 'cf', branches => { live => 'main', propagation => 'push' } }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { provider => 'github', repository => 'org/repo' }, From 244d3f286f91df6ff01b36447daa97c09b56124d Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 10:30:54 -0700 Subject: [PATCH 049/103] Remove targets and integrations validation [Improvements] - Drop early validation of `ci.targets` and `ci.integrations.source_control` in `validate_config_section`, deferring these checks to downstream schema validation and provider-specific logic - Allows configurations without pre-defined targets or source_control integrations to proceed through compilation --- lib/Genesis/CI/Compiler.pm | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index 727ff86e..d85ecabd 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -195,23 +195,6 @@ sub validate_config_section { bail("'ci' configuration in .genesis/config must be a hash") unless ref($data) eq 'HASH'; - # ci.targets must be a non-empty hash - my $targets = $data->{targets}; - bail("'ci.targets' is required and must define at least one target") - unless defined($targets) && ref($targets) eq 'HASH' && scalar(keys %$targets) > 0; - - # source_control is required — accepted at ci.integrations.source_control (nested) - # or ci.source_control (flat, per architecture config reference). - my $integrations = $data->{integrations}; - if (defined $integrations) { - bail("'ci.integrations' must be a hash") unless ref($integrations) eq 'HASH'; - } - my $sc = ($integrations || {})->{source_control} // $data->{source_control}; - bail("'ci.source_control' (or 'ci.integrations.source_control') is required") - unless defined $sc; - bail("'ci.source_control' must be a hash") - unless ref($sc) eq 'HASH'; - # Validate ci.provider section against the provider's own schema if (my $provider_data = $data->{provider}) { bail("'ci.provider' must be a hash") From 989a9612cb302da447a7c44bfb0bb0ad386a86be Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:38:01 -0400 Subject: [PATCH 050/103] Use CI_PIPELINE_CONTROL_KEY for control branch Add CI_PIPELINE_CONTROL_KEY constant in Genesis::Top and update CI compiler code to reference it instead of hardcoded 'live'. Files updated (ASTBuilder, Parser, PipelineDescriptor, Validator) now import Genesis::Top and use Genesis::Top::CI_PIPELINE_CONTROL_KEY() when accessing pipeline.branches, and the validator message was adjusted accordingly. This centralizes the pipeline control-branch key (set to 'control') so it can be changed in one place without touching callers. --- lib/Genesis/CI/Compiler/ASTBuilder.pm | 7 ++++--- lib/Genesis/CI/Compiler/Parser.pm | 5 +++-- lib/Genesis/CI/Compiler/PipelineDescriptor.pm | 9 +++++---- lib/Genesis/CI/Compiler/Validator.pm | 5 +++-- lib/Genesis/Top.pm | 5 +++-- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/Genesis/CI/Compiler/ASTBuilder.pm b/lib/Genesis/CI/Compiler/ASTBuilder.pm index c47191cd..8f24f503 100644 --- a/lib/Genesis/CI/Compiler/ASTBuilder.pm +++ b/lib/Genesis/CI/Compiler/ASTBuilder.pm @@ -3,6 +3,7 @@ use strict; use warnings; use Genesis; +use Genesis::Top (); use Genesis::CI::Compiler::AST; use JSON::PP; @@ -58,7 +59,7 @@ sub _build_from_legacy { # Branches my $branches = { - live => $p->{git}{branch} || 'master', + Genesis::Top::CI_PIPELINE_CONTROL_KEY() => $p->{git}{branch} || 'master', target_prefix => 'target/', }; @@ -216,8 +217,8 @@ sub _build_from_multi_file { # Branches my $branches = $pipeline->{branches} || { - live => 'main', - target_prefix => 'target/', + Genesis::Top::CI_PIPELINE_CONTROL_KEY() => 'main', + target_prefix => 'target/', }; # Integrations diff --git a/lib/Genesis/CI/Compiler/Parser.pm b/lib/Genesis/CI/Compiler/Parser.pm index e38584c1..5fa75f1c 100644 --- a/lib/Genesis/CI/Compiler/Parser.pm +++ b/lib/Genesis/CI/Compiler/Parser.pm @@ -3,6 +3,7 @@ use strict; use warnings; use Genesis; +use Genesis::Top (); use Genesis::CI::Layout; use JSON::PP; @@ -183,8 +184,8 @@ sub _parse_legacy_file { version => '1.0', }, branches => { - live => $p->{git}{branch} || 'master', - target_prefix => 'target/', + Genesis::Top::CI_PIPELINE_CONTROL_KEY() => $p->{git}{branch} || 'master', + target_prefix => 'target/', }, }; diff --git a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm index 7a6aa5d9..7e8f954b 100644 --- a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm +++ b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm @@ -3,6 +3,7 @@ use strict; use warnings; use Genesis; +use Genesis::Top (); use JSON::PP; ### Constructor {{{ @@ -324,7 +325,7 @@ sub _git_resource { my $sc = $ast->integrations->{source_control} || {}; my $uri = $self->_git_uri($sc); - my $branch = $sc->{default_branch} || $ast->branches->{live} || 'main'; + my $branch = $sc->{default_branch} || $ast->branches->{Genesis::Top::CI_PIPELINE_CONTROL_KEY()} || 'main'; my $source = { uri => $uri, branch => $branch }; @@ -381,7 +382,7 @@ sub _env_resources { my @resources; my $sc = $ast->integrations->{source_control} || {}; my $uri = $self->_git_uri($sc); - my $br = $sc->{default_branch} || $ast->branches->{live} || 'main'; + my $br = $sc->{default_branch} || $ast->branches->{Genesis::Top::CI_PIPELINE_CONTROL_KEY()} || 'main'; my $root = $sc->{root} || '.'; my $pr = ($root eq '.') ? '' : "$root/"; @@ -927,7 +928,7 @@ sub _task_config { CACHE_DIR => "$alias-cache", OUT_DIR => 'out/git', WORKING_DIR => "$alias-changes", - GIT_BRANCH => $sc->{default_branch} || $ast->branches->{live} || 'main', + GIT_BRANCH => $sc->{default_branch} || $ast->branches->{Genesis::Top::CI_PIPELINE_CONTROL_KEY()} || 'main', GIT_AUTHOR_NAME => ($sc->{commit_author} ? $sc->{commit_author}{name} : undef) || 'Concourse Bot', GIT_AUTHOR_EMAIL => ($sc->{commit_author} ? $sc->{commit_author}{email} : undef) @@ -1001,7 +1002,7 @@ sub _cache_task_config { CURRENT_ENV => $env, WORKING_DIR => 'out/git', OUT_DIR => 'cache-out/git', - GIT_BRANCH => $sc->{default_branch} || $ast->branches->{live} || 'main', + GIT_BRANCH => $sc->{default_branch} || $ast->branches->{Genesis::Top::CI_PIPELINE_CONTROL_KEY()} || 'main', GIT_AUTHOR_NAME => ($sc->{commit_author} ? $sc->{commit_author}{name} : undef) || 'Concourse Bot', GIT_AUTHOR_EMAIL => ($sc->{commit_author} ? $sc->{commit_author}{email} : undef) diff --git a/lib/Genesis/CI/Compiler/Validator.pm b/lib/Genesis/CI/Compiler/Validator.pm index fa1ca901..11a7a75e 100644 --- a/lib/Genesis/CI/Compiler/Validator.pm +++ b/lib/Genesis/CI/Compiler/Validator.pm @@ -3,6 +3,7 @@ use strict; use warnings; use Genesis; +use Genesis::Top (); use JSON::PP; ### Constructor {{{ @@ -532,8 +533,8 @@ sub _validate_pipeline_section { # Branches if ($pipeline->{branches}) { - $self->_error("'pipeline.branches.live' is required") - unless $pipeline->{branches}{live}; + $self->_error("'pipeline.branches." . Genesis::Top::CI_PIPELINE_CONTROL_KEY() . "' is required") + unless $pipeline->{branches}{Genesis::Top::CI_PIPELINE_CONTROL_KEY()}; } # Workflows are optional — absent means topology is derived from env files diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 0bc3c3d0..6159c3b7 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -26,8 +26,9 @@ use File::Path qw/rmtree/; # 'genesis deploy' validates its working state. Captured as a constant # (and a config key) so it can change without rippling through the # codebase; not currently exposed to end users. -use constant DEFAULT_CONTROL_BRANCH => 'control'; -use constant LATEST_CONFIG_VERSION => 3; +use constant DEFAULT_CONTROL_BRANCH => 'control'; +use constant CI_PIPELINE_CONTROL_KEY => 'control'; # key in pipeline.branches{} hash for the control branch +use constant LATEST_CONFIG_VERSION => 3; ### Config Section Delegation Registry {{{ # Modules may register themselves as handlers for specific top-level keys in From 9d92f47144100f8997c6fa9a1118b13e16aae6a5 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:00:42 -0400 Subject: [PATCH 051/103] Update ci-compiler.t --- t/ci-compiler.t | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/t/ci-compiler.t b/t/ci-compiler.t index fd2b455d..0c9e1bf6 100644 --- a/t/ci-compiler.t +++ b/t/ci-compiler.t @@ -48,7 +48,7 @@ ok !$@, "loaded Concourse provider" or diag $@; subtest 'AST - construction and accessors' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test-pipe', version => '2.0', source => 'modern' }, - branches => { live => 'main', target_prefix => 'target/' }, + branches => { control => 'main', target_prefix => 'target/' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -88,7 +88,7 @@ subtest 'AST - construction and accessors' => sub { # Accessors is $ast->metadata->{name}, 'test-pipe', "metadata name accessor works"; - is $ast->branches->{live}, 'main', "branches accessor works"; + is $ast->branches->{control}, 'main', "branches accessor works"; is $ast->integrations->{vault}{url}, 'https://vault.example.com', "integrations accessor works"; is $ast->configuration->{public}, 1, "configuration accessor works"; @@ -326,7 +326,7 @@ subtest 'ASTBuilder - legacy format' => sub { is $ast->metadata->{name}, 'my-legacy-pipeline', "legacy pipeline name preserved"; is $ast->metadata->{source}, 'legacy', "source marked as legacy"; is $ast->metadata->{source_file}, 'ci.yml', "source_file preserved"; - is $ast->branches->{live}, 'master', "branch preserved from legacy git.branch"; + is $ast->branches->{control}, 'master', "branch preserved from legacy git.branch"; ok $ast->provider_config->{concourse}{_legacy_pipeline_raw}, "legacy raw data preserved in provider_config"; }; @@ -341,7 +341,7 @@ subtest 'ASTBuilder - modern format' => sub { version => '2.0', }, branches => { - live => 'main', + control => 'main', target_prefix => 'deploy/', }, workflows => { @@ -374,7 +374,7 @@ subtest 'ASTBuilder - modern format' => sub { my $ast = $builder->build($parsed, $scripts); isa_ok $ast, 'Genesis::CI::Compiler::AST', "build returns an AST for modern format"; is $ast->metadata->{name}, 'modern-pipeline', "modern metadata preserved"; - is $ast->branches->{live}, 'main', "branches preserved from modern format"; + is $ast->branches->{control}, 'main', "branches preserved from modern format"; # Workflows should have been processed into graph form my $wf = $ast->workflows->{deploy}; @@ -399,7 +399,7 @@ subtest 'ASTBuilder - modern format populates generic fields' => sub { _source_format => 'multi-file', pipeline => { metadata => { name => 'generic-test', version => '2.0' }, - branches => { live => 'main' }, + branches => { control => 'main' }, workflows => { deploy => { type => 'deployment', @@ -436,7 +436,7 @@ subtest 'ASTBuilder - no auto-population without explicit triggers/resources' => _source_format => 'multi-file', pipeline => { metadata => { name => 'no-auto-pop-test' }, - branches => { live => 'main' }, + branches => { control => 'main' }, workflows => {}, # No explicit triggers or resources }, @@ -907,7 +907,7 @@ subtest 'Validator - DAG cycle detection' => sub { _source_format => 'multi-file', pipeline => { metadata => { name => 'test' }, - branches => { live => 'main' }, + branches => { control => 'main' }, workflows => { cycle => { graph => { @@ -941,7 +941,7 @@ subtest 'Validator - valid multi-file config' => sub { _source_format => 'multi-file', pipeline => { metadata => { name => 'test-pipeline' }, - branches => { live => 'main' }, + branches => { control => 'main' }, workflows => { deploy => { type => 'deployment', @@ -1056,7 +1056,7 @@ subtest 'Concourse - native generation from modern AST' => sub { source => 'modern', deployment_type => 'cf', }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com', @@ -1293,7 +1293,7 @@ subtest 'Concourse - output_files' => sub { subtest 'Concourse - generate_from_ast routes to native for non-legacy' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { source_control => { provider => 'github', repository => 'org/repo' }, }, @@ -1329,7 +1329,7 @@ subtest 'Concourse - locker resources generated when locker configured' => sub { source => 'modern', deployment_type => 'cf', }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com', @@ -1396,7 +1396,7 @@ subtest 'Concourse - locker resources generated when locker configured' => sub { subtest 'Concourse - locker skips bosh-lock for create-env' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'ce-test', deployment_type => 'bosh', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -1447,7 +1447,7 @@ subtest 'Concourse - locker skips bosh-lock for create-env' => sub { subtest 'Concourse - auto-update job generated' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'autoupdate-test', deployment_type => 'cf', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -1513,7 +1513,7 @@ subtest 'Concourse - auto-update job generated' => sub { subtest 'Concourse - grouped notifications' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'grouped-test', deployment_type => 'cf', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -1569,7 +1569,7 @@ subtest 'Concourse - grouped notifications' => sub { subtest 'Concourse - custom groups' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'custom-groups-test', deployment_type => 'cf', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -1630,7 +1630,7 @@ subtest 'Concourse - custom groups' => sub { subtest 'Concourse - OCFP config name support' => sub { my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'ocfp-test', deployment_type => 'cf', source => 'modern' }, - branches => { live => 'main' }, + branches => { control => 'main' }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { @@ -2561,7 +2561,7 @@ subtest 'PipelineProvider - base class check_prereqs returns 1' => sub { my $gha = Genesis::CI::GithubActions->new( ast => Genesis::CI::Compiler::AST->new( metadata => { name => 'test', version => '2.0', source => 'modern' }, - branches => { live => 'main', target_prefix => 'target/' }, + branches => { control => 'main', target_prefix => 'target/' }, integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, targets => {}, workflows => {}, @@ -2581,7 +2581,7 @@ subtest 'PipelineProvider::Concourse - check_prereqs returns 1 when fly present' } my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test', version => '2.0', source => 'modern' }, - branches => { live => 'main', target_prefix => 'target/' }, + branches => { control => 'main', target_prefix => 'target/' }, integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, targets => {}, workflows => {}, @@ -2594,7 +2594,7 @@ subtest 'PipelineProvider::Concourse - check_prereqs returns 0 when fly absent' local $ENV{PATH} = '/nonexistent'; my $ast = Genesis::CI::Compiler::AST->new( metadata => { name => 'test', version => '2.0', source => 'modern' }, - branches => { live => 'main', target_prefix => 'target/' }, + branches => { control => 'main', target_prefix => 'target/' }, integrations => { source_control => { provider => 'github', repository => 'org/repo' } }, targets => {}, workflows => {}, @@ -2797,7 +2797,7 @@ subtest 'Validator - pipeline section with metadata but no workflows is valid' = my $v = Genesis::CI::Compiler::Validator->new(); $v->validate({ _source_format => 'genesis-config', - pipeline => { name => 'cf', branches => { live => 'main', propagation => 'push' } }, + pipeline => { name => 'cf', branches => { control => 'main', propagation => 'push' } }, integrations => { vault => { url => 'https://vault.example.com' }, source_control => { provider => 'github', repository => 'org/repo' }, From 2dbb398428026723632e958346341a838ead66f9 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 12:33:36 -0700 Subject: [PATCH 052/103] Replace --ci-provider with --with-ci on repo-init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repo-init now only sets up manual provider with ci.enabled, ci.name, and ci.repo.root. Automated provider connection deferred to future `repo config ci`. Restructure ci schema: pipeline.name → ci.name, add ci.repo.root. Deduplicate _create_ci_scaffold. Include full path in legacy ci.yml bail messages. --- bin/genesis | 12 +-- lib/Genesis/Commands/Repo.pm | 88 +++++++------------- lib/Genesis/Top.pm | 22 +++-- t/unit-tests/genesis_commands_repo_ci-core.t | 4 +- 4 files changed, 52 insertions(+), 74 deletions(-) diff --git a/bin/genesis b/bin/genesis index d9f9b38f..a13eea2e 100755 --- a/bin/genesis +++ b/bin/genesis @@ -1282,12 +1282,14 @@ define_command("repo-init", { "secrets provider. You must configure one via #C{genesis secrets-provider} ". "before creating environments.", - 'ci-provider=s' => - "CI provider for pipeline automation: 'concourse', 'github-actions', ". - "or 'manual'. When specified, writes the ci: section to .genesis/config ". - "and generates the .genesis/ci/ scaffold.", + 'with-ci' => + "Enable CI pipeline support. Sets up the control branch and ". + "configures the repo for branch-based pipeline topology. ". + "Environments created with #C{genesis new} will be prompted ". + "for pipeline metadata (prior_env, gates). Connect an ". + "automated provider later with #C{genesis repo config ci}.", ], - extended_handlers => ['Genesis::Kit::Provider', 'Genesis::CI::Provider'], + extended_handlers => ['Genesis::Kit::Provider'], arguments => [ "name?" => "If the name argument is not specified, it will default to the same ". diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 9df12236..2348d5cf 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -82,13 +82,7 @@ sub _repo_init_validate { "Cannot specify both --vault and --skip-vault." ) if $opts{vault} && $opts{'skip-vault'}; - # CI provider options (--ci-provider, --ci-target, etc.) have been - # parsed by the framework's extended_handlers into the ci_provider - # slot. We extract the opts hash here for quick flag checks (e.g. - # branch validation), but defer building the actual provider object - # until step 6 (after directory/kit validation passes) so the - # interactive wizard doesn't run if we're going to bail anyway. - my %ci_provider_opts = %{$opts{ci_provider} // {}}; + my $with_ci = $opts{'with-ci'}; # --- 3. Gather data: validate local sources, detect git repo --- @@ -168,7 +162,7 @@ sub _repo_init_validate { # Without #C{--ci-provider}, no pipeline topology is being # established, so the branch name is irrelevant at this point. my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - if ($use_subdir && $ci_provider_opts{'ci-provider'}) { + if ($use_subdir && $with_ci) { my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); chomp $branch if defined $branch; if (!defined($branch) || $branch ne $control_branch) { @@ -245,28 +239,8 @@ sub _repo_init_validate { $vault_target = $vault->{name} if $vault; } - # Build the CI provider object now that all preflight checks have - # passed. If required flags (--ci-target, etc.) were omitted and - # we have a controlling terminal, fall back to the interactive - # wizard to collect them. - my $ci_provider_obj; - if ($ci_provider_opts{'ci-provider'}) { - $ci_provider_obj = eval { Genesis::CI::Provider->init(%ci_provider_opts) }; - if ($@) { - if (in_controlling_terminal) { - $ci_provider_obj = eval { - Genesis::CI::Provider->new(type => $ci_provider_opts{'ci-provider'}) - ->interactive_wizard(undef, %ci_provider_opts); - }; - bail("CI provider wizard failed: %s", $@) if $@; - } else { - bail("Could not initialize CI provider: %s", $@); - } - } - - # Verify the provider's toolchain is available before we do any work - $ci_provider_obj->check_prereqs() or exit 86; - } + # --with-ci just sets a flag; provider details come later via + # `genesis repo config ci`. No provider object needed at init time. # --- 7. Store derived values and summarize intent --- @@ -277,7 +251,7 @@ sub _repo_init_validate { _target_path => $target_path, _kit_file => $kit_file, _kit_provider => $kit_provider, - _ci_provider_obj => $ci_provider_obj, + _with_ci => $with_ci, _use_subdir => $use_subdir, _vault_target => $vault_target, _replace_existing => $replace_existing, @@ -297,7 +271,7 @@ sub _repo_init_validate { } push @plan, "vault: #C{$vault_target}" if $vault_target; push @plan, "vault: #Yi{deferred}" unless $vault_target; - push @plan, "ci provider: #C{" . $ci_provider_obj->label . "}" if $ci_provider_obj; + push @plan, "ci: #C{enabled} (manual provider)" if $with_ci; push @plan, "subdirectory of enclosing git repo: #C{yes} (no separate .git, auto-detected)" if $use_subdir; info "\nCreating #C{%s} deployment repository in #M{%s/}:", $name, $dir; info " %s", $_ for @plan; @@ -325,7 +299,7 @@ sub _repo_init_execute { $vault_target, # vault target to configure, or undef to skip vault $replace_existing, # whether to remove existing target directory if it exists $linked_dev_kit, # optional path to a local dev kit to link into the repo - $ci_provider_obj, # optional Genesis::CI::Provider object (from validation) + $with_ci, # enable CI pipeline support (manual provider) # User provided options (validated but not altered) $directory, # optional custom directory name override @@ -334,7 +308,7 @@ sub _repo_init_execute { $reason, # optional commit message override ) = get_options()->@{qw/ _name _dir _parent_dir _target_path _kit_file kit _kit_provider _use_subdir _vault_target - _replace_existing link-dev-kit _ci_provider_obj directory kits-path no-commit reason + _replace_existing link-dev-kit _with_ci directory kits-path no-commit reason /}; # Remove existing directory if validation approved it @@ -368,7 +342,7 @@ sub _repo_init_execute { # Create the repo via Top->create (pass kit_provider if we already built one) $create_opts{kit_provider} = $kit_provider if $kit_provider; my $top = Genesis::Top->create($parent_dir, $name, %create_opts, kits_path => $kit_path); - $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $ci_provider_obj; + $top->embed($ENV{GENESIS_CALLBACK_BIN} || $0) if $with_ci; my $root = $top->path; my $human_root = humanize_path($root); @@ -394,24 +368,21 @@ sub _repo_init_execute { $kit_desc = "with an empty development kit in #C{$human_root/dev}"; } - # CI provider scaffold - if ($ci_provider_obj) { - _create_ci_scaffold($top, $ci_provider_obj); + # CI: write manual provider to config + if ($with_ci) { + _create_ci_scaffold($top, root => ($use_subdir ? $dir : '.')); } # Only create a new .git when we're not already sitting inside # an enclosing git worktree; in subdir mode we share the # parent's .git and just stage into its index. In standalone - # mode, if the user is establishing pipeline topology - # (--ci-provider given), force the initial branch name to the - # control branch so the initial commit lands there instead of - # on whatever git's init.defaultBranch happens to be. When - # no CI provider is being configured, let git choose its - # default branch -- pipeline topology is not relevant yet. + # mode, if --with-ci is set, force the initial branch name to + # the control branch so the initial commit lands there instead + # of whatever git's init.defaultBranch happens to be. # #C{git symbolic-ref HEAD} works on all git versions, unlike # #C{git init -b} which requires >= 2.28. unless ($use_subdir) { - if ($ci_provider_obj) { + if ($with_ci) { my $branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); run({ onfailure => "Failed to initialize git in $human_root/" }, "git init && git symbolic-ref HEAD refs/heads/$branch"); @@ -503,7 +474,7 @@ sub _repo_init_execute { human_root => $human_root, name => $name, kit_desc => $kit_desc, - ci_provider => $ci_provider_obj ? $ci_provider_obj->label : undef, + ci_provider => $with_ci ? 'manual' : undef, vault_skipped => $vault_target ? 0 : 1, vault => $vault_target, submodule => $use_subdir, @@ -594,23 +565,24 @@ sub _select_vault_target { } sub _create_ci_scaffold { - my ($top, $provider) = @_; + my ($top, %opts) = @_; - # $provider may be a Genesis::CI::Provider object or a plain string (legacy). - my %provider_cfg = ref($provider) ? $provider->config() : (type => $provider); + my %provider_cfg; + if (ref($opts{provider})) { + %provider_cfg = $opts{provider}->config(); + } else { + %provider_cfg = (type => $opts{provider} || 'manual'); + } - $top->config->set('ci', { + my %ci = ( enabled => Genesis::Config::TRUE, provider => \%provider_cfg, - pipeline => { - name => $top->config->get('deployment_type'), - }, - }); - $top->config->save; + name => $top->config->get('deployment_type'), + ); + $ci{repo} = { root => $opts{root} } if $opts{root}; - my $ci_dir = $top->path(".genesis/ci"); - mkdir_or_fail($ci_dir); - mkfile_or_fail("$ci_dir/.keep", ""); + $top->config->set('ci', \%ci); + $top->config->save; } diff --git a/lib/Genesis/Top.pm b/lib/Genesis/Top.pm index 6159c3b7..43e7b0eb 100644 --- a/lib/Genesis/Top.pm +++ b/lib/Genesis/Top.pm @@ -1225,10 +1225,11 @@ sub _validate_config { my $ci_yml = $self->path('ci.yml'); if (-f $ci_yml && _is_legacy_ci_file($ci_yml)) { bail( - "Legacy CI configuration detected at #C{ci.yml}.\n". + "Legacy CI configuration detected at #C{%s}.\n". "Pipeline support requires a v3 config. Please run:\n\n". " genesis repo-init --upgrade\n\n". - "to migrate your CI configuration into the v3 config format." + "to migrate your CI configuration into the v3 config format.", + $ci_yml ); } @@ -1248,15 +1249,17 @@ sub _validate_config { if (-f $ci_yml && _is_legacy_ci_file($ci_yml)) { if ($self->config->get('ci.enabled') && $self->config->has('ci.provider.type')) { bail( - "Legacy #C{ci.yml} conflicts with the v3 CI configuration.\n". - "Remove #C{ci.yml} — CI is already configured in #C{.genesis/config}." + "Legacy #C{%s} conflicts with the v3 CI configuration.\n". + "Remove it — CI is already configured in #C{.genesis/config}.", + $ci_yml ); } else { bail( - "Legacy CI configuration detected at #C{ci.yml}.\n". + "Legacy CI configuration detected at #C{%s}.\n". "Pipeline support requires migrating this into the v3 config. Please run:\n\n". " genesis repo-init --upgrade\n\n". - "to migrate your CI configuration into the v3 config format." + "to migrate your CI configuration into the v3 config format.", + $ci_yml ); } } @@ -1452,11 +1455,12 @@ sub _repo_config_schema { insecure => {type => 'boolean', default => Genesis::Config::FALSE, description => 'Skip TLS verification'}, } }, - pipeline => { + name => {type => 'string', description => 'Pipeline name (defaults to deployment_type)'}, + repo => { type => 'hash', - description => 'Pipeline generation settings', + description => 'Repository layout settings', schema => { - name => {type => 'string', description => 'Pipeline name (defaults to deployment_type)'}, + root => {type => 'string', default => '.', description => 'Path to deployment root within the git repo'}, } }, } diff --git a/t/unit-tests/genesis_commands_repo_ci-core.t b/t/unit-tests/genesis_commands_repo_ci-core.t index b9ecd9b7..bd30857c 100644 --- a/t/unit-tests/genesis_commands_repo_ci-core.t +++ b/t/unit-tests/genesis_commands_repo_ci-core.t @@ -292,7 +292,7 @@ subtest 'v3 config validates with CI enabled and provider' => sub { my $dir = make_v3_repo(workdir("v3-enabled"), ci => { enabled => 'true', provider => { type => 'concourse', target => 'pipes/lmelt', url => 'https://pipes.example.com', team => 'lmelt' }, - pipeline => { name => 'bosh' }, + name => 'bosh', }); my $top = Genesis::Top->new($dir, no_vault => 1); @@ -300,7 +300,7 @@ subtest 'v3 config validates with CI enabled and provider' => sub { ok $top->ci_configured, "ci_configured is true"; is $top->config->get('ci.provider.type'), 'concourse', "provider type is concourse"; is $top->config->get('ci.provider.target'), 'pipes/lmelt', "provider target correct"; - is $top->config->get('ci.pipeline.name'), 'bosh', "pipeline name correct"; + is $top->config->get('ci.name'), 'bosh', "ci name correct"; }; subtest 'v3 config bails when enabled but no provider' => sub { From 2bbbfe37d5dbbed7171bd0580eddec5b8d09c55f Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 12:33:56 -0700 Subject: [PATCH 053/103] Skip compiler validation for manual provider The manual provider has no compiler class. Return early from validate_config_section after basic hash check. --- lib/Genesis/CI/Compiler.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/Genesis/CI/Compiler.pm b/lib/Genesis/CI/Compiler.pm index d85ecabd..0c97a697 100644 --- a/lib/Genesis/CI/Compiler.pm +++ b/lib/Genesis/CI/Compiler.pm @@ -195,10 +195,13 @@ sub validate_config_section { bail("'ci' configuration in .genesis/config must be a hash") unless ref($data) eq 'HASH'; - # Validate ci.provider section against the provider's own schema + # Validate ci.provider section against the provider's own schema. + # The 'manual' provider has no compiler class — skip validation. if (my $provider_data = $data->{provider}) { bail("'ci.provider' must be a hash") unless ref($provider_data) eq 'HASH'; + # The 'manual' provider has no compiler class — skip validation. + return if ($provider_data->{type} || '') eq 'manual'; my $type = $provider_data->{type}; bail("'ci.provider.type' is required") unless $type; From 115944ab4634b5ab732657eee052220cebf7b4dd Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 12:34:10 -0700 Subject: [PATCH 054/103] Add env-file topology to pipeline-describe/graph Build DAG directly from env files via _build_from_env_files when CI is configured, bypassing the full compiler/provider chain. Adds _describe_topology for tree output and _topology_to_mermaid_md for pipeline.md generation. --- lib/Genesis/Commands/Pipelines.pm | 122 +++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index abc274f3..da43d770 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -109,9 +109,26 @@ sub pipeline_graph { my ($layout) = @_; option_defaults(config => 'ci.yml'); - my $opts = get_options; + my $opts = get_options; + my $top = Genesis::Top->new('.'); + + # For env-file topology, build the DAG directly. + if ($top->ci_configured) { + my $env_dir = $top->path; + require Genesis::CI::Compiler::ASTBuilder; + my $builder = Genesis::CI::Compiler::ASTBuilder->new( + top => $top, + env_dir => $env_dir, + ); + my ($nodes, $edges) = $builder->_build_from_env_files($env_dir); + my $md = _topology_to_mermaid_md($top, $nodes, $edges); + mkfile_or_fail('pipeline.md', $md); + info("Wrote #C{pipeline.md}"); + exit 0; + } + + # Full compiler path for legacy/multi-file configurations my $platform = $opts->{platform} || 'concourse'; - my $top = Genesis::Top->new('.'); my $result = _compile_pipeline($top, $platform); my $ast = $result->{ast}; my $provider = $result->{provider}; @@ -132,8 +149,24 @@ sub pipeline_describe { option_defaults(config => 'ci.yml'); my $opts = get_options; - my $platform = $opts->{platform} || 'concourse'; my $top = Genesis::Top->new('.'); + + # For env-file topology (manual provider or genesis-config CI), + # build the DAG directly without the full compiler/provider chain. + if ($top->ci_configured) { + my $env_dir = $top->path; + require Genesis::CI::Compiler::ASTBuilder; + my $builder = Genesis::CI::Compiler::ASTBuilder->new( + top => $top, + env_dir => $env_dir, + ); + my ($nodes, $edges) = $builder->_build_from_env_files($env_dir); + _describe_topology($top, $nodes, $edges); + exit 0; + } + + # Full compiler path for legacy/multi-file configurations + my $platform = $opts->{platform} || 'concourse'; my $result = _compile_pipeline($top, $platform); my $ast = $result->{ast}; my $provider = $result->{provider}; @@ -817,6 +850,89 @@ sub _ast_to_mermaid_md { return "# Pipeline: $name\n\n\`\`\`mermaid\n${mermaid}\`\`\`\n"; } +# }}} +# _topology_to_mermaid_md - mermaid flowchart from nodes+edges {{{ +sub _topology_to_mermaid_md { + my ($top, $nodes, $edges) = @_; + + my $name = $top->config->get('ci.name') || $top->type; + my @lines = ( + "---", + "config:", + " flowchart:", + " useMaxWidth: false", + "---", + "flowchart TD", + ); + + # Declare nodes with explicit labels so names aren't truncated + my %declared; + for my $n (sort keys %$nodes) { + my $label = $nodes->{$n}{alias} || $n; + (my $id = $n) =~ s/[^a-zA-Z0-9_]/_/g; + push @lines, " ${id}[\"$label\"]"; + $declared{$n} = $id; + } + + for my $edge (@$edges) { + push @lines, " $declared{$edge->{from}} --> $declared{$edge->{to}}"; + } + + my $mermaid = join("\n", @lines) . "\n"; + return "# Pipeline: $name\n\n\`\`\`mermaid\n${mermaid}\`\`\`\n"; +} + +# }}} +# _describe_topology - human-readable env-file topology description {{{ +sub _describe_topology { + my ($top, $nodes, $edges) = @_; + + my $name = $top->config->get('ci.name') || $top->type; + my $provider_type = $top->config->get('ci.provider.type') || 'manual'; + output "\n#G{Pipeline}: #C{%s}", $name; + output " #Yi{Provider}: %s", $provider_type; + output ""; + + unless (%$nodes) { + output "#Yi{No environments with pipeline metadata found.}"; + output ""; + return; + } + + # Build adjacency: parent → [children] + my %children; + my %has_parent; + for my $edge (@$edges) { + push @{$children{$edge->{from}}}, $edge->{to}; + $has_parent{$edge->{to}} = 1; + } + + # Roots are nodes with no incoming edge + my @roots = sort grep { !$has_parent{$_} } keys %$nodes; + + output "#G{Environment progression}:"; + output ""; + + my $print_tree; + $print_tree = sub { + my ($env, $indent) = @_; + my $node = $nodes->{$env}; + my @flags; + push @flags, '#Y{manual}' if $node->{manual}; + push @flags, '#M{require_pr}' if $node->{require_pr}; + my $flag_str = @flags ? ' (' . join(', ', @flags) . ')' : ''; + output "%s#C{%s}%s", $indent, $env, $flag_str; + for my $child (sort @{$children{$env} || []}) { + $print_tree->($child, "$indent "); + } + }; + + for my $root (@roots) { + $print_tree->($root, ' '); + } + output ""; +} + # }}} # _describe_ast - human-readable AST description {{{ sub _describe_ast { From 18cee8648932768244f60123245032b36221bf90 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 12:51:44 -0700 Subject: [PATCH 055/103] Harden envs command against repo errors Wrap Top loading in eval so a single repo with config issues (e.g. legacy ci.yml) does not crash the entire listing. Clean up deployment root display labels. --- lib/Genesis/Commands/Info.pm | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/Genesis/Commands/Info.pm b/lib/Genesis/Commands/Info.pm index 3c0b63a5..cbcd3d53 100644 --- a/lib/Genesis/Commands/Info.pm +++ b/lib/Genesis/Commands/Info.pm @@ -537,16 +537,20 @@ sub environments { my %deployments_by_name; if (scalar @repos) { $root =~ s{/?$}{/}; + my $root_display = $label eq '@current' ? '.' + : $label eq '@parent' ? '..' + : $label =~ /^@/ ? humanize_path($root) + : $label; if ($json) { info( "\nReading %s under deployment root #C{%s}", $group_by eq 'env' ? "environments" : "repositories", - humanize_path($root, root_map => $root_map) =~ s{/?$}{/}r =~ s{>\e\[0m/$}{>\e\[0m}r + $root_display ); } else { info( - "\nDeployment root #C{%s} contains the following %s:", - humanize_path($root, root_map => $root_map) =~ s{/?$}{/}r =~ s{>\e\[0m/$}{>\e\[0m}r, + "\nDeployment root #C{%s} contains the following\n%s:\n", + $root_display, $group_by eq 'env' ? "environments" : "repositories" ); } @@ -554,9 +558,18 @@ sub environments { my ($i,$j) = (0,0); for my $repo (@repos) { __processing($i, scalar(@repos), $j) if ($show_details && ($group_by eq 'env' || $json)); - my $top = Genesis::Top->new($repo, silent_vault_check => 1, allow_no_vault => 1); + my ($top, @envs); + eval { + $top = Genesis::Top->new($repo, silent_vault_check => 1, allow_no_vault => 1); + @envs = $top->envs; + }; + if ($@) { + my $err = $@; + $err =~ s/\s+$//; + warning("Skipping #C{%s}: %s", basename($repo), $err); + next; + } my $repo_label = basename($top->path); - my @envs = $top->envs; # Do the heavy lifting to determine which files are environments @envs = grep {$_->name =~ qr($filter_env)} @envs if $filter_env; __processing(++$i, scalar(@repos), $j) if ($show_details && ($group_by eq 'env' || $json)); From f4c21df8ce5bf33784ebdb669e61d7b1831e70bd Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 15:14:42 -0700 Subject: [PATCH 056/103] WIP: genesis propagate command Cascade model with propagation_files on Env, _resolve_propagation_base, and DAG-aware targeting. Entry point logic to be reworked for file-based routing instead of root-only targeting. --- bin/genesis | 25 ++++ lib/Genesis/Commands/Pipelines.pm | 201 ++++++++++++++++++++++++++++++ lib/Genesis/Env.pm | 91 ++++++++++++++ 3 files changed, 317 insertions(+) diff --git a/bin/genesis b/bin/genesis index a13eea2e..58d87cf3 100755 --- a/bin/genesis +++ b/bin/genesis @@ -2049,6 +2049,31 @@ define_command("pipeline-graph", { ], }, 'Genesis::Commands::Pipelines::pipeline_graph'); # }}} +# genesis propagate - propagate control branch changes to environment branches {{{ +define_command("propagate", { + summary => "Propagate changes from the control branch to environment branches.", + usage => "propagate [options]", + description => + "Identifies files that have changed on the control branch since each ". + "environment branch was last synced, and copies them to the appropriate ". + "environment branches. Walks the pipeline DAG in order so upstream ". + "environments are updated before downstream ones.\n\n". + "Each propagation commit records the control branch SHA it was sourced ". + "from, enabling deterministic tracking across the pipeline chain.", + function_group => Genesis::Commands::PIPELINE, + scope => 'repo', + option_group => Genesis::Commands::REPO_OPTIONS, + options => [ + 'dry-run|n' => + "Show what would be propagated without making changes.", + + 'commit=s' => + "Control branch commit SHA to propagate from. Defaults to ". + "current HEAD when on the control branch, or the SHA recorded ". + "in the last propagation commit when on an environment branch.", + ], +}, 'Genesis::Commands::Pipelines::propagate'); +# }}} # genesis pipeline-describe - human-readable pipeline description {{{ define_command("pipeline-describe", { summary => "Describe the pipeline in human-readable form.", diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index da43d770..f168c2ad 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -103,6 +103,207 @@ sub apply { exit 0; } +# }}} +# propagate - cascade control branch changes one level down the DAG {{{ +sub propagate { + + my $opts = get_options; + my $dry_run = $opts->{'dry-run'}; + my $top = Genesis::Top->new('.'); + + bail("CI is not configured for this repository.") + unless $top->ci_configured; + + my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + + # Determine current branch + my ($current_branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); + chomp $current_branch if defined $current_branch; + bail("Cannot propagate from a detached HEAD.") + unless defined $current_branch; + + # Build the DAG + my $env_dir = $top->path; + require Genesis::CI::Compiler::ASTBuilder; + my $builder = Genesis::CI::Compiler::ASTBuilder->new( + top => $top, + env_dir => $env_dir, + ); + my ($nodes, $edges) = $builder->_build_from_env_files($env_dir); + + bail("No environments with pipeline metadata found.") + unless %$nodes; + + my %children; + my %has_parent; + for my $edge (@$edges) { + push @{$children{$edge->{from}}}, $edge->{to}; + $has_parent{$edge->{to}} = 1; + } + + # Determine the control SHA and target environments based on + # where we are in the DAG. + my ($control_sha_full, $control_sha_short, @targets); + + if ($current_branch eq $control) { + # On control: propagate to root envs (entry points) + $control_sha_full = $opts->{commit}; + unless ($control_sha_full) { + ($control_sha_full) = run({}, 'git rev-parse HEAD'); + chomp $control_sha_full; + } + ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); + chomp $control_sha_short; + + @targets = sort grep { !$has_parent{$_} } keys %$nodes; + info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}", $control, $control_sha_short; + + } elsif ($nodes->{$current_branch}) { + # On an env branch: propagate to this env's children using the + # control SHA from the last propagation commit or merge-base. + my $manual_commits; + ($control_sha_full, $manual_commits) = _resolve_propagation_base($current_branch); + bail( + "Cannot determine propagation base for branch #C{%s}.\n". + "Run #C{genesis propagate} from the #C{%s} branch first.", + $current_branch, $control + ) unless $control_sha_full; + + ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); + chomp $control_sha_short; + + @targets = sort @{$children{$current_branch} || []}; + bail("Environment #C{%s} has no downstream environments.", $current_branch) + unless @targets; + + info "\n#G{Propagating from} #C{%s} #G{(control@}#C{%s}#G{)}", $current_branch, $control_sha_short; + } else { + bail( + "Branch #C{%s} is not the control branch or a known environment branch.", + $current_branch + ); + } + + info ""; + + # Propagate to each target + my $propagated = 0; + for my $env_name (@targets) { + # Check if env branch exists + my $exists = run({ passfail => 1 }, + 'git', 'rev-parse', '--verify', $env_name); + unless ($exists) { + warning("Branch #C{%s} does not exist — skipping.", $env_name); + next; + } + + # Load the env to get its dependency files + my $env = eval { $top->load_env($env_name) }; + unless ($env) { + warning("Could not load environment #C{%s} — skipping.", $env_name); + next; + } + + my @dep_files = $env->propagation_files; + next unless @dep_files; + + # Diff: what changed between this env branch and control@SHA. + # Run from git toplevel so pathspecs resolve correctly for subdir repos. + my ($git_root) = run({}, 'git rev-parse --show-toplevel'); + chomp $git_root; + my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); + chomp $git_prefix; + my @git_dep_files = map { "${git_prefix}$_" } @dep_files; + my ($diff_out) = run({ dir => $git_root }, + 'git', 'diff', '--name-only', "$env_name", $control_sha_full, '--', @git_dep_files + ); + my @changed = grep { /\S/ } split /\n/, ($diff_out || ''); + next unless @changed; + + if ($dry_run) { + info " #C{%s}: %d file%s to propagate", $env_name, scalar(@changed), @changed == 1 ? '' : 's'; + info " %s", $_ for @changed; + $propagated++; + } else { + # Checkout env branch, update files from control@SHA, commit + run({ onfailure => "Failed to checkout $env_name" }, + 'git', 'checkout', $env_name); + + for my $file (@changed) { + my $dir = $file; + $dir =~ s{/[^/]+$}{}; + mkdir_or_fail($dir) if $dir ne $file && !-d $dir; + run({ onfailure => "Failed to checkout $file from control\@$control_sha_short" }, + 'git', 'checkout', $control_sha_full, '--', $file); + } + + run({}, 'git', 'add', @changed); + my $msg = sprintf("[pipeline] control\@%s → %s", $control_sha_short, $env_name); + run({ onfailure => "Failed to commit propagation to $env_name" }, + 'git', 'commit', '-m', $msg); + info " #G{%s}: propagated %d file%s", $env_name, scalar(@changed), @changed == 1 ? '' : 's'; + $propagated++; + } + } + + # Return to the branch we started on + unless ($dry_run) { + run({ onfailure => "Failed to return to $current_branch" }, + 'git', 'checkout', $current_branch); + } + + if ($propagated) { + info "\n#G{Done.} %s %d environment%s.", + $dry_run ? "Would propagate to" : "Propagated to", + $propagated, $propagated == 1 ? '' : 's'; + } else { + info "\n#Yi{No changes to propagate.}"; + } + exit 0; +} + +# _resolve_propagation_base - find the control SHA to diff from for an env branch {{{ +# +# Resolution order: +# 1. Scan commit log for most recent [pipeline] control@ → use that +# (warns if it's not the HEAD commit, meaning manual commits were added) +# 2. No propagation commit found → branch was spawned from control, use +# git merge-base as the starting point +# +# Returns: ($control_sha_full, $manual_commits_on_top) +sub _resolve_propagation_base { + my ($branch) = @_; + my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + + # Scan log for propagation markers + my ($log) = run({}, 'git', 'log', '--format=%H %s', $branch); + if ($log) { + my $depth = 0; + for my $line (split /\n/, $log) { + if ($line =~ /^[0-9a-f]+ \[pipeline\] control\@([0-9a-f]+)/) { + my ($full) = run({}, 'git', 'rev-parse', $1); + chomp $full if defined $full; + if ($depth > 0) { + warning( + "Branch #C{%s} has %d manual commit%s on top of the last propagation.", + $branch, $depth, $depth == 1 ? '' : 's' + ); + } + return ($full, $depth); + } + $depth++; + } + } + + # No propagation commit — use merge-base with control + my ($merge_base) = run({}, 'git', 'merge-base', $control, $branch); + chomp $merge_base if defined $merge_base; + return ($merge_base, 0) if $merge_base; + + return (undef, 0); +} +# }}} + # }}} # pipeline_graph - write pipeline.md with Mermaid flowchart {{{ sub pipeline_graph { diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 200015aa..806358be 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -1188,6 +1188,97 @@ sub actual_environment_files { return @$ref; } +# }}} +# propagation_files - list all files this environment depends on for pipeline propagation {{{ +# +# Returns a list of repo-relative paths that should be tracked when +# propagating changes from the control branch to this environment's +# branch. Includes: env file hierarchy, kit archive (or dev/), +# .genesis/config, and any reaction scripts referenced in the env. +sub propagation_files { + my ($self) = @_; + my %files; + + # Env file hierarchy (ancestors + self) + for my $f ($self->actual_environment_files) { + $f =~ s{^\./}{}; + $files{$f} = 1; + } + + # Kit source (compiled tarball or dev directory) + if ($self->kit->is_dev) { + $files{'dev/'} = 1; + } else { + my $source = $self->kit->{source}; + if ($source) { + my $rel = $source; + $rel =~ s{^\Q${\$self->path}\E/?}{}; + $files{$rel} = 1; + } + } + + # Config + $files{'.genesis/config'} = 1; + + # Reaction scripts + my $reactions = $self->lookup('genesis.reactions', {}); + if (ref($reactions) eq 'HASH') { + for my $phase (values %$reactions) { + next unless ref($phase) eq 'ARRAY'; + for my $action (@$phase) { + next unless ref($action) eq 'HASH' && $action->{script}; + $files{"bin/$action->{script}"} = 1; + } + } + } + + return sort keys %files; +} + +# }}} +# propagation_diff - files that changed between this env branch and a control commit {{{ +# +# Compares this environment's branch against a target commit on the +# control branch, filtered to only files this env depends on. +# Returns a list of repo-relative paths that need propagating. +# +# The target_sha is embedded in the propagation commit message so +# downstream environments can trace which control state they deployed. +sub propagation_diff { + my ($self, $target_sha) = @_; + $target_sha ||= 'control'; + + my @dep_files = $self->propagation_files; + return () unless @dep_files; + + my $env_branch = $self->name; + my ($diff_out, $rc) = run( + { passfail => 1 }, + 'git', 'diff', '--name-only', "$env_branch..$target_sha", '--', @dep_files + ); + return () if $rc || !$diff_out; + + return grep { /\S/ } split /\n/, $diff_out; +} + +# }}} +# last_propagated_sha - extract the control SHA from the last propagation commit {{{ +sub last_propagated_sha { + my ($self) = @_; + my ($log, $rc) = run( + { passfail => 1 }, + 'git', 'log', '--format=%s', $self->name, '--', '.' + ); + return undef if $rc || !$log; + + for my $line (split /\n/, $log) { + if ($line =~ /control\@([0-9a-f]+)/) { + return $1; + } + } + return undef; +} + # }}} # relate - get hierarchal file relationships with another environment {{{ sub relate { From bd770dfcecc2a2afed8b11c702e129d197487b46 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 16:13:15 -0700 Subject: [PATCH 057/103] Extract propagation decision logic with tests Add Genesis::CI::Propagation module with pure compute_propagation_targets function. Entry point algorithm uses ancestor overlap check: an env is a direct target only if none of its changed files are also needed by any ancestor. 16 unit tests covering all documented scenarios including cascade, fan-out, leaf-only, mixed, superseded, and scoped propagation. --- lib/Genesis/CI/Propagation.pm | 74 ++++++ lib/Genesis/Commands/Pipelines.pm | 153 ++++++----- t/unit-tests/genesis_ci_propagation-core.t | 284 +++++++++++++++++++++ 3 files changed, 450 insertions(+), 61 deletions(-) create mode 100644 lib/Genesis/CI/Propagation.pm create mode 100644 t/unit-tests/genesis_ci_propagation-core.t diff --git a/lib/Genesis/CI/Propagation.pm b/lib/Genesis/CI/Propagation.pm new file mode 100644 index 00000000..ba078815 --- /dev/null +++ b/lib/Genesis/CI/Propagation.pm @@ -0,0 +1,74 @@ +package Genesis::CI::Propagation; + +use strict; +use warnings; + +# compute_propagation_targets - determine which envs receive files directly {{{ +# +# Pure function: no git, no IO. Given a DAG topology and per-env diff +# data, returns which environments are direct entry points for propagation +# and which files each receives. +# +# An environment is a direct entry point if: +# 1. It has changed files (diff is non-empty) +# 2. NONE of its changed files overlap with any ancestor's changed files +# +# If there is overlap, the env is blocked — it must wait for its ancestor +# to deploy and cascade. This ensures commits travel as a unit through +# the chain. +# +# Args (named): +# dag_order => \@ordered_env_names (topologically sorted) +# parent_of => \%parent_map (env => parent_env or undef) +# env_changed => \%changed_files (env => \@files that differ) +# scope => \@scoped_env_names (optional: subset to consider; +# defaults to dag_order) +# +# Returns: hashref { env_name => \@files_to_propagate } +# Only entry point envs appear in the result. +sub compute_propagation_targets { + my (%args) = @_; + + my @dag_order = @{$args{dag_order} || []}; + my %parent_of = %{$args{parent_of} || {}}; + my %env_changed = %{$args{env_changed} || {}}; + my @scope = @{$args{scope} || \@dag_order}; + + # Build a set for fast scope membership check + my %in_scope = map { $_ => 1 } @scope; + + my %targets; + + for my $env_name (@scope) { + next unless $env_changed{$env_name}; + next unless @{$env_changed{$env_name}}; + + my %my_files = map { $_ => 1 } @{$env_changed{$env_name}}; + + # Walk up the DAG checking for overlap with any ancestor + my $has_ancestor_overlap = 0; + my $ancestor = $parent_of{$env_name}; + while ($ancestor) { + if ($env_changed{$ancestor} && @{$env_changed{$ancestor}}) { + for my $f (@{$env_changed{$ancestor}}) { + if ($my_files{$f}) { + $has_ancestor_overlap = 1; + last; + } + } + } + last if $has_ancestor_overlap; + $ancestor = $parent_of{$ancestor}; + } + + unless ($has_ancestor_overlap) { + $targets{$env_name} = $env_changed{$env_name}; + } + } + + return \%targets; +} + +# }}} + +1; diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index f168c2ad..0929a870 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -10,6 +10,7 @@ use Genesis::Top; use Genesis::Env; use Genesis::CI::Legacy qw//; use Genesis::CI::Compiler; +use Genesis::CI::Propagation; use Service::Vault::Remote; use File::Basename qw/dirname/; @@ -104,7 +105,17 @@ sub apply { } # }}} -# propagate - cascade control branch changes one level down the DAG {{{ +# propagate - route control branch changes to environment branches {{{ +# +# Determines which environments are the entry points for each changed +# file by walking the DAG in order. A file is "claimed" by the first +# environment that uses it — downstream envs will receive it via cascade +# after their upstream deploys. Files used only by a non-root env +# (e.g., a leaf env file) go directly to that env. +# +# Always sources files from control@SHA. Can be run from control +# (targets entry points) or from an env branch (targets children, +# using the control SHA from the last propagation commit). sub propagate { my $opts = get_options; @@ -134,49 +145,42 @@ sub propagate { bail("No environments with pipeline metadata found.") unless %$nodes; - my %children; - my %has_parent; + my (%children, %has_parent, %parent_of); for my $edge (@$edges) { push @{$children{$edge->{from}}}, $edge->{to}; $has_parent{$edge->{to}} = 1; + $parent_of{$edge->{to}} = $edge->{from}; } - # Determine the control SHA and target environments based on - # where we are in the DAG. - my ($control_sha_full, $control_sha_short, @targets); + # Topological order (BFS from roots) + my @roots = sort grep { !$has_parent{$_} } keys %$nodes; + my @dag_order; + my @queue = @roots; + my %visited; + while (@queue) { + my $env = shift @queue; + next if $visited{$env}++; + push @dag_order, $env; + push @queue, sort @{$children{$env} || []}; + } + # Resolve control SHA + my $control_sha_full = $opts->{commit}; if ($current_branch eq $control) { - # On control: propagate to root envs (entry points) - $control_sha_full = $opts->{commit}; unless ($control_sha_full) { ($control_sha_full) = run({}, 'git rev-parse HEAD'); chomp $control_sha_full; } - ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); - chomp $control_sha_short; - - @targets = sort grep { !$has_parent{$_} } keys %$nodes; - info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}", $control, $control_sha_short; - } elsif ($nodes->{$current_branch}) { - # On an env branch: propagate to this env's children using the - # control SHA from the last propagation commit or merge-base. - my $manual_commits; - ($control_sha_full, $manual_commits) = _resolve_propagation_base($current_branch); - bail( - "Cannot determine propagation base for branch #C{%s}.\n". - "Run #C{genesis propagate} from the #C{%s} branch first.", - $current_branch, $control - ) unless $control_sha_full; - - ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); - chomp $control_sha_short; - - @targets = sort @{$children{$current_branch} || []}; - bail("Environment #C{%s} has no downstream environments.", $current_branch) - unless @targets; - - info "\n#G{Propagating from} #C{%s} #G{(control@}#C{%s}#G{)}", $current_branch, $control_sha_short; + # On an env branch: use control SHA from last propagation or merge-base + unless ($control_sha_full) { + ($control_sha_full) = _resolve_propagation_base($current_branch); + bail( + "Cannot determine propagation base for branch #C{%s}.\n". + "Run #C{genesis propagate} from the #C{%s} branch first.", + $current_branch, $control + ) unless $control_sha_full; + } } else { bail( "Branch #C{%s} is not the control branch or a known environment branch.", @@ -184,52 +188,79 @@ sub propagate { ); } - info ""; + my ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); + chomp $control_sha_short; + info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}\n", $control, $control_sha_short; - # Propagate to each target - my $propagated = 0; - for my $env_name (@targets) { - # Check if env branch exists + # Git prefix for subdir repos + my ($git_root) = run({}, 'git rev-parse --show-toplevel'); + chomp $git_root; + my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); + chomp $git_prefix; + + # Determine scope: which envs are candidates for this propagation? + # On control: all envs. On an env branch: only that env's children. + my @scope; + if ($current_branch eq $control) { + @scope = @dag_order; + } else { + # Children of current env, in DAG order + my %child_set; + my @expand = @{$children{$current_branch} || []}; + while (@expand) { + my $e = shift @expand; + next if $child_set{$e}++; + push @expand, @{$children{$e} || []}; + } + @scope = grep { $child_set{$_} } @dag_order; + } + + # Build per-env dependency file lists and diffs + my %env_dep_files; + my %env_changed; + for my $env_name (@scope) { my $exists = run({ passfail => 1 }, 'git', 'rev-parse', '--verify', $env_name); - unless ($exists) { - warning("Branch #C{%s} does not exist — skipping.", $env_name); - next; - } + next unless $exists; - # Load the env to get its dependency files my $env = eval { $top->load_env($env_name) }; - unless ($env) { - warning("Could not load environment #C{%s} — skipping.", $env_name); - next; - } + next unless $env; my @dep_files = $env->propagation_files; next unless @dep_files; + $env_dep_files{$env_name} = \@dep_files; - # Diff: what changed between this env branch and control@SHA. - # Run from git toplevel so pathspecs resolve correctly for subdir repos. - my ($git_root) = run({}, 'git rev-parse --show-toplevel'); - chomp $git_root; - my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); - chomp $git_prefix; my @git_dep_files = map { "${git_prefix}$_" } @dep_files; my ($diff_out) = run({ dir => $git_root }, - 'git', 'diff', '--name-only', "$env_name", $control_sha_full, '--', @git_dep_files + 'git', 'diff', '--name-only', $env_name, $control_sha_full, '--', @git_dep_files ); my @changed = grep { /\S/ } split /\n/, ($diff_out || ''); - next unless @changed; + $env_changed{$env_name} = \@changed if @changed; + } + + # Determine entry points via pure decision logic. + my $env_propagate = Genesis::CI::Propagation::compute_propagation_targets( + dag_order => \@dag_order, + parent_of => \%parent_of, + env_changed => \%env_changed, + scope => \@scope, + ); + + # Propagate to entry point envs + my $propagated = 0; + for my $env_name (@scope) { + next unless $env_propagate->{$env_name}; + my @files = @{$env_propagate->{$env_name}}; if ($dry_run) { - info " #C{%s}: %d file%s to propagate", $env_name, scalar(@changed), @changed == 1 ? '' : 's'; - info " %s", $_ for @changed; + info " #C{%s}: %d file%s to propagate", $env_name, scalar(@files), @files == 1 ? '' : 's'; + info " %s", $_ for @files; $propagated++; } else { - # Checkout env branch, update files from control@SHA, commit run({ onfailure => "Failed to checkout $env_name" }, 'git', 'checkout', $env_name); - for my $file (@changed) { + for my $file (@files) { my $dir = $file; $dir =~ s{/[^/]+$}{}; mkdir_or_fail($dir) if $dir ne $file && !-d $dir; @@ -237,16 +268,16 @@ sub propagate { 'git', 'checkout', $control_sha_full, '--', $file); } - run({}, 'git', 'add', @changed); + run({}, 'git', 'add', @files); my $msg = sprintf("[pipeline] control\@%s → %s", $control_sha_short, $env_name); run({ onfailure => "Failed to commit propagation to $env_name" }, 'git', 'commit', '-m', $msg); - info " #G{%s}: propagated %d file%s", $env_name, scalar(@changed), @changed == 1 ? '' : 's'; + info " #G{%s}: propagated %d file%s", $env_name, scalar(@files), @files == 1 ? '' : 's'; $propagated++; } } - # Return to the branch we started on + # Return to starting branch unless ($dry_run) { run({ onfailure => "Failed to return to $current_branch" }, 'git', 'checkout', $current_branch); diff --git a/t/unit-tests/genesis_ci_propagation-core.t b/t/unit-tests/genesis_ci_propagation-core.t new file mode 100644 index 00000000..0ad24e01 --- /dev/null +++ b/t/unit-tests/genesis_ci_propagation-core.t @@ -0,0 +1,284 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use lib 't'; +use lib 'lib'; +use Test::More; +use Test::Deep; + +use_ok 'Genesis::CI::Propagation'; + +# DAG topology used across most tests: +# +# mgmt (root) +# └── lab +# └── qa +# ├── np1 → prod1 +# └── np2 → prod2 + +my @dag_order = qw(mgmt lab qa np1 np2 prod1 prod2); +my %parent_of = ( + lab => 'mgmt', + qa => 'lab', + np1 => 'qa', + np2 => 'qa', + prod1 => 'np1', + prod2 => 'np2', +); + +sub propagate { + my (%env_changed) = @_; + return Genesis::CI::Propagation::compute_propagation_targets( + dag_order => \@dag_order, + parent_of => \%parent_of, + env_changed => \%env_changed, + ); +} + +sub propagate_scoped { + my ($scope, %env_changed) = @_; + return Genesis::CI::Propagation::compute_propagation_targets( + dag_order => \@dag_order, + parent_of => \%parent_of, + env_changed => \%env_changed, + scope => $scope, + ); +} + +# ========================================================================= +# Scenario 1: Pure shared change +# A change to a shared file (used by all) should only target the root. +# ========================================================================= +subtest 'scenario 1: pure shared change targets root only' => sub { + my $targets = propagate( + mgmt => ['lmelt.yml'], + lab => ['lmelt.yml'], + qa => ['lmelt.yml'], + np1 => ['lmelt.yml'], + np2 => ['lmelt.yml'], + prod1 => ['lmelt.yml'], + prod2 => ['lmelt.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "only mgmt is an entry point for shared file"; +}; + +# ========================================================================= +# Scenario 2: Pure leaf change +# A file used only by a non-root env goes directly to that env. +# ========================================================================= +subtest 'scenario 2: pure leaf change targets leaf directly' => sub { + my $targets = propagate( + lab => ['lmelt-vsphere-canwest-1-lab.yml'], + ); + + cmp_deeply $targets, { + lab => bag('lmelt-vsphere-canwest-1-lab.yml'), + }, "lab is direct entry point for its own leaf file"; +}; + +# ========================================================================= +# Scenario 3: Mixed shared + leaf in one commit +# Shared file overlap blocks the leaf env — must wait for cascade. +# ========================================================================= +subtest 'scenario 3: mixed shared + leaf blocks downstream' => sub { + my $targets = propagate( + mgmt => ['lmelt.yml'], + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "lab is blocked because lmelt.yml overlaps with mgmt"; +}; + +# ========================================================================= +# Scenario 4: Independent changes to different tiers +# No overlap between the entry points — both propagate simultaneously. +# ========================================================================= +subtest 'scenario 4: independent changes create parallel entry points' => sub { + my $targets = propagate( + mgmt => ['lmelt-vsphere-canwest-1-mgmt.yml'], + qa => ['ops/prod-extras.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt-vsphere-canwest-1-mgmt.yml'), + qa => bag('ops/prod-extras.yml'), + }, "mgmt and qa are independent entry points"; +}; + +# ========================================================================= +# Scenario 5: New commit while previous in-flight (after mgmt caught up) +# mgmt already has the shared file (diff empty), lab gets both. +# ========================================================================= +subtest 'scenario 5: lab unblocked when mgmt is caught up' => sub { + # mgmt was already propagated — diff is empty + # lab still needs lmelt.yml (from earlier commit) + lab.yml (new commit) + my $targets = propagate( + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + ); + + cmp_deeply $targets, { + lab => bag('lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'), + }, "lab is entry point when mgmt has no changes (already caught up)"; +}; + +# ========================================================================= +# Scenario 6: Superseded propagation +# New propagation overwrites previous — latest desired state wins. +# ========================================================================= +subtest 'scenario 6: superseded propagation targets same env' => sub { + # mgmt's diff shows lmelt.yml changed (new version superseding old) + my $targets = propagate( + mgmt => ['lmelt.yml'], + lab => ['lmelt.yml'], + qa => ['lmelt.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "mgmt receives superseding change, downstream blocked"; +}; + +# ========================================================================= +# Scenario 7: The "lost no-op" — lab.yml only change with shared pending +# When using HEAD: mgmt is caught up, lab gets everything. +# ========================================================================= +subtest 'scenario 7: no-op commit picked up on next propagation' => sub { + # State: mgmt already has lmelt.yml (propagated earlier) + # lab.yml changed in a later commit + # lab still needs lmelt.yml (never propagated to lab) + # Using HEAD: mgmt diff is empty (caught up), lab gets both + my $targets = propagate( + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + ); + + cmp_deeply $targets, { + lab => bag('lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'), + }, "lab gets both files when mgmt is caught up"; +}; + +# ========================================================================= +# Scenario 8: Rapid-fire commits +# Multiple commits accumulated — single propagation captures all. +# ========================================================================= +subtest 'scenario 8: rapid-fire commits handled correctly' => sub { + # Commit A: lmelt.yml + # Commit B: lab.yml + # Commit C: lmelt.yml again + # Single propagation at HEAD (C): + my $targets = propagate( + mgmt => ['lmelt.yml'], + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + qa => ['lmelt.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "only mgmt targeted — lab/qa blocked by shared lmelt.yml overlap"; +}; + +# ========================================================================= +# After mgmt deploys (rapid-fire follow-up) +# ========================================================================= +subtest 'scenario 8b: after mgmt deploys, lab gets all accumulated changes' => sub { + # mgmt caught up (diff empty), lab still needs both files + my $targets = propagate( + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + qa => ['lmelt.yml'], + ); + + cmp_deeply $targets, { + lab => bag('lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'), + }, "lab is entry point, qa blocked by lmelt.yml overlap with lab"; +}; + +# ========================================================================= +# Scoped propagation (simulating cascade from env branch) +# After mgmt deploys: scope is mgmt's children and descendants only. +# ========================================================================= +subtest 'scoped propagation: cascade after mgmt deploy' => sub { + my $targets = propagate_scoped( + [qw(lab qa np1 np2 prod1 prod2)], + lab => ['lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'], + qa => ['lmelt.yml'], + np1 => ['lmelt.yml'], + np2 => ['lmelt.yml'], + ); + + cmp_deeply $targets, { + lab => bag('lmelt.yml', 'lmelt-vsphere-canwest-1-lab.yml'), + }, "lab is entry point within scope, qa/np blocked by overlap"; +}; + +# ========================================================================= +# Deep chain: change only affects prod1 +# ========================================================================= +subtest 'deep chain: leaf change at prod1 skips all intermediates' => sub { + my $targets = propagate( + prod1 => ['lmelt-vsphere-canwest-1-prod1.yml'], + ); + + cmp_deeply $targets, { + prod1 => bag('lmelt-vsphere-canwest-1-prod1.yml'), + }, "prod1 is direct entry point, all intermediates skipped"; +}; + +# ========================================================================= +# Fan-out: change affects both np branches independently +# ========================================================================= +subtest 'fan-out: independent changes to np1 and np2' => sub { + my $targets = propagate( + np1 => ['lmelt-vsphere-canwest-1-np1.yml'], + np2 => ['lmelt-vsphere-canwest-1-np2.yml'], + ); + + cmp_deeply $targets, { + np1 => bag('lmelt-vsphere-canwest-1-np1.yml'), + np2 => bag('lmelt-vsphere-canwest-1-np2.yml'), + }, "np1 and np2 are independent entry points (different parents)"; +}; + +# ========================================================================= +# No changes at all +# ========================================================================= +subtest 'no changes: empty result' => sub { + my $targets = propagate(); + cmp_deeply $targets, {}, "no changes → no targets"; +}; + +# ========================================================================= +# Overlap through grandparent (not just direct parent) +# ========================================================================= +subtest 'grandparent overlap: qa blocked by mgmt via shared file' => sub { + my $targets = propagate( + mgmt => ['lmelt.yml'], + qa => ['lmelt.yml', 'lmelt-vsphere-canwest-1-qa.yml'], + ); + + # qa's ancestor chain: qa → lab → mgmt + # mgmt has lmelt.yml, qa has lmelt.yml → overlap + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "qa blocked by grandparent mgmt through lmelt.yml overlap"; +}; + +# ========================================================================= +# Partial overlap: some files overlap, some don't — still blocked +# ========================================================================= +subtest 'partial overlap: any shared file blocks the env' => sub { + my $targets = propagate( + mgmt => ['lmelt.yml'], + lab => ['lmelt.yml', 'ops/lab-only.yml'], + ); + + cmp_deeply $targets, { + mgmt => bag('lmelt.yml'), + }, "lab blocked even though ops/lab-only.yml is independent — lmelt.yml overlaps"; +}; + +done_testing; From c8f65bd832ef7aaee23f3a2f0585b24cd3c3d465 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Wed, 22 Apr 2026 17:29:46 -0700 Subject: [PATCH 058/103] Refine genesis propagate command Run only from control branch with optional positional arg for cascade scoping. Always use control HEAD for diffs. Handle deletions and renames. Guard against dirty working tree (ignoring untracked files). Recover from errors by resetting partial changes and returning to starting branch. Block cascade when after_env has outstanding unpropagated changes. All git ops run from git toplevel for subdir repo support. --- bin/genesis | 23 ++-- lib/Genesis/Commands/Pipelines.pm | 219 +++++++++++++++++++++--------- 2 files changed, 169 insertions(+), 73 deletions(-) diff --git a/bin/genesis b/bin/genesis index 58d87cf3..8684bba4 100755 --- a/bin/genesis +++ b/bin/genesis @@ -2052,14 +2052,15 @@ define_command("pipeline-graph", { # genesis propagate - propagate control branch changes to environment branches {{{ define_command("propagate", { summary => "Propagate changes from the control branch to environment branches.", - usage => "propagate [options]", + usage => "propagate [options] [env]", description => - "Identifies files that have changed on the control branch since each ". - "environment branch was last synced, and copies them to the appropriate ". - "environment branches. Walks the pipeline DAG in order so upstream ". - "environments are updated before downstream ones.\n\n". - "Each propagation commit records the control branch SHA it was sourced ". - "from, enabling deterministic tracking across the pipeline chain.", + "Must be run from the control branch. Identifies files that have ". + "changed since each environment branch was last synced, and copies ". + "them to the appropriate environment branches.\n\n". + "Without arguments, determines entry points by walking the DAG and ". + "finding the first environment that uses each changed file.\n\n". + "With an environment name, scopes propagation to that environment's ". + "children (cascade after a successful deploy).", function_group => Genesis::Commands::PIPELINE, scope => 'repo', option_group => Genesis::Commands::REPO_OPTIONS, @@ -2069,8 +2070,12 @@ define_command("propagate", { 'commit=s' => "Control branch commit SHA to propagate from. Defaults to ". - "current HEAD when on the control branch, or the SHA recorded ". - "in the last propagation commit when on an environment branch.", + "HEAD of the control branch.", + ], + arguments => [ + 'env?' => + "Environment that was just deployed. Scopes propagation to ". + "its downstream children only.", ], }, 'Genesis::Commands::Pipelines::propagate'); # }}} diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 0929a870..ef74543b 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -107,16 +107,14 @@ sub apply { # }}} # propagate - route control branch changes to environment branches {{{ # -# Determines which environments are the entry points for each changed -# file by walking the DAG in order. A file is "claimed" by the first -# environment that uses it — downstream envs will receive it via cascade -# after their upstream deploys. Files used only by a non-root env -# (e.g., a leaf env file) go directly to that env. +# Must be run from the control branch. Optional positional argument +# names an environment whose children should be the propagation scope +# (cascade after deploy). Without it, all envs are candidates and +# the entry point algorithm determines which receive files. # -# Always sources files from control@SHA. Can be run from control -# (targets entry points) or from an env branch (targets children, -# using the control SHA from the last propagation commit). +# Always sources files from control HEAD (or --commit). sub propagate { + my ($after_env) = @_; my $opts = get_options; my $dry_run = $opts->{'dry-run'}; @@ -127,13 +125,25 @@ sub propagate { my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - # Determine current branch + # Must be on control branch my ($current_branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); chomp $current_branch if defined $current_branch; - bail("Cannot propagate from a detached HEAD.") - unless defined $current_branch; + bail( + "Propagation must be run from the #C{%s} branch (currently on #C{%s}).", + $control, $current_branch // '' + ) unless defined($current_branch) && $current_branch eq $control; + + # Guard: working tree must be clean before we switch branches + unless ($dry_run) { + my ($status) = run({}, 'git status --porcelain'); + my @dirty = grep { /^[^?]/ } split /\n/, ($status || ''); + bail( + "Working tree has uncommitted changes. Commit or stash them\n". + "before running propagate." + ) if @dirty; + } - # Build the DAG + # Build the DAG from control branch env files my $env_dir = $top->path; require Genesis::CI::Compiler::ASTBuilder; my $builder = Genesis::CI::Compiler::ASTBuilder->new( @@ -164,33 +174,15 @@ sub propagate { push @queue, sort @{$children{$env} || []}; } - # Resolve control SHA + # Resolve control SHA — always use HEAD my $control_sha_full = $opts->{commit}; - if ($current_branch eq $control) { - unless ($control_sha_full) { - ($control_sha_full) = run({}, 'git rev-parse HEAD'); - chomp $control_sha_full; - } - } elsif ($nodes->{$current_branch}) { - # On an env branch: use control SHA from last propagation or merge-base - unless ($control_sha_full) { - ($control_sha_full) = _resolve_propagation_base($current_branch); - bail( - "Cannot determine propagation base for branch #C{%s}.\n". - "Run #C{genesis propagate} from the #C{%s} branch first.", - $current_branch, $control - ) unless $control_sha_full; - } - } else { - bail( - "Branch #C{%s} is not the control branch or a known environment branch.", - $current_branch - ); + unless ($control_sha_full) { + ($control_sha_full) = run({}, 'git rev-parse HEAD'); + chomp $control_sha_full; } my ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); chomp $control_sha_short; - info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}\n", $control, $control_sha_short; # Git prefix for subdir repos my ($git_root) = run({}, 'git rev-parse --show-toplevel'); @@ -198,26 +190,71 @@ sub propagate { my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); chomp $git_prefix; - # Determine scope: which envs are candidates for this propagation? - # On control: all envs. On an env branch: only that env's children. + # Determine scope my @scope; - if ($current_branch eq $control) { - @scope = @dag_order; - } else { - # Children of current env, in DAG order + if ($after_env) { + bail("Environment #C{%s} is not in the pipeline topology.", $after_env) + unless $nodes->{$after_env}; + + # Verify the after_env doesn't have outstanding changes on + # control that would make it an entry point. If it does, + # those changes haven't flowed through yet — cascading past + # it is unsafe. + my ($last_sync) = _resolve_propagation_base($after_env); + if ($last_sync) { + # Check what changed on control since this env was last synced + my $after_load = eval { $top->load_env($after_env) }; + if ($after_load) { + my @after_deps = $after_load->propagation_files; + my @after_git_deps = map { "${git_prefix}$_" } @after_deps; + my ($since_diff) = run({ dir => $git_root }, + 'git', 'diff', '--name-only', $last_sync, $control_sha_full, + '--', @after_git_deps + ); + my @outstanding = grep { /\S/ } split /\n/, ($since_diff || ''); + if (@outstanding) { + bail( + "Environment #C{%s} has %d outstanding change%s on control\n". + "that have not been propagated to it yet.\n". + "Run #C{genesis propagate} without arguments first.", + $after_env, scalar(@outstanding), + @outstanding == 1 ? '' : 's' + ); + } + } + } else { + # No propagation commit and no merge-base — branch doesn't exist + # or has never been synced. + bail( + "Environment #C{%s} has never been propagated to.\n". + "Run #C{genesis propagate} without arguments first.", + $after_env + ); + } + + # Children and descendants of the named env my %child_set; - my @expand = @{$children{$current_branch} || []}; + my @expand = @{$children{$after_env} || []}; while (@expand) { my $e = shift @expand; next if $child_set{$e}++; push @expand, @{$children{$e} || []}; } @scope = grep { $child_set{$_} } @dag_order; + bail("Environment #C{%s} has no downstream environments.", $after_env) + unless @scope; + info "\n#G{Propagating from} #C{%s} #G{@} #C{%s} #G{(after %s)}\n", + $control, $control_sha_short, $after_env; + } else { + @scope = @dag_order; + info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}\n", + $control, $control_sha_short; } # Build per-env dependency file lists and diffs my %env_dep_files; my %env_changed; + my %env_changed_detail; for my $env_name (@scope) { my $exists = run({ passfail => 1 }, 'git', 'rev-parse', '--verify', $env_name); @@ -231,11 +268,31 @@ sub propagate { $env_dep_files{$env_name} = \@dep_files; my @git_dep_files = map { "${git_prefix}$_" } @dep_files; + # Use --diff-filter and --name-status to distinguish adds/modifies + # from deletes and renames. my ($diff_out) = run({ dir => $git_root }, - 'git', 'diff', '--name-only', $env_name, $control_sha_full, '--', @git_dep_files + 'git', 'diff', '--name-status', $env_name, $control_sha_full, '--', @git_dep_files ); - my @changed = grep { /\S/ } split /\n/, ($diff_out || ''); - $env_changed{$env_name} = \@changed if @changed; + my (@changed, @deleted, %renamed); + for my $line (grep { /\S/ } split /\n/, ($diff_out || '')) { + if ($line =~ /^D\t(.+)$/) { + push @deleted, $1; + } elsif ($line =~ /^R\d*\t(.+)\t(.+)$/) { + $renamed{$1} = $2; # old → new + push @deleted, $1; # remove old name + push @changed, $2; # add new name + } elsif ($line =~ /^[AMT]\t(.+)$/) { + push @changed, $1; + } + } + if (@changed || @deleted) { + $env_changed{$env_name} = [@changed, @deleted]; + $env_changed_detail{$env_name} = { + changed => \@changed, + deleted => \@deleted, + renamed => \%renamed, + }; + } } # Determine entry points via pure decision logic. @@ -248,41 +305,75 @@ sub propagate { # Propagate to entry point envs my $propagated = 0; + my $error; for my $env_name (@scope) { next unless $env_propagate->{$env_name}; my @files = @{$env_propagate->{$env_name}}; + my $detail = $env_changed_detail{$env_name} || { changed => \@files, deleted => [], renamed => {} }; + my @to_copy = @{$detail->{changed}}; + my @to_rm = @{$detail->{deleted}}; + my %renames = %{$detail->{renamed}}; + my $total = scalar(@to_copy) + scalar(@to_rm); + my $msg = sprintf("[pipeline] control\@%s -> %s", $control_sha_short, $env_name); + if ($dry_run) { - info " #C{%s}: %d file%s to propagate", $env_name, scalar(@files), @files == 1 ? '' : 's'; - info " %s", $_ for @files; + info " #C{%s}: %d file%s to propagate", $env_name, $total, $total == 1 ? '' : 's'; + for my $f (@to_copy) { + my ($old) = grep { $renames{$_} eq $f } keys %renames; + my $note = $old ? " #Yi{(renamed from $old)}" : ''; + info " #G{M} %s%s", $f, $note; + } + info " #R{D} %s", $_ for @to_rm; + info " #Yi{commit}: %s", $msg; $propagated++; } else { - run({ onfailure => "Failed to checkout $env_name" }, - 'git', 'checkout', $env_name); - - for my $file (@files) { - my $dir = $file; - $dir =~ s{/[^/]+$}{}; - mkdir_or_fail($dir) if $dir ne $file && !-d $dir; - run({ onfailure => "Failed to checkout $file from control\@$control_sha_short" }, - 'git', 'checkout', $control_sha_full, '--', $file); - } + eval { + run({ dir => $git_root, onfailure => "Failed to checkout $env_name" }, + 'git', 'checkout', $env_name); + + # Copy added/modified files from control@SHA + for my $file (@to_copy) { + my $dir = "$git_root/$file"; + $dir =~ s{/[^/]+$}{}; + mkdir_or_fail($dir) if !-d $dir; + run({ dir => $git_root, onfailure => "Failed to checkout $file from control\@$control_sha_short" }, + 'git', 'checkout', $control_sha_full, '--', $file); + } - run({}, 'git', 'add', @files); - my $msg = sprintf("[pipeline] control\@%s → %s", $control_sha_short, $env_name); - run({ onfailure => "Failed to commit propagation to $env_name" }, - 'git', 'commit', '-m', $msg); - info " #G{%s}: propagated %d file%s", $env_name, scalar(@files), @files == 1 ? '' : 's'; - $propagated++; + # Remove deleted files + for my $file (@to_rm) { + run({ dir => $git_root, passfail => 1 }, 'git', 'rm', '-f', '--', $file); + } + + run({ dir => $git_root }, 'git', 'add', @to_copy) if @to_copy; + run({ dir => $git_root, onfailure => "Failed to commit propagation to $env_name" }, + 'git', 'commit', '-m', $msg); + info " #G{%s}: propagated %d file%s", $env_name, $total, $total == 1 ? '' : 's'; + $propagated++; + }; + if ($@) { + $error = $@; + # Reset any partial changes on this branch + run({ dir => $git_root, passfail => 1 }, + 'git', 'checkout', '--', '.'); + warning( + "Propagation to #C{%s} failed: %s", + $env_name, $error =~ s/\s+$//r + ); + last; + } } } - # Return to starting branch + # Always return to starting branch after non-dry-run unless ($dry_run) { - run({ onfailure => "Failed to return to $current_branch" }, + run({ dir => $git_root, passfail => 1 }, 'git', 'checkout', $current_branch); } + bail("Propagation aborted due to error.") if $error; + if ($propagated) { info "\n#G{Done.} %s %d environment%s.", $dry_run ? "Would propagate to" : "Propagated to", From 6f279ca50cc70bb59129d6f314b150b5f178125c Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 06:28:36 -0700 Subject: [PATCH 059/103] Add prune_branch and pipeline rebuild hook Extract prune_branch on Env to remove non-dependency files from env branches. Used during genesis new and available for standalone cleanup. Guards against running from the target branch. genesis new offers to rebuild the pipeline for automated CI providers after branch creation. --- lib/Genesis/Commands/Env.pm | 28 ++++++++++++--- lib/Genesis/Env.pm | 71 +++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 3e6757d5..ddecadce 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -220,13 +220,31 @@ sub create { chomp $sha if defined $sha; info "#G{Committed} #C{%s} -- %s", $sha // '', $message; - # Create the environment branch at the current commit. - # This is the branch where future config changes and - # deploys for this environment will happen. We stay on - # the control branch. + # Create the environment branch at the current commit, + # then prune files that don't belong to this environment. run({ onfailure => "Failed to create branch '$name'" }, 'git', 'branch', $name); - info "Environment branch #C{%s} created.", $name; + + my @pruned = $env->prune_branch; + info "Environment branch #C{%s} created (%d file%s pruned).", + $name, scalar(@pruned), @pruned == 1 ? '' : 's'; + + # For automated CI providers, the pipeline needs to be + # rebuilt to include a job for the new environment branch. + my $provider_type = $top->config->get('ci.provider.type') || 'manual'; + if ($provider_type ne 'manual') { + if (in_controlling_terminal) { + if (prompt_for_boolean( + "Rebuild the CI pipeline to include #C{$name}? [y|n]", "y" + )) { + info "Rebuilding pipeline..."; + # TODO: call genesis repipe equivalent + warning("Automated pipeline rebuild not yet implemented. Run #C{genesis repipe} manually."); + } + } else { + info "Run #C{genesis repipe} to update the pipeline with the new environment."; + } + } } } diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 806358be..3c5bb33c 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -1235,6 +1235,77 @@ sub propagation_files { return sort keys %files; } +# }}} +# prune_branch - remove files from this env's branch that it doesn't depend on {{{ +# +# Checks out the environment's branch, removes tracked files not in +# propagation_files (keeping .genesis/ wholesale), commits the prune, +# and returns to the original branch. +# +# Options: +# dry_run => 1 — list files that would be removed without changing anything +# +# Returns the list of removed files (empty if nothing to prune). +sub prune_branch { + my ($self, %opts) = @_; + + my $branch = $self->name; + my @keep = $self->propagation_files; + + my ($git_root) = run({}, 'git rev-parse --show-toplevel'); + chomp $git_root; + my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); + chomp $git_prefix; + + # Must not be on the env branch — propagation_files is built from + # the current branch's files, which must be control. + unless ($opts{dry_run}) { + my ($current) = run({}, 'git rev-parse --abbrev-ref HEAD'); + chomp $current if defined $current; + bail( + "Cannot prune #C{%s} while on that branch. Switch to the control branch first.", + $branch + ) if defined($current) && $current eq $branch; + } + + # Build keep set with git-root-relative paths + my %keep_set; + for my $f (@keep) { + $keep_set{"${git_prefix}$f"} = 1; + } + + # List tracked files on the env branch under our prefix + my ($tracked) = run({ dir => $git_root }, + 'git', 'ls-tree', '-r', '--name-only', $branch, "${git_prefix}" + ); + my @to_remove; + for my $file (grep { /\S/ } split /\n/, ($tracked || '')) { + # Always keep .genesis/ contents + next if $file =~ m{^\Q${git_prefix}\E\.genesis/}; + next if $keep_set{$file}; + push @to_remove, $file; + } + + return () unless @to_remove; + return @to_remove if $opts{dry_run}; + + my ($current_branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); + chomp $current_branch; + + run({ dir => $git_root, onfailure => "Failed to checkout $branch for pruning" }, + 'git', 'checkout', $branch); + + run({ dir => $git_root, passfail => 1 }, + 'git', 'rm', '-q', '--', @to_remove); + run({ dir => $git_root, onfailure => "Failed to commit pruned branch $branch" }, + 'git', 'commit', '-m', "Prune non-dependency files from $branch"); + + run({ dir => $git_root, onfailure => "Failed to return to $current_branch" }, + 'git', 'checkout', $current_branch); + + return @to_remove; +} + # }}} # propagation_diff - files that changed between this env branch and a control commit {{{ # From ac7627db19c078930f45e6aa78379835bb31330b Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 06:28:46 -0700 Subject: [PATCH 060/103] Push propagated branches to remote After propagation, push control and env branches to the first configured remote. Warns on push failure without aborting. Add --no-push flag to skip. --- bin/genesis | 4 ++++ lib/Genesis/Commands/Pipelines.pm | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/bin/genesis b/bin/genesis index 8684bba4..4ab0e339 100755 --- a/bin/genesis +++ b/bin/genesis @@ -2071,6 +2071,10 @@ define_command("propagate", { 'commit=s' => "Control branch commit SHA to propagate from. Defaults to ". "HEAD of the control branch.", + + 'no-push' => + "Skip pushing control and env branches to the remote after ". + "propagation. By default, propagated branches are pushed.", ], arguments => [ 'env?' => diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index ef74543b..b9d81a82 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -118,6 +118,7 @@ sub propagate { my $opts = get_options; my $dry_run = $opts->{'dry-run'}; + my $no_push = $opts->{'no-push'}; my $top = Genesis::Top->new('.'); bail("CI is not configured for this repository.") @@ -305,6 +306,7 @@ sub propagate { # Propagate to entry point envs my $propagated = 0; + my @pushed_branches; my $error; for my $env_name (@scope) { next unless $env_propagate->{$env_name}; @@ -350,6 +352,7 @@ sub propagate { run({ dir => $git_root, onfailure => "Failed to commit propagation to $env_name" }, 'git', 'commit', '-m', $msg); info " #G{%s}: propagated %d file%s", $env_name, $total, $total == 1 ? '' : 's'; + push @pushed_branches, $env_name; $propagated++; }; if ($@) { @@ -374,6 +377,29 @@ sub propagate { bail("Propagation aborted due to error.") if $error; + # Push control and propagated branches to remote + if ($propagated && !$dry_run && !$no_push) { + my ($remote) = run({ passfail => 1 }, 'git', 'remote'); + if ($remote) { + chomp $remote; + $remote = (split /\n/, $remote)[0]; # use first remote + info "\n#G{Pushing} to #C{%s}...", $remote; + # Push control first (so the SHA is resolvable) + run({ dir => $git_root, passfail => 1 }, + 'git', 'push', $remote, $current_branch); + # Push propagated env branches + for my $branch (@pushed_branches) { + my $ok = run({ dir => $git_root, passfail => 1 }, + 'git', 'push', $remote, $branch); + if ($ok) { + info " #G{%s}: pushed", $branch; + } else { + warning("Failed to push #C{%s} to #C{%s}.", $branch, $remote); + } + } + } + } + if ($propagated) { info "\n#G{Done.} %s %d environment%s.", $dry_run ? "Would propagate to" : "Propagated to", From 84a558adcf1cee8ca311dc3445c74871b5c40a97 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 06:43:45 -0700 Subject: [PATCH 061/103] Add Service::Git abstraction Centralized git CLI wrapper with cached root/prefix, branch tracking with auto-restore on scope exit, structured diff_files, branch existence cache, and prefixed path helper for subdir repos. --- lib/Service/Git.pm | 407 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 lib/Service/Git.pm diff --git a/lib/Service/Git.pm b/lib/Service/Git.pm new file mode 100644 index 00000000..4281ce6a --- /dev/null +++ b/lib/Service/Git.pm @@ -0,0 +1,407 @@ +package Service::Git; + +use strict; +use warnings; + +use Genesis qw/run bail debug trace/; +use File::Basename qw/dirname/; + +### Constructor & Lifecycle {{{ + +# new - create a Git service instance for a repository {{{ +# +# Auto-detects the git root and subdir prefix from the given path +# (or cwd). Caches both for the lifetime of the object. +# +# my $git = Service::Git->new($path); # or ->new() for cwd +# my $git = Service::Git->new($path, track_branch => 1); +# +# With track_branch => 1, the current branch is saved and restored +# when the object goes out of scope (DESTROY). +sub new { + my ($class, $path, %opts) = @_; + $path ||= '.'; + + my ($root) = run({}, 'git', '-C', $path, 'rev-parse', '--show-toplevel'); + chomp $root if defined $root; + bail("Not a git repository: %s", $path) unless $root; + + my ($prefix) = run({}, 'git', '-C', $path, 'rev-parse', '--show-prefix'); + chomp $prefix if defined $prefix; + $prefix //= ''; + + my $self = bless { + root => $root, + prefix => $prefix, + _branch_cache => {}, + _track_branch => $opts{track_branch} || 0, + _original_branch => undef, + }, $class; + + if ($self->{_track_branch}) { + $self->{_original_branch} = $self->current_branch; + } + + return $self; +} + +# }}} +# DESTROY - restore original branch on scope exit {{{ +sub DESTROY { + my ($self) = @_; + if ($self->{_track_branch} && $self->{_original_branch}) { + my $current = eval { $self->current_branch }; + if ($current && $current ne $self->{_original_branch}) { + # Reset any partial changes before switching + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', '--', '.'); + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', $self->{_original_branch}); + trace("Service::Git: restored branch %s", $self->{_original_branch}); + } + } +} + +# }}} +# }}} + +### Accessors {{{ + +# root - git toplevel directory (cached) {{{ +sub root { $_[0]->{root} } + +# }}} +# prefix - subdir offset within git repo (cached, e.g., "bosh/") {{{ +sub prefix { $_[0]->{prefix} } + +# }}} +# }}} + +### Branch Operations {{{ + +# current_branch - name of HEAD branch (cached, invalidated on checkout) {{{ +sub current_branch { + my ($self) = @_; + my ($branch) = run({ dir => $self->{root} }, + 'git', 'rev-parse', '--abbrev-ref', 'HEAD'); + chomp $branch if defined $branch; + $self->{_current_branch} = $branch; + return $branch; +} + +# }}} +# checkout - switch to a branch {{{ +# +# Saves the original branch for restore_branch / DESTROY. +sub checkout { + my ($self, $branch) = @_; + $self->{_original_branch} //= $self->current_branch + if $self->{_track_branch}; + run({ dir => $self->{root}, onfailure => "Failed to checkout '$branch'" }, + 'git', 'checkout', $branch); + delete $self->{_current_branch}; + return $self; +} + +# }}} +# restore_branch - return to the branch we were on before any checkout {{{ +sub restore_branch { + my ($self) = @_; + return unless $self->{_original_branch}; + my $current = $self->current_branch; + if ($current ne $self->{_original_branch}) { + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', $self->{_original_branch}); + delete $self->{_current_branch}; + } + return $self; +} + +# }}} +# create_branch - create a new branch at the given ref (default HEAD) {{{ +sub create_branch { + my ($self, $name, $ref) = @_; + my @cmd = ('git', 'branch', $name); + push @cmd, $ref if $ref; + run({ dir => $self->{root}, onfailure => "Failed to create branch '$name'" }, + @cmd); + $self->{_branch_cache}{$name} = 1; + return $self; +} + +# }}} +# branch_exists - check if a branch exists (cached) {{{ +sub branch_exists { + my ($self, $name) = @_; + return $self->{_branch_cache}{$name} + if exists $self->{_branch_cache}{$name}; + my $exists = run({ dir => $self->{root}, passfail => 1 }, + 'git', 'rev-parse', '--verify', $name); + $self->{_branch_cache}{$name} = $exists ? 1 : 0; + return $self->{_branch_cache}{$name}; +} + +# }}} +# }}} + +### Queries {{{ + +# rev_parse - resolve a ref to a SHA {{{ +# +# Options: +# short => 1 — return abbreviated SHA +sub rev_parse { + my ($self, $ref, %opts) = @_; + my @cmd = ('git', 'rev-parse'); + push @cmd, '--short' if $opts{short}; + push @cmd, $ref; + my ($sha) = run({ dir => $self->{root} }, @cmd); + chomp $sha if defined $sha; + return $sha; +} + +# }}} +# merge_base - find the common ancestor of two refs {{{ +sub merge_base { + my ($self, $a, $b) = @_; + my ($sha) = run({ dir => $self->{root} }, 'git', 'merge-base', $a, $b); + chomp $sha if defined $sha; + return $sha; +} + +# }}} +# is_clean - true if working tree has no modified/staged/conflicted files {{{ +# +# Ignores untracked files — they don't affect branch switching. +sub is_clean { + my ($self) = @_; + my ($status) = run({ dir => $self->{root} }, 'git', 'status', '--porcelain'); + my @dirty = grep { /^[^?]/ } split /\n/, ($status || ''); + return !@dirty; +} + +# }}} +# diff_files - structured diff between two refs, filtered by pathspecs {{{ +# +# Returns a hashref: +# { +# changed => \@files, # added, modified, or type-changed +# deleted => \@files, +# renamed => \%old_to_new, +# all => \@all_files, # union of changed + deleted +# } +# +# Pathspecs are relative to git root (caller should prefix if needed). +sub diff_files { + my ($self, $from, $to, @pathspecs) = @_; + my @cmd = ('git', 'diff', '--name-status', $from, $to); + push @cmd, '--', @pathspecs if @pathspecs; + my ($out) = run({ dir => $self->{root} }, @cmd); + + my (@changed, @deleted, %renamed); + for my $line (grep { /\S/ } split /\n/, ($out || '')) { + if ($line =~ /^D\t(.+)$/) { + push @deleted, $1; + } elsif ($line =~ /^R\d*\t(.+)\t(.+)$/) { + $renamed{$1} = $2; + push @deleted, $1; + push @changed, $2; + } elsif ($line =~ /^[AMT]\t(.+)$/) { + push @changed, $1; + } + } + return { + changed => \@changed, + deleted => \@deleted, + renamed => \%renamed, + all => [@changed, @deleted], + }; +} + +# }}} +# diff_names - simple list of changed file names between two refs {{{ +sub diff_names { + my ($self, $from, $to, @pathspecs) = @_; + my @cmd = ('git', 'diff', '--name-only', $from, $to); + push @cmd, '--', @pathspecs if @pathspecs; + my ($out) = run({ dir => $self->{root} }, @cmd); + return grep { /\S/ } split /\n/, ($out || ''); +} + +# }}} +# ls_tree - list files on a ref under a prefix {{{ +sub ls_tree { + my ($self, $ref, $path) = @_; + $path //= ''; + my ($out) = run({ dir => $self->{root} }, + 'git', 'ls-tree', '-r', '--name-only', $ref, $path); + return grep { /\S/ } split /\n/, ($out || ''); +} + +# }}} +# log_subjects - return commit subjects for a branch {{{ +# +# Options: +# limit => N — max number of entries +# format => '...' — custom format (default: %H %s) +sub log_subjects { + my ($self, $branch, %opts) = @_; + my $fmt = $opts{format} || '%H %s'; + my @cmd = ('git', 'log', "--format=$fmt", $branch); + push @cmd, "-$opts{limit}" if $opts{limit}; + my ($out) = run({ dir => $self->{root} }, @cmd); + return split /\n/, ($out || ''); +} + +# }}} +# show_file - read file content from a specific ref {{{ +sub show_file { + my ($self, $ref, $path) = @_; + my ($content, $rc) = run({ dir => $self->{root}, passfail => 0 }, + 'git', 'show', "$ref:$path"); + return $content; +} + +# }}} +# }}} + +### Working Tree Operations {{{ + +# checkout_file - extract a file from a ref into the working tree {{{ +sub checkout_file { + my ($self, $ref, $file) = @_; + # Ensure parent directory exists + my $full_path = "$self->{root}/$file"; + my $dir = dirname($full_path); + if (!-d $dir) { + require File::Path; + File::Path::make_path($dir); + } + run({ dir => $self->{root}, onfailure => "Failed to checkout $file from $ref" }, + 'git', 'checkout', $ref, '--', $file); + return $self; +} + +# }}} +# add - stage files {{{ +sub add { + my ($self, @files) = @_; + return unless @files; + run({ dir => $self->{root} }, 'git', 'add', @files); + return $self; +} + +# }}} +# rm - remove files from index and working tree {{{ +sub rm { + my ($self, @files) = @_; + return unless @files; + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'rm', '-f', '-q', '--', @files); + return $self; +} + +# }}} +# commit - stage files and commit {{{ +# +# $git->commit("message"); # commit staged changes +# $git->commit("message", @files); # add files then commit +sub commit { + my ($self, $message, @files) = @_; + $self->add(@files) if @files; + run({ dir => $self->{root}, onfailure => "Failed to commit" }, + 'git', 'commit', '-m', $message); + return $self; +} + +# }}} +# reset_working_tree - discard all working tree changes to tracked files {{{ +sub reset_working_tree { + my ($self) = @_; + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', '--', '.'); + return $self; +} + +# }}} +# }}} + +### Remote Operations {{{ + +# default_remote - first configured remote name (cached) {{{ +sub default_remote { + my ($self) = @_; + return $self->{_default_remote} if exists $self->{_default_remote}; + my ($out) = run({ dir => $self->{root} }, 'git', 'remote'); + if ($out && $out =~ /\S/) { + chomp $out; + $self->{_default_remote} = (split /\n/, $out)[0]; + } else { + $self->{_default_remote} = undef; + } + return $self->{_default_remote}; +} + +# }}} +# push - push branches to a remote {{{ +# +# $git->push(@branches); # push to default remote +# $git->push($remote, @branches); # push to specific remote +# +# Returns a hashref of branch => success (1/0). +sub push { + my ($self, @args) = @_; + # If first arg looks like a remote name (not a branch we know), use it + my $remote; + if (@args && !$self->branch_exists($args[0])) { + $remote = shift @args; + } + $remote ||= $self->default_remote; + return {} unless $remote; + + my %results; + for my $branch (@args) { + my $ok = run({ dir => $self->{root}, passfail => 1 }, + 'git', 'push', $remote, $branch); + $results{$branch} = $ok ? 1 : 0; + } + return \%results; +} + +# }}} +# }}} + +### Utility {{{ + +# prefixed - prepend the git prefix to repo-relative paths {{{ +# +# Converts paths relative to the deployment repo root into paths +# relative to the git root. No-op when prefix is empty. +# +# my @git_paths = $git->prefixed(@repo_paths); +sub prefixed { + my ($self, @paths) = @_; + my $p = $self->{prefix}; + return @paths unless $p; + return map { "${p}$_" } @paths; +} + +# }}} +# in_repo - true if we're inside a subdirectory of the git root {{{ +sub in_repo { + return defined $_[0]->{root}; +} + +# }}} +# is_inside_work_tree - check if a path is inside a git work tree {{{ +sub is_inside_work_tree { + my ($class, $path) = @_; + $path ||= '.'; + return run({ passfail => 1 }, + 'git', '-C', $path, 'rev-parse', '--is-inside-work-tree'); +} + +# }}} +# }}} + +1; From a9c8a4eac31ec214bbade4498bd8037d819c9827 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 06:51:48 -0700 Subject: [PATCH 062/103] Add flyweight caching to Service::Git Cache instances by git root path so multiple callers sharing the same repo get the same object. Prevents competing branch tracking or stale caches. Upgrades track_branch on existing instances when requested. --- lib/Service/Git.pm | 52 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/lib/Service/Git.pm b/lib/Service/Git.pm index 4281ce6a..83baad35 100644 --- a/lib/Service/Git.pm +++ b/lib/Service/Git.pm @@ -6,18 +6,25 @@ use warnings; use Genesis qw/run bail debug trace/; use File::Basename qw/dirname/; +### Class State {{{ +my %_instances; # keyed by resolved git root path +# }}} + ### Constructor & Lifecycle {{{ -# new - create a Git service instance for a repository {{{ +# new - get or create a Git service instance for a repository {{{ # -# Auto-detects the git root and subdir prefix from the given path -# (or cwd). Caches both for the lifetime of the object. +# Flyweight: returns the existing instance if one already exists for +# the same .git-controlled repo. This prevents competing objects +# from stepping on each other's branch tracking or caches. # # my $git = Service::Git->new($path); # or ->new() for cwd # my $git = Service::Git->new($path, track_branch => 1); # # With track_branch => 1, the current branch is saved and restored -# when the object goes out of scope (DESTROY). +# when the object goes out of scope (DESTROY). If the instance +# already exists and track_branch is requested, it upgrades the +# existing instance. sub new { my ($class, $path, %opts) = @_; $path ||= '.'; @@ -26,6 +33,16 @@ sub new { chomp $root if defined $root; bail("Not a git repository: %s", $path) unless $root; + # Return existing instance for this repo + if (my $existing = $_instances{$root}) { + # Upgrade to track_branch if requested and not already tracking + if ($opts{track_branch} && !$existing->{_track_branch}) { + $existing->{_track_branch} = 1; + $existing->{_original_branch} //= $existing->current_branch; + } + return $existing; + } + my ($prefix) = run({}, 'git', '-C', $path, 'rev-parse', '--show-prefix'); chomp $prefix if defined $prefix; $prefix //= ''; @@ -42,24 +59,27 @@ sub new { $self->{_original_branch} = $self->current_branch; } + $_instances{$root} = $self; return $self; } # }}} -# DESTROY - restore original branch on scope exit {{{ +# DESTROY - restore original branch on process exit {{{ +# +# With flyweight caching, the instance lives for the process lifetime. +# DESTROY fires during global cleanup, restoring the branch if needed. sub DESTROY { my ($self) = @_; - if ($self->{_track_branch} && $self->{_original_branch}) { - my $current = eval { $self->current_branch }; - if ($current && $current ne $self->{_original_branch}) { - # Reset any partial changes before switching - run({ dir => $self->{root}, passfail => 1 }, - 'git', 'checkout', '--', '.'); - run({ dir => $self->{root}, passfail => 1 }, - 'git', 'checkout', $self->{_original_branch}); - trace("Service::Git: restored branch %s", $self->{_original_branch}); - } - } + return unless $self->{_track_branch} && $self->{_original_branch}; + my $current = eval { $self->current_branch }; + return unless $current && $current ne $self->{_original_branch}; + # Reset any partial changes before switching + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', '--', '.'); + run({ dir => $self->{root}, passfail => 1 }, + 'git', 'checkout', $self->{_original_branch}); + trace("Service::Git: restored branch %s", $self->{_original_branch}); + delete $_instances{$self->{root}}; } # }}} From 3aaa807e331e47e45128cf08f41b652f98de0ec7 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 07:19:35 -0700 Subject: [PATCH 063/103] Add sha method to Service::Git Semantic wrapper over rev_parse for the common case of resolving a ref to its commit hash. Supports short and full SHA. Completes the semantic method set: sha, current_branch, branch_exists, root, prefix, is_inside_work_tree, is_clean, diff_files, diff_names. --- lib/Service/Git.pm | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/Service/Git.pm b/lib/Service/Git.pm index 83baad35..94534691 100644 --- a/lib/Service/Git.pm +++ b/lib/Service/Git.pm @@ -63,6 +63,31 @@ sub new { return $self; } +# }}} +# create - initialize a new git repository and return an instance {{{ +# +# my $git = Service::Git->create($path); +# my $git = Service::Git->create($path, initial_branch => 'control'); +# +# Runs git init at the given path, optionally setting the initial +# branch name. Returns a Service::Git instance for the new repo. +sub create { + my ($class, $path, %opts) = @_; + $path ||= '.'; + + my @init = ('git', 'init'); + if ($opts{initial_branch}) { + # git symbolic-ref works on all versions (git init -b requires >= 2.28) + run({ onfailure => "Failed to initialize git in $path" }, + "cd \Q$path\E && git init && git symbolic-ref HEAD refs/heads/$opts{initial_branch}"); + } else { + run({ onfailure => "Failed to initialize git in $path" }, + 'git', '-C', $path, 'init'); + } + + return $class->new($path, %opts); +} + # }}} # DESTROY - restore original branch on process exit {{{ # @@ -166,7 +191,19 @@ sub branch_exists { ### Queries {{{ -# rev_parse - resolve a ref to a SHA {{{ +# sha - return the commit SHA for a ref {{{ +# +# $git->sha('HEAD') # full SHA +# $git->sha('HEAD', short => 1) # abbreviated +# $git->sha($branch) # resolve branch to SHA +# $git->sha($short_sha) # expand short to full +sub sha { + my ($self, $ref, %opts) = @_; + return $self->rev_parse($ref // 'HEAD', %opts); +} + +# }}} +# rev_parse - resolve a ref via git rev-parse (low-level) {{{ # # Options: # short => 1 — return abbreviated SHA From 1e289ed7ff748e9a1cedf0255f64b5362943cbd4 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 07:22:39 -0700 Subject: [PATCH 064/103] Wire Service::Git across propagate, prune, new, init Replace raw run() git calls with Service::Git methods. Use sha() for commit hash resolution, current_branch for branch detection, is_clean for dirty checks, branch_exists, checkout/restore_branch, diff_files, create_branch, commit, push, and create for git init. Flyweight instance shared across callers. --- lib/Genesis/Commands/Env.pm | 18 ++- lib/Genesis/Commands/Pipelines.pm | 228 +++++++++--------------------- lib/Genesis/Commands/Repo.pm | 45 +++--- lib/Genesis/Env.pm | 49 ++----- 4 files changed, 113 insertions(+), 227 deletions(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index ddecadce..59cb4340 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -75,10 +75,12 @@ sub create { # control branch so the topology is visible to pipeline tooling and # the environment branch can be cut from the right point. my $ci_configured = $top->ci_configured; + my $git; if ($ci_configured) { + require Service::Git; + $git = Service::Git->new('.'); my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); - chomp $branch if defined $branch; + my $branch = $git->current_branch; if (!defined($branch) || $branch ne $control) { bail( "Creating environments requires being on the #C{%s} branch, ". @@ -205,25 +207,21 @@ sub create { my %cli_opts_git = %{get_options()}; if ($ci_configured) { my $env_file = $env->file; - run({ onfailure => "Failed to stage $env_file" }, - 'git', 'add', $env_file); + $git->add($env_file); if ($cli_opts_git{'no-commit'}) { info "Skipping commit (#C{--no-commit} set); #C{%s} remains staged.", $env_file; } else { my $message = $cli_opts_git{reason} || "Add environment $name"; - run({ onfailure => "Failed to commit $env_file" }, - 'git', 'commit', '-m', $message); + $git->commit($message); - my ($sha) = run({}, 'git rev-parse --short HEAD'); - chomp $sha if defined $sha; + my $sha = $git->sha('HEAD', short => 1); info "#G{Committed} #C{%s} -- %s", $sha // '', $message; # Create the environment branch at the current commit, # then prune files that don't belong to this environment. - run({ onfailure => "Failed to create branch '$name'" }, - 'git', 'branch', $name); + $git->create_branch($name); my @pruned = $env->prune_branch; info "Environment branch #C{%s} created (%d file%s pruned).", diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index b9d81a82..55e87251 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -11,6 +11,7 @@ use Genesis::Env; use Genesis::CI::Legacy qw//; use Genesis::CI::Compiler; use Genesis::CI::Propagation; +use Service::Git; use Service::Vault::Remote; use File::Basename qw/dirname/; @@ -124,34 +125,26 @@ sub propagate { bail("CI is not configured for this repository.") unless $top->ci_configured; + my $git = Service::Git->new('.', track_branch => !$dry_run); my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - # Must be on control branch - my ($current_branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); - chomp $current_branch if defined $current_branch; bail( "Propagation must be run from the #C{%s} branch (currently on #C{%s}).", - $control, $current_branch // '' - ) unless defined($current_branch) && $current_branch eq $control; - - # Guard: working tree must be clean before we switch branches - unless ($dry_run) { - my ($status) = run({}, 'git status --porcelain'); - my @dirty = grep { /^[^?]/ } split /\n/, ($status || ''); - bail( - "Working tree has uncommitted changes. Commit or stash them\n". - "before running propagate." - ) if @dirty; - } + $control, $git->current_branch // '' + ) unless ($git->current_branch // '') eq $control; + + bail( + "Working tree has uncommitted changes. Commit or stash them\n". + "before running propagate." + ) unless $dry_run || $git->is_clean; # Build the DAG from control branch env files - my $env_dir = $top->path; require Genesis::CI::Compiler::ASTBuilder; my $builder = Genesis::CI::Compiler::ASTBuilder->new( top => $top, - env_dir => $env_dir, + env_dir => $top->path, ); - my ($nodes, $edges) = $builder->_build_from_env_files($env_dir); + my ($nodes, $edges) = $builder->_build_from_env_files($top->path); bail("No environments with pipeline metadata found.") unless %$nodes; @@ -164,9 +157,8 @@ sub propagate { } # Topological order (BFS from roots) - my @roots = sort grep { !$has_parent{$_} } keys %$nodes; my @dag_order; - my @queue = @roots; + my @queue = sort grep { !$has_parent{$_} } keys %$nodes; my %visited; while (@queue) { my $env = shift @queue; @@ -176,20 +168,8 @@ sub propagate { } # Resolve control SHA — always use HEAD - my $control_sha_full = $opts->{commit}; - unless ($control_sha_full) { - ($control_sha_full) = run({}, 'git rev-parse HEAD'); - chomp $control_sha_full; - } - - my ($control_sha_short) = run({}, 'git rev-parse --short', $control_sha_full); - chomp $control_sha_short; - - # Git prefix for subdir repos - my ($git_root) = run({}, 'git rev-parse --show-toplevel'); - chomp $git_root; - my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); - chomp $git_prefix; + my $control_sha = $opts->{commit} || $git->sha('HEAD'); + my $control_short = $git->sha($control_sha, short => 1); # Determine scope my @scope; @@ -197,35 +177,24 @@ sub propagate { bail("Environment #C{%s} is not in the pipeline topology.", $after_env) unless $nodes->{$after_env}; - # Verify the after_env doesn't have outstanding changes on - # control that would make it an entry point. If it does, - # those changes haven't flowed through yet — cascading past - # it is unsafe. - my ($last_sync) = _resolve_propagation_base($after_env); + # Verify the after_env has no outstanding changes + my ($last_sync) = _resolve_propagation_base($after_env, $git); if ($last_sync) { - # Check what changed on control since this env was last synced my $after_load = eval { $top->load_env($after_env) }; if ($after_load) { - my @after_deps = $after_load->propagation_files; - my @after_git_deps = map { "${git_prefix}$_" } @after_deps; - my ($since_diff) = run({ dir => $git_root }, - 'git', 'diff', '--name-only', $last_sync, $control_sha_full, - '--', @after_git_deps + my @outstanding = $git->diff_names( + $last_sync, $control_sha, + $git->prefixed($after_load->propagation_files) ); - my @outstanding = grep { /\S/ } split /\n/, ($since_diff || ''); - if (@outstanding) { - bail( - "Environment #C{%s} has %d outstanding change%s on control\n". - "that have not been propagated to it yet.\n". - "Run #C{genesis propagate} without arguments first.", - $after_env, scalar(@outstanding), - @outstanding == 1 ? '' : 's' - ); - } + bail( + "Environment #C{%s} has %d outstanding change%s on control\n". + "that have not been propagated to it yet.\n". + "Run #C{genesis propagate} without arguments first.", + $after_env, scalar(@outstanding), + @outstanding == 1 ? '' : 's' + ) if @outstanding; } } else { - # No propagation commit and no merge-base — branch doesn't exist - # or has never been synced. bail( "Environment #C{%s} has never been propagated to.\n". "Run #C{genesis propagate} without arguments first.", @@ -233,7 +202,6 @@ sub propagate { ); } - # Children and descendants of the named env my %child_set; my @expand = @{$children{$after_env} || []}; while (@expand) { @@ -245,58 +213,34 @@ sub propagate { bail("Environment #C{%s} has no downstream environments.", $after_env) unless @scope; info "\n#G{Propagating from} #C{%s} #G{@} #C{%s} #G{(after %s)}\n", - $control, $control_sha_short, $after_env; + $control, $control_short, $after_env; } else { @scope = @dag_order; info "\n#G{Propagating from} #C{%s} #G{@} #C{%s}\n", - $control, $control_sha_short; + $control, $control_short; } - # Build per-env dependency file lists and diffs - my %env_dep_files; - my %env_changed; - my %env_changed_detail; + # Build per-env diffs + my (%env_changed, %env_changed_detail); for my $env_name (@scope) { - my $exists = run({ passfail => 1 }, - 'git', 'rev-parse', '--verify', $env_name); - next unless $exists; - + next unless $git->branch_exists($env_name); my $env = eval { $top->load_env($env_name) }; next unless $env; my @dep_files = $env->propagation_files; next unless @dep_files; - $env_dep_files{$env_name} = \@dep_files; - my @git_dep_files = map { "${git_prefix}$_" } @dep_files; - # Use --diff-filter and --name-status to distinguish adds/modifies - # from deletes and renames. - my ($diff_out) = run({ dir => $git_root }, - 'git', 'diff', '--name-status', $env_name, $control_sha_full, '--', @git_dep_files + my $diff = $git->diff_files( + $env_name, $control_sha, + $git->prefixed(@dep_files) ); - my (@changed, @deleted, %renamed); - for my $line (grep { /\S/ } split /\n/, ($diff_out || '')) { - if ($line =~ /^D\t(.+)$/) { - push @deleted, $1; - } elsif ($line =~ /^R\d*\t(.+)\t(.+)$/) { - $renamed{$1} = $2; # old → new - push @deleted, $1; # remove old name - push @changed, $2; # add new name - } elsif ($line =~ /^[AMT]\t(.+)$/) { - push @changed, $1; - } - } - if (@changed || @deleted) { - $env_changed{$env_name} = [@changed, @deleted]; - $env_changed_detail{$env_name} = { - changed => \@changed, - deleted => \@deleted, - renamed => \%renamed, - }; + if (@{$diff->{all}}) { + $env_changed{$env_name} = $diff->{all}; + $env_changed_detail{$env_name} = $diff; } } - # Determine entry points via pure decision logic. + # Determine entry points my $env_propagate = Genesis::CI::Propagation::compute_propagation_targets( dag_order => \@dag_order, parent_of => \%parent_of, @@ -310,14 +254,14 @@ sub propagate { my $error; for my $env_name (@scope) { next unless $env_propagate->{$env_name}; - my @files = @{$env_propagate->{$env_name}}; - my $detail = $env_changed_detail{$env_name} || { changed => \@files, deleted => [], renamed => {} }; + my $detail = $env_changed_detail{$env_name} + || { changed => $env_propagate->{$env_name}, deleted => [], renamed => {} }; my @to_copy = @{$detail->{changed}}; my @to_rm = @{$detail->{deleted}}; - my %renames = %{$detail->{renamed}}; + my %renames = %{$detail->{renamed}}; my $total = scalar(@to_copy) + scalar(@to_rm); - my $msg = sprintf("[pipeline] control\@%s -> %s", $control_sha_short, $env_name); + my $msg = sprintf("[pipeline] control\@%s -> %s", $control_short, $env_name); if ($dry_run) { info " #C{%s}: %d file%s to propagate", $env_name, $total, $total == 1 ? '' : 's'; @@ -331,67 +275,36 @@ sub propagate { $propagated++; } else { eval { - run({ dir => $git_root, onfailure => "Failed to checkout $env_name" }, - 'git', 'checkout', $env_name); - - # Copy added/modified files from control@SHA - for my $file (@to_copy) { - my $dir = "$git_root/$file"; - $dir =~ s{/[^/]+$}{}; - mkdir_or_fail($dir) if !-d $dir; - run({ dir => $git_root, onfailure => "Failed to checkout $file from control\@$control_sha_short" }, - 'git', 'checkout', $control_sha_full, '--', $file); - } - - # Remove deleted files - for my $file (@to_rm) { - run({ dir => $git_root, passfail => 1 }, 'git', 'rm', '-f', '--', $file); - } - - run({ dir => $git_root }, 'git', 'add', @to_copy) if @to_copy; - run({ dir => $git_root, onfailure => "Failed to commit propagation to $env_name" }, - 'git', 'commit', '-m', $msg); + $git->checkout($env_name); + $git->checkout_file($control_sha, $_) for @to_copy; + $git->rm(@to_rm) if @to_rm; + $git->commit($msg, @to_copy); info " #G{%s}: propagated %d file%s", $env_name, $total, $total == 1 ? '' : 's'; push @pushed_branches, $env_name; $propagated++; }; if ($@) { $error = $@; - # Reset any partial changes on this branch - run({ dir => $git_root, passfail => 1 }, - 'git', 'checkout', '--', '.'); - warning( - "Propagation to #C{%s} failed: %s", - $env_name, $error =~ s/\s+$//r - ); + $git->reset_working_tree; + warning("Propagation to #C{%s} failed: %s", + $env_name, $error =~ s/\s+$//r); last; } } } - # Always return to starting branch after non-dry-run - unless ($dry_run) { - run({ dir => $git_root, passfail => 1 }, - 'git', 'checkout', $current_branch); - } - + # Restore branch explicitly (DESTROY would too, but be clear) + $git->restore_branch unless $dry_run; bail("Propagation aborted due to error.") if $error; # Push control and propagated branches to remote if ($propagated && !$dry_run && !$no_push) { - my ($remote) = run({ passfail => 1 }, 'git', 'remote'); + my $remote = $git->default_remote; if ($remote) { - chomp $remote; - $remote = (split /\n/, $remote)[0]; # use first remote info "\n#G{Pushing} to #C{%s}...", $remote; - # Push control first (so the SHA is resolvable) - run({ dir => $git_root, passfail => 1 }, - 'git', 'push', $remote, $current_branch); - # Push propagated env branches + my $results = $git->push($remote, $control, @pushed_branches); for my $branch (@pushed_branches) { - my $ok = run({ dir => $git_root, passfail => 1 }, - 'git', 'push', $remote, $branch); - if ($ok) { + if ($results->{$branch}) { info " #G{%s}: pushed", $branch; } else { warning("Failed to push #C{%s} to #C{%s}.", $branch, $remote); @@ -420,32 +333,29 @@ sub propagate { # # Returns: ($control_sha_full, $manual_commits_on_top) sub _resolve_propagation_base { - my ($branch) = @_; + my ($branch, $git) = @_; + $git ||= Service::Git->new('.'); my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); # Scan log for propagation markers - my ($log) = run({}, 'git', 'log', '--format=%H %s', $branch); - if ($log) { - my $depth = 0; - for my $line (split /\n/, $log) { - if ($line =~ /^[0-9a-f]+ \[pipeline\] control\@([0-9a-f]+)/) { - my ($full) = run({}, 'git', 'rev-parse', $1); - chomp $full if defined $full; - if ($depth > 0) { - warning( - "Branch #C{%s} has %d manual commit%s on top of the last propagation.", - $branch, $depth, $depth == 1 ? '' : 's' - ); - } - return ($full, $depth); + my @lines = $git->log_subjects($branch); + my $depth = 0; + for my $line (@lines) { + if ($line =~ /^[0-9a-f]+ \[pipeline\] control\@([0-9a-f]+)/) { + my $full = $git->sha($1); + if ($depth > 0) { + warning( + "Branch #C{%s} has %d manual commit%s on top of the last propagation.", + $branch, $depth, $depth == 1 ? '' : 's' + ); } - $depth++; + return ($full, $depth); } + $depth++; } # No propagation commit — use merge-base with control - my ($merge_base) = run({}, 'git', 'merge-base', $control, $branch); - chomp $merge_base if defined $merge_base; + my $merge_base = $git->merge_base($control, $branch); return ($merge_base, 0) if $merge_base; return (undef, 0); diff --git a/lib/Genesis/Commands/Repo.pm b/lib/Genesis/Commands/Repo.pm index 2348d5cf..477bf000 100644 --- a/lib/Genesis/Commands/Repo.pm +++ b/lib/Genesis/Commands/Repo.pm @@ -8,6 +8,7 @@ use Genesis::Commands; use Genesis::Term qw/in_controlling_terminal/; use Genesis::Top; use Genesis::UI; +use Service::Git; use Cwd qw/getcwd abs_path/; use File::Basename qw/basename dirname/; @@ -127,7 +128,7 @@ sub _repo_init_validate { # separate .git -- it will share history with the surrounding repo. # This is reported in the plan below so the user can notice an # unexpected enclosure. - my $use_subdir = run({ passfail => 1 }, 'git rev-parse --is-inside-work-tree 2>/dev/null') ? 1 : 0; + my $use_subdir = Service::Git->is_inside_work_tree('.'); # In subdir mode, require the enclosing repo to have no staged or # unstaged changes to tracked files. Untracked files are fine. @@ -136,15 +137,14 @@ sub _repo_init_validate { # #C{-F|--force} bypasses this check for users who understand the # risk (e.g. scripted environments, known-safe index). if ($use_subdir && !$opts{force}) { - my ($dirty) = run({}, 'git status --porcelain --untracked-files=no'); - if (defined($dirty) && $dirty =~ /\S/) { + my $enclosing_git = Service::Git->new('.'); + unless ($enclosing_git->is_clean) { bail( "Cannot create a Genesis deployment repository inside an ". "enclosing git repository that has uncommitted changes to ". "tracked files.\n". - " Please commit, stash, or discard the following first, ". - "or rerun with #C{-F|--force} to bypass this check:\n\n%s\n", - $dirty + " Please commit, stash, or discard them first, ". + "or rerun with #C{-F|--force} to bypass this check." ); } } @@ -163,8 +163,8 @@ sub _repo_init_validate { # established, so the branch name is irrelevant at this point. my $control_branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); if ($use_subdir && $with_ci) { - my ($branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); - chomp $branch if defined $branch; + my $enclosing_git = Service::Git->new('.'); + my $branch = $enclosing_git->current_branch; if (!defined($branch) || $branch ne $control_branch) { bail( "Configuring a CI provider requires the enclosing git ". @@ -381,18 +381,18 @@ sub _repo_init_execute { # of whatever git's init.defaultBranch happens to be. # #C{git symbolic-ref HEAD} works on all git versions, unlike # #C{git init -b} which requires >= 2.28. + my $repo_git; unless ($use_subdir) { if ($with_ci) { my $branch = Genesis::Top::DEFAULT_CONTROL_BRANCH(); - run({ onfailure => "Failed to initialize git in $human_root/" }, - "git init && git symbolic-ref HEAD refs/heads/$branch"); + $repo_git = Service::Git->create('.', initial_branch => $branch); } else { - run({ onfailure => "Failed to initialize git in $human_root/" }, - 'git init'); + $repo_git = Service::Git->create('.'); } + } else { + $repo_git = Service::Git->new('.'); } - run({ onfailure => "Failed to stage repository in $human_root/" }, - 'git add .'); + $repo_git->add('.'); # Check if the only staged changes are metadata-only (the # "Last updated" comment and/or updater/creator_version in @@ -435,15 +435,14 @@ sub _repo_init_execute { info "Skipping initial commit (#C{--no-commit} set); files remain staged."; } else { my $message = $reason || "Initial Genesis repo for $name"; - my @cmd = ('git', 'commit', '-m', $message); - push @cmd, '--', '.' if $use_subdir; - run({ onfailure => "Failed to commit initial repository in $human_root/" }, @cmd); - - # Report the commit that was just made so the user has a - # clear record of what was recorded (and, in subdir mode, - # where in the enclosing repo's history to find it). - my ($sha) = run({}, 'git rev-parse --short HEAD'); - chomp $sha if defined $sha; + if ($use_subdir) { + run({ onfailure => "Failed to commit initial repository in $human_root/" }, + 'git', 'commit', '-m', $message, '--', '.'); + } else { + $repo_git->commit($message); + } + + my $sha = $repo_git->sha('HEAD', short => 1); info "#G{Committed} #C{%s} -- %s", $sha // '', $message; } }; diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 3c5bb33c..ba05de2e 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -1249,39 +1249,27 @@ sub propagation_files { sub prune_branch { my ($self, %opts) = @_; - my $branch = $self->name; - my @keep = $self->propagation_files; + require Service::Git; + my $git = Service::Git->new('.'); + my $branch = $self->name; + my @keep = $self->propagation_files; - my ($git_root) = run({}, 'git rev-parse --show-toplevel'); - chomp $git_root; - my ($git_prefix) = run({}, 'git rev-parse --show-prefix'); - chomp $git_prefix; - - # Must not be on the env branch — propagation_files is built from - # the current branch's files, which must be control. + # Must not be on the env branch unless ($opts{dry_run}) { - my ($current) = run({}, 'git rev-parse --abbrev-ref HEAD'); - chomp $current if defined $current; bail( "Cannot prune #C{%s} while on that branch. Switch to the control branch first.", $branch - ) if defined($current) && $current eq $branch; + ) if ($git->current_branch // '') eq $branch; } # Build keep set with git-root-relative paths - my %keep_set; - for my $f (@keep) { - $keep_set{"${git_prefix}$f"} = 1; - } + my %keep_set = map { $_ => 1 } $git->prefixed(@keep); # List tracked files on the env branch under our prefix - my ($tracked) = run({ dir => $git_root }, - 'git', 'ls-tree', '-r', '--name-only', $branch, "${git_prefix}" - ); + my @tracked = $git->ls_tree($branch, $git->prefix); my @to_remove; - for my $file (grep { /\S/ } split /\n/, ($tracked || '')) { - # Always keep .genesis/ contents - next if $file =~ m{^\Q${git_prefix}\E\.genesis/}; + for my $file (@tracked) { + next if $file =~ m{^\Q@{[$git->prefix]}\E\.genesis/}; next if $keep_set{$file}; push @to_remove, $file; } @@ -1289,19 +1277,10 @@ sub prune_branch { return () unless @to_remove; return @to_remove if $opts{dry_run}; - my ($current_branch) = run({}, 'git rev-parse --abbrev-ref HEAD'); - chomp $current_branch; - - run({ dir => $git_root, onfailure => "Failed to checkout $branch for pruning" }, - 'git', 'checkout', $branch); - - run({ dir => $git_root, passfail => 1 }, - 'git', 'rm', '-q', '--', @to_remove); - run({ dir => $git_root, onfailure => "Failed to commit pruned branch $branch" }, - 'git', 'commit', '-m', "Prune non-dependency files from $branch"); - - run({ dir => $git_root, onfailure => "Failed to return to $current_branch" }, - 'git', 'checkout', $current_branch); + $git->checkout($branch); + $git->rm(@to_remove); + $git->commit("Prune non-dependency files from $branch"); + $git->restore_branch; return @to_remove; } From a4b8e82e3d345ae1224d9058466826221961cee7 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 09:30:07 -0700 Subject: [PATCH 065/103] Add pipeline-status command Shows propagation state for all environments in the DAG: last sync SHA, pending file count, and blocked status with blocker identification. Provider-agnostic. Rename Concourse-specific job health to pipeline-jobs. --- bin/genesis | 21 ++++- lib/Genesis/Commands/Pipelines.pm | 152 ++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 4 deletions(-) diff --git a/bin/genesis b/bin/genesis index 4ab0e339..b9ee87f2 100755 --- a/bin/genesis +++ b/bin/genesis @@ -2049,6 +2049,19 @@ define_command("pipeline-graph", { ], }, 'Genesis::Commands::Pipelines::pipeline_graph'); # }}} +# genesis pipeline-status - show propagation state across all environments {{{ +define_command("pipeline-status", { + summary => "Show propagation status for all pipeline environments.", + usage => "pipeline-status", + description => + "Displays the propagation state of each environment in the pipeline ". + "DAG: last sync point, number of pending changes, and whether the ". + "environment is blocked by an ancestor that needs propagation first.", + function_group => Genesis::Commands::PIPELINE, + scope => 'repo', + option_group => Genesis::Commands::REPO_OPTIONS, +}, 'Genesis::Commands::Pipelines::pipeline_status'); +# }}} # genesis propagate - propagate control branch changes to environment branches {{{ define_command("propagate", { summary => "Propagate changes from the control branch to environment branches.", @@ -2128,10 +2141,10 @@ define_command("pipeline-diff", { ], }, 'Genesis::Commands::Pipelines::diff'); # }}} -# genesis pipeline-status - show per-environment job health {{{ -define_command("pipeline-status", { - summary => "Show per-environment pipeline job health.", - usage => "pipeline-status []", +# genesis pipeline-jobs - show per-environment job health (Concourse) {{{ +define_command("pipeline-jobs", { + summary => "Show per-environment pipeline job health (Concourse).", + usage => "pipeline-jobs []", description => "Queries `fly jobs` and displays the status of each environment's ". "deployment job. Pass an environment name to filter to a single job.", diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 55e87251..15cc496e 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -105,6 +105,158 @@ sub apply { exit 0; } +# }}} +# pipeline_status - show propagation state across all environments {{{ +sub pipeline_status { + + my $top = Genesis::Top->new('.'); + bail("CI is not configured for this repository.") + unless $top->ci_configured; + + my $git = Service::Git->new('.'); + my $control = Genesis::Top::DEFAULT_CONTROL_BRANCH(); + my $head = $git->sha('HEAD'); + my $head_short = $git->sha($head, short => 1); + + # Build the DAG + require Genesis::CI::Compiler::ASTBuilder; + my $builder = Genesis::CI::Compiler::ASTBuilder->new( + top => $top, + env_dir => $top->path, + ); + my ($nodes, $edges) = $builder->_build_from_env_files($top->path); + + bail("No environments with pipeline metadata found.") + unless %$nodes; + + my (%children, %has_parent, %parent_of); + for my $edge (@$edges) { + push @{$children{$edge->{from}}}, $edge->{to}; + $has_parent{$edge->{to}} = 1; + $parent_of{$edge->{to}} = $edge->{from}; + } + + # Topological order + my @dag_order; + my @queue = sort grep { !$has_parent{$_} } keys %$nodes; + my %visited; + while (@queue) { + my $env = shift @queue; + next if $visited{$env}++; + push @dag_order, $env; + push @queue, sort @{$children{$env} || []}; + } + + # Gather state for each env + my %env_state; # env => { sync, changed, detail } + my %env_changed; # for propagation target computation + for my $env_name (@dag_order) { + my %state = ( name => $env_name ); + + unless ($git->branch_exists($env_name)) { + $state{status} = 'no-branch'; + $env_state{$env_name} = \%state; + next; + } + + my ($last_sync) = _resolve_propagation_base($env_name, $git); + $state{sync} = $last_sync ? $git->sha($last_sync, short => 1) : undef; + + my $env = eval { $top->load_env($env_name) }; + unless ($env) { + $state{status} = 'error'; + $env_state{$env_name} = \%state; + next; + } + + my @dep_files = $env->propagation_files; + my $diff = $git->diff_files( + $env_name, $head, + $git->prefixed(@dep_files) + ); + + if (@{$diff->{all}}) { + $state{changed} = $diff->{all}; + $state{count} = scalar @{$diff->{all}}; + $env_changed{$env_name} = $diff->{all}; + } else { + $state{status} = 'synced'; + } + + $env_state{$env_name} = \%state; + } + + # Run entry point algorithm to determine who's blocked + my $targets = Genesis::CI::Propagation::compute_propagation_targets( + dag_order => \@dag_order, + parent_of => \%parent_of, + env_changed => \%env_changed, + ); + + for my $env_name (@dag_order) { + my $state = $env_state{$env_name}; + next if $state->{status}; # already resolved (synced, no-branch, error) + + if ($targets->{$env_name}) { + $state->{status} = 'pending'; + } else { + # Has changes but not an entry point — blocked by ancestor + my $blocker = $parent_of{$env_name}; + while ($blocker && !$env_changed{$blocker}) { + $blocker = $parent_of{$blocker}; + } + $state->{status} = 'blocked'; + $state->{blocker} = $blocker; + } + } + + # Display + my $pipeline_name = $top->config->get('ci.name') || $top->type; + my $provider_type = $top->config->get('ci.provider.type') || 'manual'; + + output "\n#G{Pipeline}: #C{%s} #Yi{provider}: %s #Yi{control}: %s", + $pipeline_name, $provider_type, $head_short; + output ""; + + my $max_name = 0; + for (@dag_order) { + $max_name = length($_) if length($_) > $max_name; + } + + for my $env_name (@dag_order) { + my $state = $env_state{$env_name}; + my $status = $state->{status}; + my $sync = $state->{sync} || '-'; + my $indent = ''; + + # Compute depth for visual indentation + my $depth = 0; + my $p = $parent_of{$env_name}; + while ($p) { $depth++; $p = $parent_of{$p}; } + $indent = ' ' x $depth; + + my $name_pad = $max_name - length($env_name) - ($depth * 2); + $name_pad = 0 if $name_pad < 0; + my $padded_name = $env_name . (' ' x $name_pad); + + if ($status eq 'synced') { + output " %s#G{%s} %s #G{synced}", $indent, $padded_name, $sync; + } elsif ($status eq 'pending') { + output " %s#C{%s} %s #Y{%d pending}", $indent, $padded_name, $sync, $state->{count}; + } elsif ($status eq 'blocked') { + output " %s#C{%s} %s #Yi{blocked by %s} (%d files)", + $indent, $padded_name, $sync, $state->{blocker}, $state->{count}; + } elsif ($status eq 'no-branch') { + output " %s#R{%s} %s #R{no branch}", $indent, $padded_name, '-'; + } elsif ($status eq 'error') { + output " %s#R{%s} %s #R{load error}", $indent, $padded_name, '-'; + } + } + + output ""; + exit 0; +} + # }}} # propagate - route control branch changes to environment branches {{{ # From 1992fc451890a41a16c30c06e652d32bf655d5f8 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 09:36:43 -0700 Subject: [PATCH 066/103] Align pipeline-status columns Compute column width from widest indent+name across all envs so SHA and status columns align regardless of DAG depth or name length. --- lib/Genesis/Commands/Pipelines.pm | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 15cc496e..47d2432d 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -218,38 +218,39 @@ sub pipeline_status { $pipeline_name, $provider_type, $head_short; output ""; - my $max_name = 0; - for (@dag_order) { - $max_name = length($_) if length($_) > $max_name; - } - + # Compute column width: widest (indent + name) across all envs + my $col_width = 0; + my %depth_of; for my $env_name (@dag_order) { - my $state = $env_state{$env_name}; - my $status = $state->{status}; - my $sync = $state->{sync} || '-'; - my $indent = ''; - - # Compute depth for visual indentation my $depth = 0; my $p = $parent_of{$env_name}; while ($p) { $depth++; $p = $parent_of{$p}; } - $indent = ' ' x $depth; + $depth_of{$env_name} = $depth; + my $w = ($depth * 2) + length($env_name); + $col_width = $w if $w > $col_width; + } - my $name_pad = $max_name - length($env_name) - ($depth * 2); - $name_pad = 0 if $name_pad < 0; - my $padded_name = $env_name . (' ' x $name_pad); + for my $env_name (@dag_order) { + my $state = $env_state{$env_name}; + my $status = $state->{status}; + my $sync = $state->{sync} || ' - '; + my $depth = $depth_of{$env_name}; + my $indent = ' ' x $depth; + my $pad = $col_width - ($depth * 2) - length($env_name); + $pad = 0 if $pad < 0; + my $name_col = sprintf("%s%s%s", $indent, $env_name, ' ' x $pad); if ($status eq 'synced') { - output " %s#G{%s} %s #G{synced}", $indent, $padded_name, $sync; + output " #G{%s} %s #G{synced}", $name_col, $sync; } elsif ($status eq 'pending') { - output " %s#C{%s} %s #Y{%d pending}", $indent, $padded_name, $sync, $state->{count}; + output " #C{%s} %s #Y{%d pending}", $name_col, $sync, $state->{count}; } elsif ($status eq 'blocked') { - output " %s#C{%s} %s #Yi{blocked by %s} (%d files)", - $indent, $padded_name, $sync, $state->{blocker}, $state->{count}; + output " #C{%s} %s #Yi{blocked by %s} (%d files)", + $name_col, $sync, $state->{blocker}, $state->{count}; } elsif ($status eq 'no-branch') { - output " %s#R{%s} %s #R{no branch}", $indent, $padded_name, '-'; + output " #R{%s} %s #R{no branch}", $name_col, '- '; } elsif ($status eq 'error') { - output " %s#R{%s} %s #R{load error}", $indent, $padded_name, '-'; + output " #R{%s} %s #R{load error}", $name_col, '- '; } } From f73a8876aab93e49d21a8e431ba07b673ff673c8 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 09:44:00 -0700 Subject: [PATCH 067/103] Auto-checkout env branch on deploy when CI enabled When CI is configured, genesis deploy automatically switches to the environment branch before loading the env and deploying. Verifies clean working tree and branch existence. Service::Git auto-restores the original branch on exit. --- lib/Genesis/Commands/Env.pm | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 59cb4340..9d8c4e50 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -854,8 +854,32 @@ sub deploy { my %options = %{get_options()}; my @invalid_create_env_opts = grep {$options{$_}} (qw/fix dry-run fix-stemcells/); + # When CI is configured, auto-checkout the environment branch + # so the deploy reads the correct propagated state. + my $top = Genesis::Top->new('.'); + if ($top->ci_configured) { + require Service::Git; + my $git = Service::Git->new('.', track_branch => 1); + my $current = $git->current_branch // ''; + if ($current ne $env_name) { + bail( + "Working tree has uncommitted changes. Commit or stash them\n". + "before deploying." + ) unless $git->is_clean; + bail( + "Environment branch #C{%s} does not exist.\n". + "Run #C{genesis propagate} to create it.", + $env_name + ) unless $git->branch_exists($env_name); + info "\nSwitching to environment branch #C{%s}...", $env_name; + $git->checkout($env_name); + # Reload Top from the env branch + $top = Genesis::Top->new('.'); + } + } + $options{'disable-reactions'} = ! delete($options{reactions}); - my $env = Genesis::Top->new('.')->load_env($env_name)->with_vault()->with_bosh(); + my $env = $top->load_env($env_name)->with_vault()->with_bosh(); my $deployment_files = $env->deployment_cache_path_lookup('existing'); if (scalar(keys %$deployment_files)) { From 671cbf497af3ccd8bd16148a43bce30a10cbfeab Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 10:25:52 -0700 Subject: [PATCH 068/103] Deploy verification and branch auto-checkout Add git context (branch, commit, control_commit) to deployment audit data when CI is enabled. Cascade guard verifies env was deployed via exodus before allowing propagation to children. Deploy auto-checks out env branch when CI configured, strips path/suffix from env name. pipeline-status distinguishes synced from awaiting-deploy using exodus data. --- lib/Genesis/Commands/Env.pm | 15 ++-- lib/Genesis/Commands/Pipelines.pm | 107 +++++++++++++++++++++------ lib/Genesis/Env/DeploymentManager.pm | 21 ++++++ 3 files changed, 114 insertions(+), 29 deletions(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 9d8c4e50..d44f31d9 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -860,19 +860,22 @@ sub deploy { if ($top->ci_configured) { require Service::Git; my $git = Service::Git->new('.', track_branch => 1); + # Derive branch name: strip path prefix and .yml/.yaml suffix + (my $branch_name = $env_name) =~ s{^.*/}{}; + $branch_name =~ s/\.ya?ml$//; my $current = $git->current_branch // ''; - if ($current ne $env_name) { + if ($current ne $branch_name) { bail( "Working tree has uncommitted changes. Commit or stash them\n". "before deploying." ) unless $git->is_clean; bail( "Environment branch #C{%s} does not exist.\n". - "Run #C{genesis propagate} to create it.", - $env_name - ) unless $git->branch_exists($env_name); - info "\nSwitching to environment branch #C{%s}...", $env_name; - $git->checkout($env_name); + "Create it with #C{genesis new %s} on the control branch.", + $branch_name, $branch_name + ) unless $git->branch_exists($branch_name); + info "\nSwitching to environment branch #C{%s}...", $branch_name; + $git->checkout($branch_name); # Reload Top from the env branch $top = Genesis::Top->new('.'); } diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 47d2432d..4bbe86ec 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -180,7 +180,18 @@ sub pipeline_status { $state{count} = scalar @{$diff->{all}}; $env_changed{$env_name} = $diff->{all}; } else { - $state{status} = 'synced'; + # Synced — check if also deployed + my $branch_head = $git->sha($env_name); + my $deployed = 0; + my $env_v = eval { $env->with_vault }; + if ($env_v) { + my $dep = $env_v->deployments->latest_successful; + if ($dep) { + my $dep_commit = $dep->lookup('git.commit') || ''; + $deployed = 1 if !$dep_commit || $dep_commit eq $branch_head; + } + } + $state{status} = $deployed ? 'deployed' : 'awaiting-deploy'; } $env_state{$env_name} = \%state; @@ -240,8 +251,10 @@ sub pipeline_status { $pad = 0 if $pad < 0; my $name_col = sprintf("%s%s%s", $indent, $env_name, ' ' x $pad); - if ($status eq 'synced') { - output " #G{%s} %s #G{synced}", $name_col, $sync; + if ($status eq 'deployed') { + output " #G{%s} %s #G{deployed}", $name_col, $sync; + } elsif ($status eq 'awaiting-deploy') { + output " #C{%s} %s #Y{synced, pending deploy}", $name_col, $sync; } elsif ($status eq 'pending') { output " #C{%s} %s #Y{%d pending}", $name_col, $sync, $state->{count}; } elsif ($status eq 'blocked') { @@ -330,29 +343,31 @@ sub propagate { bail("Environment #C{%s} is not in the pipeline topology.", $after_env) unless $nodes->{$after_env}; - # Verify the after_env has no outstanding changes + # Verify the after_env has been propagated AND deployed my ($last_sync) = _resolve_propagation_base($after_env, $git); - if ($last_sync) { - my $after_load = eval { $top->load_env($after_env) }; - if ($after_load) { - my @outstanding = $git->diff_names( - $last_sync, $control_sha, - $git->prefixed($after_load->propagation_files) - ); - bail( - "Environment #C{%s} has %d outstanding change%s on control\n". - "that have not been propagated to it yet.\n". - "Run #C{genesis propagate} without arguments first.", - $after_env, scalar(@outstanding), - @outstanding == 1 ? '' : 's' - ) if @outstanding; - } - } else { + bail( + "Environment #C{%s} has never been propagated to.\n". + "Run #C{genesis propagate} without arguments first.", + $after_env + ) unless $last_sync; + + my $after_load = eval { $top->load_env($after_env) }; + if ($after_load) { + # 1. Check for outstanding unpropagated changes + my @outstanding = $git->diff_names( + $last_sync, $control_sha, + $git->prefixed($after_load->propagation_files) + ); bail( - "Environment #C{%s} has never been propagated to.\n". + "Environment #C{%s} has %d outstanding change%s on control\n". + "that have not been propagated to it yet.\n". "Run #C{genesis propagate} without arguments first.", - $after_env - ); + $after_env, scalar(@outstanding), + @outstanding == 1 ? '' : 's' + ) if @outstanding; + + # 2. Check that the propagated state was deployed + _verify_deployed($after_env, $after_load, $git); } my %child_set; @@ -514,6 +529,52 @@ sub _resolve_propagation_base { return (undef, 0); } # }}} +# _verify_deployed - check that an env's propagated state was deployed {{{ +# +# Compares the env branch HEAD against the git.commit field in the +# latest successful deployment's exodus audit data. Requires vault. +# Warns and allows cascade if vault is unavailable. +sub _verify_deployed { + my ($env_name, $env, $git) = @_; + + my $branch_head = $git->sha($env_name); + + # Vault access is required — soft-fail if unavailable + my $env_with_vault = eval { $env->with_vault }; + unless ($env_with_vault) { + warning( + "Could not verify deployment status for #C{%s} (vault unavailable).\n". + "Ensure it has been deployed before cascading.", + $env_name + ); + return; + } + + my $deployment = $env_with_vault->deployments->latest_successful; + bail( + "Environment #C{%s} has never been successfully deployed.\n". + "Deploy it before cascading to downstream environments.", + $env_name + ) unless $deployment; + + my $deployed_commit = $deployment->lookup('git.commit') || ''; + if ($deployed_commit && $deployed_commit ne $branch_head) { + bail( + "Environment #C{%s} has been propagated but not yet deployed\n". + "with the latest changes. Deploy it before cascading to\n". + "downstream environments.", + $env_name + ); + } elsif (!$deployed_commit) { + # Pre-pipeline deployment (no git context in exodus) — warn only + warning( + "Environment #C{%s} was deployed before pipeline tracking was enabled.\n". + "Cannot verify deployment state — ensure it has been deployed.", + $env_name + ); + } +} +# }}} # }}} # pipeline_graph - write pipeline.md with Mermaid flowchart {{{ diff --git a/lib/Genesis/Env/DeploymentManager.pm b/lib/Genesis/Env/DeploymentManager.pm index a5e4a8b3..9e97b501 100644 --- a/lib/Genesis/Env/DeploymentManager.pm +++ b/lib/Genesis/Env/DeploymentManager.pm @@ -327,6 +327,27 @@ sub _base_deployment_content { bosh => $ENV{BOSH_USERNAME} || $ENV{BOSH_USER} || $ENV{BOSH_CLIENT} || undef, }; + # Git context for pipeline propagation tracking (CI-only) + if ($env->top->ci_configured) { + eval { + require Service::Git; + my $git = Service::Git->new('.'); + $base->{git} = { + branch => $git->current_branch, + commit => $git->sha('HEAD'), + }; + # Extract control commit from last propagation commit on this branch + my @log = $git->log_subjects($git->current_branch, limit => 20); + for my $line (@log) { + if ($line =~ /\[pipeline\] control\@([0-9a-f]+)/) { + $base->{git}{control_commit} = $git->sha($1); + last; + } + } + }; + # Non-fatal — git context is supplementary + } + $base->{artifacts} = $self->_base_artifacts($action); if ($action eq 'deploy') { From 2e135e482f7da2d9e5fe2e28a2dcae74ec8831fa Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 10:31:38 -0700 Subject: [PATCH 069/103] Allow dry-run for create-env deploys Remove dry-run from the invalid create-env options blocklist. Skip bosh create-env invocation on dry-run and report the manifest path. All pre-deploy steps (manifest merge, secret checks, config validation) still run normally. --- lib/Genesis/Commands/Env.pm | 2 +- lib/Genesis/Env.pm | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index d44f31d9..a3dae02e 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -852,7 +852,7 @@ sub deploy { my ($env_name, $reason) = @_; my %options = %{get_options()}; - my @invalid_create_env_opts = grep {$options{$_}} (qw/fix dry-run fix-stemcells/); + my @invalid_create_env_opts = grep {$options{$_}} (qw/fix fix-stemcells/); # When CI is configured, auto-checkout the environment branch # so the deploy reads the correct propagated state. diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index ba05de2e..a6a6a635 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -3686,6 +3686,16 @@ sub _deploy_create_env { my @bosh_opts; push @bosh_opts, "--$_" for grep { $opts{$_} } qw/recreate skip-drain/; + # Dry-run: skip the bosh create-env call but report what would deploy. + # bosh create-env doesn't support --dry-run natively, but we've already + # merged the manifest, checked secrets, and validated configs by this point. + if ($opts{'dry-run'}) { + $self->notify("dry-run: skipping #C{bosh create-env} (not supported by BOSH CLI)"); + $self->notify("manifest: #C{%s}", $state->{manifest_path}); + $state->{results} = ['(dry-run)', 0]; + return $state->{ok} = 1; + } + # Execute deployment $state->{deploy_started} = gettimeofday; my @results = $self->bosh->create_env( From 425c927f4e0a85ee0a5be911f60f066ca3f3027b Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 12:47:56 -0700 Subject: [PATCH 070/103] Refine create-env dry-run output - Skip the 'Proceed with create-env?' prompt on dry-run - Guard completion-duration notify when deploy_started is undef (dry-run returns before setting it) - Humanize the manifest path in the dry-run summary --- lib/Genesis/Env.pm | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index a6a6a635..8ebc0a8c 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -3360,11 +3360,13 @@ sub deploy { ? $self->_deploy_create_env(%opts, noprompt => $noprompt) : $self->_deploy_to_bosh(%opts, noprompt => $noprompt); - my $deploy_completed = gettimeofday; - $self->notify( - "BOSH deployment completed in %s", - pretty_duration($deploy_completed - $self->{deployment_state}{deploy_started}, undef,undef, '','','-',1) - ); + if (defined $self->{deployment_state}{deploy_started}) { + my $deploy_completed = gettimeofday; + $self->notify( + "BOSH deployment completed in %s", + pretty_duration($deploy_completed - $self->{deployment_state}{deploy_started}, undef,undef, '','','-',1) + ); + } # Run post-deployment phase return $self->_post_deploy(%opts, noprompt => $noprompt); @@ -3655,8 +3657,10 @@ sub _deploy_create_env { info "[[ - >>no previous deployment of this environment found in the deployment archive."; } - # Confirm deployment - if (in_controlling_terminal && !$noprompt) { + # Confirm deployment (skip on dry-run — nothing destructive to confirm) + if ($opts{'dry-run'}) { + # no confirmation needed + } elsif (in_controlling_terminal && !$noprompt) { prompt_for_boolean( "Proceed with BOSH create-env for the #C{${\($self->name)}}? [y|n] ",1 ) or $self->_cleanup_and_bail("Aborted!\n"); @@ -3691,7 +3695,7 @@ sub _deploy_create_env { # merged the manifest, checked secrets, and validated configs by this point. if ($opts{'dry-run'}) { $self->notify("dry-run: skipping #C{bosh create-env} (not supported by BOSH CLI)"); - $self->notify("manifest: #C{%s}", $state->{manifest_path}); + $self->notify("manifest: #C{%s}", humanize_path($state->{manifest_path})); $state->{results} = ['(dry-run)', 0]; return $state->{ok} = 1; } From bc31735d1a2246da2b087d3febfc4c76eb09b310 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Thu, 23 Apr 2026 12:49:22 -0700 Subject: [PATCH 071/103] Fix non-OCFP environment display issues - Scale suffix is empty for non-OCFP envs; render 'targeting IaaS' instead of the malformed 'to a -scale IaaS target' - Use #R{} on the missing-secrets-detected banner so the color code renders instead of literal braces --- lib/Genesis/Commands/Env.pm | 10 ++++++---- lib/Genesis/Env.pm | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index a3dae02e..3730ade8 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -965,10 +965,12 @@ sub deploy { info " - to '#M{%s}' BOSH director at #c{%s}.", $env->bosh->{alias}, $env->bosh->{url}; } # Specify the environment iaas and scale - info( - " - to a #Y{%s}-scale #G{%s} IaaS target", - $env->scale, $env->iaas - ); + my $scale = $env->scale; + if ($scale) { + info(" - to a #Y{%s}-scale #G{%s} IaaS target", $scale, $env->iaas); + } else { + info(" - targeting #G{%s} IaaS", $env->iaas); + } # Check if the kit supports the environment's IaaS if (my $supported_iaas = $env->kit->metadata('supports')) { diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 8ebc0a8c..0316ab48 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -5256,7 +5256,7 @@ sub _check_secrets { } if ($secrets_results->{error}); if ($secrets_results->{missing}) { - my $msg = "#{missing secrets detected}"; + my $msg = "#R{missing secrets detected}"; if ($self->is_vaultified && grep {$_->{source} eq 'manifest'} ($self->secrets_plan->secrets)) { $msg .= csprintf( " (you may need to run '#g{%s add-secrets} #Y{--import}' to import them from credhub)", From b740a574a78fbe741e4f6deec8688ae4b2c6fd94 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:12:25 -0400 Subject: [PATCH 072/103] FWT-944 PR-based propagation via Service::Github --- lib/Genesis/Commands/Pipelines.pm | 244 ++++++++++++++++- lib/Service/Git.pm | 28 ++ lib/Service/Github.pm | 128 +++++++++ t/unit-tests/service_github-pr.t | 423 ++++++++++++++++++++++++++++++ 4 files changed, 819 insertions(+), 4 deletions(-) create mode 100644 t/unit-tests/service_github-pr.t diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 4bbe86ec..2b114161 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -12,6 +12,7 @@ use Genesis::CI::Legacy qw//; use Genesis::CI::Compiler; use Genesis::CI::Propagation; use Service::Git; +use Service::Github; use Service::Vault::Remote; use File::Basename qw/dirname/; @@ -221,6 +222,29 @@ sub pipeline_status { } } + # Pre-fetch open propagation PRs from GitHub if any env uses require_pr. + # One paginated API call covers all envs; non-fatal if credentials are + # absent or the call fails — display degrades to [PR required] for all. + my %gh_open_prs; # env_name => PR object for the most-recent open propagation PR + { + my $has_require_pr = grep { ($nodes->{$_}{require_pr} // 0) } keys %$nodes; + if ($has_require_pr && $ENV{GITHUB_AUTH_TOKEN}) { + my ($gh_owner, $gh_repo) = _github_owner_repo_from_remote($git); + if ($gh_owner && $gh_repo) { + my $github = Service::Github->new(org => $gh_owner); + eval { + my $prs = $github->list_prs("$gh_owner/$gh_repo", state => 'open'); + for my $pr (@$prs) { + if ($pr->{head}{ref} =~ m{^propagate/([^/]+)/}) { + $gh_open_prs{$1} //= $pr; + } + } + }; + # Silently degrade on error — status output continues without PR info + } + } + } + # Display my $pipeline_name = $top->config->get('ci.name') || $top->type; my $provider_type = $top->config->get('ci.provider.type') || 'manual'; @@ -256,7 +280,20 @@ sub pipeline_status { } elsif ($status eq 'awaiting-deploy') { output " #C{%s} %s #Y{synced, pending deploy}", $name_col, $sync; } elsif ($status eq 'pending') { - output " #C{%s} %s #Y{%d pending}", $name_col, $sync, $state->{count}; + my $req_pr = ($nodes->{$env_name} || {})->{require_pr} // 0; + if ($req_pr) { + my $open_pr = $gh_open_prs{$env_name}; + if ($open_pr) { + output " #C{%s} %s #Y{%d pending} #Yi{[PR #%d open: %s]}", + $name_col, $sync, $state->{count}, + $open_pr->{number}, $open_pr->{html_url}; + } else { + output " #C{%s} %s #Y{%d pending} #Yi{[PR required]}", + $name_col, $sync, $state->{count}; + } + } else { + output " #C{%s} %s #Y{%d pending}", $name_col, $sync, $state->{count}; + } } elsif ($status eq 'blocked') { output " #C{%s} %s #Yi{blocked by %s} (%d files)", $name_col, $sync, $state->{blocker}, $state->{count}; @@ -416,9 +453,42 @@ sub propagate { scope => \@scope, ); + # Pre-flight: build GitHub service once for all require_pr envs in scope. + # Owner/repo is resolved from the remote URL; credentials are validated + # against the API before touching any branches. With --no-push, GitHub + # is not contacted (no push, no PR) so credentials are not required. + my ($gh_owner, $gh_repo, $github); + if (!$dry_run) { + my $needs_github = grep { + $env_propagate->{$_} && ($nodes->{$_}{require_pr} // 0) + } @scope; + if ($needs_github) { + ($gh_owner, $gh_repo) = _github_owner_repo_from_remote($git); + bail( + "Could not determine GitHub owner/repo from remote URL.\n". + "Ensure the origin remote points to a GitHub repository." + ) unless $gh_owner && $gh_repo; + + unless ($no_push) { + bail( + "GitHub credentials required for PR-based propagation.\n". + "Set the GITHUB_AUTH_TOKEN environment variable." + ) unless $ENV{GITHUB_AUTH_TOKEN}; + + $github = Service::Github->new(org => $gh_owner); + my $authed_user = $github->get_authorized_user; + bail( + "GitHub credentials are invalid or lack sufficient permissions.\n". + "Verify GITHUB_AUTH_TOKEN is a valid Personal Access Token." + ) unless $authed_user; + } + } + } + # Propagate to entry point envs my $propagated = 0; my @pushed_branches; + my @pr_targets; my $error; for my $env_name (@scope) { next unless $env_propagate->{$env_name}; @@ -428,8 +498,10 @@ sub propagate { my @to_copy = @{$detail->{changed}}; my @to_rm = @{$detail->{deleted}}; my %renames = %{$detail->{renamed}}; - my $total = scalar(@to_copy) + scalar(@to_rm); - my $msg = sprintf("[pipeline] control\@%s -> %s", $control_short, $env_name); + my $total = scalar(@to_copy) + scalar(@to_rm); + my $msg = sprintf("[pipeline] control\@%s -> %s", $control_short, $env_name); + my $require_pr = $nodes->{$env_name}{require_pr} // 0; + my $prop_branch = "propagate/$env_name/$control_short"; if ($dry_run) { info " #C{%s}: %d file%s to propagate", $env_name, $total, $total == 1 ? '' : 's'; @@ -440,7 +512,67 @@ sub propagate { } info " #R{D} %s", $_ for @to_rm; info " #Yi{commit}: %s", $msg; + if ($require_pr) { + info " #Yi{PR}: would open %s -> %s", $prop_branch, $env_name; + } $propagated++; + } elsif ($require_pr) { + # PR-based propagation: commit onto a short-lived propagation branch. + # Idempotency is checked via the GitHub API when credentials are + # available (normal run), or via local branch existence when not + # (--no-push). If an open PR is found on the remote, the branch is + # fetched rather than re-created. + eval { + my $existing_pr; + + if ($github) { + my $prs = $github->list_prs("$gh_owner/$gh_repo", + head => $prop_branch, + base => $env_name, + state => 'open', + ); + ($existing_pr) = grep { $_->{head}{ref} eq $prop_branch } @$prs; + } + + if ($existing_pr) { + unless ($git->branch_exists($prop_branch)) { + $git->fetch_branch($prop_branch); + } + $git->checkout($prop_branch); + info " #Yi{%s}: PR #%d already open, reusing branch #C{%s}", + $env_name, $existing_pr->{number}, $prop_branch; + } elsif ($git->branch_exists($prop_branch)) { + # --no-push re-run: local branch exists, no PR yet + $git->checkout($prop_branch); + info " #Yi{%s}: reusing local propagation branch #C{%s}", + $env_name, $prop_branch; + } else { + $git->checkout($env_name); + $git->create_branch($prop_branch); + $git->checkout($prop_branch); + $git->checkout_file($control_sha, $_) for @to_copy; + $git->rm(@to_rm) if @to_rm; + $git->commit($msg, @to_copy); + info " #G{%s}: committed %d file%s to #C{%s}", + $env_name, $total, $total == 1 ? '' : 's', $prop_branch; + } + + push @pushed_branches, $prop_branch unless $no_push; + push @pr_targets, { + env => $env_name, + branch => $prop_branch, + detail => $detail, + existing => $existing_pr, + }; + $propagated++; + }; + if ($@) { + $error = $@; + $git->reset_working_tree; + warning("PR propagation to #C{%s} failed: %s", + $env_name, $error =~ s/\s+$//r); + last; + } } else { eval { $git->checkout($env_name); @@ -465,7 +597,7 @@ sub propagate { $git->restore_branch unless $dry_run; bail("Propagation aborted due to error.") if $error; - # Push control and propagated branches to remote + # Push control and propagated/PR branches to remote if ($propagated && !$dry_run && !$no_push) { my $remote = $git->default_remote; if ($remote) { @@ -479,6 +611,32 @@ sub propagate { } } } + + # Open or update GitHub PRs for require_pr environments. + # $github is only set when credentials were validated; @pr_targets + # is only populated for require_pr envs; both guards are needed. + if (@pr_targets && $github) { + my $owner_repo = "$gh_owner/$gh_repo"; + info "\n#G{Opening pull requests} on #C{%s}...", $owner_repo; + for my $pt (@pr_targets) { + my $pr_env = $pt->{env}; + my $pr_branch = $pt->{branch}; + my $pr_title = sprintf("[pipeline] propagate control\@%s to %s", + $control_short, $pr_env); + my $pr_body = _build_pr_body($pr_env, $control_sha, $control_short, $pt->{detail}); + eval { + my $pr = _find_or_open_pr( + $github, $owner_repo, $pr_branch, $pr_env, + $pr_title, $pr_body, $pt->{existing} + ); + info " #G{%s}: PR #%d %s", $pr_env, $pr->{number}, $pr->{html_url}; + }; + if ($@) { + warning("Failed to open/update PR for #C{%s}: %s", + $pr_env, $@ =~ s/\s+$//r); + } + } + } } if ($propagated) { @@ -575,6 +733,84 @@ sub _verify_deployed { } } # }}} +# _github_owner_repo_from_remote - parse owner and repo from the git remote URL {{{ +# +# Supports SSH (git@github.com:owner/repo.git) and HTTPS formats. +# Returns (owner, repo) or (undef, undef) on failure. +sub _github_owner_repo_from_remote { + my ($git) = @_; + my $url = $git->remote_url($git->default_remote) or return (undef, undef); + if ($url =~ m{github\.com[:/]([^/]+)/([^/.]+?)(?:\.git)?\s*$}) { + return ($1, $2); + } + return (undef, undef); +} +# }}} +# _build_pr_body - compose the pull request description {{{ +sub _build_pr_body { + my ($env_name, $control_sha, $control_short, $detail) = @_; + my @changed = @{$detail->{changed} || []}; + my @deleted = @{$detail->{deleted} || []}; + my %renames = %{$detail->{renamed} || {}}; + + my @lines = ( + "Propagating `control\@$control_short` to `$env_name`", + "", + "**Control SHA:** \`$control_sha\`", + "", + ); + + if (@changed) { + push @lines, "**Changed files:**"; + for my $f (@changed) { + my ($old) = grep { $renames{$_} eq $f } keys %renames; + push @lines, $old ? "- \`$f\` *(renamed from \`$old\`)*" : "- \`$f\`"; + } + push @lines, ""; + } + + if (@deleted) { + push @lines, "**Deleted files:**"; + push @lines, "- \`$_\`" for @deleted; + push @lines, ""; + } + + push @lines, "---"; + push @lines, "_[pipeline] control\@$control_short -> ${env_name}_"; + + return join("\n", @lines); +} +# }}} +# _find_or_open_pr - create a PR or update the existing one for this propagation branch {{{ +# +# Accepts an optional $existing PR object (pre-fetched during the propagation +# loop) to avoid a redundant list_prs call. Falls back to querying GitHub +# when $existing is not provided. +sub _find_or_open_pr { + my ($github, $owner_repo, $prop_branch, $env_name, $title, $body, $existing) = @_; + + unless ($existing) { + my $prs = $github->list_prs($owner_repo, + head => $prop_branch, + base => $env_name, + state => 'open', + ); + ($existing) = grep { $_->{head}{ref} eq $prop_branch } @$prs; + } + + return $existing + ? $github->update_pr($owner_repo, $existing->{number}, + title => $title, + body => $body, + ) + : $github->create_pr($owner_repo, + head => $prop_branch, + base => $env_name, + title => $title, + body => $body, + ); +} +# }}} # }}} # pipeline_graph - write pipeline.md with Mermaid flowchart {{{ diff --git a/lib/Service/Git.pm b/lib/Service/Git.pm index 94534691..1ee09107 100644 --- a/lib/Service/Git.pm +++ b/lib/Service/Git.pm @@ -399,6 +399,34 @@ sub default_remote { return $self->{_default_remote}; } +# }}} +# remote_url - fetch the fetch URL for a named (or default) remote {{{ +sub remote_url { + my ($self, $remote) = @_; + $remote //= $self->default_remote; + return undef unless $remote; + my ($url) = run({ dir => $self->{root} }, 'git', 'remote', 'get-url', $remote); + chomp $url if defined $url; + return $url; +} + +# }}} +# fetch_branch - fetch a specific branch from remote into a local ref {{{ +# +# Uses the forced refspec (+refs/heads/branch:refs/heads/branch) so the +# local ref is created or updated regardless of fast-forward status. +# Safe for propagation branches that are written-once and never rebased. +sub fetch_branch { + my ($self, $branch, $remote) = @_; + $remote //= $self->default_remote; + return $self unless $remote; + run({ dir => $self->{root}, onfailure => "Failed to fetch '$branch' from '$remote'" }, + 'git', 'fetch', $remote, + "+refs/heads/$branch:refs/heads/$branch"); + $self->{_branch_cache}{$branch} = 1; + return $self; +} + # }}} # push - push branches to a remote {{{ # diff --git a/lib/Service/Github.pm b/lib/Service/Github.pm index 7aa064aa..c7939a1e 100644 --- a/lib/Service/Github.pm +++ b/lib/Service/Github.pm @@ -6,6 +6,7 @@ use Genesis; use Genesis::UI; use Genesis::Term qw/csprintf/; +use JSON::PP; use Time::HiRes qw/gettimeofday/; use Digest::SHA qw/sha1_hex/; @@ -482,6 +483,133 @@ sub validate_ssh_key { return $key =~ /^(ssh-rsa|ssh-dss|ssh-ed25519|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521)\s+[A-Za-z0-9+\/]+[=]{0,3}(\s+.+)?$/; } +# }}} +# pulls_url - URL for the GitHub pull requests API {{{ +# +# $gh->pulls_url('org/repo') # list endpoint +# $gh->pulls_url('org/repo', 42) # single-PR endpoint +sub pulls_url { + my ($self, $owner_repo, $number) = @_; + my $url = sprintf("%s/repos/%s/pulls", $self->base_url, $owner_repo); + $url .= "/$number" if defined $number; + return $url; +} + +# }}} +# list_prs - list pull requests for a repository {{{ +# +# Options: +# state - 'open' (default), 'closed', or 'all' +# head - filter by head branch (bare branch name; owner: prefix added automatically) +# base - filter by base (target) branch +# +# Returns an arrayref of PR objects from the GitHub API. +sub list_prs { + my ($self, $owner_repo, %opts) = @_; + bail("Missing owner/repo for list_prs") unless $owner_repo; + + my $state = $opts{state} || 'open'; + my $url = $self->pulls_url($owner_repo) . "?state=$state&per_page=100"; + $url .= "&base=" . $opts{base} if $opts{base}; + # GitHub requires owner:branch format for cross-fork head filter + if ($opts{head}) { + my ($owner) = split m{/}, $owner_repo; + $url .= "&head=$owner:$opts{head}"; + } + + my @all_prs; + while ($url) { + my ($code, $msg, $data, $headers) = curl("GET", $url, undef, undef, 0, $self->{creds}); + bail( + "Failed to list pull requests for #C{%s}: HTTP %s - %s", + $owner_repo, $code, $msg + ) unless $code == 200; + + my $page; + eval { $page = load_json($data); 1 } + or bail("Failed to parse pull request list from GitHub: %s", $@); + push @all_prs, @$page if ref($page) eq 'ARRAY'; + + # Follow Link: ; rel="next" pagination header + $url = undef; + if ($headers) { + my ($link_hdr) = grep { s/^Link:\s*//i } split /[\r\n]+/, $headers; + if ($link_hdr) { + ($url) = grep { s/^<(.*)>; rel="next"$/$1/ } split /,\s*/, $link_hdr; + } + } + } + return \@all_prs; +} + +# }}} +# create_pr - create a pull request {{{ +# +# Required options: head (source branch), base (target branch), title +# Optional: body +# +# Returns the PR object (hashref) from the GitHub API. +sub create_pr { + my ($self, $owner_repo, %opts) = @_; + bail("Missing owner/repo for create_pr") unless $owner_repo; + bail("Missing head branch for create_pr") unless $opts{head}; + bail("Missing base branch for create_pr") unless $opts{base}; + bail("Missing title for create_pr") unless $opts{title}; + + my $payload = JSON::PP->new->encode({ + title => $opts{title}, + body => $opts{body} // '', + head => $opts{head}, + base => $opts{base}, + }); + + my ($code, $msg, $data) = curl( + "POST", $self->pulls_url($owner_repo), + {'Content-Type' => 'application/json'}, $payload, 0, $self->{creds} + ); + bail( + "Failed to create pull request for #C{%s}: HTTP %s - %s", + $owner_repo, $code, $msg + ) unless $code == 201; + + my $pr; + eval { $pr = load_json($data); 1 } + or bail("Failed to parse create_pr response from GitHub: %s", $@); + return $pr; +} + +# }}} +# update_pr - update an existing pull request {{{ +# +# opts: title, body, state ('open' or 'closed') +# +# Returns the updated PR object. +sub update_pr { + my ($self, $owner_repo, $number, %opts) = @_; + bail("Missing owner/repo for update_pr") unless $owner_repo; + bail("Missing PR number for update_pr") unless defined $number; + + my %update; + $update{title} = $opts{title} if defined $opts{title}; + $update{body} = $opts{body} if defined $opts{body}; + $update{state} = $opts{state} if defined $opts{state}; + + my $payload = JSON::PP->new->encode(\%update); + my ($code, $msg, $data) = curl( + "PATCH", $self->pulls_url($owner_repo, $number), + {'Content-Type' => 'application/json'}, $payload, 0, $self->{creds} + ); + bail( + "Failed to update pull request #%d for #C{%s}: HTTP %s - %s", + $number, $owner_repo, $code, $msg + ) unless $code == 200; + + my $pr; + eval { $pr = load_json($data); 1 } + or bail("Failed to parse update_pr response from GitHub: %s", $@); + return $pr; +} + # }}} # }}} diff --git a/t/unit-tests/service_github-pr.t b/t/unit-tests/service_github-pr.t new file mode 100644 index 00000000..b7ec4d02 --- /dev/null +++ b/t/unit-tests/service_github-pr.t @@ -0,0 +1,423 @@ +use strict; +use warnings; + +use lib 'lib'; +use lib 't'; + +use Test::More; +use Test::Exception; + +BEGIN { + use_ok 'Service::Github'; +} + +# ============================================================ +# Mock Infrastructure (same pattern as service_github-mocked.t) +# ============================================================ + +my @curl_calls; +my @curl_responses; + +{ + no warnings 'redefine'; + *Service::Github::curl = sub { + push @curl_calls, [@_]; + my $resp = shift @curl_responses; + return (500, 'No mock response queued', '', '') unless $resp; + return @$resp; + }; +} + +sub queue_curl_response { push @curl_responses, [@_] } +sub reset_mocks { @curl_calls = (); @curl_responses = () } + +{ + no warnings 'redefine'; + *Service::Github::info = sub {}; + *Service::Github::error = sub {}; + *Service::Github::trace = sub {}; + *Service::Github::debug = sub {}; + *Service::Github::warning = sub {}; +} + +sub new_gh { + my (%args) = @_; + local $ENV{GITHUB_USER} = delete $args{GITHUB_USER} if exists $args{GITHUB_USER}; + local $ENV{GITHUB_AUTH_TOKEN} = delete $args{GITHUB_AUTH_TOKEN} if exists $args{GITHUB_AUTH_TOKEN}; + return Service::Github->new(%args); +} + +require JSON::PP; +sub encode_json { JSON::PP->new->encode(shift) } +sub decode_json { JSON::PP->new->decode(shift) } + +sub make_pr { + my (%opts) = @_; + return { + number => $opts{number} // 1, + title => $opts{title} // 'Test PR', + body => $opts{body} // '', + state => $opts{state} // 'open', + html_url => $opts{html_url} // 'https://github.com/org/repo/pull/1', + head => { ref => $opts{head_ref} // 'propagate/myenv/abc1234' }, + base => { ref => $opts{base_ref} // 'myenv' }, + }; +} + +# ============================================================ +# pulls_url +# ============================================================ + +subtest 'pulls_url' => sub { + plan tests => 3; + + my $gh = Service::Github->new(org => 'myorg'); + + is( + $gh->pulls_url('myorg/myrepo'), + 'https://api.github.com/repos/myorg/myrepo/pulls', + 'list endpoint' + ); + + is( + $gh->pulls_url('myorg/myrepo', 42), + 'https://api.github.com/repos/myorg/myrepo/pulls/42', + 'single-PR endpoint with number' + ); + + is( + $gh->pulls_url('myorg/myrepo', 0), + 'https://api.github.com/repos/myorg/myrepo/pulls/0', + 'number 0 is included (defined check)' + ); +}; + +# ============================================================ +# list_prs +# ============================================================ + +subtest 'list_prs' => sub { + plan tests => 7; + + subtest 'returns arrayref of PRs on 200' => sub { + plan tests => 4; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + my @prs = (make_pr(number => 1), make_pr(number => 2)); + queue_curl_response(200, '200 OK', encode_json(\@prs), ''); + + my $result = $gh->list_prs('org/repo'); + is(ref($result), 'ARRAY', 'returns arrayref'); + is(scalar(@$result), 2, 'two PRs returned'); + is($result->[0]{number}, 1, 'first PR number'); + is($result->[1]{number}, 2, 'second PR number'); + }; + + subtest 'default state is open' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(200, '200 OK', encode_json([]), ''); + $gh->list_prs('org/repo'); + like($curl_calls[0][1], qr/state=open/, 'URL includes state=open by default'); + }; + + subtest 'filters by base branch' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(200, '200 OK', encode_json([]), ''); + $gh->list_prs('org/repo', base => 'myenv'); + like($curl_calls[0][1], qr/base=myenv/, 'URL includes base filter'); + }; + + subtest 'filters by head branch with owner prefix' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(200, '200 OK', encode_json([]), ''); + $gh->list_prs('myorg/repo', head => 'propagate/myenv/abc1234'); + like($curl_calls[0][1], qr/head=myorg%3Apropagate.*abc1234|head=myorg:propagate.*abc1234/, + 'URL includes head filter with owner prefix'); + }; + + subtest 'follows Link header pagination' => sub { + plan tests => 4; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + + my $page1 = [make_pr(number => 1), make_pr(number => 2)]; + my $page2 = [make_pr(number => 3)]; + my $next_url = 'https://api.github.com/repos/org/repo/pulls?page=2&state=open'; + + # Page 1: returns Link header with rel="next" + queue_curl_response(200, '200 OK', encode_json($page1), + "Link: <$next_url>; rel=\"next\", ; rel=\"first\"\r\n"); + # Page 2: no Link header + queue_curl_response(200, '200 OK', encode_json($page2), ''); + + my $result = $gh->list_prs('org/repo'); + is(scalar(@curl_calls), 2, 'two curl calls made (two pages)'); + is($curl_calls[1][1], $next_url, 'second call uses next URL from Link header'); + is(scalar(@$result), 3, 'all three PRs returned'); + is($result->[2]{number}, 3, 'last PR from page 2 included'); + }; + + subtest 'bails on non-200 response' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(403, 'Forbidden', '{}', ''); + dies_ok { $gh->list_prs('org/repo') } 'bails on 403'; + }; + + subtest 'bails when owner_repo missing' => sub { + plan tests => 1; + reset_mocks(); + my $gh = Service::Github->new(); + dies_ok { $gh->list_prs('') } 'bails with empty owner_repo'; + }; +}; + +# ============================================================ +# create_pr +# ============================================================ + +subtest 'create_pr' => sub { + plan tests => 6; + + subtest 'creates PR and returns PR object' => sub { + plan tests => 5; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + my $pr = make_pr(number => 7, title => 'my title', html_url => 'https://github.com/o/r/pull/7'); + queue_curl_response(201, 'Created', encode_json($pr), ''); + + my $result = $gh->create_pr('org/repo', + head => 'propagate/myenv/abc1234', + base => 'myenv', + title => 'my title', + body => 'some body', + ); + is(ref($result), 'HASH', 'returns hashref'); + is($result->{number}, 7, 'PR number'); + is($result->{title}, 'my title', 'PR title'); + is($result->{html_url}, 'https://github.com/o/r/pull/7', 'PR url'); + + is($curl_calls[0][0], 'POST', 'uses POST method'); + }; + + subtest 'payload includes required fields' => sub { + plan tests => 4; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(201, 'Created', encode_json(make_pr()), ''); + + $gh->create_pr('org/repo', + head => 'propagate/env/abc', + base => 'env', + title => 'the title', + body => 'the body', + ); + my $payload = decode_json($curl_calls[0][3]); + is($payload->{head}, 'propagate/env/abc', 'head in payload'); + is($payload->{base}, 'env', 'base in payload'); + is($payload->{title}, 'the title', 'title in payload'); + is($payload->{body}, 'the body', 'body in payload'); + }; + + subtest 'bails when head missing' => sub { + plan tests => 1; + my $gh = Service::Github->new(); + dies_ok { + $gh->create_pr('org/repo', base => 'env', title => 'T') + } 'bails without head'; + }; + + subtest 'bails when base missing' => sub { + plan tests => 1; + my $gh = Service::Github->new(); + dies_ok { + $gh->create_pr('org/repo', head => 'branch', title => 'T') + } 'bails without base'; + }; + + subtest 'bails when title missing' => sub { + plan tests => 1; + my $gh = Service::Github->new(); + dies_ok { + $gh->create_pr('org/repo', head => 'branch', base => 'env') + } 'bails without title'; + }; + + subtest 'bails on non-201 response' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(422, 'Unprocessable', '{"message":"validation failed"}', ''); + dies_ok { + $gh->create_pr('org/repo', + head => 'branch', base => 'env', title => 'T') + } 'bails on 422'; + }; +}; + +# ============================================================ +# update_pr +# ============================================================ + +subtest 'update_pr' => sub { + plan tests => 5; + + subtest 'updates PR and returns updated object' => sub { + plan tests => 4; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + my $updated = make_pr(number => 3, title => 'new title'); + queue_curl_response(200, 'OK', encode_json($updated), ''); + + my $result = $gh->update_pr('org/repo', 3, + title => 'new title', + body => 'new body', + ); + is(ref($result), 'HASH', 'returns hashref'); + is($result->{number}, 3, 'PR number'); + is($result->{title}, 'new title', 'updated title'); + is($curl_calls[0][0], 'PATCH', 'uses PATCH method'); + }; + + subtest 'PATCH URL contains PR number' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(200, 'OK', encode_json(make_pr(number => 99)), ''); + $gh->update_pr('org/repo', 99, title => 'T'); + like($curl_calls[0][1], qr{/pulls/99$}, 'URL ends with /pulls/99'); + }; + + subtest 'only sends provided fields' => sub { + plan tests => 2; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(200, 'OK', encode_json(make_pr()), ''); + $gh->update_pr('org/repo', 1, title => 'just title'); + my $payload = decode_json($curl_calls[0][3]); + ok(exists $payload->{title}, 'title included'); + ok(!exists $payload->{body}, 'body omitted when not provided'); + }; + + subtest 'bails when owner_repo missing' => sub { + plan tests => 1; + my $gh = Service::Github->new(); + dies_ok { $gh->update_pr('', 1, title => 'T') } 'bails with empty owner_repo'; + }; + + subtest 'bails on non-200 response' => sub { + plan tests => 1; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + queue_curl_response(404, 'Not Found', '{}', ''); + dies_ok { $gh->update_pr('org/repo', 999, title => 'T') } 'bails on 404'; + }; +}; + +# ============================================================ +# find_or_open_pr (via create vs update logic) +# ============================================================ + +subtest 'idempotency: open new PR when none exists' => sub { + plan tests => 3; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + + # list_prs returns empty → create_pr is called + queue_curl_response(200, 'OK', encode_json([]), ''); + queue_curl_response(201, 'Created', encode_json(make_pr(number => 5)), ''); + + # Call list_prs then create_pr manually (simulates _find_or_open_pr logic) + my $prs = $gh->list_prs('org/repo', head => 'propagate/env/abc', base => 'env'); + my ($existing) = grep { $_->{head}{ref} eq 'propagate/env/abc' } @$prs; + ok(!$existing, 'no existing PR found'); + + my $pr = $gh->create_pr('org/repo', + head => 'propagate/env/abc', + base => 'env', + title => '[pipeline] propagate control@abc to env', + body => 'some body', + ); + is($pr->{number}, 5, 'new PR number returned'); + is($curl_calls[1][0], 'POST', 'POST used to create'); +}; + +subtest 'idempotency: update existing PR when head matches' => sub { + plan tests => 3; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + + my $existing_pr = make_pr( + number => 12, + head_ref => 'propagate/env/abc1234', + base_ref => 'env', + ); + # list_prs returns the existing PR + queue_curl_response(200, 'OK', encode_json([$existing_pr]), ''); + # update_pr response + queue_curl_response(200, 'OK', encode_json(make_pr(number => 12, title => 'updated')), ''); + + my $prs = $gh->list_prs('org/repo', + head => 'propagate/env/abc1234', base => 'env'); + my ($found) = grep { $_->{head}{ref} eq 'propagate/env/abc1234' } @$prs; + ok($found, 'existing PR found'); + + my $pr = $gh->update_pr('org/repo', $found->{number}, + title => 'updated', + body => 'updated body', + ); + is($pr->{number}, 12, 'existing PR number returned'); + is($curl_calls[1][0], 'PATCH', 'PATCH used to update'); +}; + +# ============================================================ +# _find_or_open_pr: pre-passed existing skips list_prs call +# ============================================================ + +subtest '_find_or_open_pr: pre-passed existing skips list_prs' => sub { + plan tests => 3; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + + my $existing = make_pr(number => 7, head_ref => 'propagate/env/abc'); + # Only one curl call should be made (update_pr PATCH), no list_prs GET + queue_curl_response(200, 'OK', encode_json(make_pr(number => 7, title => 'updated')), ''); + + # Simulate what _find_or_open_pr does when $existing is pre-passed + my $pr = $gh->update_pr('org/repo', $existing->{number}, + title => 'updated title', + body => 'updated body', + ); + is(scalar(@curl_calls), 1, 'only one curl call (no list_prs)'); + is($curl_calls[0][0], 'PATCH', 'PATCH used directly'); + is($pr->{number}, 7, 'correct PR number'); +}; + +subtest 'pagination: two-page list with Link header' => sub { + plan tests => 3; + reset_mocks(); + my $gh = new_gh(GITHUB_AUTH_TOKEN => 'tok'); + + my $page1 = [make_pr(number => 10), make_pr(number => 11)]; + my $page2 = [make_pr(number => 12)]; + my $next = 'https://api.github.com/repos/org/repo/pulls?state=open&page=2'; + queue_curl_response(200, 'OK', encode_json($page1), + "Link: <$next>; rel=\"next\"\r\n"); + queue_curl_response(200, 'OK', encode_json($page2), ''); + + my $result = $gh->list_prs('org/repo', state => 'open'); + is(scalar(@curl_calls), 2, 'two pages fetched'); + is(scalar(@$result), 3, 'all three PRs from both pages'); + is($result->[2]{number}, 12, 'last PR from page 2 present'); +}; + +done_testing; + +# vim: ts=2 sw=2 sts=2 noet From cccbec027a401ebb5c46fd5eb3cf1870091cb5bb Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:49:34 -0400 Subject: [PATCH 073/103] FWT-947 - Per-env redeploy lane in pipeline compiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add first-class support for a per-environment "redeploy" lane in the generated pipeline — a job that redeploys an environment without content changes, triggered externally. --- docs/workflows/pipeline-propagation.md | 108 ++++ lib/Genesis/CI/Compiler/ASTBuilder.pm | 33 +- lib/Genesis/CI/Compiler/PipelineDescriptor.pm | 211 +++++++- .../genesis_ci_pipeline_descriptor-redeploy.t | 490 ++++++++++++++++++ 4 files changed, 808 insertions(+), 34 deletions(-) create mode 100644 docs/workflows/pipeline-propagation.md create mode 100644 t/unit-tests/genesis_ci_pipeline_descriptor-redeploy.t diff --git a/docs/workflows/pipeline-propagation.md b/docs/workflows/pipeline-propagation.md new file mode 100644 index 00000000..60021d7f --- /dev/null +++ b/docs/workflows/pipeline-propagation.md @@ -0,0 +1,108 @@ +# Pipeline Propagation + +Genesis propagates environment configuration through a controlled branch topology where each environment's files live on their own git branch. The `genesis propagate` command advances those branches when the upstream `control` branch changes. + +## Overview + +The pipeline uses a control branch (commonly `main` or `control`) where all shared and per-environment files are committed. `genesis propagate` walks the topology and, for each environment, commits only the files relevant to that environment onto the environment's own branch. Concourse monitors those per-env branches and triggers deployments when they change. + +## Branch Topology + +Each environment has a corresponding git branch (e.g., `sandbox`, `preprod`, `prod`). The `genesis.pipeline.prior_env` key in each environment YAML file defines the propagation order: + +```yaml +# prod.yml +genesis: + env: prod + pipeline: + prior_env: preprod +``` + +This makes `preprod` the upstream of `prod` — Concourse deploys `preprod` first, then triggers `prod`. + +## PR-Based Propagation + +For environments that require review before deployment, set `require_pr: true`: + +```yaml +# prod.yml +genesis: + env: prod + pipeline: + prior_env: preprod + require_pr: true +``` + +When `require_pr` is set, `genesis propagate` creates a `propagate//` branch instead of pushing directly to the env branch, then opens (or updates) a GitHub Pull Request for review. Merging the PR is what triggers the Concourse deploy. + +### Idempotency + +Re-running `genesis propagate` with the same control SHA updates the existing PR rather than creating a new one. The check uses the GitHub API to find an open PR with the matching head branch — this works across machines and CI runs. + +### Propagation Flags + +| Flag | Behavior | +|------|----------| +| `--dry-run` | Shows what would happen; no remote changes | +| `--no-push` | Commits locally but does not push or open PRs | + +### GitHub Credentials + +PR creation requires a GitHub token with `repo` scope. Set `GITHUB_AUTH_TOKEN` in the environment. The token is validated before any git operations begin. + +## Redeploy Lane + +The redeploy lane lets you force-redeploy an environment without any content changes — useful for recovering from a failed deployment, rotating credentials, or periodic health-checks. + +### Configuration + +In the environment's YAML file under `genesis.pipeline`: + +```yaml +genesis: + pipeline: + redeploy: manual # or: cron, signal + redeploy_cron_start: "04:00" # only required for cron mode + redeploy_cron_stop: "05:00" # only required for cron mode +``` + +### Trigger Modes + +**`manual`** — A `redeploy-` Concourse job is created with no auto-trigger. An operator triggers it via the UI or `fly trigger-job`. + +**`signal`** — Identical to `manual` in pipeline terms; the naming convention communicates that the trigger is expected from an external automated system rather than a human. + +**`cron`** — A Concourse `time` resource is created for the environment and wired as the job trigger. The job runs automatically whenever Concourse observes the time window. + +```yaml +# Daily redeploy between 04:00–05:00 UTC +genesis: + pipeline: + redeploy: cron + redeploy_cron_start: "04:00" + redeploy_cron_stop: "05:00" +``` + +When no `redeploy_cron_start` / `redeploy_cron_stop` are specified, they default to `04:00` and `05:00` UTC. + +### What the Redeploy Job Does + +- Acquires the same dual locks as a normal deploy (`-bosh-lock` and `-deployment-lock`), ensuring it coordinates with in-flight normal deploys. +- Gets the current env branch and git resource (no trigger, no change detection). +- Runs `ci-pipeline-deploy` with `PREVIOUS_ENV=~` (no cache, no propagation from upstream). +- Does **not** generate or push a cache after deploying — only normal deploys advance the cache. +- Does **not** trigger downstream environments. + +### Pipeline Groups + +Any pipeline that has at least one env with `redeploy` configured will emit a `redeploy` group in Concourse, separate from the main workflow group. This keeps the redeploy jobs visible but out of the main deployment flow. + +### Mermaid Diagram Annotation + +Environments with redeploy configured are annotated in the auto-generated pipeline Mermaid diagram: + +``` +sandbox --> preprod[(preprod\nREDEPLOY)] --> prod +``` + +Annotations can combine: `PR+MANUAL+REDEPLOY` for an env that has all three flags set. diff --git a/lib/Genesis/CI/Compiler/ASTBuilder.pm b/lib/Genesis/CI/Compiler/ASTBuilder.pm index 8f24f503..62084b98 100644 --- a/lib/Genesis/CI/Compiler/ASTBuilder.pm +++ b/lib/Genesis/CI/Compiler/ASTBuilder.pm @@ -174,8 +174,11 @@ sub _build_legacy_workflows { my ($ef_nodes) = $self->_build_from_env_files($env_dir); for my $env (keys %nodes) { my $ef = $ef_nodes->{$env} or next; - $nodes{$env}{require_pr} = $ef->{require_pr}; - $nodes{$env}{manual} = $ef->{manual}; + $nodes{$env}{require_pr} = $ef->{require_pr}; + $nodes{$env}{manual} = $ef->{manual}; + $nodes{$env}{redeploy} = $ef->{redeploy}; + $nodes{$env}{redeploy_cron_start} = $ef->{redeploy_cron_start}; + $nodes{$env}{redeploy_cron_stop} = $ef->{redeploy_cron_stop}; } } @@ -411,15 +414,22 @@ sub _build_from_env_files { for my $env (sort keys %envs_to_include) { my $data = $pipeline_data{$env} || {}; + my $rd = $data->{redeploy} || ''; + if ($rd && $rd !~ /^(manual|cron|signal)$/) { + $rd = _truthy($rd) ? 'manual' : ''; + } $nodes{$env} = { - stage_name => $env, - target_name => $env, - alias => $env, - genesis_env => $env, - auto => 0, - type => 'deployment', - require_pr => _truthy($data->{require_pr}), - manual => _truthy($data->{manual}), + stage_name => $env, + target_name => $env, + alias => $env, + genesis_env => $env, + auto => 0, + type => 'deployment', + require_pr => _truthy($data->{require_pr}), + manual => _truthy($data->{manual}), + redeploy => $rd, + redeploy_cron_start => $data->{redeploy_cron_start} || '', + redeploy_cron_stop => $data->{redeploy_cron_stop} || '', }; $prior_env_map{$env} = $data->{prior_env} if $data->{prior_env}; } @@ -484,8 +494,9 @@ sub _read_genesis_pipeline_keys { # 4-space keys under genesis.pipeline if ($in_pipeline && $line =~ /^ ([a-zA-Z_][a-zA-Z0-9_]*):\s*(.*)$/) { my ($k, $v) = ($1, $2); - $v =~ s/\s*#.*$//; # strip inline comment + $v =~ s/\s*#.*$//; # strip inline comment $v =~ s/^\s+|\s+$//g; # trim + $v =~ s/^(["'])(.*)\1$/$2/; # unquote surrounding quotes $result{$k} = $v; } } diff --git a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm index 7e8f954b..484cefb7 100644 --- a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm +++ b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm @@ -55,6 +55,7 @@ sub describe { my @wf_job_names; my @notify_job_names; + my @redeploy_job_names; for my $env (@{$wf_data->{environments}}) { my $alias = $wf_data->{aliases}{$env} || $env; my $is_auto = $wf_data->{auto}{$env}; @@ -89,6 +90,24 @@ sub describe { ); push @jobs, $dj; push @wf_job_names, $dj->{name}; + + # Redeploy lane + my $redeploy_mode = $wf_data->{redeploy}{$env} || ''; + if ($redeploy_mode) { + if ($redeploy_mode eq 'cron') { + my $start = $wf_data->{redeploy_cron_start}{$env} || '04:00'; + my $stop = $wf_data->{redeploy_cron_stop}{$env} || '05:00'; + push @resources, $self->_redeploy_resource( + $ast, $env, $alias, $start, $stop + ); + } + my $rj = $self->_redeploy_job( + $ast, $env, $alias, $deploy_type, $redeploy_mode, + $is_create_env, $wf_data + ); + push @jobs, $rj; + push @redeploy_job_names, $rj->{name}; + } } # Auto-update resources and job @@ -157,6 +176,13 @@ sub describe { jobs => ['update-genesis-assets'], }; } + + if (@redeploy_job_names) { + push @groups, { + name => 'redeploy', + jobs => [sort @redeploy_job_names], + }; + } } my $pipeline = { @@ -699,6 +725,141 @@ sub _deploy_job { }; } +# }}} +# _redeploy_resource - Concourse time resource for cron-mode redeploy {{{ +sub _redeploy_resource { + my ($self, $ast, $env, $alias, $start, $stop) = @_; + + my $config = $ast->configuration || {}; + my $tagged = $config->{tagged}; + my %to = $tagged ? (tags => [$env]) : (); + + return { + name => "$alias-redeploy-cron", + type => 'time', + icon => 'clock-outline', + %to, + source => { + start => $start, + stop => $stop, + location => 'UTC', + }, + }; +} + +# }}} +# _redeploy_job - redeploy job for an environment {{{ +# +# Redeploys an environment without change detection or cache generation. +# Trigger modes: manual (no auto-trigger), cron (time resource), signal (like manual). +sub _redeploy_job { + my ($self, $ast, $env, $alias, $deploy_type, $trigger_mode, + $is_create_env, $wf_data) = @_; + + my $config = $ast->configuration || {}; + my $sc = $ast->integrations->{source_control} || {}; + my $root = $sc->{root} || '.'; + my $tagged = $config->{tagged}; + my %to = $tagged ? (tags => [$env]) : (); + my $name = $ast->metadata->{name} || 'genesis-pipeline'; + + my $bindir = 'git'; + my $srcdir = "$alias-changes"; + $bindir .= "/$root" if $root ne '.'; + + # Resource gets — no change-detection triggers + my @gets; + if ($trigger_mode eq 'cron') { + push @gets, { + get => "$alias-redeploy-cron", + trigger => JSON::PP::true, + }; + } + push @gets, { get => "$alias-changes", trigger => JSON::PP::false }; + push @gets, { get => 'git', trigger => JSON::PP::false } + unless $config->{'require-passed-caches'}; + unless ($is_create_env) { + push @gets, { get => "$alias-cloud-config", %to, trigger => JSON::PP::false }; + push @gets, { get => "$alias-runtime-config", %to, trigger => JSON::PP::false }; + } + + # Deploy task — reuses ci-pipeline-deploy with no prior_env + my $deploy_task = { + task => 'bosh-redeploy', %to, + config => $self->_task_config($ast, $env, $alias, undef, $wf_data, + { command => 'ci-pipeline-deploy', + genesis_bindir => $bindir, + genesis_srcdir => $srcdir }), + ensure => { put => 'git', params => { repository => 'out/git' } }, + }; + my @priv = @{$config->{task}{privileged} || []}; + $deploy_task->{privileged} = JSON::PP::true if grep { $_ eq $alias } @priv; + + # Locker steps — same dual-lock pattern as the normal deploy job + my @lock_steps; + my @unlock_steps; + my $locker = $ast->integrations->{locker} || {}; + if ($locker->{url}) { + unless ($is_create_env) { + push @lock_steps, { + put => "$alias-bosh-lock", %to, + params => { + lock_op => 'lock', + key => 'dont-upgrade-bosh-on-me', + locked_by => "$env-redeploy", + }, + }; + push @unlock_steps, { + put => "$alias-bosh-lock", %to, + params => { + lock_op => 'unlock', + key => 'dont-upgrade-bosh-on-me', + locked_by => "$env-redeploy", + }, + }; + } + push @lock_steps, { + put => "$alias-deployment-lock", %to, + params => { + lock_op => 'lock', + key => 'i-need-to-deploy-myself', + locked_by => "$env-redeploy", + }, + }; + push @unlock_steps, { + put => "$alias-deployment-lock", %to, + params => { + lock_op => 'unlock', + key => 'i-need-to-deploy-myself', + locked_by => "$env-redeploy", + }, + }; + } + + my @do; + push @do, @lock_steps; + push @do, { in_parallel => \@gets }; + push @do, $deploy_task; + + my $fail = $self->_notification_step($ast, + "$name: Redeploy of $env-$deploy_type failed"); + my $succ = $self->_notification_step($ast, + "$name: Successfully redeployed $env-$deploy_type"); + + my $plan_step = {}; + $plan_step->{on_failure} = $fail if $fail; + $plan_step->{on_success} = $succ if $succ; + $plan_step->{ensure} = { do => \@unlock_steps } if @unlock_steps; + $plan_step->{do} = \@do; + + return { + name => "redeploy-$alias", + public => JSON::PP::true, + serial => JSON::PP::true, + plan => [$plan_step], + }; +} + # }}} # _auto_update_resources - build kit-release and genesis-release resources {{{ sub _auto_update_resources { @@ -1131,7 +1292,8 @@ sub _extract_workflow_data { my $nodes = $graph->{nodes} || {}; my $edges = $graph->{edges} || []; - my (%will_trigger, %triggers, %auto, %aliases, %genesis_envs); + my (%will_trigger, %triggers, %auto, %aliases, %genesis_envs, + %redeploy, %redeploy_cron_start, %redeploy_cron_stop); for my $edge (@$edges) { push @{$will_trigger{$edge->{from}}}, $edge->{to}; @@ -1139,9 +1301,12 @@ sub _extract_workflow_data { } for my $n (keys %$nodes) { my $nd = $nodes->{$n}; - $auto{$n} = 1 if $nd->{auto}; - $aliases{$n} = $nd->{alias} || $n; - $genesis_envs{$n} = $nd->{genesis_env} || $n; + $auto{$n} = 1 if $nd->{auto}; + $aliases{$n} = $nd->{alias} || $n; + $genesis_envs{$n} = $nd->{genesis_env} || $n; + $redeploy{$n} = $nd->{redeploy} || ''; + $redeploy_cron_start{$n} = $nd->{redeploy_cron_start} || ''; + $redeploy_cron_stop{$n} = $nd->{redeploy_cron_stop} || ''; } if ($workflow->{_legacy}) { @@ -1155,12 +1320,15 @@ sub _extract_workflow_data { } return { - environments => (@$edges ? [_topological_sort($graph)] : [sort keys %$nodes]), - auto => \%auto, - aliases => \%aliases, - genesis_envs => \%genesis_envs, - will_trigger => \%will_trigger, - triggers => \%triggers, + environments => (@$edges ? [_topological_sort($graph)] : [sort keys %$nodes]), + auto => \%auto, + aliases => \%aliases, + genesis_envs => \%genesis_envs, + will_trigger => \%will_trigger, + triggers => \%triggers, + redeploy => \%redeploy, + redeploy_cron_start => \%redeploy_cron_start, + redeploy_cron_stop => \%redeploy_cron_stop, }; } @@ -1270,19 +1438,16 @@ sub _mermaid_id { sub _mermaid_node_def { my ($alias, $node) = @_; $node ||= {}; - my $id = _mermaid_id($alias); - my $pr = $node->{require_pr} || 0; - my $man = $node->{manual} || 0; - - if ($pr && $man) { - return "$id([$alias\\nPR+MANUAL])"; - } elsif ($pr) { - return "$id([$alias\\nPR])"; - } elsif ($man) { - return "$id([$alias\\nMANUAL])"; - } else { - return $id; - } + my $id = _mermaid_id($alias); + + my @labels; + push @labels, 'PR' if $node->{require_pr}; + push @labels, 'MANUAL' if $node->{manual}; + push @labels, 'REDEPLOY' if $node->{redeploy}; + + return @labels + ? "$id([$alias\\n" . join('+', @labels) . "])" + : $id; } # }}} diff --git a/t/unit-tests/genesis_ci_pipeline_descriptor-redeploy.t b/t/unit-tests/genesis_ci_pipeline_descriptor-redeploy.t new file mode 100644 index 00000000..f43f32df --- /dev/null +++ b/t/unit-tests/genesis_ci_pipeline_descriptor-redeploy.t @@ -0,0 +1,490 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use lib 't'; +use lib 'lib'; +use File::Temp qw/tempdir/; +use Test::More; +use Test::Deep; + +$ENV{GENESIS_TESTING} = 'yes'; +$ENV{GENESIS_LIB} ||= 'lib'; + +use_ok 'Genesis::CI::Compiler::AST'; +use_ok 'Genesis::CI::Compiler::ASTBuilder'; +use_ok 'Genesis::CI::Compiler::PipelineDescriptor'; + +# ========================================================================= +# Helpers +# ========================================================================= + +sub _write { + my ($path, $content) = @_; + open my $fh, '>', $path or die "Cannot write $path: $!"; + print $fh $content; + close $fh; +} + +# Minimal AST with one env and configurable node attributes. +sub _ast_for { + my (%opts) = @_; + + my $env = $opts{env} || 'sandbox'; + my $alias = $opts{alias} || $env; + my $redeploy = $opts{redeploy} || ''; + my $cron_start = $opts{cron_start} || ''; + my $cron_stop = $opts{cron_stop} || ''; + my $with_locker = $opts{with_locker} // 1; + my $create_env = $opts{create_env} // 0; + + my $target_type = $create_env ? 'bosh-create-env' : 'bosh-director'; + my %target = (type => $target_type); + unless ($create_env) { + $target{connection} = { + url => 'https://bosh.example.com:25555', + ca_cert => 'fake-ca', + auth => { client_id => 'admin', client_secret => 'secret' }, + }; + } + + my %integrations = ( + source_control => { provider => 'github', repository => 'org/repo' }, + vault => { url => 'https://vault.example.com' }, + ); + if ($with_locker) { + $integrations{locker} = { + url => 'https://locker.example.com', + username => 'locker-user', + password => 'locker-pass', + }; + } + + return Genesis::CI::Compiler::AST->new( + metadata => { name => 'test-pipeline', deployment_type => 'deployment' }, + branches => { control => 'main' }, + integrations => \%integrations, + targets => { $env => \%target }, + workflows => { + default => { + name => 'default', + type => 'deployment', + graph => { + nodes => { + $env => { + alias => $alias, + stage_name => $env, + genesis_env => $env, + auto => 0, + type => 'deployment', + require_pr => 0, + manual => 0, + redeploy => $redeploy, + redeploy_cron_start => $cron_start, + redeploy_cron_stop => $cron_stop, + }, + }, + edges => [], + }, + }, + }, + configuration => { + task => { image => 'genesiscommunity/concourse', version => 'latest' }, + registry => {}, + }, + ); +} + +sub _describe { + my ($ast) = @_; + my $d = Genesis::CI::Compiler::PipelineDescriptor->new(ast => $ast); + return $d->describe(); +} + +# ========================================================================= +# 1. ASTBuilder reads redeploy attributes from env files +# ========================================================================= +subtest 'ASTBuilder reads redeploy config from env files' => sub { + my $tmp = tempdir(CLEANUP => 1); + + _write("$tmp/sandbox.yml", <<'YAML'); +--- +genesis: + env: sandbox + pipeline: + redeploy: manual +YAML + _write("$tmp/staging.yml", <<'YAML'); +--- +genesis: + env: staging + pipeline: + prior_env: sandbox + redeploy: cron + redeploy_cron_start: "04:00" + redeploy_cron_stop: "05:00" +YAML + _write("$tmp/prod.yml", <<'YAML'); +--- +genesis: + env: prod + pipeline: + prior_env: staging +YAML + + my $builder = Genesis::CI::Compiler::ASTBuilder->new(env_dir => $tmp); + my $parsed = { + _source_format => 'multi-file', + env_dir => $tmp, + pipeline => {}, + targets => {}, + integrations => {}, + scripts => {}, + provider_config => {}, + }; + my $ast = $builder->build($parsed, {}); + my $nodes = $ast->workflows->{default}{graph}{nodes}; + + is $nodes->{sandbox}{redeploy}, 'manual', 'sandbox: redeploy=manual'; + is $nodes->{sandbox}{redeploy_cron_start}, '', 'sandbox: no cron_start'; + + is $nodes->{staging}{redeploy}, 'cron', 'staging: redeploy=cron'; + is $nodes->{staging}{redeploy_cron_start}, '04:00', 'staging: cron_start=04:00'; + is $nodes->{staging}{redeploy_cron_stop}, '05:00', 'staging: cron_stop=05:00'; + + is $nodes->{prod}{redeploy}, '', 'prod: no redeploy'; +}; + +# ========================================================================= +# 2. redeploy: true / truthy shorthand normalises to 'manual' +# ========================================================================= +subtest 'ASTBuilder normalises truthy redeploy value to "manual"' => sub { + my $tmp = tempdir(CLEANUP => 1); + + _write("$tmp/sandbox.yml", <<'YAML'); +--- +genesis: + env: sandbox + pipeline: + redeploy: true +YAML + + my $builder = Genesis::CI::Compiler::ASTBuilder->new(env_dir => $tmp); + my $parsed = { + _source_format => 'multi-file', + env_dir => $tmp, + pipeline => {}, + targets => {}, + integrations => {}, + scripts => {}, + provider_config => {}, + }; + my $ast = $builder->build($parsed, {}); + my $nodes = $ast->workflows->{default}{graph}{nodes}; + + is $nodes->{sandbox}{redeploy}, 'manual', + 'truthy redeploy normalised to "manual"'; +}; + +# ========================================================================= +# 3. No redeploy configured → no redeploy job or group emitted +# ========================================================================= +subtest 'No redeploy config: no redeploy job or group emitted' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => ''); + my $pipeline = _describe($ast); + + my @redeploy_jobs = grep { $_->{name} =~ /^redeploy-/ } @{$pipeline->{jobs}}; + is scalar @redeploy_jobs, 0, 'no redeploy jobs emitted'; + + my @redeploy_groups = grep { $_->{name} eq 'redeploy' } @{$pipeline->{groups}}; + is scalar @redeploy_groups, 0, 'no redeploy group emitted'; +}; + +# ========================================================================= +# 4. Manual trigger mode +# ========================================================================= +subtest 'Manual trigger: redeploy job emitted with no auto-trigger resource' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => 'manual', with_locker => 0); + my $pipeline = _describe($ast); + + # Job is present + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job emitted'; + is $job->{serial}, 1, 'job is serial'; + + # No cron time resource + my @cron_res = grep { ($_->{name} || '') =~ /redeploy-cron/ } @{$pipeline->{resources}}; + is scalar @cron_res, 0, 'no cron time resource for manual mode'; + + # All gets in plan have trigger=false + my $plan_step = $job->{plan}[0]; + my $do = $plan_step->{do} || $plan_step; + my ($parallel) = grep { ref $_ eq 'HASH' && exists $_->{in_parallel} } + (ref $do eq 'ARRAY' ? @$do : ($do)); + ok $parallel, 'in_parallel step found in plan'; + for my $get (@{$parallel->{in_parallel}}) { + next unless ref $get eq 'HASH' && exists $get->{trigger}; + is $get->{trigger}, 0, + "get $get->{get} has trigger=false in manual mode"; + } + + # Group emitted + my ($group) = grep { $_->{name} eq 'redeploy' } @{$pipeline->{groups}}; + ok $group, 'redeploy group emitted'; + ok grep { $_ eq 'redeploy-sandbox' } @{$group->{jobs}}, + 'redeploy group contains redeploy-sandbox'; +}; + +# ========================================================================= +# 5. Signal trigger mode (same as manual from pipeline perspective) +# ========================================================================= +subtest 'Signal trigger: behaves identically to manual (no auto-trigger)' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => 'signal', with_locker => 0); + my $pipeline = _describe($ast); + + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job emitted for signal mode'; + + my @cron_res = grep { ($_->{name} || '') =~ /redeploy-cron/ } @{$pipeline->{resources}}; + is scalar @cron_res, 0, 'no cron time resource for signal mode'; +}; + +# ========================================================================= +# 6. Cron trigger mode +# ========================================================================= +subtest 'Cron trigger: time resource emitted; job gets it with trigger=true' => sub { + my $ast = _ast_for( + env => 'sandbox', + redeploy => 'cron', + cron_start => '04:00', + cron_stop => '05:00', + with_locker => 0, + ); + my $pipeline = _describe($ast); + + # Time resource + my ($cron_res) = grep { ($_->{name} || '') eq 'sandbox-redeploy-cron' } + @{$pipeline->{resources}}; + ok $cron_res, 'sandbox-redeploy-cron time resource emitted'; + is $cron_res->{type}, 'time', 'resource type is time'; + is $cron_res->{source}{start}, '04:00', 'source.start correct'; + is $cron_res->{source}{stop}, '05:00', 'source.stop correct'; + is $cron_res->{source}{location},'UTC', 'source.location is UTC'; + + # Job + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job emitted'; + + # Cron get has trigger=true + my $plan_step = $job->{plan}[0]; + my $do = $plan_step->{do} || $plan_step; + my ($parallel) = grep { ref $_ eq 'HASH' && exists $_->{in_parallel} } + (ref $do eq 'ARRAY' ? @$do : ($do)); + ok $parallel, 'in_parallel step found in plan'; + + my ($cron_get) = grep { ref $_ eq 'HASH' && ($_->{get} || '') eq 'sandbox-redeploy-cron' } + @{$parallel->{in_parallel}}; + ok $cron_get, 'cron get step present'; + is $cron_get->{trigger}, 1, 'cron get has trigger=true'; +}; + +# ========================================================================= +# 7. Cron trigger defaults (no explicit start/stop) +# ========================================================================= +subtest 'Cron trigger: defaults to 04:00-05:00 when no times specified' => sub { + my $ast = _ast_for( + env => 'sandbox', + redeploy => 'cron', + with_locker => 0, + ); + my $pipeline = _describe($ast); + + my ($cron_res) = grep { ($_->{name} || '') eq 'sandbox-redeploy-cron' } + @{$pipeline->{resources}}; + ok $cron_res, 'time resource emitted with defaults'; + is $cron_res->{source}{start}, '04:00', 'default start 04:00'; + is $cron_res->{source}{stop}, '05:00', 'default stop 05:00'; +}; + +# ========================================================================= +# 8. Dual-lock: redeploy job acquires both locks when locker configured +# ========================================================================= +subtest 'Redeploy job acquires bosh-lock and deployment-lock' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => 'manual', with_locker => 1); + my $pipeline = _describe($ast); + + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job present'; + + my $plan_step = $job->{plan}[0]; + my $do = $plan_step->{do}; + ok $do, 'plan has do block (with locks)'; + + my @lock_puts = grep { + ref $_ eq 'HASH' && $_->{put} && $_->{put} =~ /lock$/ + } @$do; + + my @bosh_locks = grep { $_->{put} eq 'sandbox-bosh-lock' } @lock_puts; + my @depl_locks = grep { $_->{put} eq 'sandbox-deployment-lock' } @lock_puts; + + is scalar @bosh_locks, 1, 'sandbox-bosh-lock acquired'; + is scalar @depl_locks, 1, 'sandbox-deployment-lock acquired'; + + is $bosh_locks[0]{params}{lock_op}, 'lock', 'bosh-lock op=lock'; + is $bosh_locks[0]{params}{locked_by}, 'sandbox-redeploy', + 'bosh-lock locked_by=sandbox-redeploy'; + is $depl_locks[0]{params}{locked_by}, 'sandbox-redeploy', + 'deployment-lock locked_by=sandbox-redeploy'; + + # ensure block releases locks + my $ensure = $plan_step->{ensure}; + ok $ensure, 'ensure block present for lock release'; + my @unlock_puts = grep { + ref $_ eq 'HASH' && $_->{put} && $_->{put} =~ /lock$/ + } @{$ensure->{do} || []}; + is scalar @unlock_puts, 2, '2 unlock steps in ensure'; +}; + +# ========================================================================= +# 9. create-env: redeploy skips bosh-lock (no BOSH director to protect) +# ========================================================================= +subtest 'Redeploy job skips bosh-lock for create-env targets' => sub { + my $ast = _ast_for( + env => 'sandbox', + redeploy => 'manual', + with_locker => 1, + create_env => 1, + ); + my $pipeline = _describe($ast); + + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job present for create-env'; + + my $plan_step = $job->{plan}[0]; + my $do = $plan_step->{do}; + + my @bosh_locks = grep { + ref $_ eq 'HASH' && ($_->{put} || '') eq 'sandbox-bosh-lock' + } @$do; + is scalar @bosh_locks, 0, 'no bosh-lock for create-env'; + + my @depl_locks = grep { + ref $_ eq 'HASH' && ($_->{put} || '') eq 'sandbox-deployment-lock' + } @$do; + is scalar @depl_locks, 1, 'deployment-lock still acquired for create-env'; +}; + +# ========================================================================= +# 10. Redeploy job uses ci-pipeline-deploy command +# ========================================================================= +subtest 'Redeploy task uses ci-pipeline-deploy command' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => 'manual', with_locker => 0); + my $pipeline = _describe($ast); + + my ($job) = grep { $_->{name} eq 'redeploy-sandbox' } @{$pipeline->{jobs}}; + ok $job, 'redeploy-sandbox job present'; + + my $plan_step = $job->{plan}[0]; + my $do = $plan_step->{do}; + my ($task_step) = grep { + ref $_ eq 'HASH' && ($_->{task} || '') eq 'bosh-redeploy' + } @$do; + ok $task_step, 'bosh-redeploy task step found'; + + my $args = $task_step->{config}{run}{args}; + ok $args && grep { $_ eq 'ci-pipeline-deploy' } @$args, + 'task runs ci-pipeline-deploy'; +}; + +# ========================================================================= +# 11. Redeploy job does NOT appear in workflow deploy group +# ========================================================================= +subtest 'Redeploy job does not pollute the main workflow group' => sub { + my $ast = _ast_for(env => 'sandbox', redeploy => 'manual', with_locker => 0); + my $pipeline = _describe($ast); + + my ($main_group) = grep { $_->{name} ne 'redeploy' } @{$pipeline->{groups}}; + ok $main_group, 'main group present'; + + my @redeploy_in_main = grep { /^redeploy-/ } @{$main_group->{jobs} || []}; + is scalar @redeploy_in_main, 0, + 'redeploy job not listed in main workflow group'; +}; + +# ========================================================================= +# 12. Multiple envs — only envs with redeploy config get redeploy jobs +# ========================================================================= +subtest 'Multiple envs: only configured envs get redeploy jobs' => sub { + my $ast = Genesis::CI::Compiler::AST->new( + metadata => { name => 'multi-test', deployment_type => 'deployment' }, + branches => { control => 'main' }, + integrations => { + source_control => { provider => 'github', repository => 'org/repo' }, + vault => { url => 'https://vault.example.com' }, + }, + targets => { + sandbox => { type => 'bosh-director', connection => { + url => 'https://bosh.sb.example.com', ca_cert => 'c', + auth => { client_id => 'u', client_secret => 's' } } }, + staging => { type => 'bosh-director', connection => { + url => 'https://bosh.st.example.com', ca_cert => 'c', + auth => { client_id => 'u', client_secret => 's' } } }, + prod => { type => 'bosh-director', connection => { + url => 'https://bosh.pd.example.com', ca_cert => 'c', + auth => { client_id => 'u', client_secret => 's' } } }, + }, + workflows => { + default => { + name => 'default', + type => 'deployment', + graph => { + nodes => { + sandbox => { alias => 'sandbox', stage_name => 'sandbox', + auto => 0, type => 'deployment', + require_pr => 0, manual => 0, + redeploy => '', redeploy_cron_start => '', redeploy_cron_stop => '' }, + staging => { alias => 'staging', stage_name => 'staging', + auto => 0, type => 'deployment', + require_pr => 0, manual => 0, + redeploy => 'manual', redeploy_cron_start => '', redeploy_cron_stop => '' }, + prod => { alias => 'prod', stage_name => 'prod', + auto => 0, type => 'deployment', + require_pr => 0, manual => 0, + redeploy => 'cron', redeploy_cron_start => '03:00', redeploy_cron_stop => '04:00' }, + }, + edges => [ + { from => 'sandbox', to => 'staging' }, + { from => 'staging', to => 'prod' }, + ], + }, + }, + }, + configuration => { + task => { image => 'genesiscommunity/concourse', version => 'latest' }, + registry => {}, + }, + ); + + my $pipeline = _describe($ast); + + my @redeploy_jobs = grep { $_->{name} =~ /^redeploy-/ } @{$pipeline->{jobs}}; + is scalar @redeploy_jobs, 2, 'only 2 redeploy jobs (staging + prod)'; + + my %rj = map { $_->{name} => 1 } @redeploy_jobs; + ok $rj{'redeploy-staging'}, 'redeploy-staging present'; + ok $rj{'redeploy-prod'}, 'redeploy-prod present'; + ok !$rj{'redeploy-sandbox'}, 'no redeploy-sandbox'; + + # Cron resource only for prod + my @cron_res = grep { ($_->{name} || '') =~ /redeploy-cron/ } @{$pipeline->{resources}}; + is scalar @cron_res, 1, 'one cron time resource (for prod only)'; + is $cron_res[0]{name}, 'prod-redeploy-cron', 'cron resource named prod-redeploy-cron'; + + # Redeploy group lists both + my ($rg) = grep { $_->{name} eq 'redeploy' } @{$pipeline->{groups}}; + ok $rg, 'redeploy group emitted'; + is_deeply [sort @{$rg->{jobs}}], ['redeploy-prod', 'redeploy-staging'], + 'redeploy group lists staging and prod'; +}; + +done_testing; From 4ecd36bd45ca2d90f2d1899f3f9c16e0a8b1f517 Mon Sep 17 00:00:00 2001 From: "Dennis J. Bell" Date: Fri, 24 Apr 2026 11:20:26 -0700 Subject: [PATCH 074/103] Add required_files for propagation dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Environments can declare additional paths that must travel with their branch via genesis.pipeline.required_files. Entries are deployment-root-relative, support placeholder and globs, and inherit through the env file hierarchy — pipeline-wide defaults can be declared once in the top env file. Path safety rejects absolute paths, home-dir references, and path traversal. propagation_files now returns one unified, git-root-relative pathspec so callers don't juggle prefixing, and Service::Git::unprefixed restores the user-facing paths for propagation output. --- lib/Genesis/Commands/Pipelines.pm | 18 +-- lib/Genesis/Env.pm | 128 ++++++++++++++-- lib/Service/Git.pm | 16 ++ .../genesis_env_required_files-core.t | 140 ++++++++++++++++++ 4 files changed, 280 insertions(+), 22 deletions(-) create mode 100644 t/unit-tests/genesis_env_required_files-core.t diff --git a/lib/Genesis/Commands/Pipelines.pm b/lib/Genesis/Commands/Pipelines.pm index 2b114161..2cdf6431 100644 --- a/lib/Genesis/Commands/Pipelines.pm +++ b/lib/Genesis/Commands/Pipelines.pm @@ -171,10 +171,7 @@ sub pipeline_status { } my @dep_files = $env->propagation_files; - my $diff = $git->diff_files( - $env_name, $head, - $git->prefixed(@dep_files) - ); + my $diff = $git->diff_files($env_name, $head, @dep_files); if (@{$diff->{all}}) { $state{changed} = $diff->{all}; @@ -393,7 +390,7 @@ sub propagate { # 1. Check for outstanding unpropagated changes my @outstanding = $git->diff_names( $last_sync, $control_sha, - $git->prefixed($after_load->propagation_files) + $after_load->propagation_files, ); bail( "Environment #C{%s} has %d outstanding change%s on control\n". @@ -436,8 +433,7 @@ sub propagate { next unless @dep_files; my $diff = $git->diff_files( - $env_name, $control_sha, - $git->prefixed(@dep_files) + $env_name, $control_sha, @dep_files ); if (@{$diff->{all}}) { $env_changed{$env_name} = $diff->{all}; @@ -507,10 +503,12 @@ sub propagate { info " #C{%s}: %d file%s to propagate", $env_name, $total, $total == 1 ? '' : 's'; for my $f (@to_copy) { my ($old) = grep { $renames{$_} eq $f } keys %renames; - my $note = $old ? " #Yi{(renamed from $old)}" : ''; - info " #G{M} %s%s", $f, $note; + my ($disp_f) = $git->unprefixed($f); + my ($disp_old) = $old ? $git->unprefixed($old) : (); + my $note = $old ? " #Yi{(renamed from $disp_old)}" : ''; + info " #G{M} %s%s", $disp_f, $note; } - info " #R{D} %s", $_ for @to_rm; + info " #R{D} %s", $_ for $git->unprefixed(@to_rm); info " #Yi{commit}: %s", $msg; if ($require_pr) { info " #Yi{PR}: would open %s -> %s", $prop_branch, $env_name; diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 0316ab48..467dbf4b 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -1189,17 +1189,21 @@ sub actual_environment_files { } # }}} -# propagation_files - list all files this environment depends on for pipeline propagation {{{ +# propagation_files - git-root-relative paths this env depends on for pipeline propagation {{{ # -# Returns a list of repo-relative paths that should be tracked when -# propagating changes from the control branch to this environment's -# branch. Includes: env file hierarchy, kit archive (or dev/), -# .genesis/config, and any reaction scripts referenced in the env. +# Returns the unified pathspec that propagation and pruning operate +# over: kit dependencies (env file hierarchy, kit archive, config, +# reaction scripts) prefixed with the git prefix so they're +# git-root-relative, plus any `genesis.pipeline.required_files` +# declared on the env (already git-root-relative). +# +# All returned paths are relative to the git root — no caller +# should apply `Service::Git->prefixed` on top. sub propagation_files { my ($self) = @_; my %files; - # Env file hierarchy (ancestors + self) + # Env file hierarchy (ancestors + self) — kit-relative for my $f ($self->actual_environment_files) { $f =~ s{^\./}{}; $files{$f} = 1; @@ -1232,9 +1236,109 @@ sub propagation_files { } } - return sort keys %files; + require Service::Git; + my $git = Service::Git->new('.'); + + # Prefix kit-relative paths to git-root-relative + my %out = map { $_ => 1 } $git->prefixed(sort keys %files); + + # Merge required_files — already git-root-relative, validated + $out{$_} = 1 for $self->required_files; + + return sort keys %out; } +# }}} +# required_files - list additional paths that should travel with this env's branch {{{ +# +# Reads `genesis.pipeline.required_files` (inherited via env file +# hierarchy — so pipeline-wide defaults can be declared in a parent +# env file like `lmelt.yml` and apply to every child that doesn't +# override). +# +# Path templates (user-facing, Top-root-relative): +# - `` is substituted with the environment's name +# - Glob metacharacters (`*`, `?`, `[`) are expanded against the +# deployment top root (i.e., the kit subdirectory) +# - Anything else is used verbatim as a Top-root-relative path +# +# Returns git-root-relative paths for internal use — callers pass +# them straight to git. User-facing output should strip the git +# prefix before display. +sub required_files { + my ($self) = @_; + + my $entries = $self->lookup('genesis.pipeline.required_files', []); + return () unless ref($entries) eq 'ARRAY' && @$entries; + + require Service::Git; + my $git = Service::Git->new('.'); + my $root = $git->root; + return () unless $root; + + # Top root = git root + git prefix (e.g., "$repo/bosh" for a bosh/ kit) + my $top_root = $root; + if (my $prefix = $git->prefix) { + (my $p = $prefix) =~ s{/$}{}; + $top_root = "$root/$p" if length $p; + } + + my @top_relative = __PACKAGE__->_resolve_required_files( + $entries, $self->name, $top_root + ); + return $git->prefixed(@top_relative); +} + +# _resolve_required_files - pure helper for required_files path resolution {{{ +# +# Takes a list of raw path templates, an env name, and a root path +# (conceptually the deployment Top root). Returns sorted unique +# paths relative to that root: +# - `` is substituted with the env name +# - Glob patterns are expanded against the filesystem under the +# provided root +# - Literal paths are passed through verbatim +# +# Rejects entries that would escape the root (absolute paths, +# `~/...`, or any `..` segment). Bail()s on violation so config +# errors surface loudly. +# +# Exposed as a class method so unit tests can exercise the logic +# without constructing an Env or touching Service::Git. +sub _resolve_required_files { + my ($class, $entries, $env_name, $root) = @_; + return () unless ref($entries) eq 'ARRAY' && @$entries; + return () unless defined $root && length $root; + + require File::Glob; + + my %out; + for my $raw (@$entries) { + next unless defined $raw && length $raw; + (my $path = $raw) =~ s//$env_name/g; + + bail( + "genesis.pipeline.required_files entry #C{%s} escapes the deployment root.\n". + " Paths must be relative to the deployment root — no #R{/}, #R{~/}, or #R{..} allowed.", + $raw + ) if $path =~ m{^/} || $path =~ m{^~} || $path =~ m{(?:^|/)\.\.(?:/|$)}; + + if ($path =~ m{[*?\[]}) { + my @matches = File::Glob::bsd_glob("$root/$path"); + for my $abs (@matches) { + (my $rel = $abs) =~ s{^\Q$root/\E}{}; + $out{$rel} = 1 if length $rel; + } + } else { + $out{$path} = 1; + } + } + + return sort keys %out; +} + +# }}} + # }}} # prune_branch - remove files from this env's branch that it doesn't depend on {{{ # @@ -1262,8 +1366,8 @@ sub prune_branch { ) if ($git->current_branch // '') eq $branch; } - # Build keep set with git-root-relative paths - my %keep_set = map { $_ => 1 } $git->prefixed(@keep); + # propagation_files returns git-root-relative paths already + my %keep_set = map { $_ => 1 } @keep; # List tracked files on the env branch under our prefix my @tracked = $git->ls_tree($branch, $git->prefix); @@ -1298,13 +1402,13 @@ sub propagation_diff { my ($self, $target_sha) = @_; $target_sha ||= 'control'; - my @dep_files = $self->propagation_files; - return () unless @dep_files; + my @pathspec = $self->propagation_files; + return () unless @pathspec; my $env_branch = $self->name; my ($diff_out, $rc) = run( { passfail => 1 }, - 'git', 'diff', '--name-only', "$env_branch..$target_sha", '--', @dep_files + 'git', 'diff', '--name-only', "$env_branch..$target_sha", '--', @pathspec ); return () if $rc || !$diff_out; diff --git a/lib/Service/Git.pm b/lib/Service/Git.pm index 1ee09107..ea977d51 100644 --- a/lib/Service/Git.pm +++ b/lib/Service/Git.pm @@ -471,6 +471,22 @@ sub prefixed { return map { "${p}$_" } @paths; } +# }}} +# unprefixed - strip the git prefix from git-root-relative paths {{{ +# +# Inverse of `prefixed`. Converts git-root-relative paths into +# Top-root-relative paths for user-facing display. Paths that don't +# start with the prefix are left untouched (so sibling-dir paths +# outside the deployment Top root remain identifiable). +# +# my @user_paths = $git->unprefixed(@git_paths); +sub unprefixed { + my ($self, @paths) = @_; + my $p = $self->{prefix}; + return @paths unless $p; + return map { my $q = $_; $q =~ s{^\Q$p\E}{}; $q } @paths; +} + # }}} # in_repo - true if we're inside a subdirectory of the git root {{{ sub in_repo { diff --git a/t/unit-tests/genesis_env_required_files-core.t b/t/unit-tests/genesis_env_required_files-core.t new file mode 100644 index 00000000..225f1506 --- /dev/null +++ b/t/unit-tests/genesis_env_required_files-core.t @@ -0,0 +1,140 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use lib 't'; +use lib 'lib'; + +use Test::More; +use Test::Deep; +use Test::Fatal; +use File::Temp qw/tempdir/; +use File::Path qw/make_path/; + +use_ok 'Genesis::Env'; + +# Build a small scratch tree for glob resolution +my $root = tempdir(CLEANUP => 1); +make_path("$root/cloud-config"); +make_path("$root/overrides"); +for my $rel (qw( + cloud-config/lmelt.yml + cloud-config/lmelt-vsphere-canwest-1-mgmt.yml + cloud-config/lmelt-vsphere-canwest-1-lab.yml + overrides/net.yml + overrides/storage.yml +)) { + open my $fh, '>', "$root/$rel" or die $!; + close $fh; +} + +subtest 'empty / malformed inputs return empty' => sub { + is_deeply([Genesis::Env->_resolve_required_files(undef, 'x', $root)], []); + is_deeply([Genesis::Env->_resolve_required_files([], 'x', $root)], []); + is_deeply([Genesis::Env->_resolve_required_files(['foo'], 'x', undef)], []); + is_deeply([Genesis::Env->_resolve_required_files(['foo'], 'x', '')], []); + is_deeply([Genesis::Env->_resolve_required_files( + [undef, '', 'keep'], 'x', $root + )], ['keep'], 'skips undef/empty entries'); +}; + +subtest ' placeholder substitution' => sub { + is_deeply( + [Genesis::Env->_resolve_required_files( + ['cloud-config/.yml'], + 'lmelt-vsphere-canwest-1-mgmt', + $root, + )], + ['cloud-config/lmelt-vsphere-canwest-1-mgmt.yml'], + ); +}; + +subtest 'literal paths pass through verbatim even if nonexistent' => sub { + is_deeply( + [Genesis::Env->_resolve_required_files( + ['cloud-config/absent.yml', 'overrides/net.yml'], + 'any', + $root, + )], + ['cloud-config/absent.yml', 'overrides/net.yml'], + 'literal paths returned whether or not they exist', + ); +}; + +subtest 'glob expansion is relative to git root' => sub { + my @got = Genesis::Env->_resolve_required_files( + ['overrides/*.yml'], + 'whatever', + $root, + ); + is_deeply(\@got, ['overrides/net.yml', 'overrides/storage.yml']); +}; + +subtest 'placeholder + glob combine naturally' => sub { + # expanded first, then glob + my @got = Genesis::Env->_resolve_required_files( + ['cloud-config/lmelt-vsphere-canwest-1-*.yml'], + 'ignored', + $root, + ); + is_deeply(\@got, [ + 'cloud-config/lmelt-vsphere-canwest-1-lab.yml', + 'cloud-config/lmelt-vsphere-canwest-1-mgmt.yml', + ]); +}; + +subtest 'duplicates are de-duplicated; output sorted' => sub { + my @got = Genesis::Env->_resolve_required_files( + [ + 'overrides/net.yml', + 'overrides/*.yml', # also matches overrides/net.yml + 'cloud-config/.yml', + ], + 'lmelt-vsphere-canwest-1-mgmt', + $root, + ); + is_deeply(\@got, [ + 'cloud-config/lmelt-vsphere-canwest-1-mgmt.yml', + 'overrides/net.yml', + 'overrides/storage.yml', + ]); +}; + +subtest 'unmatched glob returns nothing' => sub { + is_deeply( + [Genesis::Env->_resolve_required_files( + ['no-such-dir/*.yml'], + 'x', + $root, + )], + [], + ); +}; + +subtest 'path-safety rule bails on escape attempts' => sub { + for my $bad ( + '/etc/passwd', + '/absolute/cloud-config.yml', + '~/secrets.yml', + '../escape.yml', + 'cloud-config/../../escape.yml', + 'a/b/../c/../../d.yml', + ) { + like( + exception { + Genesis::Env->_resolve_required_files([$bad], 'x', $root); + }, + qr/escapes/, + "rejects $bad", + ); + } + + # '..' embedded in a filename (not a segment) is allowed + is_deeply( + [Genesis::Env->_resolve_required_files(['foo..bar.yml'], 'x', $root)], + ['foo..bar.yml'], + "'..' inside a filename is not treated as traversal", + ); +}; + +done_testing; From fa7fa23293bb2b62b1cf914399db6ba035e51d29 Mon Sep 17 00:00:00 2001 From: "Tristan Poland (Trident_For_U)" <34868944+tristanpoland@users.noreply.github.com> Date: Fri, 24 Apr 2026 14:27:29 -0400 Subject: [PATCH 075/103] FWY-948 - Deployment status signal abstraction Adds a generic "deployment status signal" abstraction so pipelines can emit and consume deploy success/failure/abort/error events across environments and external tools. --- docs/workflows/deployment-status-signals.md | 179 ++++++ lib/Genesis/CI/Compiler/ASTBuilder.pm | 10 + lib/Genesis/CI/Compiler/PipelineDescriptor.pm | 210 ++++++- .../CI/Compiler/Providers/Concourse.pm | 75 ++- .../genesis_ci_pipeline_descriptor-signal.t | 562 ++++++++++++++++++ 5 files changed, 1004 insertions(+), 32 deletions(-) create mode 100644 docs/workflows/deployment-status-signals.md create mode 100644 t/unit-tests/genesis_ci_pipeline_descriptor-signal.t diff --git a/docs/workflows/deployment-status-signals.md b/docs/workflows/deployment-status-signals.md new file mode 100644 index 00000000..526c55bd --- /dev/null +++ b/docs/workflows/deployment-status-signals.md @@ -0,0 +1,179 @@ +# Deployment Status Signals + +Genesis pipelines can emit structured deployment status events — success, failure, abort, and error — through a generic signal abstraction backed by the `cfcommunity/shuttle-resource` Concourse custom resource type. + +## Overview + +When `status_signal` is configured, every deploy and redeploy job in the pipeline emits a Concourse put step for each of its four outcome hooks (`on_success`, `on_failure`, `on_abort`, `on_error`). External consumers — other pipelines, automation, dashboards — can get these signals to gate work on upstream deployment outcomes. + +The signal resource for each environment is named `-signal`. The shuttle resource writes a small status record to a backing store (S3, GCS, or local file), which Concourse polls as a versioned resource. + +## Configuration + +### Global configuration + +Set `status_signal` in the `configuration:` block of your `ci.yml`: + +```yaml +configuration: + status_signal: + backend: s3 + bucket: genesis-signals + region: us-east-1 + access_key_id: ((aws-access-key-id)) + secret_access_key: ((aws-secret-access-key)) +``` + +This applies to every environment in the pipeline unless overridden or disabled per-env. + +### Backends + +**`file`** — Writes signal records to the Concourse worker filesystem. Useful for local testing or single-worker setups. + +```yaml +status_signal: + backend: file + path: /tmp/genesis-signals # defaults to /tmp/genesis-signals +``` + +**`s3`** — Writes to an S3 bucket. Suitable for multi-worker pipelines and cross-pipeline consumers. + +```yaml +status_signal: + backend: s3 + bucket: genesis-signals + region: us-east-1 + access_key_id: ((aws-access-key-id)) + secret_access_key: ((aws-secret-access-key)) + endpoint: https://s3.example.com # optional, for S3-compatible stores +``` + +**`gcs`** — Writes to a Google Cloud Storage bucket. + +```yaml +status_signal: + backend: gcs + bucket: genesis-signals + json_key: ((gcs-service-account-json)) +``` + +### Optional global keys + +| Key | Default | Description | +|-----|---------|-------------| +| `prefix` | *(empty)* | Prepended to the per-env prefix: `/`. If unset, prefix is just the env name. | +| `image` | `cfcommunity/shuttle-resource` | Override the shuttle resource Docker image. | +| `image_tag` | `latest` | Override the image tag. | + +### Per-environment overrides + +In each environment's YAML file under `genesis.pipeline`: + +```yaml +# sandbox.yml +genesis: + env: sandbox + pipeline: + status_signal: s3 # override backend for this env only + signal_prefix: acme/sandbox # override prefix for this env +``` + +To disable signals for a specific environment: + +```yaml +genesis: + env: prod + pipeline: + status_signal: false +``` + +| Value | Meaning | +|-------|---------| +| *(absent)* | Uses global config | +| `file`, `s3`, `gcs` | Enables signals, overrides backend | +| `true` / `1` | Enables signals, uses global backend | +| `false` / `no` / `0` | Disables signals for this env | + +## Signal resource prefix + +The per-environment signal resource stores records under a prefix path. The effective prefix is resolved as follows: + +1. `signal_prefix` in the env's YAML — used verbatim +2. Global `prefix` + `/` + env name (e.g., `mypipeline/sandbox`) +3. Just the env name if no global prefix is set + +## Outcome hooks + +Each deploy and redeploy job emits a put step to `-signal` for all four Concourse outcome hooks: + +| Hook | Signal `status` | +|------|----------------| +| `on_success` | `success` | +| `on_failure` | `failure` | +| `on_abort` | `abort` | +| `on_error` | `error` | + +The put params are: + +```yaml +put: sandbox-signal +params: + status: success + deployment: sandbox-deployment +``` + +When Slack/email notifications are also configured, the `on_success` and `on_failure` hooks combine both steps in an `in_parallel` block. The `on_abort` and `on_error` hooks emit the signal put directly (no notification for abort/error). + +## Cross-pipeline gating + +A downstream pipeline can gate on upstream deploy success by getting the signal resource: + +```yaml +resources: + - name: sandbox-signal + type: shuttle + source: + backend: s3 + bucket: genesis-signals + region: us-east-1 + access_key_id: ((aws-access-key-id)) + secret_access_key: ((aws-secret-access-key)) + prefix: sandbox + +jobs: + - name: integration-tests + plan: + - get: sandbox-signal + trigger: true + version: { status: success } + - task: run-tests + ... +``` + +## Redeploy signal trigger + +When an environment has `redeploy: signal` set, the `redeploy-` job automatically triggers off the signal resource instead of running manually: + +```yaml +genesis: + env: prod + pipeline: + redeploy: signal + status_signal: s3 +``` + +This creates a `get: prod-signal, trigger: true, version: {status: success}` step at the top of the redeploy job plan. The redeploy fires whenever a successful deployment signal arrives for that env — useful for chaining a post-deploy health-check or forcing a redeploy after upstream changes. + +If `redeploy: signal` is set but no global `status_signal` is configured, the job falls back to manual trigger behaviour (no auto-trigger resource is wired). + +## Pipeline description annotation + +The `genesis ci describe` output annotates each environment with its active features: + +``` +Pipeline: cf +Workflow: cf + sandbox sandbox-deployment [auto] [REDEPLOY:CRON, SIGNAL:s3] + preprod preprod-deployment [manual] (triggered by sandbox) [SIGNAL:s3] + prod prod-deployment [manual] (triggered by preprod) [REDEPLOY:SIGNAL, SIGNAL:s3] +``` diff --git a/lib/Genesis/CI/Compiler/ASTBuilder.pm b/lib/Genesis/CI/Compiler/ASTBuilder.pm index 62084b98..b51ba7e2 100644 --- a/lib/Genesis/CI/Compiler/ASTBuilder.pm +++ b/lib/Genesis/CI/Compiler/ASTBuilder.pm @@ -179,6 +179,8 @@ sub _build_legacy_workflows { $nodes{$env}{redeploy} = $ef->{redeploy}; $nodes{$env}{redeploy_cron_start} = $ef->{redeploy_cron_start}; $nodes{$env}{redeploy_cron_stop} = $ef->{redeploy_cron_stop}; + $nodes{$env}{status_signal} = $ef->{status_signal}; + $nodes{$env}{signal_prefix} = $ef->{signal_prefix}; } } @@ -418,6 +420,12 @@ sub _build_from_env_files { if ($rd && $rd !~ /^(manual|cron|signal)$/) { $rd = _truthy($rd) ? 'manual' : ''; } + my $ss = $data->{status_signal} || ''; + if ($ss && $ss !~ /^(false|no|0|file|s3|gcs)$/i && _truthy($ss)) { + $ss = '1'; # truthy shorthand → "enabled, use global config" + } elsif ($ss && $ss =~ /^(false|no|0)$/i) { + $ss = '0'; # normalized disabled + } $nodes{$env} = { stage_name => $env, target_name => $env, @@ -430,6 +438,8 @@ sub _build_from_env_files { redeploy => $rd, redeploy_cron_start => $data->{redeploy_cron_start} || '', redeploy_cron_stop => $data->{redeploy_cron_stop} || '', + status_signal => $ss, + signal_prefix => $data->{signal_prefix} || '', }; $prior_env_map{$env} = $data->{prior_env} if $data->{prior_env}; } diff --git a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm index 484cefb7..e49b0975 100644 --- a/lib/Genesis/CI/Compiler/PipelineDescriptor.pm +++ b/lib/Genesis/CI/Compiler/PipelineDescriptor.pm @@ -74,6 +74,14 @@ sub describe { )}; } + # Signal resource + my $signal_cfg = $self->_env_signal_config($ast, $env, $wf_data); + if ($signal_cfg) { + push @resources, $self->_signal_resource( + $ast, $env, $alias, $signal_cfg + ); + } + unless ($is_auto) { my $nj = $self->_notify_job( $ast, $env, $alias, $deploy_type, $trigger_from, @@ -296,6 +304,8 @@ sub description { push @lines, sprintf("Workflow: %s", $wf_name eq 'default' ? $name : $wf_name); + my $signal_global = ($ast->configuration || {})->{status_signal}; + for my $env (@order) { my $alias = $wf_data->{aliases}{$env} || $env; my $is_auto = $wf_data->{auto}{$env}; @@ -305,8 +315,22 @@ sub description { my $mode = $is_auto ? 'auto' : 'manual'; my $source = $trigger_a ? " (triggered by $trigger_a)" : ''; - push @lines, sprintf(" %-20s %s-$deploy_type [%s]%s", - $alias, $alias, $mode, $source); + # Annotations: REDEPLOY, SIGNAL + my @annot; + my $redeploy_mode = $wf_data->{redeploy}{$env} || ''; + push @annot, 'REDEPLOY:' . uc($redeploy_mode) if $redeploy_mode; + if ($signal_global) { + my $ss = $wf_data->{status_signal}{$env} // ''; + unless ($ss =~ /^(false|no|0)$/i) { + my $backend = ($ss =~ /^(file|s3|gcs)$/) ? $ss + : ($signal_global->{backend} || 'file'); + push @annot, "SIGNAL:$backend"; + } + } + my $annot_str = @annot ? ' [' . join(', ', @annot) . ']' : ''; + + push @lines, sprintf(" %-20s %s-$deploy_type [%s]%s%s", + $alias, $alias, $mode, $source, $annot_str); } push @lines, ""; } @@ -322,7 +346,8 @@ sub description { sub _resource_types { my ($self, $ast) = @_; - my $registry = ($ast->configuration || {})->{registry} || {}; + my $config = $ast->configuration || {}; + my $registry = $config->{registry} || {}; my $prefix = $registry->{uri} ? "$registry->{uri}/" : ''; my %rs; @@ -331,7 +356,7 @@ sub _resource_types { $rs{password} = _unwrap_ref($registry->{password}); } - return [map {{ + my @types = map {{ name => $_->[0], type => 'registry-image', source => { %rs, repository => "$prefix$_->[1]" }, @@ -341,7 +366,22 @@ sub _resource_types { ['slack-notification', 'cfcommunity/slack-notification-resource'], ['bosh-config', 'cfcommunity/bosh-config-resource'], ['locker', 'cfcommunity/locker-resource'], - )]; + ); + + # Shuttle resource type for deployment status signals + if (my $signal = $config->{status_signal}) { + my $img = (ref $signal eq 'HASH' ? $signal->{image} : undef) + || 'cfcommunity/shuttle-resource'; + my $tag = (ref $signal eq 'HASH' ? $signal->{image_tag} : undef) + || 'latest'; + push @types, { + name => 'shuttle', + type => 'registry-image', + source => { %rs, repository => "$prefix$img", tag => $tag }, + }; + } + + return \@types; } # }}} @@ -706,14 +746,9 @@ sub _deploy_job { } } - my $fail = $self->_notification_step($ast, - "$name: Deployment to $env-$deploy_type failed"); - my $succ = $self->_notification_step($ast, - "$name: Successfully deployed $env-$deploy_type"); - + my %hooks = $self->_outcome_hooks($ast, $env, $alias, $deploy_type, $wf_data, 0); my $plan_step = {}; - $plan_step->{on_failure} = $fail if $fail; - $plan_step->{on_success} = $succ if $succ; + $plan_step->{$_} = $hooks{$_} for sort keys %hooks; $plan_step->{ensure} = { do => \@unlock_steps } if @unlock_steps; $plan_step->{do} = \@do; @@ -768,12 +803,19 @@ sub _redeploy_job { $bindir .= "/$root" if $root ne '.'; # Resource gets — no change-detection triggers + my $signal_cfg = $self->_env_signal_config($ast, $env, $wf_data); my @gets; if ($trigger_mode eq 'cron') { push @gets, { get => "$alias-redeploy-cron", trigger => JSON::PP::true, }; + } elsif ($trigger_mode eq 'signal' && $signal_cfg) { + push @gets, { + get => "$alias-signal", + trigger => JSON::PP::true, + version => { status => 'success' }, + }; } push @gets, { get => "$alias-changes", trigger => JSON::PP::false }; push @gets, { get => 'git', trigger => JSON::PP::false } @@ -841,14 +883,9 @@ sub _redeploy_job { push @do, { in_parallel => \@gets }; push @do, $deploy_task; - my $fail = $self->_notification_step($ast, - "$name: Redeploy of $env-$deploy_type failed"); - my $succ = $self->_notification_step($ast, - "$name: Successfully redeployed $env-$deploy_type"); - + my %hooks = $self->_outcome_hooks($ast, $env, $alias, $deploy_type, $wf_data, 1); my $plan_step = {}; - $plan_step->{on_failure} = $fail if $fail; - $plan_step->{on_success} = $succ if $succ; + $plan_step->{$_} = $hooks{$_} for sort keys %hooks; $plan_step->{ensure} = { do => \@unlock_steps } if @unlock_steps; $plan_step->{do} = \@do; @@ -860,6 +897,126 @@ sub _redeploy_job { }; } +# }}} +# _env_signal_config - resolve effective signal config for an env {{{ +# +# Returns a hashref with the merged global+per-env signal config, or undef +# if signals are not configured or explicitly disabled for this env. +sub _env_signal_config { + my ($self, $ast, $env, $wf_data) = @_; + + my $global = ($ast->configuration || {})->{status_signal}; + return undef unless $global && ref($global) eq 'HASH'; + + my $env_setting = $wf_data->{status_signal}{$env} // ''; + + # Explicitly disabled for this env + return undef if $env_setting =~ /^(false|no|0)$/i; + + # Effective backend: per-env backend override or global + my $backend = $global->{backend} || 'file'; + $backend = $env_setting if $env_setting =~ /^(file|s3|gcs)$/; + + # Effective prefix: per-env override, or global-base/env, or just env name + my $env_prefix = $wf_data->{signal_prefix}{$env} || ''; + my $global_prefix = $global->{prefix} || ''; + my $prefix = $env_prefix + || ($global_prefix ? "$global_prefix/$env" : $env); + + return { %$global, backend => $backend, prefix => $prefix }; +} + +# }}} +# _signal_resource - build per-env signal (shuttle) resource {{{ +sub _signal_resource { + my ($self, $ast, $env, $alias, $signal_cfg) = @_; + + my $config = $ast->configuration || {}; + my $tagged = $config->{tagged}; + my %to = $tagged ? (tags => [$env]) : (); + my $backend = $signal_cfg->{backend} || 'file'; + my $prefix = $signal_cfg->{prefix} || $env; + + my %source = (backend => $backend, prefix => $prefix); + + if ($backend eq 's3') { + $source{bucket} = $signal_cfg->{bucket} || 'genesis-signals'; + $source{region} = $signal_cfg->{region} || 'us-east-1'; + $source{access_key_id} = _unwrap_ref($signal_cfg->{access_key_id}); + $source{secret_access_key} = _unwrap_ref($signal_cfg->{secret_access_key}); + $source{endpoint} = $signal_cfg->{endpoint} + if $signal_cfg->{endpoint}; + } elsif ($backend eq 'gcs') { + $source{bucket} = $signal_cfg->{bucket} || 'genesis-signals'; + $source{json_key} = _unwrap_ref($signal_cfg->{json_key}); + } elsif ($backend eq 'file') { + my $base = $signal_cfg->{path} || '/tmp/genesis-signals'; + $source{path} = "$base/$prefix"; + } + + return { + name => "$alias-signal", + type => 'shuttle', + icon => 'broadcast', + %to, + source => \%source, + }; +} + +# }}} +# _signal_put_step - build a put step to emit a deployment status signal {{{ +sub _signal_put_step { + my ($self, $alias, $status, $env, $deploy_type) = @_; + return { + put => "$alias-signal", + params => { + status => $status, + deployment => "$env-$deploy_type", + }, + }; +} + +# }}} +# _outcome_hooks - build on_success/on_failure/on_abort/on_error plan hooks {{{ +# +# Combines notification steps (success/failure) with signal put steps (all four +# outcomes). Returns a hash with only the keys that have actual content. +sub _outcome_hooks { + my ($self, $ast, $env, $alias, $deploy_type, $wf_data, $is_redeploy) = @_; + + my $name = $ast->metadata->{name} || 'genesis-pipeline'; + my $action = $is_redeploy ? 'Redeploy of' : 'Deployment to'; + my $ok_action = $is_redeploy ? 'redeployed' : 'deployed'; + my $ok_msg = "$name: Successfully ${ok_action} $env-$deploy_type"; + my $fail_msg = "$name: $action $env-$deploy_type failed"; + + my $signal_cfg = $self->_env_signal_config($ast, $env, $wf_data); + + my %hooks; + for my $outcome (qw(success failure abort error)) { + my @steps; + + # Notification: success and failure only + if ($outcome eq 'success') { + my $notif = $self->_notification_step($ast, $ok_msg); + push @steps, @{$notif->{in_parallel}} if $notif; + } elsif ($outcome eq 'failure') { + my $notif = $self->_notification_step($ast, $fail_msg); + push @steps, @{$notif->{in_parallel}} if $notif; + } + + # Signal: all four outcomes + if ($signal_cfg) { + push @steps, $self->_signal_put_step($alias, $outcome, $env, $deploy_type); + } + + next unless @steps; + $hooks{"on_$outcome"} = @steps == 1 ? $steps[0] : { in_parallel => \@steps }; + } + + return %hooks; +} + # }}} # _auto_update_resources - build kit-release and genesis-release resources {{{ sub _auto_update_resources { @@ -1293,7 +1450,8 @@ sub _extract_workflow_data { my $edges = $graph->{edges} || []; my (%will_trigger, %triggers, %auto, %aliases, %genesis_envs, - %redeploy, %redeploy_cron_start, %redeploy_cron_stop); + %redeploy, %redeploy_cron_start, %redeploy_cron_stop, + %status_signal, %signal_prefix); for my $edge (@$edges) { push @{$will_trigger{$edge->{from}}}, $edge->{to}; @@ -1301,12 +1459,14 @@ sub _extract_workflow_data { } for my $n (keys %$nodes) { my $nd = $nodes->{$n}; - $auto{$n} = 1 if $nd->{auto}; - $aliases{$n} = $nd->{alias} || $n; - $genesis_envs{$n} = $nd->{genesis_env} || $n; - $redeploy{$n} = $nd->{redeploy} || ''; + $auto{$n} = 1 if $nd->{auto}; + $aliases{$n} = $nd->{alias} || $n; + $genesis_envs{$n} = $nd->{genesis_env} || $n; + $redeploy{$n} = $nd->{redeploy} || ''; $redeploy_cron_start{$n} = $nd->{redeploy_cron_start} || ''; $redeploy_cron_stop{$n} = $nd->{redeploy_cron_stop} || ''; + $status_signal{$n} = $nd->{status_signal} // ''; + $signal_prefix{$n} = $nd->{signal_prefix} || ''; } if ($workflow->{_legacy}) { @@ -1329,6 +1489,8 @@ sub _extract_workflow_data { redeploy => \%redeploy, redeploy_cron_start => \%redeploy_cron_start, redeploy_cron_stop => \%redeploy_cron_stop, + status_signal => \%status_signal, + signal_prefix => \%signal_prefix, }; } diff --git a/lib/Genesis/CI/Compiler/Providers/Concourse.pm b/lib/Genesis/CI/Compiler/Providers/Concourse.pm index af250c37..8cd38ed2 100644 --- a/lib/Genesis/CI/Compiler/Providers/Concourse.pm +++ b/lib/Genesis/CI/Compiler/Providers/Concourse.pm @@ -640,15 +640,70 @@ Genesis::CI::Concourse - Concourse CI provider implementation =head1 DESCRIPTION Genesis::CI::Concourse implements the Genesis::CI trait interface for -Concourse CI. It handles parsing ci.yml configuration, generating Concourse +Concourse CI. It handles parsing ci.yml configuration, generating Concourse pipeline YAML, and deploying pipelines via the fly CLI. -This implementation currently delegates to Genesis::CI::Legacy for the heavy -lifting, but provides a clean interface for future refactoring. +The native pipeline compiler (C) produces fully-resolved +Concourse YAML with support for: + +=over 4 + +=item Deployment propagation + +Environments form a directed graph; each env's branch is updated after its +upstream deploys successfully. + +=item Redeploy lane (C) + +A separate CenvE> Concourse job that redeploys an +environment without change detection or cache promotion. Three trigger modes: + +=over 4 + +=item C — operator triggers via Concourse UI or C. + +=item C — a Concourse C