From 2bb8b15afc827aa2a3a16fff9beaa2799eff608f Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Thu, 28 Aug 2025 14:47:25 +1000 Subject: [PATCH 01/80] Add a credential helper to avoid plain-text storage warnings --- .../FeatureToggles/OctopusFeatureToggle.cs | 4 +- .../Download/DockerCredentialHelper.cs | 301 ++++++++++++++ .../Download/DockerImagePackageDownloader.cs | 59 ++- .../Packages/Download/ScriptExtractor.cs | 35 ++ .../Scripts/docker-credential-octopus.ps1 | 50 +++ .../Scripts/docker-credential-octopus.sh | 14 + .../Packages/DockerCredentialHelperFixture.cs | 263 ++++++++++++ .../DockerCredentialScriptsFixture.cs | 386 ++++++++++++++++++ ...ackageDownloaderCredentialHelperFixture.cs | 259 ++++++++++++ .../Commands/DockerCredentialCommand.cs | 110 +++++ 10 files changed, 1446 insertions(+), 35 deletions(-) create mode 100644 source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs create mode 100644 source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.ps1 create mode 100644 source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.sh create mode 100644 source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialHelperFixture.cs create mode 100644 source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs create mode 100644 source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderCredentialHelperFixture.cs create mode 100644 source/Calamari/Commands/DockerCredentialCommand.cs diff --git a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs index d025f5d2ed..6dcb2aa8c1 100644 --- a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs +++ b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs @@ -9,11 +9,13 @@ public static class KnownSlugs public const string KubernetesObjectManifestInspection = "kubernetes-object-manifest-inspection"; public const string KOSForHelm = "kos-for-helm"; public const string ArgoCDCreatePullRequestFeatureToggle = "argocd-create-pull-request"; + public const string UseDockerCredentialHelper = "calamari-use-docker-credential-helper"; }; public static readonly OctopusFeatureToggle KubernetesObjectManifestInspectionFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KubernetesObjectManifestInspection); public static readonly OctopusFeatureToggle KOSForHelmFeatureToggle = new OctopusFeatureToggle(KnownSlugs.KOSForHelm); public static readonly OctopusFeatureToggle ArgoCDCreatePullRequestFeatureToggle = new OctopusFeatureToggle(KnownSlugs.ArgoCDCreatePullRequestFeatureToggle); + public static readonly OctopusFeatureToggle UseDockerCredentialHelperFeatureToggle = new OctopusFeatureToggle(KnownSlugs.UseDockerCredentialHelper); public class OctopusFeatureToggle { @@ -30,4 +32,4 @@ public bool IsEnabled(IVariables variables) } } } -} \ No newline at end of file +} diff --git a/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs b/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs new file mode 100644 index 0000000000..91941ec82c --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing; +using Calamari.Common.Plumbing.Extensions; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.Variables; +using Newtonsoft.Json; +using System; +using System.Linq; + +namespace Calamari.Integration.Packages.Download +{ + public class DockerCredentialHelper + { + readonly ICalamariFileSystem fileSystem; + readonly ILog log; + const string CredentialsDirectory = "credentials"; + + public DockerCredentialHelper(ICalamariFileSystem fileSystem, ILog log) + { + this.fileSystem = fileSystem; + this.log = log; + } + + public void StoreCredentials(string serverUrl, string username, string password, string encryptionPassword, string dockerConfigPath) + { + var credentialsDir = Path.Combine(dockerConfigPath, CredentialsDirectory); + Directory.CreateDirectory(credentialsDir); + + var credential = new DockerCredential + { + Username = username, + Secret = password + }; + + var credentialJson = JsonConvert.SerializeObject(credential); + var encryptor = AesEncryption.ForScripts(encryptionPassword); + var encryptedBytes = encryptor.Encrypt(credentialJson); + + var fileName = GetCredentialFileName(serverUrl); + var filePath = Path.Combine(credentialsDir, fileName); + + File.WriteAllBytes(filePath, encryptedBytes); + log.Verbose($"Stored encrypted credentials for {serverUrl}"); + } + + public DockerCredential? GetCredentials(string serverUrl, string encryptionPassword, string dockerConfigPath) + { + var fileName = GetCredentialFileName(serverUrl); + var filePath = Path.Combine(dockerConfigPath, CredentialsDirectory, fileName); + + if (!File.Exists(filePath)) + { + log.Verbose($"No stored credentials found for {serverUrl}"); + return null; + } + + try + { + var encryptedBytes = File.ReadAllBytes(filePath); + var encryptor = AesEncryption.ForScripts(encryptionPassword); + var credentialJson = encryptor.Decrypt(encryptedBytes); + + var credential = JsonConvert.DeserializeObject(credentialJson); + log.Verbose($"Retrieved credentials for {serverUrl}"); + return credential; + } + catch (Exception ex) + { + log.Verbose($"Failed to decrypt credentials for {serverUrl}: {ex.Message}"); + return null; + } + } + + public void EraseCredentials(string serverUrl, string dockerConfigPath) + { + var fileName = GetCredentialFileName(serverUrl); + var filePath = Path.Combine(dockerConfigPath, CredentialsDirectory, fileName); + + if (File.Exists(filePath)) + { + File.Delete(filePath); + log.Verbose($"Erased credentials for {serverUrl}"); + } + } + + public string CreateDockerConfig(string dockerConfigPath, Dictionary credHelpers) + { + var config = new DockerConfig + { + CredHelpers = credHelpers + }; + + var configJson = JsonConvert.SerializeObject(config, Formatting.Indented); + var configFilePath = Path.Combine(dockerConfigPath, "config.json"); + + File.WriteAllText(configFilePath, configJson); + return configFilePath; + } + + public void CleanupCredentials(string dockerConfigPath) + { + var credentialsDir = Path.Combine(dockerConfigPath, CredentialsDirectory); + if (Directory.Exists(credentialsDir)) + { + try + { + Directory.Delete(credentialsDir, recursive: true); + log.Verbose("Cleaned up Docker credential files"); + } + catch (Exception ex) + { + log.Verbose($"Failed to cleanup credential files: {ex.Message}"); + } + } + } + + static string GetCredentialFileName(string serverUrl) + { + var serverBytes = Encoding.UTF8.GetBytes(serverUrl); + var base64Server = Convert.ToBase64String(serverBytes) + .Replace("/", "_") + .Replace("+", "-") + .Replace("=", ""); + return $"{base64Server}.cred"; + } + + public bool SetupCredentialHelper(Dictionary environmentVariables, + IVariables variables, + Uri feedUri, + string username, + string password, + string dockerHubRegistry) + { + try + { + var dockerConfigPath = environmentVariables["DOCKER_CONFIG"]; + Directory.CreateDirectory(dockerConfigPath); + + // Get the encryption password from sensitive variables + var encryptionPassword = variables.Get("Octopus.Action.Package.DownloadOnTentacle") ?? + variables.Get("SensitiveVariablesPassword") ?? + "DefaultFallbackPassword"; + + // Deploy credential helper scripts + DeployCredentialHelperScript(environmentVariables, variables); + + // Store credentials using the helper + var serverUrl = GetServerUrlForCredentialHelper(feedUri, dockerHubRegistry); + StoreCredentials(serverUrl, username, password, encryptionPassword, dockerConfigPath); + + // Create Docker config with credential helper configuration + var credHelpers = new Dictionary(); + if (feedUri.Host.Equals(dockerHubRegistry)) + { + credHelpers["index.docker.io"] = "octopus"; + credHelpers["docker.io"] = "octopus"; + credHelpers["registry-1.docker.io"] = "octopus"; + } + else + { + credHelpers[feedUri.Host] = "octopus"; + if (feedUri.Port != -1 && feedUri.Port != 80 && feedUri.Port != 443) + { + credHelpers[$"{feedUri.Host}:{feedUri.Port}"] = "octopus"; + } + } + + CreateDockerConfig(dockerConfigPath, credHelpers); + log.Verbose($"Configured Docker credential helper for {serverUrl}"); + return true; + } + catch (Exception ex) + { + log.Warn($"Failed to setup credential helper: {ex.Message}"); + return false; + } + } + + static string GetCalamariExecutablePath() + { + // First try to get the entry assembly (works in production) + var entryAssembly = System.Reflection.Assembly.GetEntryAssembly(); + if (entryAssembly != null) + { + var entryLocation = entryAssembly.Location; + var entryName = Path.GetFileNameWithoutExtension(entryLocation); + + // If the entry assembly is Calamari itself, use it + if (entryName.Equals("Calamari", StringComparison.OrdinalIgnoreCase)) + { + return entryLocation; + } + } + + // Fallback for test scenarios: look for Calamari executable in the same directory as this assembly + var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly(); + var currentDirectory = Path.GetDirectoryName(currentAssembly.Location); + + if (!string.IsNullOrEmpty(currentDirectory)) + { + // Try different possible names + var possibleNames = new[] { "Calamari.exe", "Calamari" }; + + foreach (var name in possibleNames) + { + var candidatePath = Path.Combine(currentDirectory, name); + if (File.Exists(candidatePath)) + { + return candidatePath; + } + } + } + + // Last resort: use "Calamari" and hope it's in PATH + return "Calamari"; + } + + void DeployCredentialHelperScript(Dictionary environmentVariables, IVariables variables) + { + var dockerConfigPath = environmentVariables["DOCKER_CONFIG"]; + var scriptName = "docker-credential-octopus"; + var helperScript = ScriptExtractor.GetScript(fileSystem, scriptName); + + // Make the script executable on Unix systems + if (CalamariEnvironment.IsRunningOnNix || CalamariEnvironment.IsRunningOnMac) + { + var result = SilentProcessRunner.ExecuteCommand("chmod", $"+x {helperScript}", ".", new Dictionary(), _ => { }, _ => { }); + if (result.ExitCode != 0) + { + log.Verbose($"Failed to make credential helper script executable: {result.ExitCode}"); + } + } + + // Add the script directory to PATH for Docker to find the helper + var scriptDir = Path.GetDirectoryName(Path.GetFullPath(helperScript)); + var currentPath = Environment.GetEnvironmentVariable("PATH") ?? ""; + var pathSeparator = CalamariEnvironment.IsRunningOnWindows ? ";" : ":"; + + if (!currentPath.Split(pathSeparator.ToCharArray()).Contains(scriptDir)) + { + environmentVariables["PATH"] = $"{scriptDir}{pathSeparator}{currentPath}"; + } + + // Set environment variables for the credential helper + var encryptionPassword = variables.Get("Octopus.Action.Package.DownloadOnTentacle") ?? + variables.Get("SensitiveVariablesPassword") ?? + "DefaultFallbackPassword"; + + // Pass the Calamari executable path to the credential helper script + var calamariExecutable = GetCalamariExecutablePath(); + if (!string.IsNullOrEmpty(calamariExecutable)) + { + environmentVariables["OCTOPUS_CALAMARI_EXECUTABLE"] = calamariExecutable; + } + + environmentVariables["OCTOPUS_CREDENTIAL_PASSWORD"] = encryptionPassword; + } + + public void CleanupCredentialHelper(Dictionary environmentVariables) + { + try + { + var dockerConfigPath = environmentVariables["DOCKER_CONFIG"]; + CleanupCredentials(dockerConfigPath); + } + catch (Exception ex) + { + log.Verbose($"Failed to cleanup credential helper files: {ex.Message}"); + } + } + + public static string GetServerUrlForCredentialHelper(Uri feedUri, string dockerHubRegistry) + { + // Docker credential helpers expect specific server URLs + if (feedUri.Host.Equals(dockerHubRegistry)) + { + return "https://index.docker.io/v1/"; + } + + return feedUri.GetLeftPart(UriPartial.Authority); + } + + } + + public class DockerCredential + { + public string Username { get; set; } = string.Empty; + public string Secret { get; set; } = string.Empty; + } + + public class DockerConfig + { + [JsonProperty("credHelpers")] + public Dictionary CredHelpers { get; set; } = new Dictionary(); + } +} diff --git a/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs index 28a995eab0..ceabb6911c 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs @@ -1,25 +1,20 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Reflection; using Calamari.Common.Commands; -using Calamari.Common.Features.EmbeddedResources; using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; using Calamari.Common.Features.Scripting; -using Calamari.Common.Features.Scripts; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; +using Calamari.Deployment; using Newtonsoft.Json.Linq; using Octopus.Versioning; namespace Calamari.Integration.Packages.Download { - // Note about moving this class: GetScript method uses the namespace of this class as part of the - // get Embedded Resource to find the DockerLogin and DockerPull scripts. If you move this file, be sure look at that method - // and make sure it can still find the scripts public class DockerImagePackageDownloader : IPackageDownloader { readonly IScriptEngine scriptEngine; @@ -92,11 +87,23 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, //Always try re-pull image, docker engine can take care of the rest var fullImageName = GetFullImageName(packageId, version, feedUri); + var dockerCredentialHelper = new DockerCredentialHelper(fileSystem, log); var feedHost = GetFeedHost(feedUri); var strategy = PackageDownloaderRetryUtils.CreateRetryStrategy(maxDownloadAttempts, downloadAttemptBackoff, log); - strategy.Execute(() => PerformLogin(username, password, feedHost)); + + var useCredentialHelper = OctopusFeatureToggles.UseDockerCredentialHelperFeatureToggle.IsEnabled(variables); + var helperSetupSuccessfully = false; + if (useCredentialHelper && !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) + { + helperSetupSuccessfully = strategy.Execute(() + => dockerCredentialHelper.SetupCredentialHelper(environmentVariables, variables, feedUri, username, password, DockerHubRegistry)); + } + if (!helperSetupSuccessfully) + { + strategy.Execute(() => PerformLogin(username, password, feedHost)); + } const string cachedWorkerToolsShortLink = "https://g.octopushq.com/CachedWorkerToolsImages"; var imageNotCachedMessage = @@ -110,6 +117,13 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, strategy.Execute(() => PerformPull(fullImageName)); var (hash, size) = GetImageDetails(fullImageName); + + // Cleanup credential helper files if used + if (useCredentialHelper) + { + dockerCredentialHelper.CleanupCredentialHelper(environmentVariables); + } + return new PackagePhysicalFileMetadata(new PackageFileNameMetadata(packageId, version, version, ""), string.Empty, hash, size); } @@ -149,7 +163,7 @@ void PerformLogin(string? username, string? password, string feed) if (result.ExitCode != 0) throw new CommandException("Unable to log in Docker registry"); } - + bool IsImageCached(string fullImageName) { var cachedDigests = GetCachedImageDigests(); @@ -179,7 +193,7 @@ void PerformPull(string fullImageName) CommandResult ExecuteScript(string scriptName, Dictionary envVars) { - var file = GetScript(scriptName); + var file = ScriptExtractor.GetScript(fileSystem, scriptName); using (new TemporaryFile(file)) { var clone = variables.Clone(); @@ -265,28 +279,5 @@ CommandResult ExecuteScript(string scriptName, Dictionary envVa return null; } } - - string GetScript(string scriptName) - { - var syntax = ScriptSyntaxHelper.GetPreferredScriptSyntaxForEnvironment(); - - string contextFile; - switch (syntax) - { - case ScriptSyntax.Bash: - contextFile = $"{scriptName}.sh"; - break; - case ScriptSyntax.PowerShell: - contextFile = $"{scriptName}.ps1"; - break; - default: - throw new InvalidOperationException("No kubernetes context wrapper exists for " + syntax); - } - - var scriptFile = Path.Combine(".", $"Octopus.{contextFile}"); - var contextScript = new AssemblyEmbeddedResources().GetEmbeddedResourceText(Assembly.GetExecutingAssembly(), $"{typeof(DockerImagePackageDownloader).Namespace}.Scripts.{contextFile}"); - fileSystem.OverwriteFile(scriptFile, contextScript); - return scriptFile; - } } -} \ No newline at end of file +} diff --git a/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs b/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs new file mode 100644 index 0000000000..b0cf9fa30b --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; +using System.Reflection; +using Calamari.Common.Features.EmbeddedResources; +using Calamari.Common.Features.Scripts; +using Calamari.Common.Plumbing.FileSystem; + +namespace Calamari.Integration.Packages.Download +{ + public static class ScriptExtractor + { + internal static string GetScript(ICalamariFileSystem fileSystem, string scriptName) + { + var syntax = ScriptSyntaxHelper.GetPreferredScriptSyntaxForEnvironment(); + + string contextFile; + switch (syntax) + { + case ScriptSyntax.Bash: + contextFile = $"{scriptName}.sh"; + break; + case ScriptSyntax.PowerShell: + contextFile = $"{scriptName}.ps1"; + break; + default: + throw new InvalidOperationException("No script wrapper exists for " + syntax); + } + + var scriptFile = Path.Combine(".", $"Octopus.{contextFile}"); + var contextScript = new AssemblyEmbeddedResources().GetEmbeddedResourceText(Assembly.GetExecutingAssembly(), $"{typeof(DockerImagePackageDownloader).Namespace}.Scripts.{contextFile}"); + fileSystem.OverwriteFile(scriptFile, contextScript); + return scriptFile; + } + } +} \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.ps1 b/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.ps1 new file mode 100644 index 0000000000..3a3082f48c --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.ps1 @@ -0,0 +1,50 @@ +param( + [Parameter(Position=0)] + [string]$Operation +) + +$ErrorActionPreference = "Stop" + +# Get the Calamari executable path from environment variable +$calamariExe = $env:OCTOPUS_CALAMARI_EXECUTABLE +if (-not $calamariExe) { + Write-Error "OCTOPUS_CALAMARI_EXECUTABLE environment variable not set" + exit 1 +} + +# Execute Calamari docker-credential command, passing stdin/stdout through +$arguments = @("docker-credential", "--operation=$Operation") + +$psi = New-Object System.Diagnostics.ProcessStartInfo +$psi.FileName = $calamariExe +$psi.Arguments = ($arguments -join " ") +$psi.UseShellExecute = $false +$psi.RedirectStandardInput = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true + +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $psi +$process.Start() + +# Forward stdin to process +$input = [Console]::In.ReadToEnd() +if ($input) { + $process.StandardInput.Write($input) +} +$process.StandardInput.Close() + +# Forward stdout/stderr back +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() + +$process.WaitForExit() + +if ($stdout) { + Write-Output $stdout +} +if ($stderr) { + Write-Error $stderr +} + +exit $process.ExitCode \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.sh b/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.sh new file mode 100644 index 0000000000..6e0aa6566b --- /dev/null +++ b/source/Calamari.Shared/Integration/Packages/Download/Scripts/docker-credential-octopus.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +OPERATION="$1" + +# Get the Calamari executable path from environment variable +if [ -z "${OCTOPUS_CALAMARI_EXECUTABLE:-}" ]; then + echo "OCTOPUS_CALAMARI_EXECUTABLE environment variable not set" >&2 + exit 1 +fi + +CALAMARI_EXE="$OCTOPUS_CALAMARI_EXECUTABLE" +exec "$CALAMARI_EXE" docker-credential --operation="$OPERATION" \ No newline at end of file diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialHelperFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialHelperFixture.cs new file mode 100644 index 0000000000..39c57fe4e4 --- /dev/null +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialHelperFixture.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Integration.Packages.Download; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; + +namespace Calamari.Tests.Fixtures.Integration.Packages +{ + [TestFixture] + public class DockerCredentialHelperFixture + { + string tempDirectory; + string dockerConfigPath; + const string TestEncryptionPassword = "TestPassword123!"; + const string TestServerUrl = "https://index.docker.io/v1/"; + const string TestUsername = "testuser"; + const string TestPassword = "testpass"; + readonly ICalamariFileSystem fileSystem = CalamariPhysicalFileSystem.GetPhysicalFileSystem(); + + [SetUp] + public void Setup() + { + tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + dockerConfigPath = Path.Combine(tempDirectory, "docker-config"); + Directory.CreateDirectory(dockerConfigPath); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + + [Test] + public void StoreCredentials_CreatesEncryptedCredentialFile() + { + // Act + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + + // Assert + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.Exists(credentialsDir).Should().BeTrue(); + + var credentialFiles = Directory.GetFiles(credentialsDir, "*.cred"); + credentialFiles.Should().HaveCount(1); + + var credentialFile = credentialFiles[0]; + var encryptedBytes = File.ReadAllBytes(credentialFile); + encryptedBytes.Should().NotBeEmpty(); + } + + [Test] + public void GetCredentials_RetrievesStoredCredentials() + { + // Arrange + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + + // Act + var retrievedCredential = credentialHelper.GetCredentials(TestServerUrl, TestEncryptionPassword, dockerConfigPath); + + // Assert + retrievedCredential.Should().NotBeNull(); + retrievedCredential.Username.Should().Be(TestUsername); + retrievedCredential.Secret.Should().Be(TestPassword); + } + + [Test] + public void GetCredentials_WithWrongPassword_ReturnsNull() + { + // Arrange + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + + // Act + var retrievedCredential = credentialHelper.GetCredentials(TestServerUrl, "WrongPassword", dockerConfigPath); + + // Assert + retrievedCredential.Should().BeNull(); + } + + [Test] + public void GetCredentials_WithNonExistentCredentials_ReturnsNull() + { + // Act + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + var retrievedCredential = credentialHelper.GetCredentials("https://nonexistent.registry.com", TestEncryptionPassword, dockerConfigPath); + + // Assert + retrievedCredential.Should().BeNull(); + } + + [Test] + public void EraseCredentials_RemovesStoredCredentials() + { + // Arrange + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + + // Act + credentialHelper.EraseCredentials(TestServerUrl, dockerConfigPath); + + // Assert + Directory.GetFiles(credentialsDir, "*.cred").Should().BeEmpty(); + } + + [Test] + public void CreateDockerConfig_CreatesValidConfigFile() + { + // Arrange + var credHelpers = new Dictionary + { + ["index.docker.io"] = "octopus", + ["docker.io"] = "octopus", + ["myregistry.com"] = "octopus" + }; + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + + // Act + var configPath = credentialHelper.CreateDockerConfig(dockerConfigPath, credHelpers); + + // Assert + File.Exists(configPath).Should().BeTrue(); + configPath.Should().Be(Path.Combine(dockerConfigPath, "config.json")); + + var configContent = File.ReadAllText(configPath); + configContent.Should().Contain("\"credHelpers\""); + configContent.Should().Contain("\"index.docker.io\": \"octopus\""); + configContent.Should().Contain("\"docker.io\": \"octopus\""); + configContent.Should().Contain("\"myregistry.com\": \"octopus\""); + } + + [Test] + public void CleanupCredentials_RemovesCredentialsDirectory() + { + // Arrange + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.Exists(credentialsDir).Should().BeTrue(); + + // Act + credentialHelper.CleanupCredentials(dockerConfigPath); + + // Assert + Directory.Exists(credentialsDir).Should().BeFalse(); + } + + [Test] + public void StoreAndRetrieveMultipleCredentials_WorksCorrectly() + { + // Arrange + var servers = new[] + { + ("https://index.docker.io/v1/", "dockeruser", "dockerpass"), + ("https://myregistry.com", "myuser", "mypass"), + ("https://another.registry.io", "anotheruser", "anotherpass") + }; + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + + // Act - Store multiple credentials + foreach (var (serverUrl, username, password) in servers) + { + credentialHelper.StoreCredentials(serverUrl, username, password, TestEncryptionPassword, dockerConfigPath); + } + + // Assert - Retrieve and verify each credential + foreach (var (serverUrl, expectedUsername, expectedPassword) in servers) + { + var credential = credentialHelper.GetCredentials(serverUrl, TestEncryptionPassword, dockerConfigPath); + credential.Should().NotBeNull(); + credential.Username.Should().Be(expectedUsername); + credential.Secret.Should().Be(expectedPassword); + } + + // Verify separate files were created + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(3); + } + + [Test] + public void ServerUrlEncoding_HandlesSpecialCharacters() + { + // Arrange + var serverUrls = new[] + { + "https://registry.with-dashes.com", + "https://registry.with_underscores.com:8080", + "https://registry/with/slashes", + "https://registry.with.dots.and:8443/path" + }; + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + + // Act & Assert + foreach (var serverUrl in serverUrls) + { + credentialHelper.StoreCredentials(serverUrl, TestUsername, TestPassword, TestEncryptionPassword, dockerConfigPath); + var credential = credentialHelper.GetCredentials(serverUrl, TestEncryptionPassword, dockerConfigPath); + + credential.Should().NotBeNull(); + credential.Username.Should().Be(TestUsername); + credential.Secret.Should().Be(TestPassword); + } + } + + [Test] + public void EncryptionIsSecure_PlaintextNotVisibleInFile() + { + // Arrange + const string sensitivePassword = "SuperSecretPassword123!@#"; + var credentialHelper = new DockerCredentialHelper(fileSystem, Substitute.For()); + + // Act + credentialHelper.StoreCredentials(TestServerUrl, TestUsername, sensitivePassword, TestEncryptionPassword, dockerConfigPath); + + // Assert + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + var credentialFiles = Directory.GetFiles(credentialsDir, "*.cred"); + var credentialFile = credentialFiles[0]; + var fileContent = File.ReadAllText(credentialFile); + + // Verify sensitive data is not visible in plaintext + fileContent.Should().NotContain(TestUsername); + fileContent.Should().NotContain(sensitivePassword); + fileContent.Should().NotContain(TestServerUrl); + } + + + [Test] + public void GetServerUrlForCredentialHelper_HandlesDockerHubCorrectly() + { + // Arrange & Act + var dockerHubUri = new Uri("https://index.docker.io"); + + var result = DockerCredentialHelper.GetServerUrlForCredentialHelper(dockerHubUri, "index.docker.io"); + + // Assert + result.Should().Be("https://index.docker.io/v1/"); + } + + [Test] + public void GetServerUrlForCredentialHelper_HandlesCustomRegistryCorrectly() + { + // Arrange & Act + var customUri = new Uri("https://myregistry.com:8080"); + + var result = DockerCredentialHelper.GetServerUrlForCredentialHelper(customUri, "index.docker.io"); + + // Assert + result.Should().Be("https://myregistry.com:8080"); + } + } +} diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs new file mode 100644 index 0000000000..622661667c --- /dev/null +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs @@ -0,0 +1,386 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Calamari.Common.Features.EmbeddedResources; +using Calamari.Common.Features.Processes; +using Calamari.Integration.Packages.Download; +using Calamari.Testing.Requirements; +using FluentAssertions; +using Newtonsoft.Json; +using NUnit.Framework; + +namespace Calamari.Tests.Fixtures.Integration.Packages +{ + [TestFixture] + public class DockerCredentialScriptsFixture + { + string tempDirectory; + string dockerConfigPath; + string powershellScript; + string bashScript; + string calamariExecutable; + + const string TestEncryptionPassword = "TestPassword123!"; + const string TestServerUrl = "https://index.docker.io/v1/"; + const string TestUsername = "testuser"; + const string TestPassword = "testpass"; + + [SetUp] + public void Setup() + { + tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + dockerConfigPath = Path.Combine(tempDirectory, "docker-config"); + Directory.CreateDirectory(dockerConfigPath); + + // Create the credential helper scripts in temp directory + CreateCredentialHelperScripts(); + + // Find or create a mock Calamari executable + SetupCalamariExecutable(); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + + void CreateCredentialHelperScripts() + { + // Use the same embedded resources that the real DockerImagePackageDownloader uses + var embeddedResources = new AssemblyEmbeddedResources(); + var dockerImagePackageDownloaderAssembly = typeof(DockerImagePackageDownloader).Assembly; + var scriptsNamespace = $"{typeof(DockerImagePackageDownloader).Namespace}.Scripts"; + + // Create PowerShell script from embedded resource + powershellScript = Path.Combine(tempDirectory, "docker-credential-octopus.ps1"); + var powershellContent = embeddedResources.GetEmbeddedResourceText(dockerImagePackageDownloaderAssembly, $"{scriptsNamespace}.docker-credential-octopus.ps1"); + File.WriteAllText(powershellScript, powershellContent); + + // Create Bash script from embedded resource + bashScript = Path.Combine(tempDirectory, "docker-credential-octopus.sh"); + var bashContent = embeddedResources.GetEmbeddedResourceText(dockerImagePackageDownloaderAssembly, $"{scriptsNamespace}.docker-credential-octopus.sh"); + File.WriteAllText(bashScript, bashContent); + + // Make bash script executable on Unix systems + if (Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX) + { + SilentProcessRunner.ExecuteCommand("chmod", $"+x {bashScript}", ".", new Dictionary(), _ => { }, _ => { }); + } + } + + void SetupCalamariExecutable() + { + // Use the real Calamari executable for true integration testing + var testAssemblyLocation = System.Reflection.Assembly.GetExecutingAssembly().Location; + var testBinDirectory = Path.GetDirectoryName(testAssemblyLocation); + + // Try to find Calamari executable in the same directory as the test assembly + // On Unix systems it's "Calamari", on Windows it's "Calamari.exe" + var executableNames = Environment.OSVersion.Platform == PlatformID.Win32NT + ? new[] { "Calamari.exe", "Calamari" } + : new[] { "Calamari", "Calamari.exe" }; + + foreach (var executableName in executableNames) + { + calamariExecutable = Path.Combine(testBinDirectory, executableName); + if (File.Exists(calamariExecutable)) + { + return; + } + } + + throw new InvalidOperationException($"Could not find Calamari executable in {testBinDirectory}. Tried: {string.Join(", ", executableNames)}. Make sure the project is built."); + } + + + [Test] + [WindowsTest] + public void PowerShellScript_WithStoreOperation_CallsCalamariCorrectly() + { + // Arrange + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + + // Act + var result = ExecutePowerShellScript("store", credentialJson); + + // Assert + result.ExitCode.Should().Be(0); + + // Verify credentials were actually stored by the real command + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.Exists(credentialsDir).Should().BeTrue(); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + } + + [Test] + [WindowsTest] + public void PowerShellScript_WithGetOperation_ReturnsCredentials() + { + // Arrange - First store credentials + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + ExecutePowerShellScript("store", credentialJson); + + // Act + var result = ExecutePowerShellScript("get", TestServerUrl); + + // Assert + result.ExitCode.Should().Be(0); + + // Parse and verify the returned JSON credentials + // Extract the JSON line from Calamari output (skip verbose logging) + var outputLines = result.Output.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + var jsonLine = outputLines.FirstOrDefault(line => line.Trim().StartsWith("{")); + jsonLine.Should().NotBeNull("Expected JSON response from credential get operation"); + + var responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); + ((string)responseJson.Username).Should().Be(TestUsername); + ((string)responseJson.Secret).Should().Be(TestPassword); + } + + [Test] + [WindowsTest] + public void PowerShellScript_WithEraseOperation_CallsCalamariCorrectly() + { + // Arrange - First store credentials + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + ExecutePowerShellScript("store", credentialJson); + + // Verify credential exists + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + + // Act + var result = ExecutePowerShellScript("erase", TestServerUrl); + + // Assert + result.ExitCode.Should().Be(0); + + // Verify credential was actually erased + Directory.GetFiles(credentialsDir, "*.cred").Should().BeEmpty(); + } + + [Test] + [NonWindowsTest] + public void BashScript_WithStoreOperation_CallsCalamariCorrectly() + { + // Arrange + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + + // Act + var result = ExecuteBashScript("store", credentialJson); + + // Assert + result.ExitCode.Should().Be(0); + + // Verify credentials were actually stored by the real command + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.Exists(credentialsDir).Should().BeTrue(); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + } + + [Test] + [NonWindowsTest] + public void BashScript_WithGetOperation_ReturnsCredentials() + { + // Arrange - First store credentials + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + ExecuteBashScript("store", credentialJson); + + // Act + var result = ExecuteBashScript("get", TestServerUrl); + + // Assert + result.ExitCode.Should().Be(0); + + // Parse and verify the returned JSON credentials + // Extract the JSON line from Calamari output (skip verbose logging) + var outputLines = result.Output.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + var jsonLine = outputLines.FirstOrDefault(line => line.Trim().StartsWith("{")); + jsonLine.Should().NotBeNull("Expected JSON response from credential get operation"); + + var responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); + ((string)responseJson.Username).Should().Be(TestUsername); + ((string)responseJson.Secret).Should().Be(TestPassword); + } + + [Test] + [NonWindowsTest] + public void BashScript_WithEraseOperation_CallsCalamariCorrectly() + { + // Arrange - First store credentials + var credentialJson = JsonConvert.SerializeObject(new + { + ServerURL = TestServerUrl, + Username = TestUsername, + Secret = TestPassword + }); + ExecuteBashScript("store", credentialJson); + + // Verify credential exists + var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); + Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + + // Act + var result = ExecuteBashScript("erase", TestServerUrl); + + // Assert + result.ExitCode.Should().Be(0); + + // Verify credential was actually erased + Directory.GetFiles(credentialsDir, "*.cred").Should().BeEmpty(); + } + + [Test] + public void ScriptGeneration_CreatesValidPowerShellScript() + { + // Assert + File.Exists(powershellScript).Should().BeTrue(); + var content = File.ReadAllText(powershellScript); + + content.Should().Contain("param("); + content.Should().Contain("$Operation"); + content.Should().Contain("OCTOPUS_CALAMARI_EXECUTABLE"); + content.Should().Contain("docker-credential"); + content.Should().Contain("--operation="); + } + + [Test] + public void ScriptGeneration_CreatesValidBashScript() + { + // Assert + File.Exists(bashScript).Should().BeTrue(); + var content = File.ReadAllText(bashScript); + + content.Should().StartWith("#!/bin/bash"); + content.Should().Contain("OPERATION=\"$1\""); + content.Should().Contain("OCTOPUS_CALAMARI_EXECUTABLE"); + content.Should().Contain("docker-credential"); + content.Should().Contain("--operation="); + } + + ScriptExecutionResult ExecutePowerShellScript(string operation, string input = null) + { + var psi = new ProcessStartInfo + { + FileName = "powershell.exe", + Arguments = $"-ExecutionPolicy Bypass -File \"{powershellScript}\" {operation}", + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = tempDirectory + }; + + // Set the environment variable for the real Calamari executable + psi.EnvironmentVariables["OCTOPUS_CALAMARI_EXECUTABLE"] = calamariExecutable; + + // Set up environment for real docker-credential command + psi.EnvironmentVariables["DOCKER_CONFIG"] = dockerConfigPath; + psi.EnvironmentVariables["OCTOPUS_CREDENTIAL_PASSWORD"] = TestEncryptionPassword; + + using (var process = new System.Diagnostics.Process { StartInfo = psi }) + { + process.Start(); + + if (input != null) + { + process.StandardInput.WriteLine(input); + } + process.StandardInput.Close(); + + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + return new ScriptExecutionResult + { + ExitCode = process.ExitCode, + Output = output, + Error = error + }; + } + } + + ScriptExecutionResult ExecuteBashScript(string operation, string input = null) + { + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"\"{bashScript}\" {operation}", + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = tempDirectory + }; + + // Set the environment variable for the real Calamari executable + psi.EnvironmentVariables["OCTOPUS_CALAMARI_EXECUTABLE"] = calamariExecutable; + + // Set up environment for real docker-credential command + psi.EnvironmentVariables["DOCKER_CONFIG"] = dockerConfigPath; + psi.EnvironmentVariables["OCTOPUS_CREDENTIAL_PASSWORD"] = TestEncryptionPassword; + + using (var process = new System.Diagnostics.Process { StartInfo = psi }) + { + process.Start(); + + if (input != null) + { + process.StandardInput.WriteLine(input); + } + process.StandardInput.Close(); + + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + return new ScriptExecutionResult + { + ExitCode = process.ExitCode, + Output = output, + Error = error + }; + } + } + + + class ScriptExecutionResult + { + public int ExitCode { get; set; } + public string Output { get; set; } + public string Error { get; set; } + } + } +} diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderCredentialHelperFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderCredentialHelperFixture.cs new file mode 100644 index 0000000000..d3a2151b25 --- /dev/null +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderCredentialHelperFixture.cs @@ -0,0 +1,259 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Calamari.Common.Commands; +using Calamari.Common.Features.Processes; +using Calamari.Common.Features.Scripting; +using Calamari.Common.FeatureToggles; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.Variables; +using Calamari.Integration.Packages.Download; +using Calamari.Testing; +using Calamari.Testing.Helpers; +using Calamari.Testing.Requirements; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Versioning.Semver; + +namespace Calamari.Tests.Fixtures.Integration.Packages +{ + [TestFixture] + public class DockerImagePackageDownloaderCredentialHelperFixture + { + static readonly string DockerHubFeedUri = "https://index.docker.io"; + static readonly string DockerTestUsername = "octopustestaccount"; + static string dockerTestPassword; + static readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource(); + readonly CancellationToken cancellationToken = CancellationTokenSource.Token; + static readonly string Home = Path.GetTempPath(); + + [OneTimeSetUp] + public async Task TestFixtureSetUp() + { + dockerTestPassword = await ExternalVariables.Get(ExternalVariable.DockerReaderPassword, cancellationToken); + Environment.SetEnvironmentVariable("TentacleHome", Home); + } + + [OneTimeTearDown] + public void TestFixtureTearDown() + { + Environment.SetEnvironmentVariable("TentacleHome", null); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_EnabledByDefault_UsesCredentialHelper() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + variables.Set(KnownVariables.EnabledFeatureToggles, OctopusFeatureToggles.KnownSlugs.UseDockerCredentialHelper); + var downloader = GetDownloader(log, variables); + + // Act + var pkg = downloader.DownloadPackage("octopusdeploy/octo-prerelease", + new SemanticVersion("7.3.7-alpine"), "docker-feed", + new Uri(DockerHubFeedUri), DockerTestUsername, dockerTestPassword, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert + pkg.Should().NotBeNull(); + pkg.PackageId.Should().Be("octopusdeploy/octo-prerelease"); + + // Verify credential helper was configured + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("Cleaned up Docker credential files")); + + // Verify no unencrypted credential warnings in the log + log.Messages.Should().NotContain(m => m.FormattedMessage.Contains("credentials are stored unencrypted")); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_ExplicitlyDisabled_UsesFallbackLogin() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + + var downloader = GetDownloader(log, variables); + + // Act + var pkg = downloader.DownloadPackage("octopusdeploy/octo-prerelease", + new SemanticVersion("7.3.7-alpine"), "docker-feed", + new Uri(DockerHubFeedUri), DockerTestUsername, dockerTestPassword, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert + pkg.Should().NotBeNull(); + pkg.PackageId.Should().Be("octopusdeploy/octo-prerelease"); + + // Verify credential helper was NOT used + log.Messages.Should().NotContain(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + log.Messages.Should().NotContain(m => m.FormattedMessage.Contains("Cleaned up Docker credential files")); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_WithoutCredentials_SkipsCredentialHelper() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + variables.Set(KnownVariables.EnabledFeatureToggles, OctopusFeatureToggles.KnownSlugs.UseDockerCredentialHelper); + var downloader = GetDownloader(log, variables); + + // Act + var pkg = downloader.DownloadPackage("alpine", + new SemanticVersion("3.6.5"), "docker-feed", + new Uri(DockerHubFeedUri), null, null, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert + pkg.Should().NotBeNull(); + pkg.PackageId.Should().Be("alpine"); + + // Verify credential helper was NOT used when no credentials provided + log.Messages.Should().NotContain(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_CreatesCorrectDockerConfig_ForDockerHub() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + variables.Set(KnownVariables.EnabledFeatureToggles, OctopusFeatureToggles.KnownSlugs.UseDockerCredentialHelper); + var downloader = GetDownloader(log, variables); + + // Act + var pkg = downloader.DownloadPackage("octopusdeploy/octo-prerelease", + new SemanticVersion("7.3.7-alpine"), "docker-feed", + new Uri(DockerHubFeedUri), DockerTestUsername, dockerTestPassword, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert - Extract docker config path from logs for verification + var dockerConfigLogMessage = log.Messages.FirstOrDefault(m => m.FormattedMessage.Contains("DOCKER_CONFIG")); + dockerConfigLogMessage.Should().NotBeNull(); + + // The Docker config should have been created and then cleaned up + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("index.docker.io")); + + pkg.Should().NotBeNull(); + pkg.PackageId.Should().Be("octopusdeploy/octo-prerelease"); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_WithCustomRegistry_CreatesCorrectConfig() + { + // Arrange - Using a public registry that doesn't require auth for this test + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + variables.Set(KnownVariables.EnabledFeatureToggles, OctopusFeatureToggles.KnownSlugs.UseDockerCredentialHelper); + var downloader = GetDownloader(log, variables); + + var customRegistryUri = new Uri("https://quay.io"); + + // Act - Download a public image to test config creation (even though auth isn't needed) + try + { + // This will attempt to set up credential helper but fall back gracefully + downloader.DownloadPackage("coreos/etcd", + new SemanticVersion("v3.5.0"), "docker-feed", + customRegistryUri, "dummyuser", "dummypass", true, 1, + TimeSpan.FromSeconds(10)); + } + catch (CommandException) + { + // Expected - dummy credentials will fail, but we can still verify config setup + } + + // Assert + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("Failed to setup credential helper, falling back to direct login") || + m.FormattedMessage.Contains("Configured Docker credential helper")); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_FailureFallback_ContinuesWithDirectLogin() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + + // Simulate a scenario where credential helper setup might fail + // by using an invalid encryption password format + variables.Set("SensitiveVariablesPassword", ""); // Empty password might cause issues + var downloader = GetDownloader(log, variables); + + // Act + var pkg = downloader.DownloadPackage("octopusdeploy/octo-prerelease", + new SemanticVersion("7.3.7-alpine"), "docker-feed", + new Uri(DockerHubFeedUri), DockerTestUsername, dockerTestPassword, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert + pkg.Should().NotBeNull(); + pkg.PackageId.Should().Be("octopusdeploy/octo-prerelease"); + + // Should either succeed with credential helper or fall back gracefully + var hasCredentialHelperMessage = log.Messages.Any(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + var hasFallbackMessage = log.Messages.Any(m => m.FormattedMessage.Contains("Failed to setup credential helper, falling back")); + + (hasCredentialHelperMessage || hasFallbackMessage).Should().BeTrue("Either credential helper should work or fallback should occur"); + } + + [Test] + [RequiresDockerInstalled] + public void CredentialHelper_MultipleRegistries_HandlesCorrectly() + { + // Arrange + var log = new InMemoryLog(); + var variables = new CalamariVariables(); + variables.Set(KnownVariables.EnabledFeatureToggles, OctopusFeatureToggles.KnownSlugs.UseDockerCredentialHelper); + var downloader = GetDownloader(log, variables); + + + // Act - Download from Docker Hub first + var pkg1 = downloader.DownloadPackage("octopusdeploy/octo-prerelease", + new SemanticVersion("7.3.7-alpine"), "docker-feed", + new Uri(DockerHubFeedUri), DockerTestUsername, dockerTestPassword, true, 1, + TimeSpan.FromSeconds(10)); + + // Then download from the same registry again (should reuse or recreate config) + var pkg2 = downloader.DownloadPackage("alpine", + new SemanticVersion("3.6.5"), "docker-feed", + new Uri(DockerHubFeedUri), null, null, true, 1, + TimeSpan.FromSeconds(10)); + + // Assert + pkg1.Should().NotBeNull(); + pkg2.Should().NotBeNull(); + + // First download should use credential helper + log.Messages.Should().Contain(m => m.FormattedMessage.Contains("Configured Docker credential helper")); + + // Both downloads should succeed + pkg1.PackageId.Should().Be("octopusdeploy/octo-prerelease"); + pkg2.PackageId.Should().Be("alpine"); + } + + static DockerImagePackageDownloader GetDownloader(ILog log, IVariables variables) + { + var runner = new CommandLineRunner(log, variables); + return new DockerImagePackageDownloader( + new ScriptEngine(Enumerable.Empty(), log), + CalamariPhysicalFileSystem.GetPhysicalFileSystem(), + runner, + variables, + log, + new FeedLoginDetailsProviderFactory()); + } + } +} diff --git a/source/Calamari/Commands/DockerCredentialCommand.cs b/source/Calamari/Commands/DockerCredentialCommand.cs new file mode 100644 index 0000000000..7b01edab53 --- /dev/null +++ b/source/Calamari/Commands/DockerCredentialCommand.cs @@ -0,0 +1,110 @@ +using System; +using Calamari.Commands.Support; +using Calamari.Common.Commands; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Integration.Packages.Download; +using Newtonsoft.Json; + +namespace Calamari.Commands +{ + [Command("docker-credential", Description = "Docker credential helper operations for secure credential storage")] + public class DockerCredentialCommand : Command + { + readonly ILog log; + string operation = string.Empty; + readonly ICalamariFileSystem fileSystem = CalamariPhysicalFileSystem.GetPhysicalFileSystem(); + + public DockerCredentialCommand(ILog log) + { + this.log = log; + + Options.Add("operation=", "The credential operation to perform (store, get, erase)", v => operation = v); + } + + public override int Execute(string[] commandLineArguments) + { + Options.Parse(commandLineArguments); + + if (string.IsNullOrEmpty(operation)) + { + log.Error("Operation parameter is required (store, get, erase)"); + return 1; + } + + var encryptionPassword = Environment.GetEnvironmentVariable("OCTOPUS_CREDENTIAL_PASSWORD"); + var dockerConfigPath = Environment.GetEnvironmentVariable("DOCKER_CONFIG"); + + if (string.IsNullOrEmpty(encryptionPassword)) + { + log.Error("OCTOPUS_CREDENTIAL_PASSWORD environment variable not set"); + return 1; + } + + if (string.IsNullOrEmpty(dockerConfigPath)) + { + log.Error("DOCKER_CONFIG environment variable not set"); + return 1; + } + + var dockerCredentialHelper = new DockerCredentialHelper(fileSystem, log); + + try + { + switch (operation.ToLower()) + { + case "store": + return StoreCredential(dockerCredentialHelper, encryptionPassword, dockerConfigPath); + case "get": + return GetCredential(dockerCredentialHelper, encryptionPassword, dockerConfigPath); + case "erase": + return EraseCredential(dockerCredentialHelper, dockerConfigPath); + default: + log.Error($"Invalid operation: {operation}. Valid operations are: store, get, erase"); + return 1; + } + } + catch (Exception ex) + { + log.Error($"Docker credential operation failed: {ex.Message}"); + return 1; + } + } + + int StoreCredential(DockerCredentialHelper dockerCredentialHelper, string encryptionPassword, string dockerConfigPath) + { + var inputJson = Console.In.ReadToEnd(); + var credentialRequest = JsonConvert.DeserializeObject(inputJson); + + var serverUrl = (string)credentialRequest.ServerURL; + var username = (string)credentialRequest.Username; + var secret = (string)credentialRequest.Secret; + + dockerCredentialHelper.StoreCredentials(serverUrl, username, secret, encryptionPassword, dockerConfigPath); + return 0; + } + + int GetCredential(DockerCredentialHelper dockerCredentialHelper, string encryptionPassword, string dockerConfigPath) + { + var serverUrl = Console.ReadLine(); + var credential = dockerCredentialHelper.GetCredentials(serverUrl, encryptionPassword, dockerConfigPath); + + if (credential == null) + { + Console.Error.WriteLine("credentials not found in native keychain"); + return 1; + } + + var response = new { Username = credential.Username, Secret = credential.Secret }; + Console.WriteLine(JsonConvert.SerializeObject(response)); + return 0; + } + + int EraseCredential(DockerCredentialHelper dockerCredentialHelper, string dockerConfigPath) + { + var serverUrl = Console.ReadLine(); + dockerCredentialHelper.EraseCredentials(serverUrl, dockerConfigPath); + return 0; + } + } +} From 4a485887e4c4707e85d2336c1d15618b8af50816 Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Fri, 29 Aug 2025 08:45:50 +1000 Subject: [PATCH 02/80] Improve debugability --- .../DockerCredentialScriptsFixture.cs | 116 +++++++++++++----- 1 file changed, 84 insertions(+), 32 deletions(-) diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs index 622661667c..e81d419cdb 100644 --- a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerCredentialScriptsFixture.cs @@ -5,9 +5,11 @@ using System.Linq; using Calamari.Common.Features.EmbeddedResources; using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing; using Calamari.Integration.Packages.Download; using Calamari.Testing.Requirements; using FluentAssertions; +using FluentAssertions.Execution; using Newtonsoft.Json; using NUnit.Framework; @@ -31,6 +33,7 @@ public class DockerCredentialScriptsFixture public void Setup() { tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(tempDirectory); dockerConfigPath = Path.Combine(tempDirectory, "docker-config"); Directory.CreateDirectory(dockerConfigPath); @@ -68,7 +71,7 @@ void CreateCredentialHelperScripts() File.WriteAllText(bashScript, bashContent); // Make bash script executable on Unix systems - if (Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX) + if (CalamariEnvironment.IsRunningOnNix || CalamariEnvironment.IsRunningOnMac) { SilentProcessRunner.ExecuteCommand("chmod", $"+x {bashScript}", ".", new Dictionary(), _ => { }, _ => { }); } @@ -80,22 +83,39 @@ void SetupCalamariExecutable() var testAssemblyLocation = System.Reflection.Assembly.GetExecutingAssembly().Location; var testBinDirectory = Path.GetDirectoryName(testAssemblyLocation); - // Try to find Calamari executable in the same directory as the test assembly + // Look for Calamari executable in multiple potential locations + var searchDirectories = new[] + { + testBinDirectory, // Same directory as test assembly + Path.Combine(testBinDirectory, "..", "..", "..", "..", "bin", "Debug", "net462"), // Relative path to main bin + Path.Combine(testBinDirectory, "..", "..", "..", "..", "..", "bin", "Debug", "net462") // Another potential path + }; + // On Unix systems it's "Calamari", on Windows it's "Calamari.exe" - var executableNames = Environment.OSVersion.Platform == PlatformID.Win32NT + var executableNames = CalamariEnvironment.IsRunningOnWindows ? new[] { "Calamari.exe", "Calamari" } : new[] { "Calamari", "Calamari.exe" }; - foreach (var executableName in executableNames) + foreach (var searchDirectory in searchDirectories) { - calamariExecutable = Path.Combine(testBinDirectory, executableName); - if (File.Exists(calamariExecutable)) + foreach (var executableName in executableNames) { - return; + var candidatePath = Path.Combine(searchDirectory, executableName); + if (File.Exists(candidatePath)) + { + calamariExecutable = Path.GetFullPath(candidatePath); + + // Make sure the executable has execute permissions on Unix systems + if (CalamariEnvironment.IsRunningOnNix || CalamariEnvironment.IsRunningOnMac) + { + SilentProcessRunner.ExecuteCommand("chmod", $"+x \"{calamariExecutable}\"", ".", new Dictionary(), _ => { }, _ => { }); + } + return; + } } } - throw new InvalidOperationException($"Could not find Calamari executable in {testBinDirectory}. Tried: {string.Join(", ", executableNames)}. Make sure the project is built."); + throw new InvalidOperationException($"Could not find Calamari executable. Searched in: {string.Join(", ", searchDirectories)}. Tried names: {string.Join(", ", executableNames)}. Make sure the project is built."); } @@ -115,12 +135,13 @@ public void PowerShellScript_WithStoreOperation_CallsCalamariCorrectly() var result = ExecutePowerShellScript("store", credentialJson); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Verify credentials were actually stored by the real command var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); - Directory.Exists(credentialsDir).Should().BeTrue(); - Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + Directory.Exists(credentialsDir).Should().BeTrue($"Expected credentials directory '{credentialsDir}' to exist"); + var credFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + credFiles.Should().HaveCount(1, $"Expected exactly 1 credential file in '{credentialsDir}', but found: {string.Join(", ", credFiles)}"); } [Test] @@ -140,17 +161,26 @@ public void PowerShellScript_WithGetOperation_ReturnsCredentials() var result = ExecutePowerShellScript("get", TestServerUrl); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Parse and verify the returned JSON credentials // Extract the JSON line from Calamari output (skip verbose logging) var outputLines = result.Output.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); var jsonLine = outputLines.FirstOrDefault(line => line.Trim().StartsWith("{")); - jsonLine.Should().NotBeNull("Expected JSON response from credential get operation"); + jsonLine.Should().NotBeNull($"Expected JSON response from credential get operation. Full output: {result.Output}, Error: {result.Error}"); + + dynamic responseJson; + try + { + responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); + } + catch (Exception ex) + { + throw new AssertionFailedException($"Failed to parse JSON response '{jsonLine}'. Error: {ex.Message}. Full output: {result.Output}"); + } - var responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); - ((string)responseJson.Username).Should().Be(TestUsername); - ((string)responseJson.Secret).Should().Be(TestPassword); + ((string)responseJson.Username).Should().Be(TestUsername, $"Username mismatch in response: {jsonLine}"); + ((string)responseJson.Secret).Should().Be(TestPassword, $"Password mismatch in response: {jsonLine}"); } [Test] @@ -166,18 +196,20 @@ public void PowerShellScript_WithEraseOperation_CallsCalamariCorrectly() }); ExecutePowerShellScript("store", credentialJson); - // Verify credential exists + // Verify credential exists before erase var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); - Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + var existingFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + existingFiles.Should().HaveCount(1, $"Expected exactly 1 credential file before erase in '{credentialsDir}', but found: {string.Join(", ", existingFiles)}"); // Act var result = ExecutePowerShellScript("erase", TestServerUrl); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Verify credential was actually erased - Directory.GetFiles(credentialsDir, "*.cred").Should().BeEmpty(); + var remainingFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + remainingFiles.Should().BeEmpty($"Expected no credential files after erase, but found: {string.Join(", ", remainingFiles)}"); } [Test] @@ -196,12 +228,13 @@ public void BashScript_WithStoreOperation_CallsCalamariCorrectly() var result = ExecuteBashScript("store", credentialJson); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Verify credentials were actually stored by the real command var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); - Directory.Exists(credentialsDir).Should().BeTrue(); - Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + Directory.Exists(credentialsDir).Should().BeTrue($"Expected credentials directory '{credentialsDir}' to exist"); + var credFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + credFiles.Should().HaveCount(1, $"Expected exactly 1 credential file in '{credentialsDir}', but found: {string.Join(", ", credFiles)}"); } [Test] @@ -221,17 +254,26 @@ public void BashScript_WithGetOperation_ReturnsCredentials() var result = ExecuteBashScript("get", TestServerUrl); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Parse and verify the returned JSON credentials // Extract the JSON line from Calamari output (skip verbose logging) var outputLines = result.Output.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); var jsonLine = outputLines.FirstOrDefault(line => line.Trim().StartsWith("{")); - jsonLine.Should().NotBeNull("Expected JSON response from credential get operation"); + jsonLine.Should().NotBeNull($"Expected JSON response from credential get operation. Full output: {result.Output}, Error: {result.Error}"); - var responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); - ((string)responseJson.Username).Should().Be(TestUsername); - ((string)responseJson.Secret).Should().Be(TestPassword); + dynamic responseJson; + try + { + responseJson = JsonConvert.DeserializeObject(jsonLine.Trim()); + } + catch (Exception ex) + { + throw new AssertionFailedException($"Failed to parse JSON response '{jsonLine}'. Error: {ex.Message}. Full output: {result.Output}"); + } + + ((string)responseJson.Username).Should().Be(TestUsername, $"Username mismatch in response: {jsonLine}"); + ((string)responseJson.Secret).Should().Be(TestPassword, $"Password mismatch in response: {jsonLine}"); } [Test] @@ -247,18 +289,20 @@ public void BashScript_WithEraseOperation_CallsCalamariCorrectly() }); ExecuteBashScript("store", credentialJson); - // Verify credential exists + // Verify credential exists before erase var credentialsDir = Path.Combine(dockerConfigPath, "credentials"); - Directory.GetFiles(credentialsDir, "*.cred").Should().HaveCount(1); + var existingFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + existingFiles.Should().HaveCount(1, $"Expected exactly 1 credential file before erase in '{credentialsDir}', but found: {string.Join(", ", existingFiles)}"); // Act var result = ExecuteBashScript("erase", TestServerUrl); // Assert - result.ExitCode.Should().Be(0); + result.ExitCode.Should().Be(0, $"Script execution failed. Output: {result.Output}, Error: {result.Error}"); // Verify credential was actually erased - Directory.GetFiles(credentialsDir, "*.cred").Should().BeEmpty(); + var remainingFiles = Directory.Exists(credentialsDir) ? Directory.GetFiles(credentialsDir, "*.cred") : new string[0]; + remainingFiles.Should().BeEmpty($"Expected no credential files after erase, but found: {string.Join(", ", remainingFiles)}"); } [Test] @@ -291,6 +335,10 @@ public void ScriptGeneration_CreatesValidBashScript() ScriptExecutionResult ExecutePowerShellScript(string operation, string input = null) { + Console.WriteLine($"[TEST DEBUG] Executing PowerShell script: {powershellScript} with operation: {operation}"); + Console.WriteLine($"[TEST DEBUG] Calamari executable: {calamariExecutable}"); + Console.WriteLine($"[TEST DEBUG] Docker config path: {dockerConfigPath}"); + var psi = new ProcessStartInfo { FileName = "powershell.exe", @@ -334,6 +382,10 @@ ScriptExecutionResult ExecutePowerShellScript(string operation, string input = n ScriptExecutionResult ExecuteBashScript(string operation, string input = null) { + Console.WriteLine($"[TEST DEBUG] Executing Bash script: {bashScript} with operation: {operation}"); + Console.WriteLine($"[TEST DEBUG] Calamari executable: {calamariExecutable}"); + Console.WriteLine($"[TEST DEBUG] Docker config path: {dockerConfigPath}"); + var psi = new ProcessStartInfo { FileName = "/bin/bash", From 91e8425250a9be08c8befef5b722b0490424e482 Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Fri, 29 Aug 2025 17:38:05 +1000 Subject: [PATCH 03/80] Make it works --- .../Calamari.Common/CalamariFlavourProgram.cs | 4 +- .../IWantCustomHandlingOfDeferredLogs.cs | 13 ++ .../Features/Processes/CommandLineRunner.cs | 8 +- .../Features/Processes/CommandResult.cs | 9 +- .../Processes/InMemoryCommandOutputSink.cs | 23 ++++ .../Plumbing/Logging/DeferredLogger.cs | 118 ++++++++++++++++++ .../Download/DockerCredentialHelper.cs | 52 ++++++-- .../Download/DockerImagePackageDownloader.cs | 62 +++++---- .../Packages/Download/ScriptExtractor.cs | 6 +- .../Processes/LibraryCallRunner.cs | 6 +- .../Calamari.Testing/EnvironmentVariables.cs | 9 +- .../PackagedScriptConventionFixture.cs | 4 +- ...ackageDownloaderCredentialHelperFixture.cs | 27 ++-- .../DockerImagePackageDownloaderFixture.cs | 66 +++++++--- .../Commands/DockerCredentialCommand.cs | 27 ++-- source/Calamari/Program.cs | 32 ++++- 16 files changed, 375 insertions(+), 91 deletions(-) create mode 100644 source/Calamari.Common/Commands/IWantCustomHandlingOfDeferredLogs.cs create mode 100644 source/Calamari.Common/Features/Processes/InMemoryCommandOutputSink.cs create mode 100644 source/Calamari.Common/Plumbing/Logging/DeferredLogger.cs diff --git a/source/Calamari.Common/CalamariFlavourProgram.cs b/source/Calamari.Common/CalamariFlavourProgram.cs index 7d55d63adf..1d7b26dcb9 100644 --- a/source/Calamari.Common/CalamariFlavourProgram.cs +++ b/source/Calamari.Common/CalamariFlavourProgram.cs @@ -27,7 +27,7 @@ namespace Calamari.Common { public abstract class CalamariFlavourProgram { - readonly ILog log; + protected readonly ILog log; protected CalamariFlavourProgram(ILog log) { @@ -154,4 +154,4 @@ protected virtual IEnumerable GetAllAssembliesToRegister() yield return typeof(CalamariFlavourProgram).Assembly; // Calamari.Common } } -} \ No newline at end of file +} diff --git a/source/Calamari.Common/Commands/IWantCustomHandlingOfDeferredLogs.cs b/source/Calamari.Common/Commands/IWantCustomHandlingOfDeferredLogs.cs new file mode 100644 index 0000000000..6befdab3e7 --- /dev/null +++ b/source/Calamari.Common/Commands/IWantCustomHandlingOfDeferredLogs.cs @@ -0,0 +1,13 @@ +using System; + +namespace Calamari.Common.Commands +{ + /// + /// We defer logs from startup, until we know what command we're going to run + /// For classes that do not implement this interface, any deferred logs will be flushed before Execute is called + /// Classes that implement this interface are responsible for flushing deferred logs themselves + /// + public interface IWantCustomHandlingOfDeferredLogs + { + } +} diff --git a/source/Calamari.Common/Features/Processes/CommandLineRunner.cs b/source/Calamari.Common/Features/Processes/CommandLineRunner.cs index 64fb20ee23..d4a5b873e2 100644 --- a/source/Calamari.Common/Features/Processes/CommandLineRunner.cs +++ b/source/Calamari.Common/Features/Processes/CommandLineRunner.cs @@ -21,7 +21,10 @@ public CommandLineRunner(ILog log, IVariables variables) public CommandResult Execute(CommandLineInvocation invocation) { - var commandOutput = new SplitCommandInvocationOutputSink(GetCommandOutputs(invocation)); + var outputSinks = GetCommandOutputs(invocation); + var inMemoryOutputLog = new InMemoryCommandOutputSink(); + outputSinks.Add(inMemoryOutputLog); + var commandOutput = new SplitCommandInvocationOutputSink(outputSinks); try { @@ -38,6 +41,7 @@ public CommandResult Execute(CommandLineInvocation invocation) return new CommandResult( invocation.ToString(), exitCode.ExitCode, + inMemoryOutputLog.StdOut, exitCode.ErrorOutput, invocation.WorkingDirectory); } @@ -79,4 +83,4 @@ public static string ConstructWin32ExceptionMessage(string executable) $"Unable to execute {executable}, please ensure that {executable} is installed and is in the PATH.{Environment.NewLine}"; } } -} \ No newline at end of file +} diff --git a/source/Calamari.Common/Features/Processes/CommandResult.cs b/source/Calamari.Common/Features/Processes/CommandResult.cs index 5ee1609bd4..9febfa8e2a 100644 --- a/source/Calamari.Common/Features/Processes/CommandResult.cs +++ b/source/Calamari.Common/Features/Processes/CommandResult.cs @@ -9,16 +9,23 @@ public class CommandResult readonly string? workingDirectory; public CommandResult(string command, int exitCode, string? additionalErrors = null, string? workingDirectory = null) + : this(command, exitCode, null, additionalErrors, workingDirectory) + { + } + + public CommandResult(string command, int exitCode, string? output, string? additionalErrors = null, string? workingDirectory = null) { this.command = command; ExitCode = exitCode; Errors = additionalErrors; this.workingDirectory = workingDirectory; + this.Output = output; } public int ExitCode { get; } public string? Errors { get; } + public string? Output { get; } public bool HasErrors => !string.IsNullOrWhiteSpace(Errors) && ErrorsExcludeServiceMessages(Errors); @@ -37,4 +44,4 @@ static bool ErrorsExcludeServiceMessages(string s) => .Where(s => !string.IsNullOrWhiteSpace(s)) .Any(s => !s.StartsWith("##octopus")); } -} \ No newline at end of file +} diff --git a/source/Calamari.Common/Features/Processes/InMemoryCommandOutputSink.cs b/source/Calamari.Common/Features/Processes/InMemoryCommandOutputSink.cs new file mode 100644 index 0000000000..5005128633 --- /dev/null +++ b/source/Calamari.Common/Features/Processes/InMemoryCommandOutputSink.cs @@ -0,0 +1,23 @@ +using System; +using System.Text; +using Calamari.Common.Plumbing.Commands; + +namespace Calamari.Common.Features.Processes +{ + public class InMemoryCommandOutputSink : ICommandInvocationOutputSink + { + readonly StringBuilder stdOut = new StringBuilder(); + readonly StringBuilder stdErr = new StringBuilder(); + public string StdOut => stdOut.ToString(); + + public void WriteInfo(string line) + { + stdOut.AppendLine(line); + } + + public void WriteError(string line) + { + stdErr.AppendLine(line); + } + } +} \ No newline at end of file diff --git a/source/Calamari.Common/Plumbing/Logging/DeferredLogger.cs b/source/Calamari.Common/Plumbing/Logging/DeferredLogger.cs new file mode 100644 index 0000000000..ed8071b23f --- /dev/null +++ b/source/Calamari.Common/Plumbing/Logging/DeferredLogger.cs @@ -0,0 +1,118 @@ +using System.Collections.Generic; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.ServiceMessages; + +namespace Calamari +{ + public class DeferredLogger : AbstractLog + { + readonly ILog log; + readonly List deferredActions = new List(); + bool releaseDeferredLogs; + + public DeferredLogger(ILog log) + { + this.log = log; + } + + protected override void StdOut(string message) + { + if (releaseDeferredLogs) + log.Info(message); + } + + protected override void StdErr(string message) + { + if (releaseDeferredLogs) + log.Error(message); + } + + public override void Verbose(string message) + { + if (releaseDeferredLogs) + log.Verbose(message); + else + deferredActions.Add(new LogAction { Type = LogActionType.Verbose, Message = message }); + } + + public override void Info(string message) + { + if (releaseDeferredLogs) + log.Info(message); + else + deferredActions.Add(new LogAction { Type = LogActionType.Info, Message = message }); + } + + public override void Warn(string message) + { + if (releaseDeferredLogs) + log.Warn(message); + else + deferredActions.Add(new LogAction { Type = LogActionType.Warn, Message = message }); + } + + public override void Error(string message) + { + if (releaseDeferredLogs) + log.Error(message); + else + deferredActions.Add(new LogAction { Type = LogActionType.Error, Message = message }); + } + + public override void WriteServiceMessage(ServiceMessage serviceMessage) + { + if (releaseDeferredLogs) + log.WriteServiceMessage(serviceMessage); + else + deferredActions.Add(new LogAction { Type = LogActionType.ServiceMessage, ServiceMessage = serviceMessage }); + } + + public void FlushDeferredLogs() + { + if (!releaseDeferredLogs) + { + releaseDeferredLogs = true; + + foreach (var action in deferredActions) + { + switch (action.Type) + { + case LogActionType.Verbose: + log.Verbose(action.Message); + break; + case LogActionType.Info: + log.Info(action.Message); + break; + case LogActionType.Warn: + log.Warn(action.Message); + break; + case LogActionType.Error: + log.Error(action.Message); + break; + case LogActionType.ServiceMessage: + log.WriteServiceMessage(action.ServiceMessage); + break; + } + } + + deferredActions.Clear(); + } + } + + private class LogAction + { + public LogActionType Type { get; set; } + public string Message { get; set; } + public ServiceMessage ServiceMessage { get; set; } + } + + private enum LogActionType + { + Verbose, + Info, + Warn, + Error, + ServiceMessage + } + } +} diff --git a/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs b/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs index 91941ec82c..fb1cdc53ba 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/DockerCredentialHelper.cs @@ -147,8 +147,9 @@ public bool SetupCredentialHelper(Dictionary environmentVariable "DefaultFallbackPassword"; // Deploy credential helper scripts - DeployCredentialHelperScript(environmentVariables, variables); - + var credentialHelperScriptName = DeployCredentialHelperScript(environmentVariables, variables); + var credentialHelperName = credentialHelperScriptName.Replace("docker-credential-", ""); + // Store credentials using the helper var serverUrl = GetServerUrlForCredentialHelper(feedUri, dockerHubRegistry); StoreCredentials(serverUrl, username, password, encryptionPassword, dockerConfigPath); @@ -157,16 +158,17 @@ public bool SetupCredentialHelper(Dictionary environmentVariable var credHelpers = new Dictionary(); if (feedUri.Host.Equals(dockerHubRegistry)) { - credHelpers["index.docker.io"] = "octopus"; - credHelpers["docker.io"] = "octopus"; - credHelpers["registry-1.docker.io"] = "octopus"; + credHelpers["index.docker.io"] = credentialHelperName; + credHelpers["docker.io"] = credentialHelperName; + credHelpers["registry-1.docker.io"] = credentialHelperName; + credHelpers["https://index.docker.io/v1/"] = credentialHelperName; } else { - credHelpers[feedUri.Host] = "octopus"; + credHelpers[feedUri.Host] = credentialHelperName; if (feedUri.Port != -1 && feedUri.Port != 80 && feedUri.Port != 443) { - credHelpers[$"{feedUri.Host}:{feedUri.Port}"] = "octopus"; + credHelpers[$"{feedUri.Host}:{feedUri.Port}"] = credentialHelperName; } } @@ -220,9 +222,8 @@ static string GetCalamariExecutablePath() return "Calamari"; } - void DeployCredentialHelperScript(Dictionary environmentVariables, IVariables variables) + string DeployCredentialHelperScript(Dictionary environmentVariables, IVariables variables) { - var dockerConfigPath = environmentVariables["DOCKER_CONFIG"]; var scriptName = "docker-credential-octopus"; var helperScript = ScriptExtractor.GetScript(fileSystem, scriptName); @@ -259,6 +260,7 @@ void DeployCredentialHelperScript(Dictionary environmentVariable } environmentVariables["OCTOPUS_CREDENTIAL_PASSWORD"] = encryptionPassword; + return fileSystem.GetFileName(helperScript); } public void CleanupCredentialHelper(Dictionary environmentVariables) @@ -267,6 +269,38 @@ public void CleanupCredentialHelper(Dictionary environmentVariab { var dockerConfigPath = environmentVariables["DOCKER_CONFIG"]; CleanupCredentials(dockerConfigPath); + + // Cleanup config.json + var configFilePath = Path.Combine(dockerConfigPath, "config.json"); + if (File.Exists(configFilePath)) + { + File.Delete(configFilePath); + log.Verbose("Cleaned up Docker config.json file"); + } + + // Remove the credential helper script file + var scriptName = "docker-credential-octopus"; + var helperScript = ScriptExtractor.GetScript(fileSystem, scriptName); + if (File.Exists(helperScript)) + { + File.Delete(helperScript); + log.Verbose("Cleaned up Docker credential helper script"); + } + + // Remove the credential helper script from PATH if we added it + if (environmentVariables.ContainsKey("PATH")) + { + var scriptDir = Path.GetDirectoryName(Path.GetFullPath(helperScript)); + var currentPath = environmentVariables["PATH"]; + var pathSeparator = CalamariEnvironment.IsRunningOnWindows ? ";" : ":"; + var pathParts = currentPath.Split(pathSeparator.ToCharArray()).ToList(); + + if (pathParts.Remove(scriptDir)) + { + environmentVariables["PATH"] = string.Join(pathSeparator, pathParts); + log.Verbose("Removed credential helper script directory from PATH"); + } + } } catch (Exception ex) { diff --git a/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs index ceabb6911c..79cb1d87ae 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/DockerImagePackageDownloader.cs @@ -9,7 +9,6 @@ using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; -using Calamari.Deployment; using Newtonsoft.Json.Linq; using Octopus.Versioning; @@ -23,6 +22,8 @@ public class DockerImagePackageDownloader : IPackageDownloader readonly IVariables variables; readonly ILog log; readonly IFeedLoginDetailsProviderFactory feedLoginDetailsProviderFactory; + readonly bool useCredentialHelper; + readonly DockerCredentialHelper dockerCredentialHelper; const string DockerHubRegistry = "index.docker.io"; static readonly HashSet SupportedLoginDetailsFeedTypes = new HashSet @@ -32,11 +33,13 @@ public class DockerImagePackageDownloader : IPackageDownloader FeedType.GoogleContainerRegistry }; + const string DockerConfigFolder = "./octo-docker-configs"; + // Ensures that any credential details are only available for the duration of the acquisition readonly Dictionary environmentVariables = new Dictionary() { { - "DOCKER_CONFIG", "./octo-docker-configs" + "DOCKER_CONFIG", DockerConfigFolder } }; @@ -53,6 +56,8 @@ public DockerImagePackageDownloader(IScriptEngine scriptEngine, this.variables = variables; this.log = log; this.feedLoginDetailsProviderFactory = feedLoginDetailsProviderFactory; + this.useCredentialHelper = OctopusFeatureToggles.UseDockerCredentialHelperFeatureToggle.IsEnabled(variables); + this.dockerCredentialHelper = new DockerCredentialHelper(fileSystem, log); } (string Username, string Password, Uri FeedUri) GetContainerRegistryLoginDetails(string feedTypeStr, string username, string password, Uri feedUri) @@ -87,23 +92,16 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, //Always try re-pull image, docker engine can take care of the rest var fullImageName = GetFullImageName(packageId, version, feedUri); - var dockerCredentialHelper = new DockerCredentialHelper(fileSystem, log); var feedHost = GetFeedHost(feedUri); var strategy = PackageDownloaderRetryUtils.CreateRetryStrategy(maxDownloadAttempts, downloadAttemptBackoff, log); - var useCredentialHelper = OctopusFeatureToggles.UseDockerCredentialHelperFeatureToggle.IsEnabled(variables); - var helperSetupSuccessfully = false; if (useCredentialHelper && !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) { - helperSetupSuccessfully = strategy.Execute(() - => dockerCredentialHelper.SetupCredentialHelper(environmentVariables, variables, feedUri, username, password, DockerHubRegistry)); + strategy.Execute(() => dockerCredentialHelper.SetupCredentialHelper(environmentVariables, variables, feedUri, username, password, DockerHubRegistry)); } - if (!helperSetupSuccessfully) - { - strategy.Execute(() => PerformLogin(username, password, feedHost)); - } + strategy.Execute(() => PerformLogin(username, password, feedHost, environmentVariables)); const string cachedWorkerToolsShortLink = "https://g.octopushq.com/CachedWorkerToolsImages"; var imageNotCachedMessage = @@ -114,7 +112,7 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, log.InfoFormat(imageNotCachedMessage, fullImageName); } - strategy.Execute(() => PerformPull(fullImageName)); + strategy.Execute(() => PerformPull(fullImageName, environmentVariables)); var (hash, size) = GetImageDetails(fullImageName); @@ -123,6 +121,9 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId, { dockerCredentialHelper.CleanupCredentialHelper(environmentVariables); } + + if (fileSystem.DirectoryExists(DockerConfigFolder)) + fileSystem.DeleteDirectory(DockerConfigFolder); return new PackagePhysicalFileMetadata(new PackageFileNameMetadata(packageId, version, version, ""), string.Empty, hash, size); } @@ -149,19 +150,27 @@ static string GetFeedHost(Uri feedUri) return $"{feedUri.Host}:{feedUri.Port}"; } - void PerformLogin(string? username, string? password, string feed) + void PerformLogin(string? username, string? password, string feed, Dictionary dictionary) { - var result = ExecuteScript("DockerLogin", - new Dictionary - { - ["DockerUsername"] = username, - ["DockerPassword"] = password, - ["FeedUri"] = feed - }); + var envVars = new Dictionary(dictionary); + envVars["DockerUsername"] = username; + envVars["DockerPassword"] = password; + envVars["FeedUri"] = feed; + + var result = ExecuteScript("DockerLogin", envVars); if (result == null) throw new CommandException("Null result attempting to log in Docker registry"); if (result.ExitCode != 0) + { + if (useCredentialHelper && result.Errors != null && result.Output.Contains("Error saving credentials")) + { + log.Verbose("Docker login failed due to credential helper error, retrying without credential helper"); + dockerCredentialHelper.CleanupCredentialHelper(environmentVariables); + PerformLogin(username, password, feed, dictionary); + return; + } throw new CommandException("Unable to log in Docker registry"); + } } bool IsImageCached(string fullImageName) @@ -178,13 +187,12 @@ bool IsImageCached(string fullImageName) return cachedDigests.Intersect(selectedDigests).Any(); } - void PerformPull(string fullImageName) + void PerformPull(string fullImageName, Dictionary dictionary) { - var result = ExecuteScript("DockerPull", - new Dictionary - { - ["Image"] = fullImageName - }); + var envVars = new Dictionary(dictionary); + envVars["Image"] = fullImageName; + + var result = ExecuteScript("DockerPull", envVars); if (result == null) throw new CommandException("Null result attempting to pull Docker image"); if (result.ExitCode != 0) @@ -193,7 +201,7 @@ void PerformPull(string fullImageName) CommandResult ExecuteScript(string scriptName, Dictionary envVars) { - var file = ScriptExtractor.GetScript(fileSystem, scriptName); + var file = ScriptExtractor.GetScript(fileSystem, scriptName, "Octopus."); using (new TemporaryFile(file)) { var clone = variables.Clone(); diff --git a/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs b/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs index b0cf9fa30b..79d7cc8d73 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/ScriptExtractor.cs @@ -9,7 +9,7 @@ namespace Calamari.Integration.Packages.Download { public static class ScriptExtractor { - internal static string GetScript(ICalamariFileSystem fileSystem, string scriptName) + internal static string GetScript(ICalamariFileSystem fileSystem, string scriptName, string? outputFileNamePrefix = null) { var syntax = ScriptSyntaxHelper.GetPreferredScriptSyntaxForEnvironment(); @@ -26,10 +26,10 @@ internal static string GetScript(ICalamariFileSystem fileSystem, string scriptNa throw new InvalidOperationException("No script wrapper exists for " + syntax); } - var scriptFile = Path.Combine(".", $"Octopus.{contextFile}"); + var scriptFile = Path.Combine(".", $"{outputFileNamePrefix}{contextFile}"); var contextScript = new AssemblyEmbeddedResources().GetEmbeddedResourceText(Assembly.GetExecutingAssembly(), $"{typeof(DockerImagePackageDownloader).Namespace}.Scripts.{contextFile}"); fileSystem.OverwriteFile(scriptFile, contextScript); return scriptFile; } } -} \ No newline at end of file +} diff --git a/source/Calamari.Shared/Integration/Processes/LibraryCallRunner.cs b/source/Calamari.Shared/Integration/Processes/LibraryCallRunner.cs index e6db1d5763..ef96f2147b 100644 --- a/source/Calamari.Shared/Integration/Processes/LibraryCallRunner.cs +++ b/source/Calamari.Shared/Integration/Processes/LibraryCallRunner.cs @@ -11,14 +11,14 @@ public CommandResult Execute(LibraryCallInvocation invocation) { var exitCode = invocation.Executable(invocation.Arguments); - return new CommandResult(invocation.ToString(), exitCode, null); + return new CommandResult(invocation.ToString(), exitCode); } catch (Exception ex) { Console.Error.WriteLine(ex); Console.Error.WriteLine("The command that caused the exception was: " + invocation); - return new CommandResult(invocation.ToString(), -1, ex.ToString()); + return new CommandResult(invocation.ToString(), -1, additionalErrors: ex.ToString()); } } } -} \ No newline at end of file +} diff --git a/source/Calamari.Testing/EnvironmentVariables.cs b/source/Calamari.Testing/EnvironmentVariables.cs index 509ae22b00..6a39a09886 100644 --- a/source/Calamari.Testing/EnvironmentVariables.cs +++ b/source/Calamari.Testing/EnvironmentVariables.cs @@ -41,8 +41,11 @@ public enum ExternalVariable [EnvironmentVariable("Artifactory_Admin_PAT", "op://Calamari Secrets for Tests/Artifactory Admin PAT/PAT")] ArtifactoryE2EPassword, - [EnvironmentVariable("DockerHub_TestReaderAccount_Password", "op://Calamari Secrets for Tests/DockerHub Test Reader Account/password")] - DockerReaderPassword, + [EnvironmentVariable("DockerHub_TestReaderAccount_Username", "op://Calamari Secrets for Tests/DockerHub Test Reader Org Access Token/Token Username")] + DockerHubOrgAccessUsername, + + [EnvironmentVariable("DockerHub_TestReaderAccount_Username", "op://Calamari Secrets for Tests/DockerHub Test Reader Org Access Token/API Token")] + DockerHubOrgAccessToken, [EnvironmentVariable("AWS_E2E_AccessKeyId", "op://Calamari Secrets for Tests/AWS E2E Test User Keys/AccessKeyId")] AwsCloudFormationAndS3AccessKey, @@ -184,4 +187,4 @@ public EnvironmentVariableAttribute(string name, string? secretReference = null, return GetCustomAttribute(mi[0], typeof(EnvironmentVariableAttribute)) as EnvironmentVariableAttribute; } } -} \ No newline at end of file +} diff --git a/source/Calamari.Tests/Fixtures/Deployment/Conventions/PackagedScriptConventionFixture.cs b/source/Calamari.Tests/Fixtures/Deployment/Conventions/PackagedScriptConventionFixture.cs index 82e366e40f..127038b16e 100644 --- a/source/Calamari.Tests/Fixtures/Deployment/Conventions/PackagedScriptConventionFixture.cs +++ b/source/Calamari.Tests/Fixtures/Deployment/Conventions/PackagedScriptConventionFixture.cs @@ -33,7 +33,7 @@ public void SetUp() fileSystem = Substitute.For(); fileSystem.EnumerateFiles(Arg.Any(), Arg.Any()).Returns(new[] {TestEnvironment.ConstructRootedPath("App", "MyApp", "Hello.ps1"), TestEnvironment.ConstructRootedPath("App", "MyApp", "Deploy.ps1"), TestEnvironment.ConstructRootedPath("App", "MyApp", "Deploy.csx"), TestEnvironment.ConstructRootedPath("App", "MyApp", "PreDeploy.ps1"), TestEnvironment.ConstructRootedPath("App", "MyApp", "PreDeploy.sh"), TestEnvironment.ConstructRootedPath("App", "MyApp", "PostDeploy.ps1"), TestEnvironment.ConstructRootedPath("App", "MyApp", "PostDeploy.sh"), TestEnvironment.ConstructRootedPath("App", "MyApp", "DeployFailed.ps1"), TestEnvironment.ConstructRootedPath("App", "MyApp", "DeployFailed.sh")}); - commandResult = new CommandResult("PowerShell.exe foo bar", 0, null); + commandResult = new CommandResult("PowerShell.exe foo bar", 0); scriptEngine = Substitute.For(); scriptEngine.Execute(Arg.Any