From 988e0500908c8f4139b6545671c9d05f78cb52c4 Mon Sep 17 00:00:00 2001 From: Nick Josevski Date: Thu, 28 May 2026 22:16:48 +1000 Subject: [PATCH 1/5] Use file creation time for GetNearestPackages when version is non-SemVer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For SemVer versions the existing behaviour is unchanged: filter to versions <= target and order by version descending so the closest lower version is offered as the delta base. For non-SemVer versions (Docker tags, build numbers, date-stamps) the SemVer comparison is unreliable and may return no candidates or the wrong ones. Fall back to ordering all cached packages for the same package ID by file creation time descending, so the most recently cached package is offered as the delta candidate. This is a performance-only change — if no suitable delta candidate is found Calamari already falls back to downloading the full package. The fix improves delta compression quality for deployments using the MostRecentlyPublished channel ordering strategy (OctopusDeploy/OctopusDeploy#43750). Co-Authored-By: Claude Sonnet 4.6 --- .../Certificates/FileSystem/PackageStore.cs | 17 +++++-- .../FileSystem/PackageStoreFixture.cs | 49 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs b/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs index 7c20699540..01a6eaed9f 100644 --- a/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs +++ b/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs @@ -71,12 +71,21 @@ public IEnumerable GetNearestPackages(string packag { fileSystem.EnsureDirectoryExists(GetPackagesDirectory()); - var zipPackages = + var allMatchingPackages = from filePath in PackageFiles(packageId) let zip = PackageMetadata(filePath) - where zip != null && string.Equals(zip.PackageId, packageId, StringComparison.OrdinalIgnoreCase) && zip.Version.CompareTo(version) <= 0 - orderby zip.Version descending - select new {zip, filePath}; + where zip != null && string.Equals(zip.PackageId, packageId, StringComparison.OrdinalIgnoreCase) + select new { zip, filePath }; + + // For SemVer versions, find packages with version <= target ordered by version descending (closest lower version first). + // For non-SemVer versions (Docker tags, build numbers etc.) SemVer comparison is meaningless, so fall back to + // ordering by file creation time — the most recently cached package is the best delta candidate. + var zipPackages = version.Format == VersionFormat.Semver + ? allMatchingPackages + .Where(x => x.zip.Version.CompareTo(version) <= 0) + .OrderByDescending(x => x.zip.Version) + : allMatchingPackages + .OrderByDescending(x => fileSystem.GetCreationTime(x.filePath)); return from zipPackage in zipPackages.Take(take) diff --git a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs index 322208b198..d440756734 100644 --- a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs @@ -8,6 +8,7 @@ using Calamari.Testing.Helpers; using Calamari.Tests.Fixtures.Deployment.Packages; using NUnit.Framework; +using Octopus.Versioning; using Octopus.Versioning.Semver; namespace Calamari.Tests.Fixtures.Integration.FileSystem @@ -90,6 +91,54 @@ public void IgnoresInvalidFiles() } } + [Test] + public void NonSemVerVersionsReturnPackagesOrderedByCreationTime() + { + // For non-SemVer tags (Docker tags, build numbers etc.) version comparison is unreliable. + // GetNearestPackages should fall back to file creation time so the most recently + // cached package is offered as the delta candidate. + var v1Path = CreatePackageWithDockerTag("feature-login-100"); + var v2Path = CreatePackageWithDockerTag("main-9999"); + var v3Path = CreatePackageWithDockerTag("feature-signup-42"); + + // Set deterministic creation times: v2 is most recent, then v3, then v1 + File.SetCreationTimeUtc(v1Path, DateTime.UtcNow.AddMinutes(-30)); + File.SetCreationTimeUtc(v3Path, DateTime.UtcNow.AddMinutes(-15)); + File.SetCreationTimeUtc(v2Path, DateTime.UtcNow.AddMinutes(-5)); + + using (new TemporaryFile(v1Path)) + using (new TemporaryFile(v2Path)) + using (new TemporaryFile(v3Path)) + { + var store = new PackageStore( + CreatePackageExtractor(), + CalamariPhysicalFileSystem.GetPhysicalFileSystem() + ); + + var target = VersionFactory.TryCreateDockerTag("feature-new-99"); + var packages = store.GetNearestPackages("Acme.Web", target).ToList(); + + // All three are returned (no version filter), ordered by creation time descending + Assert.That(packages.Count, Is.EqualTo(3)); + Assert.That(packages[0].Version.ToString(), Is.EqualTo("main-9999")); + Assert.That(packages[1].Version.ToString(), Is.EqualTo("feature-signup-42")); + Assert.That(packages[2].Version.ToString(), Is.EqualTo("feature-login-100")); + } + } + + private string CreatePackageWithDockerTag(string version) + { + var dockerVersion = VersionFactory.TryCreateDockerTag(version)!; + var sourcePackage = PackageBuilder.BuildSamplePackage("Acme.Web", version, true); + var destinationPath = Path.Combine(PackagePath, PackageName.ToCachedFileName("Acme.Web", dockerVersion, ".nupkg")); + + if (File.Exists(destinationPath)) + File.Delete(destinationPath); + + File.Move(sourcePackage, destinationPath); + return destinationPath; + } + private string CreateEmptyFile(string version) { var destinationPath = Path.Combine(PackagePath, PackageName.ToCachedFileName("Acme.Web", new SemanticVersion(version), ".nupkg")); From 450f95c4f3cb95790b2a14b821b28699ad227b0f Mon Sep 17 00:00:00 2001 From: Nick Josevski Date: Fri, 29 May 2026 07:24:05 +1000 Subject: [PATCH 2/5] Fix cross-platform build: add GetLastWriteTime to IFile/ICalamariFileSystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File.SetCreationTimeUtc is a no-op on Linux — the test was using it to set deterministic ordering which caused failures across all Linux CI targets. GetLastWriteTime is fully settable on all platforms via File.SetLastWriteTimeUtc. Added GetLastWriteTime to IFile, StandardFile, ICalamariFileSystem, CalamariPhysicalFileSystem, and TestFile. Updated PackageStore to use fileSystem.GetLastWriteTime and the test to use File.SetLastWriteTimeUtc. Co-Authored-By: Claude Sonnet 4.6 --- .../Plumbing/FileSystem/CalamariPhysicalFileSystem.cs | 5 +++++ .../Plumbing/FileSystem/FileOperations.cs | 6 ++++++ .../Plumbing/FileSystem/ICalamariFileSystem.cs | 1 + .../Certificates/FileSystem/PackageStore.cs | 2 +- .../Integration/FileSystem/PackageStoreFixture.cs | 10 ++++++---- .../Fixtures/Integration/FileSystem/TestFile.cs | 1 + 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/source/Calamari.Common/Plumbing/FileSystem/CalamariPhysicalFileSystem.cs b/source/Calamari.Common/Plumbing/FileSystem/CalamariPhysicalFileSystem.cs index b7d00560d0..eab210d884 100644 --- a/source/Calamari.Common/Plumbing/FileSystem/CalamariPhysicalFileSystem.cs +++ b/source/Calamari.Common/Plumbing/FileSystem/CalamariPhysicalFileSystem.cs @@ -693,6 +693,11 @@ public DateTime GetCreationTime(string filePath) return File.GetCreationTime(filePath); } + public DateTime GetLastWriteTime(string filePath) + { + return File.GetLastWriteTime(filePath); + } + public string GetFileName(string filePath) { return new FileInfo(filePath).Name; diff --git a/source/Calamari.Common/Plumbing/FileSystem/FileOperations.cs b/source/Calamari.Common/Plumbing/FileSystem/FileOperations.cs index 0e6d09c2f7..1ed1caf40e 100644 --- a/source/Calamari.Common/Plumbing/FileSystem/FileOperations.cs +++ b/source/Calamari.Common/Plumbing/FileSystem/FileOperations.cs @@ -15,6 +15,7 @@ public interface IFile void Move(string sourceFile, string destination); void SetAttributes(string path, FileAttributes normal); DateTime GetCreationTime(string filePath); + DateTime GetLastWriteTime(string filePath); Stream Open(string filePath, FileMode fileMode, FileAccess fileAccess, FileShare none); } @@ -73,6 +74,11 @@ public DateTime GetCreationTime(string path) return File.GetCreationTime(path); } + public DateTime GetLastWriteTime(string path) + { + return File.GetLastWriteTime(path); + } + public Stream Open(string path, FileMode mode, FileAccess access, FileShare share) { return File.Open(path, mode, access, share); diff --git a/source/Calamari.Common/Plumbing/FileSystem/ICalamariFileSystem.cs b/source/Calamari.Common/Plumbing/FileSystem/ICalamariFileSystem.cs index a38a517b30..131a14f945 100644 --- a/source/Calamari.Common/Plumbing/FileSystem/ICalamariFileSystem.cs +++ b/source/Calamari.Common/Plumbing/FileSystem/ICalamariFileSystem.cs @@ -51,6 +51,7 @@ public interface ICalamariFileSystem string GetRelativePath(string fromFile, string toFile); Stream OpenFileExclusively(string filePath, FileMode fileMode, FileAccess fileAccess); DateTime GetCreationTime(string filePath); + DateTime GetLastWriteTime(string filePath); string GetFileName(string filePath); string GetDirectoryName(string directoryPath); byte[] ReadAllBytes(string filePath); diff --git a/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs b/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs index 01a6eaed9f..6ec5299a78 100644 --- a/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs +++ b/source/Calamari.Shared/Integration/Certificates/FileSystem/PackageStore.cs @@ -85,7 +85,7 @@ from filePath in PackageFiles(packageId) .Where(x => x.zip.Version.CompareTo(version) <= 0) .OrderByDescending(x => x.zip.Version) : allMatchingPackages - .OrderByDescending(x => fileSystem.GetCreationTime(x.filePath)); + .OrderByDescending(x => fileSystem.GetLastWriteTime(x.filePath)); return from zipPackage in zipPackages.Take(take) diff --git a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs index d440756734..fabe10494a 100644 --- a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs @@ -101,10 +101,12 @@ public void NonSemVerVersionsReturnPackagesOrderedByCreationTime() var v2Path = CreatePackageWithDockerTag("main-9999"); var v3Path = CreatePackageWithDockerTag("feature-signup-42"); - // Set deterministic creation times: v2 is most recent, then v3, then v1 - File.SetCreationTimeUtc(v1Path, DateTime.UtcNow.AddMinutes(-30)); - File.SetCreationTimeUtc(v3Path, DateTime.UtcNow.AddMinutes(-15)); - File.SetCreationTimeUtc(v2Path, DateTime.UtcNow.AddMinutes(-5)); + // Set deterministic last-write times: v2 is most recent, then v3, then v1. + // Using SetLastWriteTimeUtc rather than SetCreationTimeUtc because creation time + // is read-only on Linux filesystems. + File.SetLastWriteTimeUtc(v1Path, DateTime.UtcNow.AddMinutes(-30)); + File.SetLastWriteTimeUtc(v3Path, DateTime.UtcNow.AddMinutes(-15)); + File.SetLastWriteTimeUtc(v2Path, DateTime.UtcNow.AddMinutes(-5)); using (new TemporaryFile(v1Path)) using (new TemporaryFile(v2Path)) diff --git a/source/Calamari.Tests/Fixtures/Integration/FileSystem/TestFile.cs b/source/Calamari.Tests/Fixtures/Integration/FileSystem/TestFile.cs index 93f42681d3..a9be97ae41 100644 --- a/source/Calamari.Tests/Fixtures/Integration/FileSystem/TestFile.cs +++ b/source/Calamari.Tests/Fixtures/Integration/FileSystem/TestFile.cs @@ -15,6 +15,7 @@ public class TestFile : IFile public bool Exists(string path) => File.Exists(WithBase(path)); public byte[] ReadAllBytes(string path) => File.ReadAllBytes(WithBase(path)); public DateTime GetCreationTime(string filePath) => File.GetCreationTime(WithBase(filePath)); + public DateTime GetLastWriteTime(string filePath) => File.GetLastWriteTime(WithBase(filePath)); public Stream Open(string filePath, FileMode fileMode, FileAccess fileAccess, FileShare none) => File.Open(WithBase(filePath), fileMode, fileAccess, none); From 187088cbce0e6d7af95edc06cbabc6b8f4d51c7d Mon Sep 17 00:00:00 2001 From: Nick Josevski Date: Fri, 29 May 2026 09:36:01 +1000 Subject: [PATCH 3/5] Fix test: use .zip + dummy file instead of dotnet pack for Docker tag versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous test used PackageBuilder.BuildSamplePackage which runs `dotnet pack /p:Version=` — but dotnet pack rejects non-SemVer versions like 'feature-login-100' and 'main-9999', causing the test to fail across all Linux/Windows/macOS Calamari netcore test runners. Switched to using .zip extension with dummy file content. The PackageStore only reads file metadata (hash, size, last-write time) for delta candidate selection, so the file content is irrelevant. Using .zip also avoids PackageName.FromFile attempting to parse NuGet metadata when the extension is .nupkg. Test passes locally now. Verified with: dotnet test --filter FullyQualifiedName~PackageStoreFixture Co-Authored-By: Claude Sonnet 4.6 --- .../FileSystem/PackageStoreFixture.cs | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs index fabe10494a..7c458eeed5 100644 --- a/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/FileSystem/PackageStoreFixture.cs @@ -92,14 +92,18 @@ public void IgnoresInvalidFiles() } [Test] - public void NonSemVerVersionsReturnPackagesOrderedByCreationTime() + public void NonSemVerVersionsReturnPackagesOrderedByLastWriteTime() { // For non-SemVer tags (Docker tags, build numbers etc.) version comparison is unreliable. - // GetNearestPackages should fall back to file creation time so the most recently + // GetNearestPackages should fall back to file last-write time so the most recently // cached package is offered as the delta candidate. - var v1Path = CreatePackageWithDockerTag("feature-login-100"); - var v2Path = CreatePackageWithDockerTag("main-9999"); - var v3Path = CreatePackageWithDockerTag("feature-signup-42"); + // + // We bypass PackageBuilder.BuildSamplePackage here because `dotnet pack /p:Version=...` + // rejects non-SemVer versions. We also use .zip rather than .nupkg so PackageName.FromFile + // does not try to read NuGet metadata from the file content. + var v1Path = CreateDummyCachedFile("feature-login-100"); + var v2Path = CreateDummyCachedFile("main-9999"); + var v3Path = CreateDummyCachedFile("feature-signup-42"); // Set deterministic last-write times: v2 is most recent, then v3, then v1. // Using SetLastWriteTimeUtc rather than SetCreationTimeUtc because creation time @@ -120,7 +124,7 @@ public void NonSemVerVersionsReturnPackagesOrderedByCreationTime() var target = VersionFactory.TryCreateDockerTag("feature-new-99"); var packages = store.GetNearestPackages("Acme.Web", target).ToList(); - // All three are returned (no version filter), ordered by creation time descending + // All three are returned (no version filter), ordered by last-write time descending Assert.That(packages.Count, Is.EqualTo(3)); Assert.That(packages[0].Version.ToString(), Is.EqualTo("main-9999")); Assert.That(packages[1].Version.ToString(), Is.EqualTo("feature-signup-42")); @@ -128,16 +132,14 @@ public void NonSemVerVersionsReturnPackagesOrderedByCreationTime() } } - private string CreatePackageWithDockerTag(string version) + private string CreateDummyCachedFile(string version) { var dockerVersion = VersionFactory.TryCreateDockerTag(version)!; - var sourcePackage = PackageBuilder.BuildSamplePackage("Acme.Web", version, true); - var destinationPath = Path.Combine(PackagePath, PackageName.ToCachedFileName("Acme.Web", dockerVersion, ".nupkg")); - - if (File.Exists(destinationPath)) - File.Delete(destinationPath); - - File.Move(sourcePackage, destinationPath); + // Use .zip rather than .nupkg — .nupkg files trigger NuGet metadata parsing inside + // PackageName.FromFile, which would fail on these dummy files. The PackageStore only + // reads file metadata (hash, size, last-write time), not the file contents themselves. + var destinationPath = Path.Combine(PackagePath, PackageName.ToCachedFileName("Acme.Web", dockerVersion, ".zip")); + File.WriteAllText(destinationPath, "dummy content for " + version); return destinationPath; } From d7de7bef27c5aab8fb49bb5a21f473a546609f51 Mon Sep 17 00:00:00 2001 From: Nick Josevski Date: Fri, 29 May 2026 11:37:27 +1000 Subject: [PATCH 4/5] =?UTF-8?q?ci:=20retry=20=E2=80=94=20Azure=20SCM=20502?= =?UTF-8?q?=20on=20Ubuntu=20AzureAppService=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 6db6993329e4c4e57b4b50d3d6a31d209bcfd785 Mon Sep 17 00:00:00 2001 From: Nick Josevski Date: Tue, 2 Jun 2026 16:13:29 +1000 Subject: [PATCH 5/5] ci: retrigger build (empty commit) Trigger a fresh Calamari chain for the rebased non-semver package cache ordering branch so the pre-release package publishes to the internal feed. Co-Authored-By: Claude Opus 4.8 (1M context)