From 261879e5f0d06d7006f1a6b7366ec8c24d43bcc5 Mon Sep 17 00:00:00 2001 From: George Rybakov Date: Thu, 18 Jun 2026 15:53:20 +0300 Subject: [PATCH 1/2] backup: implement manifest aggregation Add backup manifest data types, fragment decoding, aggregation, validation, warnings, and unit tests with testdata fixtures. Closes TNTP-8208 --- lib/backup/aggregation.go | 255 ++++++++++++++++ lib/backup/aggregation_test.go | 195 +++++++++++++ lib/backup/fixtures_test.go | 69 +++++ lib/backup/manifest.go | 59 ++++ lib/backup/testdata/cluster_manifest.json | 109 +++++++ lib/backup/testdata/fragment_a.json | 18 ++ lib/backup/testdata/fragment_b.json | 18 ++ .../fragment_with_empty_recovery_points.json | 15 + .../fragment_without_recovery_points.json | 14 + lib/backup/types.go | 29 ++ lib/backup/validation.go | 179 ++++++++++++ lib/backup/validation_test.go | 274 ++++++++++++++++++ lib/backup/warnings.go | 67 +++++ 13 files changed, 1301 insertions(+) create mode 100644 lib/backup/aggregation.go create mode 100644 lib/backup/aggregation_test.go create mode 100644 lib/backup/fixtures_test.go create mode 100644 lib/backup/manifest.go create mode 100644 lib/backup/testdata/cluster_manifest.json create mode 100644 lib/backup/testdata/fragment_a.json create mode 100644 lib/backup/testdata/fragment_b.json create mode 100644 lib/backup/testdata/fragment_with_empty_recovery_points.json create mode 100644 lib/backup/testdata/fragment_without_recovery_points.json create mode 100644 lib/backup/types.go create mode 100644 lib/backup/validation.go create mode 100644 lib/backup/validation_test.go create mode 100644 lib/backup/warnings.go diff --git a/lib/backup/aggregation.go b/lib/backup/aggregation.go new file mode 100644 index 000000000..f3d26bda0 --- /dev/null +++ b/lib/backup/aggregation.go @@ -0,0 +1,255 @@ +package backup + +import ( + "encoding/json" + "fmt" + "time" +) + +const artifactCompression = "zstd" + +// RecoveryPoint describes one engine recovery point returned by Tarantool. +type RecoveryPoint struct { + UUID string `json:"uuid"` + ReplicaID uint32 `json:"replica_id"` + LSN uint64 `json:"lsn"` + Timestamp int64 `json:"timestamp"` // unix-time +} + +// Fragment is a per-replicaset backup description stored in an instance archive. +type Fragment struct { + ReplicasetUUID string `json:"replicaset_uuid"` + InstanceUUID string `json:"instance_uuid"` + InstanceName string `json:"instance_name"` + Hostname string `json:"hostname"` + Type BackupType `json:"type"` + VclockBegin Vclock `json:"vclock_begin"` + VclockEnd Vclock `json:"vclock_end"` + Files []string `json:"files"` + ChecksumSHA256 string `json:"checksum_sha256"` + RecoveryPoints []*RecoveryPoint `json:"recovery_points,omitempty"` +} + +// AggregateInput contains all external data needed to build a manifest. +type AggregateInput struct { + BackupID BackupID + BaseFullBackupID BackupID + PreviousBackupID BackupID + CreationTime time.Time + CreationDuration time.Duration + Topology Topology + Shards []*ShardInput +} + +// ShardInput describes one expected replicaset backup result. +type ShardInput struct { + ReplicasetUUID string + Fragment *Fragment + Location *ArtifactLocation + Err error +} + +// ArtifactLocation identifies an already uploaded shard archive. +type ArtifactLocation struct { + Path string + SizeBytes int64 +} + +// DecodeFragment decodes and validates one instance_backup.json payload. +func DecodeFragment(data []byte) (*Fragment, error) { + var fragment Fragment + + if err := json.Unmarshal(data, &fragment); err != nil { + return nil, fmt.Errorf("decode fragment: %w", err) + } + + if err := fragment.Validate(); err != nil { + return nil, fmt.Errorf("validate fragment: %w", err) + } + + return &fragment, nil +} + +// NewAggregateInput collects backup metadata, topology and shard inputs. +func NewAggregateInput( + backupID BackupID, + previousBackupID BackupID, + baseFullBackupID BackupID, + creationTime time.Time, + creationDuration time.Duration, + topology Topology, + shards []*ShardInput, +) AggregateInput { + return AggregateInput{ + BackupID: backupID, + PreviousBackupID: previousBackupID, + BaseFullBackupID: baseFullBackupID, + CreationTime: creationTime, + CreationDuration: creationDuration, + Topology: topology, + Shards: shards, + } +} + +// Aggregate builds and validates a cluster manifest from shard fragments. +func Aggregate(in AggregateInput) (*ClusterManifest, error) { + manifest := newClusterManifest(in) + + for _, shardInput := range in.Shards { + if err := aggregateShard(manifest, shardInput); err != nil { + return nil, fmt.Errorf("aggregate shard: %w", err) + } + } + + manifest.Status = calculateStatus(manifest) + if err := manifest.Validate(); err != nil { + return nil, fmt.Errorf("validate cluster manifest: %w", err) + } + + return manifest, nil +} + +// newClusterManifest initializes a manifest with immutable aggregate metadata. +func newClusterManifest(in AggregateInput) *ClusterManifest { + return &ClusterManifest{ + SchemaVersion: SchemaVersion, + BackupID: in.BackupID, + PreviousBackupID: in.PreviousBackupID, + BaseFullBackupID: in.BaseFullBackupID, + Status: StatusFailed, + CreationTime: in.CreationTime, + CreationDuration: in.CreationDuration, + Shards: make(map[string]Shard, len(in.Shards)), + Topology: in.Topology, + Warnings: make([]Warning, 0), + } +} + +// aggregateShard adds one shard input to the manifest. +func aggregateShard(manifest *ClusterManifest, shardInput *ShardInput) error { + replicasetUUID := shardInput.ReplicasetUUID + + if shardInput.Fragment == nil { + aggregateFailedShard(manifest, replicasetUUID, shardInput.Err) + return nil + } + + if shardInput.Err != nil { + manifest.Shards[replicasetUUID] = Shard{Error: shardInput.Err.Error()} + manifest.Warnings = append( + manifest.Warnings, + NewShardPartialWarning( + replicasetUUID, + shardInput.Fragment.InstanceUUID, + shardInput.Err.Error(), + ), + ) + return nil + } + + if shardInput.Fragment.ReplicasetUUID != replicasetUUID { + return fmt.Errorf( + "fragment replicaset_uuid %q does not match shard input replicaset_uuid %q", + shardInput.Fragment.ReplicasetUUID, + replicasetUUID, + ) + } + + aggregateSuccessfulShard(manifest, replicasetUUID, shardInput) + return nil +} + +// aggregateFailedShard adds an error result for a shard. +func aggregateFailedShard(manifest *ClusterManifest, replicasetUUID string, err error) { + if err == nil { + manifest.Shards[replicasetUUID] = Shard{Error: "shard unreachable"} + manifest.Warnings = append(manifest.Warnings, + NewShardUnreachableWarning(replicasetUUID)) + return + } + + manifest.Shards[replicasetUUID] = Shard{Error: err.Error()} +} + +// aggregateSuccessfulShard adds an instance result for a shard. +func aggregateSuccessfulShard( + manifest *ClusterManifest, + replicasetUUID string, + shardInput *ShardInput, +) { + fragment := shardInput.Fragment + location := ArtifactLocation{} + if shardInput.Location != nil { + location = *shardInput.Location + } + + manifest.Shards[replicasetUUID] = Shard{ + Instance: &ShardInstance{ + InstanceUUID: fragment.InstanceUUID, + InstanceName: fragment.InstanceName, + Hostname: fragment.Hostname, + VclockBegin: fragment.VclockBegin, + VclockEnd: fragment.VclockEnd, + Artifact: Artifact{ + Path: location.Path, + SizeBytes: location.SizeBytes, + ChecksumSHA256: fragment.ChecksumSHA256, + Compression: artifactCompression, + Files: append([]string(nil), fragment.Files...), + RecoveryPoints: recoveryPointsFromFragment(manifest, replicasetUUID, fragment), + Type: fragment.Type, + }, + }, + } +} + +// recoveryPointsFromFragment converts optional fragment recovery points. +func recoveryPointsFromFragment( + manifest *ClusterManifest, + replicasetUUID string, + fragment *Fragment, +) []RecoveryPoint { + recoveryPoints := make([]RecoveryPoint, 0) + if fragment.RecoveryPoints == nil { + manifest.Warnings = append(manifest.Warnings, + NewRecoveryPointsUnavailableWarning(replicasetUUID, "recovery points unavailable")) + return recoveryPoints + } + + for _, point := range fragment.RecoveryPoints { + if point != nil { + recoveryPoints = append(recoveryPoints, *point) + } + } + return recoveryPoints +} + +// calculateStatus derives cluster backup health from shard results and warnings. +func calculateStatus(manifest *ClusterManifest) Status { + successful := 0 + failed := 0 + + for _, shard := range manifest.Shards { + if shard.Instance != nil { + successful++ + } + if shard.Error != "" { + failed++ + } + } + + if successful == 0 { + return StatusFailed + } + if isDegraded(manifest, successful, failed) { + return StatusDegraded + } + return StatusOK +} + +// isDegraded reports whether a partially useful manifest has issues. +func isDegraded(manifest *ClusterManifest, successful, failed int) bool { + return failed > 0 || + len(manifest.Warnings) > 0 || + successful < len(manifest.Topology.Replicasets) +} diff --git a/lib/backup/aggregation_test.go b/lib/backup/aggregation_test.go new file mode 100644 index 000000000..d81a38b9d --- /dev/null +++ b/lib/backup/aggregation_test.go @@ -0,0 +1,195 @@ +package backup + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestAggregateSuccessfulManifest(t *testing.T) { + fragment := mustDecodeFragment(t, fixtureFragmentA) + + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: 2300 * time.Millisecond, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{ + { + ReplicasetUUID: testRSA, + Fragment: &fragment, + Location: &ArtifactLocation{Path: "data/rs-a.tar.zst", SizeBytes: 42}, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, SchemaVersion, manifest.SchemaVersion) + require.Equal(t, StatusOK, manifest.Status) + require.Empty(t, manifest.Warnings) + + shard := manifest.Shards[testRSA] + require.NotNil(t, shard.Instance) + require.Equal(t, "data/rs-a.tar.zst", shard.Instance.Artifact.Path) + require.Equal(t, int64(42), shard.Instance.Artifact.SizeBytes) + require.Equal(t, "zstd", shard.Instance.Artifact.Compression) + require.Len(t, shard.Instance.Artifact.RecoveryPoints, 2) + require.Equal(t, 2300*time.Millisecond, manifest.CreationDuration) +} + +func TestAggregateUnavailableShard(t *testing.T) { + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{{ReplicasetUUID: testRSA}}, + }) + require.NoError(t, err) + require.Equal(t, StatusFailed, manifest.Status) + require.Equal(t, "shard unreachable", manifest.Shards[testRSA].Error) + require.Len(t, manifest.Warnings, 1) + require.Equal(t, WarnShardUnreachable, manifest.Warnings[0].Code) +} + +func TestAggregateNilRecoveryPointsAddsWarningAndEmptySlice(t *testing.T) { + fragment := mustDecodeFragment(t, fixtureFragmentWithoutRecoveryPoints) + + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{{ReplicasetUUID: testRSA, Fragment: &fragment}}, + }) + require.NoError(t, err) + require.Equal(t, StatusDegraded, manifest.Status) + require.Len(t, manifest.Warnings, 1) + require.Equal(t, WarnRecoveryPointsUnavailable, manifest.Warnings[0].Code) + require.NotNil(t, manifest.Shards[testRSA].Instance.Artifact.RecoveryPoints) + require.Empty(t, manifest.Shards[testRSA].Instance.Artifact.RecoveryPoints) +} + +func TestAggregateEmptyRecoveryPointsDoesNotAddWarning(t *testing.T) { + fragment := mustDecodeFragment(t, fixtureFragmentWithEmptyRecoveryPoints) + + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{{ReplicasetUUID: testRSA, Fragment: &fragment}}, + }) + require.NoError(t, err) + require.Equal(t, StatusOK, manifest.Status) + require.Empty(t, manifest.Warnings) + require.NotNil(t, manifest.Shards[testRSA].Instance.Artifact.RecoveryPoints) +} + +func TestAggregateShardErrorUsesErrorShard(t *testing.T) { + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{{ + ReplicasetUUID: testRSA, + Err: errors.New("timeout: replicaset unreachable"), + }}, + }) + require.NoError(t, err) + require.Equal(t, "timeout: replicaset unreachable", manifest.Shards[testRSA].Error) + require.Equal(t, StatusFailed, manifest.Status) + require.Empty(t, manifest.Warnings) +} + +func TestAggregateRejectsInvalidFragment(t *testing.T) { + fragment := mustDecodeFragment(t, fixtureFragmentA) + fragment.Type = BackupType("bad") + + _, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSA), + Shards: []*ShardInput{{ReplicasetUUID: testRSA, Fragment: &fragment}}, + }) + require.ErrorContains(t, err, "invalid backup type") +} + +func TestAggregateRejectsReplicasetMismatch(t *testing.T) { + fragment := mustDecodeFragment(t, fixtureFragmentA) + + _, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: time.Second, + Topology: topologyFromClusterManifestFixture(t, testRSB), + Shards: []*ShardInput{{ReplicasetUUID: testRSB, Fragment: &fragment}}, + }) + require.ErrorContains(t, err, "does not match shard input") +} + +func TestAggregateBuildsClusterManifest(t *testing.T) { + fragmentA := mustDecodeFragment(t, fixtureFragmentA) + fragmentB := mustDecodeFragment(t, fixtureFragmentB) + + manifest, err := Aggregate(AggregateInput{ + BackupID: testBackupID, + BaseFullBackupID: testBackupID, + CreationTime: testCreationTime(), + CreationDuration: 2300 * time.Millisecond, + Topology: topologyFromClusterManifestFixture(t, testRSA, testRSB, testRSC), + Shards: []*ShardInput{ + { + ReplicasetUUID: testRSA, + Fragment: &fragmentA, + Location: &ArtifactLocation{ + Path: "20260312T120000Z-replicaset_A_uuid.tar.zst", + SizeBytes: 104857600, + }, + }, + { + ReplicasetUUID: testRSB, + Fragment: &fragmentB, + Location: &ArtifactLocation{ + Path: "20260312T120000Z-replicaset_B_uuid.tar.zst", + SizeBytes: 98304000, + }, + }, + {ReplicasetUUID: testRSC, Err: errors.New("timeout: replicaset unreachable")}, + }, + }) + require.NoError(t, err) + require.NoError(t, manifest.Validate()) + require.Equal(t, StatusDegraded, manifest.Status) + require.Equal(t, "timeout: replicaset unreachable", manifest.Shards[testRSC].Error) + require.Equal( + t, + manifest.Shards[testRSA].Instance.Artifact.RecoveryPoints[0].UUID, + manifest.Shards[testRSB].Instance.Artifact.RecoveryPoints[0].UUID, + ) +} + +func topologyFromClusterManifestFixture(t *testing.T, replicasetUUIDs ...string) Topology { + t.Helper() + + manifest := mustDecodeClusterManifest(t, fixtureClusterManifest) + topology := Topology{Replicasets: make(map[string][]TopologyInstance, len(replicasetUUIDs))} + for _, replicasetUUID := range replicasetUUIDs { + topology.Replicasets[replicasetUUID] = manifest.Topology.Replicasets[replicasetUUID] + } + return topology +} + +func testCreationTime() time.Time { + return time.Date(2026, 3, 12, 12, 0, 2, 456000000, time.UTC) +} diff --git a/lib/backup/fixtures_test.go b/lib/backup/fixtures_test.go new file mode 100644 index 000000000..797203089 --- /dev/null +++ b/lib/backup/fixtures_test.go @@ -0,0 +1,69 @@ +package backup + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + fixtureClusterManifest = "cluster_manifest.json" + fixtureFragmentA = "fragment_a.json" + fixtureFragmentB = "fragment_b.json" + fixtureFragmentWithoutRecoveryPoints = "fragment_without_recovery_points.json" + fixtureFragmentWithEmptyRecoveryPoints = "fragment_with_empty_recovery_points.json" +) + +func mustReadFixture(t *testing.T, name string) []byte { + t.Helper() + + data, err := os.ReadFile(filepath.Join("testdata", name)) + require.NoError(t, err) + return data +} + +func mustDecodeClusterManifest(t *testing.T, name string) ClusterManifest { + t.Helper() + + type clusterManifestJSON struct { + SchemaVersion int `json:"schema_version"` + BackupID BackupID `json:"backup_id"` + PreviousBackupID BackupID `json:"previous_backup_id"` + BaseFullBackupID BackupID `json:"base_full_backup_id"` + Status Status `json:"status"` + CreationTime time.Time `json:"creation_time"` + CreationDuration string `json:"creation_duration"` + Shards map[string]Shard `json:"shards"` + Topology Topology `json:"topology"` + Warnings []Warning `json:"warnings"` + } + + var raw clusterManifestJSON + require.NoError(t, json.Unmarshal(mustReadFixture(t, name), &raw)) + duration, err := time.ParseDuration(raw.CreationDuration) + require.NoError(t, err) + return ClusterManifest{ + SchemaVersion: raw.SchemaVersion, + BackupID: raw.BackupID, + PreviousBackupID: raw.PreviousBackupID, + BaseFullBackupID: raw.BaseFullBackupID, + Status: raw.Status, + CreationTime: raw.CreationTime, + CreationDuration: duration, + Shards: raw.Shards, + Topology: raw.Topology, + Warnings: raw.Warnings, + } +} + +func mustDecodeFragment(t *testing.T, name string) Fragment { + t.Helper() + + var fragment Fragment + require.NoError(t, json.Unmarshal(mustReadFixture(t, name), &fragment)) + return fragment +} diff --git a/lib/backup/manifest.go b/lib/backup/manifest.go new file mode 100644 index 000000000..4fa5913c5 --- /dev/null +++ b/lib/backup/manifest.go @@ -0,0 +1,59 @@ +package backup + +import "time" + +// BackupID is an opaque identifier of a backup in a chain. +type BackupID string + +// ClusterManifest is the complete cluster-level backup manifest. +type ClusterManifest struct { + SchemaVersion int `json:"schema_version"` + BackupID BackupID `json:"backup_id"` + PreviousBackupID BackupID `json:"previous_backup_id"` + BaseFullBackupID BackupID `json:"base_full_backup_id"` + Status Status `json:"status"` + CreationTime time.Time `json:"creation_time"` + CreationDuration time.Duration `json:"creation_duration"` + Shards map[string]Shard `json:"shards"` // Key is replicaset. + Topology Topology `json:"topology"` + Warnings []Warning `json:"warnings"` // Empty is [] +} + +// Shard is one replicaset result: either a backed-up instance or an error. +type Shard struct { + Instance *ShardInstance `json:"instance,omitempty"` + Error string `json:"error,omitempty"` +} + +// ShardInstance contains successful backup metadata for one instance. +type ShardInstance struct { + InstanceUUID string `json:"instance_uuid"` + InstanceName string `json:"instance_name"` + Hostname string `json:"hostname"` + VclockBegin Vclock `json:"vclock_begin"` + VclockEnd Vclock `json:"vclock_end"` + Artifact Artifact `json:"artifact"` +} + +// Artifact describes the stored archive produced for a shard. +type Artifact struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + ChecksumSHA256 string `json:"checksum_sha256"` + Compression string `json:"compression"` + Files []string `json:"files"` + RecoveryPoints []RecoveryPoint `json:"recovery_points"` + Type BackupType `json:"type"` +} + +// Topology lists expected instances grouped by replicaset UUID. +type Topology struct { + Replicasets map[string][]TopologyInstance `json:"replicasets"` +} + +// TopologyInstance is one expected instance in cluster topology. +type TopologyInstance struct { + InstanceUUID string `json:"instance_uuid"` + InstanceName string `json:"instance_name"` + Hostname string `json:"hostname"` +} diff --git a/lib/backup/testdata/cluster_manifest.json b/lib/backup/testdata/cluster_manifest.json new file mode 100644 index 000000000..646f251b7 --- /dev/null +++ b/lib/backup/testdata/cluster_manifest.json @@ -0,0 +1,109 @@ +{ + "schema_version": 1, + "backup_id": "20260312T120000Z", + "previous_backup_id": null, + "base_full_backup_id": "20260312T120000Z", + "status": "OK", + "creation_time": "2026-03-12T12:00:02.456Z", + "creation_duration": "2.3s", + "shards": { + "11111111-1111-1111-1111-111111111111": { + "instance": { + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000001", + "instance_name": "router-001", + "hostname": "tarantool-node-01.prod.example.com", + "vclock_begin": {"1": 1500, "2": 230}, + "vclock_end": {"1": 1502, "2": 230}, + "artifact": { + "path": "20260312T120000Z-replicaset_A_uuid.tar.zst", + "size_bytes": 104857600, + "checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92400000000000000000000000000000000", + "compression": "zstd", + "files": [ + "00000000000000001500.snap", + "00000000000000001500.xlog" + ], + "recovery_points": [ + {"uuid": "550e8400-e29b-41d4-a716-446655440000", "replica_id": 1, "lsn": 1501, "timestamp": 1741780500}, + {"uuid": "6ba7b810-9dad-11d1-80b4-00c04fd430c8", "replica_id": 1, "lsn": 1502, "timestamp": 1741780780} + ], + "type": "full" + } + } + }, + "22222222-2222-2222-2222-222222222222": { + "instance": { + "instance_uuid": "bbbbbbbb-0000-0000-0000-000000000001", + "instance_name": "storage-001", + "hostname": "tarantool-node-03.prod.example.com", + "vclock_begin": {"3": 800, "4": 110}, + "vclock_end": {"3": 801, "4": 110}, + "artifact": { + "path": "20260312T120000Z-replicaset_B_uuid.tar.zst", + "size_bytes": 98304000, + "checksum_sha256": "d7a8fbb307d7809469d49b82694fae5700000000000000000000000000000000", + "compression": "zstd", + "files": [ + "00000000000000000800.snap", + "00000000000000000800.xlog" + ], + "recovery_points": [ + {"uuid": "550e8400-e29b-41d4-a716-446655440000", "replica_id": 3, "lsn": 800, "timestamp": 1741780501}, + {"uuid": "6ba7b810-9dad-11d1-80b4-00c04fd430c8", "replica_id": 3, "lsn": 801, "timestamp": 1741780781} + ], + "type": "full" + } + } + }, + "33333333-3333-3333-3333-333333333333": { + "error": "timeout: replicaset unreachable" + } + }, + "topology": { + "replicasets": { + "11111111-1111-1111-1111-111111111111": [ + { + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000001", + "instance_name": "router-001", + "hostname": "tarantool-node-01.prod.example.com" + }, + { + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000002", + "instance_name": "router-002", + "hostname": "tarantool-node-02.prod.example.com" + } + ], + "22222222-2222-2222-2222-222222222222": [ + { + "instance_uuid": "bbbbbbbb-0000-0000-0000-000000000001", + "instance_name": "storage-001", + "hostname": "tarantool-node-03.prod.example.com" + }, + { + "instance_uuid": "bbbbbbbb-0000-0000-0000-000000000002", + "instance_name": "storage-002", + "hostname": "tarantool-node-04.prod.example.com" + } + ], + "33333333-3333-3333-3333-333333333333": [ + { + "instance_uuid": "cccccccc-0000-0000-0000-000000000001", + "instance_name": "storage-003", + "hostname": "tarantool-node-05.prod.example.com" + }, + { + "instance_uuid": "cccccccc-0000-0000-0000-000000000002", + "instance_name": "storage-004", + "hostname": "tarantool-node-06.prod.example.com" + } + ] + } + }, + "warnings": [ + { + "code": "shard_unreachable", + "message": "replicaset C did not respond within timeout", + "details": {"replicaset_uuid": "33333333-3333-3333-3333-333333333333"} + } + ] +} diff --git a/lib/backup/testdata/fragment_a.json b/lib/backup/testdata/fragment_a.json new file mode 100644 index 000000000..654f7aa04 --- /dev/null +++ b/lib/backup/testdata/fragment_a.json @@ -0,0 +1,18 @@ +{ + "replicaset_uuid": "11111111-1111-1111-1111-111111111111", + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000001", + "instance_name": "router-001", + "hostname": "tarantool-node-01.prod.example.com", + "type": "full", + "vclock_begin": {"1": 1500, "2": 230}, + "vclock_end": {"1": 1502, "2": 230}, + "files": [ + "00000000000000001500.snap", + "00000000000000001500.xlog" + ], + "checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92400000000000000000000000000000000", + "recovery_points": [ + {"uuid": "550e8400-e29b-41d4-a716-446655440000", "replica_id": 1, "lsn": 1501, "timestamp": 1741780500}, + {"uuid": "6ba7b810-9dad-11d1-80b4-00c04fd430c8", "replica_id": 1, "lsn": 1502, "timestamp": 1741780780} + ] +} diff --git a/lib/backup/testdata/fragment_b.json b/lib/backup/testdata/fragment_b.json new file mode 100644 index 000000000..ed0db2117 --- /dev/null +++ b/lib/backup/testdata/fragment_b.json @@ -0,0 +1,18 @@ +{ + "replicaset_uuid": "22222222-2222-2222-2222-222222222222", + "instance_uuid": "bbbbbbbb-0000-0000-0000-000000000001", + "instance_name": "storage-001", + "hostname": "tarantool-node-03.prod.example.com", + "type": "full", + "vclock_begin": {"3": 800, "4": 110}, + "vclock_end": {"3": 801, "4": 110}, + "files": [ + "00000000000000000800.snap", + "00000000000000000800.xlog" + ], + "checksum_sha256": "d7a8fbb307d7809469d49b82694fae5700000000000000000000000000000000", + "recovery_points": [ + {"uuid": "550e8400-e29b-41d4-a716-446655440000", "replica_id": 3, "lsn": 800, "timestamp": 1741780501}, + {"uuid": "6ba7b810-9dad-11d1-80b4-00c04fd430c8", "replica_id": 3, "lsn": 801, "timestamp": 1741780781} + ] +} diff --git a/lib/backup/testdata/fragment_with_empty_recovery_points.json b/lib/backup/testdata/fragment_with_empty_recovery_points.json new file mode 100644 index 000000000..0e4785a92 --- /dev/null +++ b/lib/backup/testdata/fragment_with_empty_recovery_points.json @@ -0,0 +1,15 @@ +{ + "replicaset_uuid": "11111111-1111-1111-1111-111111111111", + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000001", + "instance_name": "router-001", + "hostname": "tarantool-node-01.prod.example.com", + "type": "full", + "vclock_begin": {"0": 12, "1": 1500}, + "vclock_end": {"0": 12, "1": 1502}, + "files": [ + "00000000000000001500.snap", + "00000000000000001500.xlog" + ], + "checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92400000000000000000000000000000000", + "recovery_points": [] +} diff --git a/lib/backup/testdata/fragment_without_recovery_points.json b/lib/backup/testdata/fragment_without_recovery_points.json new file mode 100644 index 000000000..d5de47526 --- /dev/null +++ b/lib/backup/testdata/fragment_without_recovery_points.json @@ -0,0 +1,14 @@ +{ + "replicaset_uuid": "11111111-1111-1111-1111-111111111111", + "instance_uuid": "aaaaaaaa-0000-0000-0000-000000000001", + "instance_name": "router-001", + "hostname": "tarantool-node-01.prod.example.com", + "type": "full", + "vclock_begin": {"0": 12, "1": 1500}, + "vclock_end": {"0": 12, "1": 1502}, + "files": [ + "00000000000000001500.snap", + "00000000000000001500.xlog" + ], + "checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92400000000000000000000000000000000" +} diff --git a/lib/backup/types.go b/lib/backup/types.go new file mode 100644 index 000000000..4a9943308 --- /dev/null +++ b/lib/backup/types.go @@ -0,0 +1,29 @@ +package backup + +// SchemaVersion is the current cluster manifest JSON schema version. +const SchemaVersion = 1 + +const ( + // BackupTypeFull marks a complete backup chain starting point. + BackupTypeFull BackupType = "full" + // BackupTypeIncremental marks a backup based on a previous one. + BackupTypeIncremental BackupType = "incremental" +) + +const ( + // StatusOK means all expected shards were backed up without warnings. + StatusOK Status = "OK" + // StatusDegraded means some data exists, but the backup has warnings or shard errors. + StatusDegraded Status = "degraded" + // StatusFailed means no shard was successfully backed up. + StatusFailed Status = "failed" +) + +// Vclock maps replica IDs to their LSNs, including replica 0. +type Vclock map[uint32]uint64 + +// BackupType is the backup mode: full or incremental. +type BackupType string + +// Status is the aggregate health of the cluster backup. +type Status string diff --git a/lib/backup/validation.go b/lib/backup/validation.go new file mode 100644 index 000000000..8c450c4a6 --- /dev/null +++ b/lib/backup/validation.go @@ -0,0 +1,179 @@ +package backup + +import "fmt" + +// Validate checks that a fragment has required structural fields. +func (fragment Fragment) Validate() error { + if fragment.ReplicasetUUID == "" { + return fmt.Errorf("replicaset_uuid is empty") + } + if fragment.InstanceUUID == "" { + return fmt.Errorf("instance_uuid is empty") + } + if fragment.InstanceName == "" { + return fmt.Errorf("instance_name is empty") + } + if fragment.Hostname == "" { + return fmt.Errorf("hostname is empty") + } + if !isValidBackupType(fragment.Type) { + return fmt.Errorf("invalid backup type %q", fragment.Type) + } + if len(fragment.VclockBegin) == 0 { + return fmt.Errorf("vclock_begin is empty") + } + if len(fragment.VclockEnd) == 0 { + return fmt.Errorf("vclock_end is empty") + } + + return nil +} + +// Validate checks manifest structure without chain or storage verification. +func (manifest ClusterManifest) Validate() error { + if err := manifest.validateHeader(); err != nil { + return fmt.Errorf("validate manifest header: %w", err) + } + if err := manifest.validateShards(); err != nil { + return fmt.Errorf("validate manifest shards: %w", err) + } + if err := manifest.validateWarnings(); err != nil { + return fmt.Errorf("validate manifest warnings: %w", err) + } + return nil +} + +// validateHeader checks top-level manifest fields. +func (manifest ClusterManifest) validateHeader() error { + if manifest.SchemaVersion != SchemaVersion { + return fmt.Errorf("unsupported schema_version %d", manifest.SchemaVersion) + } + if manifest.BackupID == "" { + return fmt.Errorf("backup_id is empty") + } + if manifest.BaseFullBackupID == "" { + return fmt.Errorf("base_full_backup_id is empty") + } + if !isValidStatus(manifest.Status) { + return fmt.Errorf("invalid status %q", manifest.Status) + } + if manifest.Shards == nil { + return fmt.Errorf("shards is nil") + } + if manifest.Topology.Replicasets == nil { + return fmt.Errorf("topology.replicasets is nil") + } + if manifest.Warnings == nil { + return fmt.Errorf("warnings is nil") + } + return nil +} + +// validateShards checks shard keys and shard payloads. +func (manifest ClusterManifest) validateShards() error { + for replicasetUUID, shard := range manifest.Shards { + if err := manifest.validateShard(replicasetUUID, shard); err != nil { + return fmt.Errorf("validate shard %q: %w", replicasetUUID, err) + } + } + return nil +} + +// validateShard checks one shard entry. +func (manifest ClusterManifest) validateShard(replicasetUUID string, shard Shard) error { + if replicasetUUID == "" { + return fmt.Errorf("shards contains empty replicaset uuid") + } + if _, ok := manifest.Topology.Replicasets[replicasetUUID]; !ok { + return fmt.Errorf("shard %q is not present in topology", replicasetUUID) + } + + hasInstance := shard.Instance != nil + hasError := shard.Error != "" + if hasInstance == hasError { + return fmt.Errorf( + "shard %q must contain exactly one of instance or error", + replicasetUUID, + ) + } + if hasInstance { + if err := validateShardInstance(replicasetUUID, *shard.Instance); err != nil { + return fmt.Errorf("validate shard %q instance: %w", replicasetUUID, err) + } + } + return nil +} + +// validateWarnings checks warning codes. +func (manifest ClusterManifest) validateWarnings() error { + for i, warning := range manifest.Warnings { + if !isValidWarningCode(warning.Code) { + return fmt.Errorf("warnings[%d] has invalid code %q", i, warning.Code) + } + } + return nil +} + +// validateShardInstance checks successful shard metadata. +func validateShardInstance(replicasetUUID string, instance ShardInstance) error { + if instance.InstanceUUID == "" { + return fmt.Errorf("shard %q instance_uuid is empty", replicasetUUID) + } + if instance.InstanceName == "" { + return fmt.Errorf("shard %q instance_name is empty", replicasetUUID) + } + if instance.Hostname == "" { + return fmt.Errorf("shard %q hostname is empty", replicasetUUID) + } + if len(instance.VclockBegin) == 0 { + return fmt.Errorf("shard %q vclock_begin is empty", replicasetUUID) + } + if len(instance.VclockEnd) == 0 { + return fmt.Errorf("shard %q vclock_end is empty", replicasetUUID) + } + if !isValidBackupType(instance.Artifact.Type) { + return fmt.Errorf( + "shard %q has invalid backup type %q", + replicasetUUID, + instance.Artifact.Type, + ) + } + if instance.Artifact.RecoveryPoints == nil { + return fmt.Errorf("shard %q artifact.recovery_points is nil", replicasetUUID) + } + + return nil +} + +// isValidBackupType reports whether backupType is known by this schema. +func isValidBackupType(backupType BackupType) bool { + switch backupType { + case BackupTypeFull, BackupTypeIncremental: + return true + default: + return false + } +} + +// isValidStatus reports whether status is known by this schema. +func isValidStatus(status Status) bool { + switch status { + case StatusOK, StatusDegraded, StatusFailed: + return true + default: + return false + } +} + +// isValidWarningCode reports whether code is known by this schema. +func isValidWarningCode(code WarningCode) bool { + switch code { + case WarnShardPartial, + WarnShardUnreachable, + WarnRecoveryPointsUnavailable, + WarnStoragePartialUpload: + return true + default: + return false + } +} diff --git a/lib/backup/validation_test.go b/lib/backup/validation_test.go new file mode 100644 index 000000000..c8c02eeba --- /dev/null +++ b/lib/backup/validation_test.go @@ -0,0 +1,274 @@ +package backup + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + testBackupID BackupID = "20260312T120000Z" + testRSA string = "11111111-1111-1111-1111-111111111111" + testRSB string = "22222222-2222-2222-2222-222222222222" + testRSC string = "33333333-3333-3333-3333-333333333333" +) + +func TestFragmentValidate(t *testing.T) { + valid := mustDecodeFragment(t, fixtureFragmentA) + + tests := []struct { + name string + fragment Fragment + wantError string + }{ + { + name: "valid", + fragment: valid, + }, + { + name: "empty replicaset uuid", + fragment: func() Fragment { + fragment := valid + fragment.ReplicasetUUID = "" + return fragment + }(), + wantError: "replicaset_uuid is empty", + }, + { + name: "empty instance uuid", + fragment: func() Fragment { + fragment := valid + fragment.InstanceUUID = "" + return fragment + }(), + wantError: "instance_uuid is empty", + }, + { + name: "invalid type", + fragment: func() Fragment { + fragment := valid + fragment.Type = BackupType("snapshot") + return fragment + }(), + wantError: "invalid backup type", + }, + { + name: "empty vclock begin", + fragment: func() Fragment { + fragment := valid + fragment.VclockBegin = nil + return fragment + }(), + wantError: "vclock_begin is empty", + }, + { + name: "empty vclock end", + fragment: func() Fragment { + fragment := valid + fragment.VclockEnd = Vclock{} + return fragment + }(), + wantError: "vclock_end is empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.fragment.Validate() + if tt.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantError) + } + }) + } +} + +func TestClusterManifestValidate(t *testing.T) { + valid := func() ClusterManifest { return testClusterManifest(t, StatusOK) } + + tests := []struct { + name string + manifest ClusterManifest + wantError string + }{ + { + name: "valid", + manifest: valid(), + }, + { + name: "foreign schema version", + manifest: func() ClusterManifest { + manifest := valid() + manifest.SchemaVersion = 2 + return manifest + }(), + wantError: "unsupported schema_version 2", + }, + { + name: "empty backup id", + manifest: func() ClusterManifest { + manifest := valid() + manifest.BackupID = "" + return manifest + }(), + wantError: "backup_id is empty", + }, + { + name: "invalid status", + manifest: func() ClusterManifest { + manifest := valid() + manifest.Status = Status("partial") + return manifest + }(), + wantError: "invalid status", + }, + { + name: "shard has both instance and error", + manifest: func() ClusterManifest { + manifest := valid() + shard := manifest.Shards[testRSA] + shard.Error = "unexpected error" + manifest.Shards[testRSA] = shard + return manifest + }(), + wantError: "must contain exactly one of instance or error", + }, + { + name: "shard has neither instance nor error", + manifest: func() ClusterManifest { + manifest := valid() + manifest.Shards[testRSA] = Shard{} + return manifest + }(), + wantError: "must contain exactly one of instance or error", + }, + { + name: "orphan shard outside topology", + manifest: func() ClusterManifest { + manifest := valid() + manifest.Shards["99999999-9999-9999-9999-999999999999"] = manifest.Shards[testRSA] + return manifest + }(), + wantError: "is not present in topology", + }, + { + name: "successful shard with empty vclock begin", + manifest: func() ClusterManifest { + manifest := valid() + shard := manifest.Shards[testRSA] + shard.Instance.VclockBegin = nil + manifest.Shards[testRSA] = shard + return manifest + }(), + wantError: "vclock_begin is empty", + }, + { + name: "invalid artifact type", + manifest: func() ClusterManifest { + manifest := valid() + shard := manifest.Shards[testRSA] + shard.Instance.Artifact.Type = BackupType("bad") + manifest.Shards[testRSA] = shard + return manifest + }(), + wantError: "invalid backup type", + }, + { + name: "invalid warning code", + manifest: func() ClusterManifest { + manifest := valid() + manifest.Warnings = []Warning{{Code: WarningCode("unknown")}} + return manifest + }(), + wantError: "invalid code", + }, + { + name: "nil warnings", + manifest: func() ClusterManifest { + manifest := valid() + manifest.Warnings = nil + return manifest + }(), + wantError: "warnings is nil", + }, + { + name: "nil recovery points", + manifest: func() ClusterManifest { + manifest := valid() + shard := manifest.Shards[testRSA] + shard.Instance.Artifact.RecoveryPoints = nil + manifest.Shards[testRSA] = shard + return manifest + }(), + wantError: "artifact.recovery_points is nil", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.manifest.Validate() + if tt.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantError) + } + }) + } +} + +func TestClusterManifestIsValidAndSerializable(t *testing.T) { + manifest := mustDecodeClusterManifest(t, fixtureClusterManifest) + require.NoError(t, manifest.Validate()) + + data, err := json.Marshal(manifest) + require.NoError(t, err) + + jsonText := string(data) + for _, want := range []string{ + `"schema_version":1`, + `"previous_backup_id":""`, + `"creation_duration":2300000000`, + `"status":"OK"`, + `"550e8400-e29b-41d4-a716-446655440000"`, + `"6ba7b810-9dad-11d1-80b4-00c04fd430c8"`, + `"error":"timeout: replicaset unreachable"`, + `"code":"shard_unreachable"`, + } { + require.Contains(t, jsonText, want) + } + + type clusterManifestRoundTrip struct { + SchemaVersion int `json:"schema_version"` + BackupID BackupID `json:"backup_id"` + PreviousBackupID BackupID `json:"previous_backup_id"` + BaseFullBackupID BackupID `json:"base_full_backup_id"` + Status Status `json:"status"` + CreationTime time.Time `json:"creation_time"` + CreationDuration time.Duration `json:"creation_duration"` + Shards map[string]Shard `json:"shards"` + Topology Topology `json:"topology"` + Warnings []Warning `json:"warnings"` + } + + var roundTrip clusterManifestRoundTrip + require.NoError(t, json.Unmarshal(data, &roundTrip)) + decoded := ClusterManifest(roundTrip) + require.NoError(t, decoded.Validate()) + require.Equal(t, 2300*time.Millisecond, decoded.CreationDuration) +} + +func testClusterManifest(t *testing.T, status Status) ClusterManifest { + t.Helper() + + manifest := mustDecodeClusterManifest(t, fixtureClusterManifest) + manifest.Status = status + delete(manifest.Shards, testRSB) + delete(manifest.Shards, testRSC) + delete(manifest.Topology.Replicasets, testRSB) + delete(manifest.Topology.Replicasets, testRSC) + manifest.Warnings = []Warning{} + return manifest +} diff --git a/lib/backup/warnings.go b/lib/backup/warnings.go new file mode 100644 index 000000000..3d6ab6745 --- /dev/null +++ b/lib/backup/warnings.go @@ -0,0 +1,67 @@ +package backup + +// WarningCode identifies a non-fatal manifest issue. +type WarningCode string + +const ( + // WarnShardPartial marks a shard backup that completed only partially. + WarnShardPartial WarningCode = "shard_partial" + // WarnShardUnreachable marks a shard that did not produce a fragment. + WarnShardUnreachable WarningCode = "shard_unreachable" + // WarnRecoveryPointsUnavailable marks a missing recovery_points field. + WarnRecoveryPointsUnavailable WarningCode = "recovery_points_unavailable" + // WarnStoragePartialUpload marks archives that were not fully uploaded. + WarnStoragePartialUpload WarningCode = "storage_partial_upload" +) + +// Warning is a typed, serializable non-fatal backup issue. +type Warning struct { + Code WarningCode `json:"code"` + Message string `json:"message"` + Details map[string]any `json:"details"` +} + +// NewShardPartialWarning reports a partial shard backup. +func NewShardPartialWarning(replicasetUUID, instanceUUID, reason string) Warning { + return Warning{ + Code: WarnShardPartial, + Message: reason, + Details: map[string]any{ + "replicaset_uuid": replicasetUUID, + "instance_uuid": instanceUUID, + }, + } +} + +// NewShardUnreachableWarning reports a missing shard fragment. +func NewShardUnreachableWarning(replicasetUUID string) Warning { + return Warning{ + Code: WarnShardUnreachable, + Message: "shard unreachable", + Details: map[string]any{ + "replicaset_uuid": replicasetUUID, + }, + } +} + +// NewRecoveryPointsUnavailableWarning reports unavailable recovery points. +func NewRecoveryPointsUnavailableWarning(replicasetUUID, errMsg string) Warning { + return Warning{ + Code: WarnRecoveryPointsUnavailable, + Message: errMsg, + Details: map[string]any{ + "replicaset_uuid": replicasetUUID, + }, + } +} + +// NewStoragePartialUploadWarning reports storage keys that failed to upload. +func NewStoragePartialUploadWarning(keys []string) Warning { + return Warning{ + Code: WarnStoragePartialUpload, + Message: "storage partial upload", + Details: map[string]any{ + "keys": keys, + }, + } +} From 95376cf76cf06a09909a998fe96f1b5c1322c94f Mon Sep 17 00:00:00 2001 From: sssciel Date: Mon, 22 Jun 2026 14:09:37 +0300 Subject: [PATCH 2/2] backup: add storage interface and fs/s3 impl Implement local filesystem and S3-compatible backends. Closes TNTP-8210 --- .cspell_project-words.txt | 2 + go.mod | 44 +++- go.sum | 94 +++++-- lib/backup/storage/fs/fs.go | 263 +++++++++++++++++++ lib/backup/storage/fs/fs_test.go | 198 ++++++++++++++ lib/backup/storage/keys.go | 23 ++ lib/backup/storage/s3/s3.go | 196 ++++++++++++++ lib/backup/storage/s3/s3_integration_test.go | 209 +++++++++++++++ lib/backup/storage/s3/s3_test.go | 137 ++++++++++ lib/backup/storage/storage.go | 116 ++++++++ lib/backup/storage/storage_test.go | 150 +++++++++++ test/integration/aeon/server/go.mod | 6 +- test/integration/aeon/server/go.sum | 12 +- 13 files changed, 1404 insertions(+), 46 deletions(-) create mode 100644 lib/backup/storage/fs/fs.go create mode 100644 lib/backup/storage/fs/fs_test.go create mode 100644 lib/backup/storage/keys.go create mode 100644 lib/backup/storage/s3/s3.go create mode 100644 lib/backup/storage/s3/s3_integration_test.go create mode 100644 lib/backup/storage/s3/s3_test.go create mode 100644 lib/backup/storage/storage.go create mode 100644 lib/backup/storage/storage_test.go diff --git a/.cspell_project-words.txt b/.cspell_project-words.txt index 7b47e0449..fe1999d4a 100644 --- a/.cspell_project-words.txt +++ b/.cspell_project-words.txt @@ -21,6 +21,7 @@ demoter disbalance distfile* drwxr +dxflrs equalf errf etcd* @@ -30,6 +31,7 @@ falsef finalizer* flightrec getfixturevalue +GKTTBACKUPTEST golangci heywoodlh iconfig diff --git a/go.mod b/go.mod index 941062da7..14733543a 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,11 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/mattn/go-isatty v0.0.20 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d + github.com/minio/minio-go/v7 v7.0.95 github.com/mitchellh/mapstructure v1.5.0 github.com/moby/go-archive v0.2.0 - github.com/moby/moby/api v1.54.0 - github.com/moby/moby/client v0.3.0 + github.com/moby/moby/api v1.54.2 + github.com/moby/moby/client v0.4.0 github.com/moby/term v0.5.2 github.com/nxadm/tail v1.4.11 github.com/otiai10/copy v1.14.1 @@ -36,15 +37,16 @@ require ( github.com/tarantool/tt/lib/connect v0.0.0-0 github.com/tarantool/tt/lib/dial v0.0.0-0 github.com/tarantool/tt/lib/integrity v0.0.0 + github.com/testcontainers/testcontainers-go v0.43.0 github.com/vmihailenco/msgpack/v5 v5.4.1 github.com/yuin/gopher-lua v1.1.1 go.etcd.io/etcd/client/pkg/v3 v3.6.8 go.etcd.io/etcd/client/v3 v3.6.8 go.etcd.io/etcd/tests/v3 v3.6.8 - golang.org/x/crypto v0.49.0 + golang.org/x/crypto v0.51.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa - golang.org/x/sys v0.42.0 - golang.org/x/term v0.41.0 + golang.org/x/sys v0.45.0 + golang.org/x/term v0.43.0 google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 @@ -52,6 +54,7 @@ require ( ) require ( + dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/FZambia/tarantool v0.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -61,6 +64,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/c-bata/go-prompt v0.2.5 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cheggaaa/pb/v3 v3.1.6 // indirect @@ -69,20 +73,25 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect @@ -97,13 +106,18 @@ require ( github.com/hpcloud/tail v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/klauspost/compress v1.18.4 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect github.com/mattn/go-tty v0.0.7 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect @@ -113,14 +127,18 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/otiai10/mint v1.6.3 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/robfig/config v0.0.0-20141207224736-0f78529c8c7e // indirect + github.com/rs/xid v1.6.0 // indirect github.com/shirou/gopsutil v3.21.2+incompatible // indirect + github.com/shirou/gopsutil/v4 v4.26.5 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/sirupsen/logrus v1.9.4 // indirect @@ -131,11 +149,13 @@ require ( github.com/tarantool/go-openssl v1.2.2 // indirect github.com/tarantool/go-option v1.0.0 // indirect github.com/tarantool/go-tlsdialer v1.0.2 // indirect - github.com/tklauser/go-sysconf v0.3.4 // indirect - github.com/tklauser/numcpus v0.2.1 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/etcdctl/v3 v3.6.8 // indirect @@ -157,9 +177,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools/godoc v0.1.0-deprecated // indirect gonum.org/v1/gonum v0.17.0 // indirect diff --git a/go.sum b/go.sum index 9ccb1dc81..e8689727b 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -35,6 +37,8 @@ github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE5 github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/briandowns/spinner v1.23.2 h1:Zc6ecUnI+YzLmJniCfDNaMbW0Wid1d5+qcTq4L2FW8w= github.com/briandowns/spinner v1.23.2/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -63,10 +67,14 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -86,6 +94,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -101,6 +111,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -109,9 +121,11 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gojuno/minimock/v3 v3.4.7 h1:vhE5zpniyPDRT0DXd5s3DbtZJVlcbmC5k80izYtj9lY= @@ -134,6 +148,7 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -165,8 +180,11 @@ github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7X github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -179,8 +197,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -207,18 +229,24 @@ github.com/mattn/go-tty v0.0.7/go.mod h1:f2i5ZOvXBU/tCABmLmOfzLz9azMo5wdAaElRNnJ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/moby/api v1.54.0 h1:7kbUgyiKcoBhm0UrWbdrMs7RX8dnwzURKVbZGy2GnL0= -github.com/moby/moby/api v1.54.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc= -github.com/moby/moby/client v0.3.0 h1:UUGL5okry+Aomj3WhGt9Aigl3ZOxZGqR7XPo+RLPlKs= -github.com/moby/moby/client v0.3.0/go.mod h1:HJgFbJRvogDQjbM8fqc1MCEm4mIAGMLjXbgwoZp6jCQ= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -249,6 +277,8 @@ github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -257,6 +287,8 @@ github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiK github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -271,10 +303,14 @@ github.com/robfig/config v0.0.0-20141207224736-0f78529c8c7e/go.mod h1:Zerq1qYbCK github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.2+incompatible h1:U+YvJfjCh6MslYlIAXvPtzhW3YZEtc9uncueUNpD/0A= github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v4 v4.26.5 h1:RPcBXkpz7kOj9PqGFQOlBPZHsyaPvPVQc098y9RmCNM= +github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= @@ -301,6 +337,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -331,6 +369,10 @@ github.com/tarantool/go-tarantool/v2 v2.4.2 h1:rkzYtFhLJLA9RDIhjzN93MJBN5PBxHW4+ github.com/tarantool/go-tarantool/v2 v2.4.2/go.mod h1:MTbhdjFc3Jl63Lgi/UJr5D+QbT+QegqOzsNJGmaw7VM= github.com/tarantool/go-tlsdialer v1.0.2 h1:TiOkihvC2ufLbOqJcJLuQ9I7W5bsZtmnT7KHF/t8n4s= github.com/tarantool/go-tlsdialer v1.0.2/go.mod h1:ztE9J5oh4YzNQkSqlpjA0OkY89zHy/BiJJbbSFm+IiQ= +github.com/testcontainers/testcontainers-go v0.43.0 h1:oEQx5MW2DGd9z3AeEQfB2lPM0eLs7ztyaGRu75bFo5A= +github.com/testcontainers/testcontainers-go v0.43.0/go.mod h1:+VxkT2NQnKOZPKi6praMuMKYHYyOGXr0XSBSlSMCzFo= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -338,10 +380,10 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= -github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= -github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= -github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= @@ -358,6 +400,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM= @@ -421,8 +465,8 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= @@ -447,8 +491,8 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -475,8 +519,8 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -488,20 +532,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/lib/backup/storage/fs/fs.go b/lib/backup/storage/fs/fs.go new file mode 100644 index 000000000..aacfe317c --- /dev/null +++ b/lib/backup/storage/fs/fs.go @@ -0,0 +1,263 @@ +package fs + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/tarantool/tt/lib/backup/storage" +) + +// tempFilePattern is the glob pattern for temporary files created during Put. +// Files with names starting with ".tt-backup-" are excluded from List results. +const tempFilePattern = ".tt-backup-*" + +var errPathRequired = errors.New("fs storage path is required") + +// Config describes local filesystem storage configuration. +type Config struct { + Path string + Prefix string +} + +// Storage is a local filesystem backup storage backend. +type Storage struct { + root string +} + +// New opens local filesystem backup storage. +// The root directory is created lazily on the first Put call. +func New(cfg Config) (*Storage, error) { + root := strings.TrimSpace(cfg.Path) + if root == "" { + return nil, errPathRequired + } + + if cfg.Prefix != "" { + prefix, err := storage.CleanPrefix(cfg.Prefix) + if err != nil { + return nil, fmt.Errorf("failed to clean storage prefix %q: %w", cfg.Prefix, err) + } + if prefix != "" { + root = filepath.Join(root, filepath.FromSlash(strings.TrimRight(prefix, "/"))) + } + } + + return &Storage{root: root}, nil +} + +func (s *Storage) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) { + prefix, err := storage.CleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("failed to clean list prefix %q: %w", prefix, err) + } + + root := filepath.Join(s.root, scanDir(prefix)) + + var objects []storage.ObjectInfo + if _, err := os.Stat(root); err != nil { + if errors.Is(err, os.ErrNotExist) { + return objects, nil + } + + return nil, fmt.Errorf("failed to stat prefix %q: %w", prefix, err) + } + + if err := s.walkDir(ctx, root, prefix, &objects); err != nil { + return nil, fmt.Errorf("failed to list prefix %q: %w", prefix, err) + } + + sort.Slice(objects, func(i, j int) bool { + return objects[i].Key < objects[j].Key + }) + + return objects, nil +} + +func (s *Storage) walkDir( + ctx context.Context, + root string, + prefix string, + objects *[]storage.ObjectInfo, +) error { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if d.IsDir() || strings.HasPrefix(d.Name(), + strings.TrimSuffix(tempFilePattern, "*"), + ) { + + return nil + } + + info, err := d.Info() + if err != nil { + return fmt.Errorf("failed to get file info for %q: %w", path, err) + } + + rel, err := filepath.Rel(s.root, path) + if err != nil { + return fmt.Errorf("failed to get relative path for %q: %w", path, err) + } + + key := filepath.ToSlash(rel) + if !strings.HasPrefix(key, prefix) { + return nil + } + + *objects = append(*objects, storage.ObjectInfo{ + Key: key, + Size: info.Size(), + LastModified: info.ModTime(), + }) + + return nil + }) + if err != nil { + return fmt.Errorf("walkdir failed: %w", err) + } + + return nil +} + +func (s *Storage) Get(_ context.Context, key string) (io.ReadCloser, error) { + path, err := s.objectPath(key) + if err != nil { + return nil, fmt.Errorf("failed to resolve object path %q: %w", key, err) + } + + f, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, storage.ErrKeyNotFound + } + return nil, fmt.Errorf("failed to open object %q: %w", key, err) + } + return f, nil +} + +// Put stores an object to the filesystem. The size parameter is unused; +// it exists to satisfy the storage.Storage interface. +func (s *Storage) Put(ctx context.Context, key string, r io.Reader, _ int64) error { + path, err := s.objectPath(key) + if err != nil { + return fmt.Errorf("failed to resolve object path %q: %w", key, err) + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("failed to put object %q: %w", key, err) + } + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("failed to create object directory for %q: %w", key, err) + } + + tmp, err := os.CreateTemp(dir, tempFilePattern) + if err != nil { + return fmt.Errorf("failed to create temporary object %q: %w", key, err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + + if _, err := io.Copy(tmp, ctxReader{ctx, r}); err != nil { + _ = tmp.Close() + return fmt.Errorf("failed to write object %q: %w", key, err) + } + + if err := tmp.Close(); err != nil { + return fmt.Errorf("failed to close object %q: %w", key, err) + } + + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("failed to store object %q: %w", key, err) + } + + return nil +} + +func (s *Storage) Delete(_ context.Context, key string) error { + path, err := s.objectPath(key) + if err != nil { + return fmt.Errorf("failed to resolve object path %q: %w", key, err) + } + + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to delete object %q: %w", key, err) + } + + return nil +} + +func (s *Storage) objectPath(key string) (string, error) { + key, err := storage.CleanKey(key) + if err != nil { + return "", fmt.Errorf("failed to clean object key %q: %w", key, err) + } + + path := filepath.Join(s.root, filepath.FromSlash(key)) + cleanRoot, err := filepath.Abs(s.root) + if err != nil { + return "", fmt.Errorf("failed to resolve storage root %q: %w", s.root, err) + } + + cleanPath, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to resolve object path %q: %w", path, err) + } + + if cleanPath != cleanRoot && !strings.HasPrefix(cleanPath, cleanRoot+string(os.PathSeparator)) { + return "", fmt.Errorf("storage key %q escapes storage root", key) + } + + return path, nil +} + +// ctxReader wraps an io.Reader and checks context cancellation before each Read call. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (cr ctxReader) Read(p []byte) (int, error) { + if err := cr.ctx.Err(); err != nil { + return 0, fmt.Errorf("context canceled: %w", err) + } + + n, err := cr.r.Read(p) + if err != nil { + if err == io.EOF { + return n, io.EOF + } + + return n, fmt.Errorf("read failed: %w", err) + } + + return n, nil +} + +func scanDir(prefix string) string { + if prefix == "" { + return "" + } + + prefixNoTrailingSlash := strings.TrimRight(prefix, "/") + if strings.HasSuffix(prefix, "/") { + return filepath.FromSlash(prefixNoTrailingSlash) + } + if slash := strings.LastIndex(prefixNoTrailingSlash, "/"); slash >= 0 { + return filepath.FromSlash(prefixNoTrailingSlash[:slash]) + } + + return "" +} diff --git a/lib/backup/storage/fs/fs_test.go b/lib/backup/storage/fs/fs_test.go new file mode 100644 index 000000000..c04b3a668 --- /dev/null +++ b/lib/backup/storage/fs/fs_test.go @@ -0,0 +1,198 @@ +package fs + +import ( + "bytes" + "context" + "errors" + "io" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tarantool/tt/lib/backup/storage" +) + +func TestPut(t *testing.T) { + ctx := t.Context() + root := t.TempDir() + s := newTestStorage(t, root) + + key := storage.ArchiveKey("put", "rs1") + data := []byte("archive") + require.NoError(t, s.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + require.FileExists(t, filepath.Join(root, "cluster", "production", key)) +} + +func TestPutWithoutPrefix(t *testing.T) { + ctx := t.Context() + root := t.TempDir() + + s, err := New(Config{Path: root}) + require.NoError(t, err) + + key := storage.ManifestKey("no-prefix") + data := []byte(`{"ok":true}`) + require.NoError(t, s.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + require.FileExists(t, filepath.Join(root, key)) +} + +func TestPutCancelledContext(t *testing.T) { + root := t.TempDir() + s := newTestStorage(t, root) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + key := storage.ManifestKey("cancelled") + err := s.Put(ctx, key, bytes.NewReader([]byte("data")), 4) + require.True(t, errors.Is(err, context.Canceled)) +} + +func TestGet(t *testing.T) { + ctx := t.Context() + s := newTestStorage(t, t.TempDir()) + + key := storage.ManifestKey("get") + data := []byte(`{"ok":true}`) + require.NoError(t, s.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + reader, err := s.Get(ctx, key) + require.NoError(t, err) + defer reader.Close() + + actual, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, data, actual) +} + +func TestGetNotFound(t *testing.T) { + s := newTestStorage(t, t.TempDir()) + + _, err := s.Get(t.Context(), storage.ManifestKey("missing")) + require.True(t, errors.Is(err, storage.ErrKeyNotFound)) +} + +func TestList(t *testing.T) { + ctx := t.Context() + s := newTestStorage(t, t.TempDir()) + + manifestKey := storage.ManifestKey("list") + archiveKey := storage.ArchiveKey("list", "rs1") + manifest := []byte(`{"ok":true}`) + archive := []byte("archive") + require.NoError(t, s.Put(ctx, manifestKey, bytes.NewReader(manifest), int64(len(manifest)))) + require.NoError(t, s.Put(ctx, archiveKey, bytes.NewReader(archive), int64(len(archive)))) + + objects, err := s.List(ctx, storage.ManifestsPrefix()) + require.NoError(t, err) + require.Len(t, objects, 1) + require.Equal(t, manifestKey, objects[0].Key) + require.Equal(t, int64(len(manifest)), objects[0].Size) + require.False(t, objects[0].LastModified.IsZero()) +} + +func TestListEmpty(t *testing.T) { + s := newTestStorage(t, t.TempDir()) + + objects, err := s.List(t.Context(), storage.ManifestsPrefix()) + require.NoError(t, err) + require.Empty(t, objects) +} + +func TestListWithObjectPrefix(t *testing.T) { + ctx := t.Context() + s := newTestStorage(t, t.TempDir()) + + matchingKey := storage.ArchiveKey("backup", "rs1") + otherKey := storage.ArchiveKey("other", "rs1") + require.NoError( + t, + s.Put(ctx, matchingKey, bytes.NewReader([]byte("matching")), int64(len("matching"))), + ) + require.NoError(t, s.Put(ctx, otherKey, bytes.NewReader([]byte("other")), int64(len("other")))) + + objects, err := s.List(ctx, "data/backup") + require.NoError(t, err) + require.Len(t, objects, 1) + require.Equal(t, matchingKey, objects[0].Key) +} + +func TestDelete(t *testing.T) { + ctx := t.Context() + s := newTestStorage(t, t.TempDir()) + + key := storage.ManifestKey("delete") + data := []byte(`{"ok":true}`) + require.NoError(t, s.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + require.NoError(t, s.Delete(ctx, key)) + _, err := s.Get(ctx, key) + require.True(t, errors.Is(err, storage.ErrKeyNotFound)) +} + +func TestDeleteNotFound(t *testing.T) { + s := newTestStorage(t, t.TempDir()) + + err := s.Delete(t.Context(), storage.ManifestKey("missing")) + require.NoError(t, err) +} + +func TestStorageRejectsInvalidKey(t *testing.T) { + s := newTestStorage(t, t.TempDir()) + + err := s.Put(t.Context(), "../escape", nil, 0) + require.True(t, errors.Is(err, storage.ErrInvalidKey)) + + err = s.Put(t.Context(), "data//archive.tar.zst", nil, 0) + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestNewRejectsInvalidPrefix(t *testing.T) { + _, err := New(Config{Path: t.TempDir(), Prefix: "../escape"}) + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestNewRejectsEmptyPath(t *testing.T) { + _, err := New(Config{Path: ""}) + require.True(t, errors.Is(err, errPathRequired)) +} + +func TestConcurrentPut(t *testing.T) { + ctx := t.Context() + s := newTestStorage(t, t.TempDir()) + + const n = 10 + var wg sync.WaitGroup + wg.Add(n) + + for range n { + go func() { + defer wg.Done() + key := storage.ManifestKey("concurrent") + data := []byte("data") + _ = s.Put(ctx, key, bytes.NewReader(data), int64(len(data))) + }() + } + wg.Wait() + + reader, err := s.Get(ctx, storage.ManifestKey("concurrent")) + require.NoError(t, err) + defer reader.Close() + + actual, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, []byte("data"), actual) +} + +func newTestStorage(t *testing.T, root string) *Storage { + t.Helper() + + s, err := New(Config{ + Path: root, + Prefix: "cluster/production/", + }) + require.NoError(t, err) + return s +} diff --git a/lib/backup/storage/keys.go b/lib/backup/storage/keys.go new file mode 100644 index 000000000..9314fcc1e --- /dev/null +++ b/lib/backup/storage/keys.go @@ -0,0 +1,23 @@ +package storage + +import "fmt" + +// ManifestsPrefix returns the relative key prefix for backup manifests. +func ManifestsPrefix() string { + return "manifests/" +} + +// DataPrefix returns the relative key prefix for backup archives. +func DataPrefix() string { + return "data/" +} + +// ManifestKey returns a relative key for a cluster manifest object. +func ManifestKey(backupID string) string { + return fmt.Sprintf("%s%s.json", ManifestsPrefix(), backupID) +} + +// ArchiveKey returns a relative key for a replicaset backup archive object. +func ArchiveKey(backupID, replicasetUUID string) string { + return fmt.Sprintf("%s%s-%s.tar.zst", DataPrefix(), backupID, replicasetUUID) +} diff --git a/lib/backup/storage/s3/s3.go b/lib/backup/storage/s3/s3.go new file mode 100644 index 000000000..3e08229c8 --- /dev/null +++ b/lib/backup/storage/s3/s3.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/tarantool/tt/lib/backup/storage" +) + +var ( + errEndpointRequired = errors.New("s3 endpoint is required") + errBucketRequired = errors.New("s3 bucket is required") + errAccessKeyIDRequired = errors.New("s3 access_key_id is required") + errSecretAccessKeyRequired = errors.New("s3 secret_access_key is required") + errNegativeObjectSize = errors.New("s3 object size must be non-negative") +) + +// Config describes S3-compatible storage configuration. +type Config struct { + Endpoint string + Bucket string + Region string + AccessKeyID string + SecretAccessKey string + UseSSL bool + Prefix string +} + +// Storage is an S3-compatible backup storage backend. +type Storage struct { + client *minio.Client + bucket string + prefix string +} + +// New opens S3-compatible backup storage using minio-go. +func New(cfg Config) (*Storage, error) { + if err := validateConfig(cfg); err != nil { + return nil, fmt.Errorf("failed to validate s3 config: %w", err) + } + + client, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""), + Secure: cfg.UseSSL, + Region: cfg.Region, + }) + if err != nil { + return nil, fmt.Errorf("failed to create s3 client: %w", err) + } + + prefix, err := storage.CleanPrefix(cfg.Prefix) + if err != nil { + return nil, fmt.Errorf("failed to clean storage prefix %q: %w", cfg.Prefix, err) + } + + return &Storage{ + client: client, + bucket: cfg.Bucket, + prefix: storage.PrefixWithSlash(prefix), + }, nil +} + +func (s *Storage) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) { + prefix, err := storage.CleanPrefix(prefix) + if err != nil { + return nil, fmt.Errorf("failed to clean list prefix %q: %w", prefix, err) + } + + objectPrefix := s.objectName(prefix) + + objectsCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: objectPrefix, + Recursive: true, + }) + + objects := make([]storage.ObjectInfo, 0) + + for obj := range objectsCh { + if obj.Err != nil { + return nil, fmt.Errorf("failed to list s3 prefix %q: %w", prefix, obj.Err) + } + key := strings.TrimPrefix(obj.Key, s.prefix) + + objects = append(objects, storage.ObjectInfo{ + Key: key, + Size: obj.Size, + LastModified: obj.LastModified, + }) + } + + sort.Slice(objects, func(i, j int) bool { + return objects[i].Key < objects[j].Key + }) + + return objects, nil +} + +func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) { + key, err := storage.CleanKey(key) + if err != nil { + return nil, fmt.Errorf("failed to clean object key %q: %w", key, err) + } + + objectName := s.objectName(key) + + object, err := s.client.GetObject(ctx, s.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get s3 object %q: %w", key, err) + } + + if _, err := object.Stat(); err != nil { + _ = object.Close() + if isNotFound(err) { + return nil, storage.ErrKeyNotFound + } + + return nil, fmt.Errorf("failed to get s3 object %q: %w", key, err) + } + + return object, nil +} + +func (s *Storage) Put(ctx context.Context, key string, r io.Reader, size int64) error { + key, err := storage.CleanKey(key) + if err != nil { + return fmt.Errorf("failed to clean object key %q: %w", key, err) + } + + if size < 0 { + return errNegativeObjectSize + } + + objectName := s.objectName(key) + + _, err = s.client.PutObject(ctx, s.bucket, objectName, r, size, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("failed to put s3 object %q: %w", key, err) + } + + return nil +} + +func (s *Storage) Delete(ctx context.Context, key string) error { + key, err := storage.CleanKey(key) + if err != nil { + return fmt.Errorf("failed to clean object key %q: %w", key, err) + } + + objectName := s.objectName(key) + + err = s.client.RemoveObject(ctx, s.bucket, objectName, minio.RemoveObjectOptions{}) + if err != nil { + if isNotFound(err) { + return nil + } + + return fmt.Errorf("failed to delete s3 object %q: %w", key, err) + } + + return nil +} + +func (s *Storage) objectName(key string) string { + return s.prefix + key +} + +func validateConfig(cfg Config) error { + switch { + case strings.TrimSpace(cfg.Endpoint) == "": + return errEndpointRequired + case strings.TrimSpace(cfg.Bucket) == "": + return errBucketRequired + case strings.TrimSpace(cfg.AccessKeyID) == "": + return errAccessKeyIDRequired + case strings.TrimSpace(cfg.SecretAccessKey) == "": + return errSecretAccessKeyRequired + default: + return nil + } +} + +func isNotFound(err error) bool { + resp := minio.ToErrorResponse(err) + switch resp.Code { + case "NoSuchKey", "NoSuchBucket": + return true + default: + return false + } +} diff --git a/lib/backup/storage/s3/s3_integration_test.go b/lib/backup/storage/s3/s3_integration_test.go new file mode 100644 index 000000000..ad914d39f --- /dev/null +++ b/lib/backup/storage/s3/s3_integration_test.go @@ -0,0 +1,209 @@ +//go:build integration + +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tarantool/tt/lib/backup/storage" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func TestGaragePut(t *testing.T) { + ctx := t.Context() + st := newGarageStorage(ctx, t) + + key := storage.ArchiveKey("garage-put", "rs1") + data := []byte("archive payload") + + require.NoError(t, st.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + objects, err := st.List(ctx, storage.DataPrefix()) + require.NoError(t, err) + require.Len(t, objects, 1) + require.Equal(t, key, objects[0].Key) + require.Equal(t, int64(len(data)), objects[0].Size) +} + +func TestGarageGet(t *testing.T) { + ctx := t.Context() + st := newGarageStorage(ctx, t) + + key := storage.ManifestKey("garage-get") + data := []byte(`{"status":"ok"}`) + require.NoError(t, st.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + reader, err := st.Get(ctx, key) + require.NoError(t, err) + defer reader.Close() + + actual, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, data, actual) +} + +func TestGarageList(t *testing.T) { + ctx := t.Context() + st := newGarageStorage(ctx, t) + + manifestKey := storage.ManifestKey("garage-list") + archiveKey := storage.ArchiveKey("garage-list", "rs1") + manifest := []byte(`{"status":"ok"}`) + archive := []byte("archive") + require.NoError(t, st.Put(ctx, manifestKey, bytes.NewReader(manifest), int64(len(manifest)))) + require.NoError(t, st.Put(ctx, archiveKey, bytes.NewReader(archive), int64(len(archive)))) + + objects, err := st.List(ctx, storage.ManifestsPrefix()) + require.NoError(t, err) + require.Len(t, objects, 1) + require.Equal(t, manifestKey, objects[0].Key) + require.False(t, objects[0].LastModified.IsZero()) +} + +func TestGarageDelete(t *testing.T) { + ctx := t.Context() + st := newGarageStorage(ctx, t) + + key := storage.ManifestKey("garage-delete") + data := []byte(`{"status":"ok"}`) + require.NoError(t, st.Put(ctx, key, bytes.NewReader(data), int64(len(data)))) + + require.NoError(t, st.Delete(ctx, key)) + _, err := st.Get(ctx, key) + require.True(t, errors.Is(err, storage.ErrKeyNotFound)) +} + +type garageInstance struct { + endpoint string + bucket string + region string + accessKey string + secretKey string +} + +func newGarageStorage(ctx context.Context, t *testing.T) *Storage { + t.Helper() + testcontainers.SkipIfProviderIsNotHealthy(t) + + garage := startGarage(ctx, t) + st, err := New(Config{ + Endpoint: garage.endpoint, + Bucket: garage.bucket, + Region: garage.region, + AccessKeyID: garage.accessKey, + SecretAccessKey: garage.secretKey, + Prefix: "payments-cluster/production/", + }) + require.NoError(t, err) + return st +} + +func startGarage(ctx context.Context, t *testing.T) garageInstance { + t.Helper() + + inst := garageInstance{ + bucket: "tt-backup-test", + region: "garage", + accessKey: "GKTTBACKUPTEST000000000000000000", + secretKey: "tt-backup-test-secret-key", + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "dxflrs/garage:v2.3.0", + ExposedPorts: []string{"3900/tcp"}, + Cmd: []string{"/garage", "server", "--single-node", "--default-bucket"}, + Env: map[string]string{ + "GARAGE_DEFAULT_ACCESS_KEY": inst.accessKey, + "GARAGE_DEFAULT_SECRET_KEY": inst.secretKey, + "GARAGE_DEFAULT_BUCKET": inst.bucket, + }, + Files: []testcontainers.ContainerFile{ + { + Reader: strings.NewReader(garageConfig), + ContainerFilePath: "/etc/garage.toml", + FileMode: 0o644, + }, + }, + WaitingFor: wait.ForListeningPort("3900/tcp").WithStartupTimeout(time.Minute), + }, + Started: true, + }) + testcontainers.CleanupContainer(t, container) + require.NoError(t, err) + + inst.endpoint, err = container.PortEndpoint(ctx, "3900/tcp", "") + require.NoError(t, err) + + waitForDefaultBucket(ctx, t, container, inst.bucket) + return inst +} + +func waitForDefaultBucket( + ctx context.Context, + t *testing.T, + container testcontainers.Container, + bucket string, +) { + t.Helper() + + deadline := time.Now().Add(time.Minute) + var lastOutput string + for time.Now().Before(deadline) { + exitCode, output, err := container.Exec(ctx, []string{ + "/garage", "bucket", "info", bucket, + }) + if err == nil && exitCode == 0 { + return + } + if output != nil { + data, readErr := io.ReadAll(output) + if readErr == nil { + lastOutput = string(data) + } + } + time.Sleep(time.Second) + } + + logs := readContainerLogs(ctx, container) + t.Fatalf("Garage default bucket was not created, last exec output:\n%s\ncontainer logs:\n%s", + lastOutput, logs) +} + +func readContainerLogs(ctx context.Context, container testcontainers.Container) string { + logs, err := container.Logs(ctx) + if err != nil { + return fmt.Sprintf("failed to read logs: %v", err) + } + defer logs.Close() + + data, err := io.ReadAll(logs) + if err != nil { + return fmt.Sprintf("failed to read logs: %v", err) + } + return string(data) +} + +const garageConfig = `metadata_dir = "/tmp/meta" +data_dir = "/tmp/data" +db_engine = "sqlite" +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000" + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" +` diff --git a/lib/backup/storage/s3/s3_test.go b/lib/backup/storage/s3/s3_test.go new file mode 100644 index 000000000..c1fdcfbb3 --- /dev/null +++ b/lib/backup/storage/s3/s3_test.go @@ -0,0 +1,137 @@ +package s3 + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tarantool/tt/lib/backup/storage" +) + +func TestObjectName(t *testing.T) { + s := &Storage{prefix: "base/prefix/"} + + require.Equal(t, "base/prefix/manifests/", s.objectName("manifests/")) + require.Equal(t, "base/prefix/manifests/backup.json", s.objectName("manifests/backup.json")) +} + +func TestNewUsesPrefixWithSlash(t *testing.T) { + s, err := New(Config{ + Endpoint: "localhost:3900", + Bucket: "backups", + AccessKeyID: "key", + SecretAccessKey: "secret", + Prefix: "cluster/prod", + }) + require.NoError(t, err) + require.Equal(t, "cluster/prod/", s.prefix) +} + +func TestNewEmptyPrefix(t *testing.T) { + s, err := New(Config{ + Endpoint: "localhost:3900", + Bucket: "backups", + AccessKeyID: "key", + SecretAccessKey: "secret", + }) + require.NoError(t, err) + require.Equal(t, "", s.prefix) +} + +func TestNewPrefixWithTrailingSlash(t *testing.T) { + s, err := New(Config{ + Endpoint: "localhost:3900", + Bucket: "backups", + AccessKeyID: "key", + SecretAccessKey: "secret", + Prefix: "cluster/prod/", + }) + require.NoError(t, err) + require.Equal(t, "cluster/prod/", s.prefix) +} + +func TestValidateConfig(t *testing.T) { + valid := Config{ + Endpoint: "localhost:3900", + Bucket: "backups", + AccessKeyID: "key", + SecretAccessKey: "secret", + } + + testCases := []struct { + name string + cfg Config + err error + }{ + {name: "empty endpoint", cfg: Config{}, err: errEndpointRequired}, + {name: "empty bucket", cfg: Config{Endpoint: valid.Endpoint}, err: errBucketRequired}, + { + name: "empty access key id", + cfg: Config{Endpoint: valid.Endpoint, Bucket: valid.Bucket}, + err: errAccessKeyIDRequired, + }, + { + name: "empty secret access key", + cfg: Config{ + Endpoint: valid.Endpoint, + Bucket: valid.Bucket, + AccessKeyID: valid.AccessKeyID, + }, + err: errSecretAccessKeyRequired, + }, + {name: "valid", cfg: valid}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateConfig(tc.cfg) + if tc.err == nil { + require.NoError(t, err) + } else { + require.True(t, errors.Is(err, tc.err)) + } + }) + } +} + +func TestPutRejectsNegativeSize(t *testing.T) { + s := &Storage{} + err := s.Put(t.Context(), "key", strings.NewReader("data"), -1) + require.True(t, errors.Is(err, errNegativeObjectSize)) +} + +func TestPutRejectsInvalidKey(t *testing.T) { + s := &Storage{} + err := s.Put(t.Context(), "../escape", strings.NewReader("data"), 4) + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestGetRejectsInvalidKey(t *testing.T) { + s := &Storage{} + _, err := s.Get(t.Context(), "../escape") + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestDeleteRejectsInvalidKey(t *testing.T) { + s := &Storage{} + err := s.Delete(t.Context(), "../escape") + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestListRejectsInvalidPrefix(t *testing.T) { + s := &Storage{} + _, err := s.List(t.Context(), "../escape") + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} + +func TestNewRejectsInvalidPrefix(t *testing.T) { + _, err := New(Config{ + Endpoint: "localhost:3900", + Bucket: "backups", + AccessKeyID: "key", + SecretAccessKey: "secret", + Prefix: "../escape", + }) + require.True(t, errors.Is(err, storage.ErrInvalidKey)) +} diff --git a/lib/backup/storage/storage.go b/lib/backup/storage/storage.go new file mode 100644 index 000000000..9a05bb6bb --- /dev/null +++ b/lib/backup/storage/storage.go @@ -0,0 +1,116 @@ +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "time" +) + +// Storage describes a backup object storage backend. +type Storage interface { + List(ctx context.Context, prefix string) ([]ObjectInfo, error) + Get(ctx context.Context, key string) (io.ReadCloser, error) + Put(ctx context.Context, key string, r io.Reader, size int64) error + Delete(ctx context.Context, key string) error +} + +// ObjectInfo describes one stored object. +type ObjectInfo struct { + Key string + Size int64 + LastModified time.Time +} + +var ( + // ErrKeyNotFound is returned when a requested object does not exist. + ErrKeyNotFound = errors.New("storage: key not found") + // ErrInvalidKey is returned when a key is not a canonical storage key. + ErrInvalidKey = errors.New("storage: invalid key") +) + +// GetBytes reads a small object into memory. +func GetBytes(ctx context.Context, s Storage, key string) ([]byte, error) { + r, err := s.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("failed to get object %q: %w", key, err) + } + defer r.Close() + + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("failed to read object %q: %w", key, err) + } + + return data, nil +} + +// PutBytes stores a small object from memory. +func PutBytes(ctx context.Context, s Storage, key string, b []byte) error { + if err := s.Put(ctx, key, bytes.NewReader(b), int64(len(b))); err != nil { + return fmt.Errorf("failed to put object %q: %w", key, err) + } + + return nil +} + +// CleanKey returns a canonical storage object key. +func CleanKey(key string) (string, error) { + key = strings.Trim(key, "/") + if key == "" { + return "", ErrInvalidKey + } + + if err := validatePathParts(key); err != nil { + return "", fmt.Errorf("failed to validate key path parts: %w", err) + } + + return key, nil +} + +// CleanPrefix returns a canonical storage list prefix. +func CleanPrefix(prefix string) (string, error) { + prefix = strings.TrimLeft(prefix, "/") + if prefix == "" { + return "", nil + } + + hasTrailingSlash := strings.HasSuffix(prefix, "/") + prefix = strings.TrimRight(prefix, "/") + + if err := validatePathParts(prefix); err != nil { + return "", fmt.Errorf("failed to validate prefix path parts: %w", err) + } + + if hasTrailingSlash { + return prefix + "/", nil + } + + return prefix, nil +} + +// PrefixWithSlash ensures the prefix ends with a trailing slash, unless it is empty. +func PrefixWithSlash(prefix string) string { + if prefix == "" || strings.HasSuffix(prefix, "/") { + return prefix + } + + return prefix + "/" +} + +func validatePathParts(key string) error { + if key == "" || strings.Contains(key, "\\") { + return ErrInvalidKey + } + + for _, part := range strings.Split(key, "/") { + if part == "" || part == "." || part == ".." { + return ErrInvalidKey + } + } + + return nil +} diff --git a/lib/backup/storage/storage_test.go b/lib/backup/storage/storage_test.go new file mode 100644 index 000000000..05fd4e13e --- /dev/null +++ b/lib/backup/storage/storage_test.go @@ -0,0 +1,150 @@ +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPrefixWithSlash(t *testing.T) { + testCases := map[string]string{ + "": "", + "data": "data/", + "data/": "data/", + "a/b/c": "a/b/c/", + "a/b/c/": "a/b/c/", + } + for input, expected := range testCases { + t.Run(input, func(t *testing.T) { + require.Equal(t, expected, PrefixWithSlash(input)) + }) + } +} + +func TestGetBytesNotFound(t *testing.T) { + ctx := t.Context() + s := newMemoryStorage() + + _, err := GetBytes(ctx, s, "missing") + require.True(t, errors.Is(err, ErrKeyNotFound)) +} + +func TestKeyHelpers(t *testing.T) { + require.Equal(t, "manifests/", ManifestsPrefix()) + require.Equal(t, "data/", DataPrefix()) + require.Equal(t, "manifests/20260102T030405Z.json", ManifestKey("20260102T030405Z")) + require.Equal(t, + "data/20260102T030405Z-550e8400-e29b-41d4-a716-446655440000.tar.zst", + ArchiveKey("20260102T030405Z", "550e8400-e29b-41d4-a716-446655440000"), + ) +} + +func TestErrKeyNotFoundComparable(t *testing.T) { + require.True(t, errors.Is(ErrKeyNotFound, ErrKeyNotFound)) +} + +func TestCleanKey(t *testing.T) { + key, err := CleanKey("/data/backup-rs1.tar.zst/") + require.NoError(t, err) + require.Equal(t, "data/backup-rs1.tar.zst", key) + + invalid := []string{ + "", + "////", + "data//backup-rs1.tar.zst", + "data/../backup-rs1.tar.zst", + "data/./backup-rs1.tar.zst", + `data\backup-rs1.tar.zst`, + } + for _, key := range invalid { + t.Run(key, func(t *testing.T) { + _, err := CleanKey(key) + require.True(t, errors.Is(err, ErrInvalidKey)) + }) + } +} + +func TestCleanPrefix(t *testing.T) { + testCases := map[string]string{ + "": "", + "/data/": "data/", + "data//": "data/", + "/data/backup": "data/backup", + } + for prefix, expected := range testCases { + t.Run(prefix, func(t *testing.T) { + actual, err := CleanPrefix(prefix) + require.NoError(t, err) + require.Equal(t, expected, actual) + }) + } + + invalid := []string{ + "data//backup", + "data/../", + "data/./", + `data\`, + } + for _, prefix := range invalid { + t.Run(prefix, func(t *testing.T) { + _, err := CleanPrefix(prefix) + require.True(t, errors.Is(err, ErrInvalidKey)) + }) + } +} + +func TestPutBytes(t *testing.T) { + ctx := t.Context() + s := newMemoryStorage() + + require.NoError(t, PutBytes(ctx, s, "key", []byte("value"))) + require.Equal(t, []byte("value"), s.objects["key"]) +} + +func TestGetBytes(t *testing.T) { + ctx := t.Context() + s := newMemoryStorage() + s.objects["key"] = []byte("value") + + data, err := GetBytes(ctx, s, "key") + require.NoError(t, err) + require.Equal(t, []byte("value"), data) +} + +type memoryStorage struct { + objects map[string][]byte +} + +func newMemoryStorage() *memoryStorage { + return &memoryStorage{objects: make(map[string][]byte)} +} + +func (s *memoryStorage) List(context.Context, string) ([]ObjectInfo, error) { + return nil, nil +} + +func (s *memoryStorage) Get(_ context.Context, key string) (io.ReadCloser, error) { + data, ok := s.objects[key] + if !ok { + return nil, ErrKeyNotFound + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (s *memoryStorage) Put(_ context.Context, key string, r io.Reader, _ int64) error { + data, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("failed to read object %q: %w", key, err) + } + s.objects[key] = data + return nil +} + +func (s *memoryStorage) Delete(context.Context, string) error { + return nil +} diff --git a/test/integration/aeon/server/go.mod b/test/integration/aeon/server/go.mod index 6e4f5732a..c90f9324d 100644 --- a/test/integration/aeon/server/go.mod +++ b/test/integration/aeon/server/go.mod @@ -8,9 +8,9 @@ require ( ) require ( - golang.org/x/net v0.52.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/test/integration/aeon/server/go.sum b/test/integration/aeon/server/go.sum index e1e6e156f..2c98a2290 100644 --- a/test/integration/aeon/server/go.sum +++ b/test/integration/aeon/server/go.sum @@ -22,12 +22,12 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=