From 19e10dda719024f9d2f17a253fc47195b181ba99 Mon Sep 17 00:00:00 2001 From: reshke kirill Date: Sun, 31 May 2026 18:06:27 +0000 Subject: [PATCH] GUCify storage concurrency for copy on restore --- config/instance.go | 12 ++++++++---- config/storage.go | 3 ++- pkg/proc/interaction.go | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/config/instance.go b/config/instance.go index 59de60b..702ac69 100644 --- a/config/instance.go +++ b/config/instance.go @@ -69,10 +69,11 @@ func initInstanceConfig(file *os.File, cfgInstance *Instance) error { } const ( - DefaultStorageConcurrency = 100 - DefaultStatPort = 7432 - DefaultPsqlPort = 8432 - DefaultMetricsPort = 2112 + DefaultStorageConcurrency = 100 + DefaultCopyStorageConcurrency = 200 + DefaultStatPort = 7432 + DefaultPsqlPort = 8432 + DefaultMetricsPort = 2112 DefaultEndpointSourceScheme = "https" @@ -89,6 +90,9 @@ func EmbedDefaults(cfgInstance *Instance) { if cfgInstance.StorageCnf.StorageConcurrency == 0 { cfgInstance.StorageCnf.StorageConcurrency = DefaultStorageConcurrency } + if cfgInstance.StorageCnf.CopyStorageConcurrency == 0 { + cfgInstance.StorageCnf.CopyStorageConcurrency = DefaultCopyStorageConcurrency + } if cfgInstance.BackupStorageCnf.StorageType == "" { cfgInstance.BackupStorageCnf.StorageType = "s3" } diff --git a/config/storage.go b/config/storage.go index 0a2cf3b..32d85a8 100644 --- a/config/storage.go +++ b/config/storage.go @@ -24,7 +24,8 @@ type Storage struct { TablespaceMap map[string]string `json:"tablespace_map" toml:"tablespace_map" yaml:"tablespace_map"` // how many concurrent connection acquire allowed - StorageConcurrency int64 `json:"storage_concurrency" toml:"storage_concurrency" yaml:"storage_concurrency"` + StorageConcurrency int64 `json:"storage_concurrency" toml:"storage_concurrency" yaml:"storage_concurrency"` + CopyStorageConcurrency int64 `json:"copy_storage_concurrency" toml:"copy_storage_concurrency" yaml:"copy_storage_concurrency"` // default will be false EnableRateLimiter bool `json:"enable_rate_limiter" toml:"enable_rate_limiter" yaml:"enable_rate_limiter"` diff --git a/pkg/proc/interaction.go b/pkg/proc/interaction.go index 785f6df..aba2f42 100644 --- a/pkg/proc/interaction.go +++ b/pkg/proc/interaction.go @@ -287,7 +287,7 @@ func ProcessCopyExtended( for len(objectMetas) > 0 && retryCount < 10 { retryCount++ - sem := semaphore.NewWeighted(200) + sem := semaphore.NewWeighted(config.InstanceConfig().StorageCnf.CopyStorageConcurrency) wg := sync.WaitGroup{}