From cce620a3ab15e879cfd1de943b1ee250b434ca43 Mon Sep 17 00:00:00 2001 From: Shane Delamore Date: Fri, 17 Apr 2026 17:04:41 +0200 Subject: [PATCH] patch/fix: delete command! --- source/S3Parallel/S3P.caf | 57 ++++++++++++ source/S3Parallel/S3PCli.caf | 22 +++++ source/S3Parallel/S3PCliCommands.caf | 1 + source/test/Delete.test.caf | 125 +++++++++++++++++++++++++++ source/test/MinioTestHelper.caf | 19 +++- 5 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 source/test/Delete.test.caf diff --git a/source/S3Parallel/S3P.caf b/source/S3Parallel/S3P.caf index 14041a1..8451fba 100644 --- a/source/S3Parallel/S3P.caf +++ b/source/S3Parallel/S3P.caf @@ -213,6 +213,63 @@ class S3P @_copyWrapper options, (updatedOptions) => S3C.eachPromises updatedOptions + ## + IN: options: + bucket: required - bucket to delete FROM + confirmDeleteItemsFromBucket: required - must === bucket (safety) + prefix, filter, pattern, etc. standard S3Comprehensions.each options + pretend: if true, lists but does NOT delete + deleteConcurrency: [500] max in-flight delete calls + OUT: + finalStats: + deletedFiles + deletedBytes + ... + @delete: (options) => + options extract + bucket + confirmDeleteItemsFromBucket + dryrun + pretend = dryrun + verbose + stats + deleteConcurrency = 500 + deletePwp + region, endpoint, useAccelerateEndpoint + s3 + + unless pretend || confirmDeleteItemsFromBucket == bucket + throw new Error "" + confirm-delete-items-from-bucket (#{confirmDeleteItemsFromBucket}) + must exactly match bucket (#{bucket}) + (hint: use --dryrun or --pretend to skip this check) + + deletePwp ?= new PromiseWorkerPool deleteConcurrency + s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint + + stats ?= {} + stats.deletedFiles = 0 + stats.deletedBytes = 0 + + S3C.eachPromises merge options, + stats + map: ({Key, Size}) -> + deletePwp.queue -> + if pretend + Promise.then -> + log "#{if pretend then 'PRETEND ' else ''}delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose + stats.deletedFiles++ + stats.deletedBytes += Size + else + s3.delete {} bucket, key: Key + .then -> + log "delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose + stats.deletedFiles++ + stats.deletedBytes += Size + + .then (info) -> + finalStats: S3P.getStatsWithHumanByteSizes merge info, stats + @sync: (options) => @_copyWrapper options, (options2) -> options2 extract diff --git a/source/S3Parallel/S3PCli.caf b/source/S3Parallel/S3PCli.caf index d2f614b..f749379 100644 --- a/source/S3Parallel/S3PCli.caf +++ b/source/S3Parallel/S3PCli.caf @@ -186,6 +186,28 @@ cliCommands = bucket: :my-bucket to-bucket: :my-to-bucket "" Copy everything from my-bucket to my-to-bucket + delete: + description: + """ + Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject. + + SAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed. + options: merge allCommandOptions, advancedOptionsForAll, + confirm-delete-items-from-bucket: [] + :bucket-name + "" + REQUIRED. Must exactly match --bucket. Guards against accidental deletion. + dryrun: "" Will not modify anything. List everything that would be deleted. + pretend: "" alias for --dryrun + delete-concurrency: advanced: true argument: :500 description: "" Maximum number of simultaneous delete operations. + args: standardFromArgs + examples: + bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket prefix: :old/ + "" Delete every key under 'old/' from my-bucket. + + bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket dryrun: true + "" Dry-run: list what would be deleted but delete nothing. + each: description: "" Create your own iteration. Specify a --map or --map-list option. options: merge allCommandOptions, advancedOptionsForAll, diff --git a/source/S3Parallel/S3PCliCommands.caf b/source/S3Parallel/S3PCliCommands.caf index fb599e1..483630a 100644 --- a/source/S3Parallel/S3PCliCommands.caf +++ b/source/S3Parallel/S3PCliCommands.caf @@ -40,5 +40,6 @@ wrapCommandWithArgsProcessing = (command) -> compare: wrapCommandWithArgsProcessing &S3P.compare cp: wrapCommandWithArgsProcessing &S3P.copy sync: wrapCommandWithArgsProcessing &S3P.sync + delete: wrapCommandWithArgsProcessing &S3P.delete each: wrapCommandWithArgsProcessing &S3Comprehensions.each map: wrapCommandWithArgsProcessing &S3Comprehensions.map diff --git a/source/test/Delete.test.caf b/source/test/Delete.test.caf new file mode 100644 index 0000000..59506f2 --- /dev/null +++ b/source/test/Delete.test.caf @@ -0,0 +1,125 @@ +import &StandardImport, &MinioTestHelper + +sourceObjects = + {} Key: :del/alpha.txt Body: :AAAA + {} Key: :del/beta.txt Body: :BB + {} Key: :del/gamma.txt Body: :CCCCCCCC + {} Key: :keep/one.txt Body: :XYZ + +setupTestScenario :delete sourceObjects, ({endpoint, testBucket}) -> + + baseOptions = {} + bucket: testBucket + endpoint: endpoint + quiet: true + + test "delete without confirm-delete-items-from-bucket throws" -> + assert.throws -> S3P.delete merge baseOptions, prefix: :del/ + + test "delete with wrong confirm-delete-items-from-bucket throws" -> + assert.throws -> + S3P.delete merge + baseOptions + prefix: :del/ + confirmDeleteItemsFromBucket: :wrong-bucket + + test "delete with matching confirm deletes the matching items" -> + # re-seed to guarantee keys exist (prior tests may have run first) + putTestObjects sourceObjects + + .then -> + S3P.delete merge + baseOptions + prefix: :del/ + confirmDeleteItemsFromBucket: testBucket + + .then ({finalStats}) -> + assert.eq finalStats.deletedFiles, 3 + assert.eq finalStats.deletedBytes, 4 + 2 + 8 + listBucketKeys() + + .then (keys) -> + assert.ok !(:del/alpha.txt in keys), "" del/alpha.txt should be gone + assert.ok !(:del/beta.txt in keys), "" del/beta.txt should be gone + assert.ok !(:del/gamma.txt in keys), "" del/gamma.txt should be gone + assert.ok :keep/one.txt in keys, "" keep/one.txt should remain + + test "delete with filter only deletes matching items" -> + putTestObjects sourceObjects + + .then -> + S3P.delete merge + baseOptions + prefix: :del/ + filter: ({Size}) -> Size >= 4 + confirmDeleteItemsFromBucket: testBucket + + .then ({finalStats}) -> + assert.eq finalStats.deletedFiles, 2 + listBucketKeys :del/ + + .then (keys) -> + assert.eq keys, [] :del/beta.txt + + test "delete with pretend deletes nothing" -> + putTestObjects sourceObjects + + .then -> + S3P.delete merge + baseOptions + prefix: :del/ + pretend: true + confirmDeleteItemsFromBucket: testBucket + + .then ({finalStats}) -> + assert.eq finalStats.deletedFiles, 3 + listBucketKeys :del/ + + .then (keys) -> + assert.eq keys.sort(), [] :del/alpha.txt :del/beta.txt :del/gamma.txt + + test "delete with pretend does NOT require confirm" -> + putTestObjects sourceObjects + + .then -> + # no confirmDeleteItemsFromBucket — but pretend makes it safe + S3P.delete merge baseOptions, prefix: :del/, pretend: true + + .then ({finalStats}) -> + assert.eq finalStats.deletedFiles, 3 + listBucketKeys :del/ + .then (keys) -> + assert.eq keys.sort(), [] :del/alpha.txt :del/beta.txt :del/gamma.txt + + test "delete with dryrun does NOT require confirm" -> + putTestObjects sourceObjects + + .then -> + S3P.delete merge baseOptions, prefix: :del/, dryrun: true + + .then ({finalStats}) -> + assert.eq finalStats.deletedFiles, 3 + + test "delete with verbose logs each deleted key" -> + putTestObjects sourceObjects + + .then -> + logged = [] + originalLog = console.log + console.log = (args...) -> logged.push args.join ' ' + + S3P.delete merge + baseOptions + prefix: :del/ + verbose: true + confirmDeleteItemsFromBucket: testBucket + + .finally -> + console.log = originalLog + + .then -> + combined = logged.join :\n + each key in-array [] :del/alpha.txt :del/beta.txt :del/gamma.txt + assert.ok + combined.indexOf(key) >= 0 + "" expected log to include #{key}, got: #{combined} diff --git a/source/test/MinioTestHelper.caf b/source/test/MinioTestHelper.caf index 5c4fbba..2777b56 100644 --- a/source/test/MinioTestHelper.caf +++ b/source/test/MinioTestHelper.caf @@ -33,6 +33,21 @@ deleteTestObjects = (keys) -> Bucket: testBucket Key: Key +## listBucketKeys + IN: prefix: optional - only keys starting with prefix + OUT: Promise +listBucketKeys = (prefix) -> + s3Client.send new ListObjectsV2Command + Bucket: testBucket + Prefix: prefix + .then ({Contents}) -> array {Key} in-array (Contents ? []) with Key + +## deleteAllBucketObjects + Clean the entire test bucket. Use in afterAll when tests create dynamic keys. +deleteAllBucketObjects = -> + listBucketKeys() + .then deleteTestObjects + ## setupTestScenario Sets up a describe block with beforeAll/afterAll that creates and cleans up test objects. IN: @@ -44,7 +59,7 @@ setupTestScenario = (name, keys, fn) -> if endpoint describe name, -> beforeAll -> putTestObjects keys - afterAll -> deleteTestObjects keys + afterAll -> deleteAllBucketObjects() fn {} endpoint, testBucket else test "#{name} skipped (no S3_ENDPOINT)" -> @@ -55,4 +70,6 @@ setupTestScenario = (name, keys, fn) -> s3Client putTestObjects deleteTestObjects + listBucketKeys + deleteAllBucketObjects setupTestScenario