patch/fix: delete command!

source/S3Parallel/S3P.caf

@@ -213,6 +213,63 @@ class S3P
     @_copyWrapper options, (updatedOptions) =>
       S3C.eachPromises updatedOptions
 
+  ##
+    IN: options:
+      bucket:                         <String> required - bucket to delete FROM
+      confirmDeleteItemsFromBucket:   <String> required - must === bucket (safety)
+      prefix, filter, pattern, etc.   standard S3Comprehensions.each options
+      pretend:                        <Boolean> if true, lists but does NOT delete
+      deleteConcurrency:              <Number>  [500] max in-flight delete calls
+    OUT:
+      finalStats:
+        deletedFiles
+        deletedBytes
+        ...
+  @delete: (options) =>
+    options extract
+      bucket
+      confirmDeleteItemsFromBucket
+      dryrun
+      pretend = dryrun
+      verbose
+      stats
+      deleteConcurrency = 500
+      deletePwp
+      region, endpoint, useAccelerateEndpoint
+      s3
+
+    unless pretend || confirmDeleteItemsFromBucket == bucket
+      throw new Error ""
+        confirm-delete-items-from-bucket (#{confirmDeleteItemsFromBucket})
+        must exactly match bucket (#{bucket})
+        (hint: use --dryrun or --pretend to skip this check)
+
+    deletePwp ?= new PromiseWorkerPool deleteConcurrency
+    s3 ?= new &Lib/S3 {} region, endpoint, useAccelerateEndpoint
+
+    stats ?= {}
+    stats.deletedFiles = 0
+    stats.deletedBytes = 0
+
+    S3C.eachPromises merge options,
+      stats
+      map: ({Key, Size}) ->
+        deletePwp.queue ->
+          if pretend
+            Promise.then ->
+              log "#{if pretend then 'PRETEND ' else ''}delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose
+              stats.deletedFiles++
+              stats.deletedBytes += Size
+          else
+            s3.delete {} bucket, key: Key
+            .then ->
+              log "delete s3://#{bucket}/#{Key} # #{humanByteSize Size}" if verbose
+              stats.deletedFiles++
+              stats.deletedBytes += Size
+
+    .then (info) ->
+      finalStats: S3P.getStatsWithHumanByteSizes merge info, stats
+
   @sync: (options) =>
     @_copyWrapper options, (options2) ->
       options2 extract

source/S3Parallel/S3PCli.caf

@@ -186,6 +186,28 @@ cliCommands =
       bucket: :my-bucket to-bucket: :my-to-bucket
       "" Copy everything from my-bucket to my-to-bucket
 
+  delete:
+    description:
+      """
+        Delete all matching files from a bucket. Uses s3.listObjectsV2 and s3.deleteObject.
+
+        SAFETY: --confirm-delete-items-from-bucket MUST be provided and MUST exactly match --bucket. Without this, s3p refuses to proceed.
+    options: merge allCommandOptions, advancedOptionsForAll,
+      confirm-delete-items-from-bucket: []
+        :bucket-name
+        ""
+          REQUIRED. Must exactly match --bucket. Guards against accidental deletion.
+      dryrun:                   "" Will not modify anything. List everything that would be deleted.
+      pretend:                  "" alias for --dryrun
+      delete-concurrency:       advanced: true argument: :500 description: "" Maximum number of simultaneous delete operations.
+    args: standardFromArgs
+    examples:
+      bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket prefix: :old/
+      "" Delete every key under 'old/' from my-bucket.
+
+      bucket: :my-bucket confirm-delete-items-from-bucket: :my-bucket dryrun: true
+      "" Dry-run: list what would be deleted but delete nothing.
+
   each:
     description:  "" Create your own iteration. Specify a --map or --map-list option.
     options:      merge allCommandOptions, advancedOptionsForAll,

source/S3Parallel/S3PCliCommands.caf

@@ -40,5 +40,6 @@ wrapCommandWithArgsProcessing = (command) ->
   compare:      wrapCommandWithArgsProcessing &S3P.compare
   cp:           wrapCommandWithArgsProcessing &S3P.copy
   sync:         wrapCommandWithArgsProcessing &S3P.sync
+  delete:       wrapCommandWithArgsProcessing &S3P.delete
   each:         wrapCommandWithArgsProcessing &S3Comprehensions.each
   map:          wrapCommandWithArgsProcessing &S3Comprehensions.map

source/test/Delete.test.caf

@@ -0,0 +1,125 @@
+import &StandardImport, &MinioTestHelper
+
+sourceObjects =
+  {} Key: :del/alpha.txt  Body: :AAAA
+  {} Key: :del/beta.txt   Body: :BB
+  {} Key: :del/gamma.txt  Body: :CCCCCCCC
+  {} Key: :keep/one.txt   Body: :XYZ
+
+setupTestScenario :delete sourceObjects, ({endpoint, testBucket}) ->
+
+  baseOptions = {}
+    bucket:   testBucket
+    endpoint: endpoint
+    quiet:    true
+
+  test "delete without confirm-delete-items-from-bucket throws" ->
+    assert.throws -> S3P.delete merge baseOptions, prefix: :del/
+
+  test "delete with wrong confirm-delete-items-from-bucket throws" ->
+    assert.throws ->
+      S3P.delete merge
+        baseOptions
+        prefix: :del/
+        confirmDeleteItemsFromBucket: :wrong-bucket
+
+  test "delete with matching confirm deletes the matching items" ->
+    # re-seed to guarantee keys exist (prior tests may have run first)
+    putTestObjects sourceObjects
+
+    .then ->
+      S3P.delete merge
+        baseOptions
+        prefix: :del/
+        confirmDeleteItemsFromBucket: testBucket
+
+    .then ({finalStats}) ->
+      assert.eq finalStats.deletedFiles, 3
+      assert.eq finalStats.deletedBytes, 4 + 2 + 8
+      listBucketKeys()
+
+    .then (keys) ->
+      assert.ok !(:del/alpha.txt in keys), "" del/alpha.txt should be gone
+      assert.ok !(:del/beta.txt  in keys), "" del/beta.txt should be gone
+      assert.ok !(:del/gamma.txt in keys), "" del/gamma.txt should be gone
+      assert.ok   :keep/one.txt  in keys,  "" keep/one.txt should remain
+
+  test "delete with filter only deletes matching items" ->
+    putTestObjects sourceObjects
+
+    .then ->
+      S3P.delete merge
+        baseOptions
+        prefix: :del/
+        filter: ({Size}) -> Size >= 4
+        confirmDeleteItemsFromBucket: testBucket
+
+    .then ({finalStats}) ->
+      assert.eq finalStats.deletedFiles, 2
+      listBucketKeys :del/
+
+    .then (keys) ->
+      assert.eq keys, [] :del/beta.txt
+
+  test "delete with pretend deletes nothing" ->
+    putTestObjects sourceObjects
+
+    .then ->
+      S3P.delete merge
+        baseOptions
+        prefix: :del/
+        pretend: true
+        confirmDeleteItemsFromBucket: testBucket
+
+    .then ({finalStats}) ->
+      assert.eq finalStats.deletedFiles, 3
+      listBucketKeys :del/
+
+    .then (keys) ->
+      assert.eq keys.sort(), [] :del/alpha.txt :del/beta.txt :del/gamma.txt
+
+  test "delete with pretend does NOT require confirm" ->
+    putTestObjects sourceObjects
+
+    .then ->
+      # no confirmDeleteItemsFromBucket — but pretend makes it safe
+      S3P.delete merge baseOptions, prefix: :del/, pretend: true
+
+    .then ({finalStats}) ->
+      assert.eq finalStats.deletedFiles, 3
+      listBucketKeys :del/
+    .then (keys) ->
+      assert.eq keys.sort(), [] :del/alpha.txt :del/beta.txt :del/gamma.txt
+
+  test "delete with dryrun does NOT require confirm" ->
+    putTestObjects sourceObjects
+
+    .then ->
+      S3P.delete merge baseOptions, prefix: :del/, dryrun: true
+
+    .then ({finalStats}) ->
+      assert.eq finalStats.deletedFiles, 3
+
+  test "delete with verbose logs each deleted key" ->
+    putTestObjects sourceObjects
+
+    .then ->
+      logged = []
+      originalLog = console.log
+      console.log = (args...) -> logged.push args.join ' '
+
+      S3P.delete merge
+        baseOptions
+        prefix:   :del/
+        verbose:  true
+        confirmDeleteItemsFromBucket: testBucket
+
+      .finally ->
+        console.log = originalLog
+
+      .then ->
+        combined = logged.join :\n
+        each key in-array [] :del/alpha.txt :del/beta.txt :del/gamma.txt
+          assert.ok
+            combined.indexOf(key) >= 0
+            "" expected log to include #{key}, got: #{combined}

source/test/MinioTestHelper.caf

@@ -33,6 +33,21 @@ deleteTestObjects = (keys) ->
       Bucket: testBucket
       Key:    Key
 
+## listBucketKeys
+  IN: prefix: optional - only keys starting with prefix
+  OUT: Promise<array of strings>
+listBucketKeys = (prefix) ->
+  s3Client.send new ListObjectsV2Command
+    Bucket: testBucket
+    Prefix: prefix
+  .then ({Contents}) -> array {Key} in-array (Contents ? []) with Key
+
+## deleteAllBucketObjects
+  Clean the entire test bucket. Use in afterAll when tests create dynamic keys.
+deleteAllBucketObjects = ->
+  listBucketKeys()
+  .then deleteTestObjects
+
 ## setupTestScenario
   Sets up a describe block with beforeAll/afterAll that creates and cleans up test objects.
   IN:
@@ -44,7 +59,7 @@ setupTestScenario = (name, keys, fn) ->
   if endpoint
     describe name, ->
       beforeAll -> putTestObjects keys
-      afterAll  -> deleteTestObjects keys
+      afterAll  -> deleteAllBucketObjects()
       fn {} endpoint, testBucket
   else
     test "#{name} skipped (no S3_ENDPOINT)" ->
@@ -55,4 +70,6 @@ setupTestScenario = (name, keys, fn) ->
   s3Client
   putTestObjects
   deleteTestObjects
+  listBucketKeys
+  deleteAllBucketObjects
   setupTestScenario