From 977cced46967d62a3c0747918a12810053a921b4 Mon Sep 17 00:00:00 2001 From: Mark Olliver Date: Thu, 13 Jun 2024 12:14:06 +0000 Subject: [PATCH 1/4] masking clean commit --- config/test/bloblang/masking.yaml | 48 +++++++++++ internal/impl/pure/bloblang_mask.go | 82 +++++++++++++++++++ internal/impl/pure/bloblang_mask_test.go | 100 +++++++++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 config/test/bloblang/masking.yaml create mode 100644 internal/impl/pure/bloblang_mask.go create mode 100644 internal/impl/pure/bloblang_mask_test.go diff --git a/config/test/bloblang/masking.yaml b/config/test/bloblang/masking.yaml new file mode 100644 index 000000000..44f7a59b5 --- /dev/null +++ b/config/test/bloblang/masking.yaml @@ -0,0 +1,48 @@ +pipeline: + processors: + - bloblang: | + root = if env("MASK") == "FIXED_5" { this.value.mask(5)} else if env("MASK") == "LEFT_5" { this.value.mask(5, "left")} else if env("MASK") == "RIGHT_4" {this.value.mask(4,"right")} else if env("MASK") == "RIGHT_4_HASH" {this.value.mask(4,"right", "#")} else { this.value.mask() } + +tests: + - name: Fixed mask 5 + target_processors: /pipeline/processors + environment: + MASK: FIXED_5 + input_batch: + - content: '{"value": "this is a happy cat meow"}' + output_batches: + - - content_equals: '*****' + + - name: Left mask 5 + target_processors: /pipeline/processors + environment: + MASK: LEFT_5 + input_batch: + - content: '{"value": "this is a happy cat meow"}' + output_batches: + - - content_equals: 'this *******************' + + - name: Right mask 4 + target_processors: /pipeline/processors + environment: + MASK: RIGHT_4 + input_batch: + - content: '{"value": "this is a happy cat meow"}' + output_batches: + - - content_equals: '********************meow' + + - name: Right mask 4 with hash + target_processors: /pipeline/processors + environment: + MASK: RIGHT_4_HASH + input_batch: + - content: '{"value": "this is a happy cat meow"}' + output_batches: + - - content_equals: '####################meow' + + - name: default mask + target_processors: /pipeline/processors + input_batch: + - content: '{"value": "this is a happy cat meow"}' + output_batches: + - - content_equals: '************************' \ No newline at end of file diff --git a/internal/impl/pure/bloblang_mask.go b/internal/impl/pure/bloblang_mask.go new file mode 100644 index 000000000..a6c8aa06e --- /dev/null +++ b/internal/impl/pure/bloblang_mask.go @@ -0,0 +1,82 @@ +package pure + +import ( + "errors" + "strings" + + "github.com/redpanda-data/benthos/v4/internal/bloblang/query" + "github.com/redpanda-data/benthos/v4/public/bloblang" +) + +func init() { + if err := bloblang.RegisterMethodV2("mask", + bloblang.NewPluginSpec(). + Category(query.MethodCategoryParsing). + Description(`Masks a string using the given character, leaving X number of characters unmasked and returns a string.`). + Param(bloblang.NewInt64Param("count").Description("the number of characters that will not be masked on the left or right hand side, in the case of a all mask, it is the number of mask characters to return giving a fixed length string, default is 0 which will return all characters masked.").Optional().Default(0)). + Param(bloblang.NewStringParam("direction").Description("the direction to mask, left, right or all, default is all").Optional().Default("all")). + Param(bloblang.NewStringParam("char").Description("the character used for masking, default is *").Optional().Default("*")). + Example("Mask the first 13 characters", `root.body_mask = this.body.mask(13, right)`, + [2]string{ + `{"body":"the cat goes meow"}`, + `{"body_mask":"*************meow"}`, + }, + ), + func(args *bloblang.ParsedParams) (bloblang.Method, error) { + count, err := args.GetOptionalInt64("count") + if err != nil { + return nil, errors.New("failed to get count as int: " + err.Error()) + } + + char, err := args.GetString("char") + if err != nil { + return nil, errors.New("failed to get masking char as string: " + err.Error()) + } + + direction, err := args.GetString("direction") + if err != nil { + return nil, errors.New("failed to get direction as string: " + err.Error()) + } + + direction = strings.ToLower(direction) + if direction != "left" && direction != "right" && direction != "all" { + return nil, errors.New("direction must be one of left, right or all") + } + + return bloblang.StringMethod(func(s string) (any, error) { + return maskString(s, char, direction, int(*count)), nil + }), nil + }); err != nil { + panic(err) + } +} + +// maskString masks the string based on the given parameters. +// `s` is the input string +// `char` is the character used for masking, +// `direction` determines whether the start ("left") or the end ("right") of the string is masked or all for the whole string, +// `count` is the number of characters that will not be masked on the left or right hand side, in the case of a all mask, it is the number of mask characters to return giving a fixed length string. +func maskString(s string, char string, direction string, count int) string { + sLength := len(s) + if count == 0 { + count = sLength + } + fixedMask := count + if count > sLength { + count = sLength + } + + switch direction { + case "left": + unmasked := s[:count] + masked := strings.Repeat(char, sLength-count) + return unmasked + masked + case "right": + unmasked := s[sLength-count:] + masked := strings.Repeat(char, sLength-count) + return masked + unmasked + default: + masked := strings.Repeat(char, fixedMask) + return masked + } +} diff --git a/internal/impl/pure/bloblang_mask_test.go b/internal/impl/pure/bloblang_mask_test.go new file mode 100644 index 000000000..901eb6cd7 --- /dev/null +++ b/internal/impl/pure/bloblang_mask_test.go @@ -0,0 +1,100 @@ +package pure + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/internal/bloblang/query" + "github.com/redpanda-data/benthos/v4/internal/value" +) + +func TestMask(t *testing.T) { + testCases := []struct { + name string + method string + target any + args []any + exp any + err string + }{ + { + name: "default fixed string", + method: "mask", + target: "this is a test", + args: []any{}, + exp: "**************", + err: "", + }, + { + name: "default fixed string length 5", + method: "mask", + target: "this is a test", + args: []any{int64(5)}, + exp: "*****", + err: "", + }, + { + name: "Mask left leave left hand four chars unmasked", + method: "mask", + target: "this is a test", + args: []any{int64(4), "left"}, + exp: "this**********", + err: "", + }, + { + name: "Mask right leave right hand four chars unmasked", + method: "mask", + target: "this is a test", + args: []any{int64(6), "right"}, + exp: "********a test", + err: "", + }, + { + name: "Mask right leave right hand four chars unmasked, mask with '%' char", + method: "mask", + target: "this is a test", + args: []any{int64(6), "right", "%"}, + exp: "%%%%%%%%a test", + err: "", + }, + { + name: "invalid direction", + method: "mask", + target: "this is a test", + args: []any{int64(5), "Fred", "*"}, + exp: nil, + err: "direction must be one of left, right or all", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.name, func(t *testing.T) { + targetClone := value.IClone(test.target) + argsClone := value.IClone(test.args).([]any) + + fn, err := query.InitMethodHelper(test.method, query.NewLiteralFunction("", targetClone), argsClone...) + + if test.err != "" { + require.Error(t, err) + assert.EqualError(t, err, test.err) + return + } + + require.NoError(t, err) + + res, err := fn.Exec(query.FunctionContext{ + Maps: map[string]query.Function{}, + Index: 0, + MsgBatch: nil, + }) + require.NoError(t, err) + + assert.Equal(t, test.exp, res) + assert.Equal(t, test.target, targetClone) + assert.Equal(t, test.args, argsClone) + }) + } +} From 71926fea9f899175bc512b5db4a84510a27284f6 Mon Sep 17 00:00:00 2001 From: Mark Olliver Date: Mon, 17 Jun 2024 14:46:29 +0000 Subject: [PATCH 2/4] fix count type --- internal/impl/pure/bloblang_mask.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/impl/pure/bloblang_mask.go b/internal/impl/pure/bloblang_mask.go index a6c8aa06e..87dd26697 100644 --- a/internal/impl/pure/bloblang_mask.go +++ b/internal/impl/pure/bloblang_mask.go @@ -16,17 +16,18 @@ func init() { Param(bloblang.NewInt64Param("count").Description("the number of characters that will not be masked on the left or right hand side, in the case of a all mask, it is the number of mask characters to return giving a fixed length string, default is 0 which will return all characters masked.").Optional().Default(0)). Param(bloblang.NewStringParam("direction").Description("the direction to mask, left, right or all, default is all").Optional().Default("all")). Param(bloblang.NewStringParam("char").Description("the character used for masking, default is *").Optional().Default("*")). - Example("Mask the first 13 characters", `root.body_mask = this.body.mask(13, right)`, + Example("Mask the first 13 characters", `root.body_mask = this.body.mask(13, "right")`, [2]string{ `{"body":"the cat goes meow"}`, `{"body_mask":"*************meow"}`, }, ), func(args *bloblang.ParsedParams) (bloblang.Method, error) { - count, err := args.GetOptionalInt64("count") + countPtr, err := args.GetOptionalInt64("count") if err != nil { return nil, errors.New("failed to get count as int: " + err.Error()) } + count := int(*countPtr) char, err := args.GetString("char") if err != nil { @@ -44,7 +45,7 @@ func init() { } return bloblang.StringMethod(func(s string) (any, error) { - return maskString(s, char, direction, int(*count)), nil + return maskString(s, char, direction, int(count)), nil }), nil }); err != nil { panic(err) @@ -52,10 +53,6 @@ func init() { } // maskString masks the string based on the given parameters. -// `s` is the input string -// `char` is the character used for masking, -// `direction` determines whether the start ("left") or the end ("right") of the string is masked or all for the whole string, -// `count` is the number of characters that will not be masked on the left or right hand side, in the case of a all mask, it is the number of mask characters to return giving a fixed length string. func maskString(s string, char string, direction string, count int) string { sLength := len(s) if count == 0 { From c2ed284255e22133c2c41a7217d4c2b60afba58d Mon Sep 17 00:00:00 2001 From: Mark Olliver Date: Fri, 12 Jul 2024 13:25:33 +0000 Subject: [PATCH 3/4] fix up examples and test --- config/test/bloblang/masking.yaml | 11 +++++++++-- internal/impl/pure/bloblang_mask.go | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/test/bloblang/masking.yaml b/config/test/bloblang/masking.yaml index 44f7a59b5..c4dcb1e4f 100644 --- a/config/test/bloblang/masking.yaml +++ b/config/test/bloblang/masking.yaml @@ -1,7 +1,14 @@ pipeline: processors: - - bloblang: | - root = if env("MASK") == "FIXED_5" { this.value.mask(5)} else if env("MASK") == "LEFT_5" { this.value.mask(5, "left")} else if env("MASK") == "RIGHT_4" {this.value.mask(4,"right")} else if env("MASK") == "RIGHT_4_HASH" {this.value.mask(4,"right", "#")} else { this.value.mask() } + - mapping: | + root = this + root = match env("MASK") { + "FIXED_5" => root.value.mask(5) + "LEFT_5" => root.value.mask(5, "left") + "RIGHT_4" => root.value.mask(4,"right") + "RIGHT_4_HASH" => root.value.mask(4,"right", "#") + _ => root.value.mask() + } tests: - name: Fixed mask 5 diff --git a/internal/impl/pure/bloblang_mask.go b/internal/impl/pure/bloblang_mask.go index 87dd26697..5a3082820 100644 --- a/internal/impl/pure/bloblang_mask.go +++ b/internal/impl/pure/bloblang_mask.go @@ -16,7 +16,7 @@ func init() { Param(bloblang.NewInt64Param("count").Description("the number of characters that will not be masked on the left or right hand side, in the case of a all mask, it is the number of mask characters to return giving a fixed length string, default is 0 which will return all characters masked.").Optional().Default(0)). Param(bloblang.NewStringParam("direction").Description("the direction to mask, left, right or all, default is all").Optional().Default("all")). Param(bloblang.NewStringParam("char").Description("the character used for masking, default is *").Optional().Default("*")). - Example("Mask the first 13 characters", `root.body_mask = this.body.mask(13, "right")`, + Example("Mask the first 13 characters", `root.body_mask = this.body.mask(4, "right")`, [2]string{ `{"body":"the cat goes meow"}`, `{"body_mask":"*************meow"}`, From 2edcf4c62f1c0690c3a0b774accc2862a6b905e0 Mon Sep 17 00:00:00 2001 From: Mark Olliver Date: Fri, 12 Jul 2024 13:29:55 +0000 Subject: [PATCH 4/4] fix lint --- internal/impl/pure/bloblang_mask.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/pure/bloblang_mask.go b/internal/impl/pure/bloblang_mask.go index 5a3082820..5134c2791 100644 --- a/internal/impl/pure/bloblang_mask.go +++ b/internal/impl/pure/bloblang_mask.go @@ -45,7 +45,7 @@ func init() { } return bloblang.StringMethod(func(s string) (any, error) { - return maskString(s, char, direction, int(count)), nil + return maskString(s, char, direction, count), nil }), nil }); err != nil { panic(err)