dont use empty mask re in common regex

ozontech · Dec 14, 2023 · 960bc56 · 960bc56
1 parent c83b720
commit 960bc56
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 14 deletions.
diff --git a/plugin/README.md b/plugin/README.md
@@ -319,6 +319,8 @@ pipelines:
     actions:
     - type: mask
       metric_subsystem_name: "some_name"
+      ignore_fields:
+      - trace_id
       masks:
       - mask:
         re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"

diff --git a/plugin/action/README.md b/plugin/action/README.md
@@ -154,6 +154,8 @@ pipelines:
     actions:
     - type: mask
       metric_subsystem_name: "some_name"
+      ignore_fields:
+      - trace_id
       masks:
       - mask:
         re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"

diff --git a/plugin/action/mask/README.md b/plugin/action/mask/README.md
@@ -10,6 +10,8 @@ pipelines:
     actions:
     - type: mask
       metric_subsystem_name: "some_name"
+      ignore_fields:
+      - trace_id
       masks:
       - mask:
         re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"

diff --git a/plugin/action/mask/mask.go b/plugin/action/mask/mask.go
@@ -26,6 +26,8 @@ pipelines:
     actions:
     - type: mask
       metric_subsystem_name: "some_name"
+      ignore_fields:
+      - trace_id
       masks:
       - mask:
         re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
@@ -189,7 +191,9 @@ func compileMasks(masks []Mask, logger *zap.Logger) ([]Mask, *regexp.Regexp) {
 	patterns := make([]string, 0, len(masks))
 	for i := range masks {
 		compileMask(&masks[i], logger)
-		patterns = append(patterns, masks[i].Re)
+		if masks[i].Re != "" {
+			patterns = append(patterns, masks[i].Re)
+		}
 	}
 
 	combinedPattern := strings.Join(patterns, "|")
@@ -440,14 +444,17 @@ func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult {
 	p.valueNodes = getValueNodeList(root, p.valueNodes, p.ignoredFields)
 	for _, v := range p.valueNodes {
 		value := v.AsBytes()
-		if p.config.SkipMismatched && !p.matchRe.Match(value) {
-			continue
-		}
+		valueIsCommonMatched := p.matchRe.Match(value)
 
 		p.sourceBuf = append(p.sourceBuf[:0], value...)
 		p.maskBuf = append(p.maskBuf[:0], p.sourceBuf...)
 		for i := range p.config.Masks {
 			mask := &p.config.Masks[i]
+			if mask.Re != "" && p.config.SkipMismatched && !valueIsCommonMatched {
+				// skips messages not matched common regex
+				continue
+			}
+
 			p.maskBuf, locApplied = p.maskValue(mask, p.sourceBuf, p.maskBuf)
 			p.sourceBuf = append(p.sourceBuf[:0], p.maskBuf...)
 			if !locApplied {

diff --git a/plugin/action/mask/mask_test.go b/plugin/action/mask/mask_test.go
@@ -482,6 +482,7 @@ func TestPlugin(t *testing.T) {
 	}
 
 	config := test.NewConfig(&Config{
+		SkipMismatched: true,
 		Masks: []Mask{
 			{
 				Re:     `a(x*)b`,
@@ -528,25 +529,76 @@ func TestPlugin(t *testing.T) {
 	}
 }
 
-func createConfig() Config {
-	config := Config{
+func TestWithEmptyRegex(t *testing.T) {
+	suits := []struct {
+		name     string
+		input    []string
+		expected []string
+		comment  string
+	}{
+		{
+			name:     "ID&card",
+			input:    []string{`{"field1":"Индивидуальный предприниматель Иванов Иван Иванович"}`},
+			expected: []string{`{"field1":"Индивидуальный предприниматель Иванов Иван Иванович","access_token_leaked":"personal_data_leak"}`},
+			comment:  "Add field access_token_leaked",
+		},
+	}
+
+	config := test.NewConfig(&Config{
 		SkipMismatched: true,
 		Masks: []Mask{
 			{
-				Re:     `a(x*)b`,
-				Groups: []int{0},
+				MatchRules: []matchrule.RuleSet{
+					{
+						Rules: []matchrule.Rule{
+							{
+								Values:          []string{"Индивидуальный предприниматель"},
+								Mode:            matchrule.ModeContains,
+								CaseInsensitive: false,
+							},
+						},
+					},
+				},
+				AppliedField: "access_token_leaked",
+				AppliedValue: "personal_data_leak",
+				MetricName:   "sec_dataleak_predprinimatel",
+				MetricLabels: []string{"service"},
 			},
 			{
 				Re:     kDefaultCardRegExp,
 				Groups: []int{1, 2, 3, 4},
 			},
-			{
-				Re:     kDefaultIDRegExp,
-				Groups: []int{0},
-			},
 		},
+	}, nil)
+
+	for _, s := range suits {
+		t.Run(s.name, func(t *testing.T) {
+			sut, input, output := test.NewPipelineMock(
+				test.NewActionPluginStaticInfo(factory, config,
+					pipeline.MatchModeAnd,
+					nil,
+					false))
+			wg := sync.WaitGroup{}
+			wg.Add(len(s.input))
+
+			outEvents := make([]string, 0, len(s.expected))
+			output.SetOutFn(func(e *pipeline.Event) {
+				outEvents = append(outEvents, e.Root.EncodeToString())
+				wg.Done()
+			})
+
+			for _, in := range s.input {
+				input.In(0, "test.log", 0, []byte(in))
+			}
+
+			wg.Wait()
+			sut.Stop()
+
+			for i := range s.expected {
+				assert.Equal(t, s.expected[i], outEvents[i], s.comment)
+			}
+		})
 	}
-	return config
 }
 
 //nolint:funlen
@@ -707,7 +759,7 @@ func TestPluginWithComplexMasks(t *testing.T) {
 	for _, s := range suits {
 		t.Run(s.name, func(t *testing.T) {
 			config := test.NewConfig(&Config{
-        SkipMismatched:      true,
+				SkipMismatched:      true,
 				Masks:               s.masks,
 				AppliedMetricName:   s.metricName,
 				AppliedMetricLabels: s.metricLabels,