diff --git a/plugin/README.md b/plugin/README.md index db6385eeb..94a86977f 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -319,6 +319,8 @@ pipelines: actions: - type: mask metric_subsystem_name: "some_name" + ignore_fields: + - trace_id masks: - mask: re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b" diff --git a/plugin/action/README.md b/plugin/action/README.md index 40374c9b5..72425e058 100755 --- a/plugin/action/README.md +++ b/plugin/action/README.md @@ -154,6 +154,8 @@ pipelines: actions: - type: mask metric_subsystem_name: "some_name" + ignore_fields: + - trace_id masks: - mask: re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b" diff --git a/plugin/action/mask/README.md b/plugin/action/mask/README.md index 56adb5394..718644c41 100644 --- a/plugin/action/mask/README.md +++ b/plugin/action/mask/README.md @@ -10,6 +10,8 @@ pipelines: actions: - type: mask metric_subsystem_name: "some_name" + ignore_fields: + - trace_id masks: - mask: re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b" diff --git a/plugin/action/mask/mask.go b/plugin/action/mask/mask.go index e029f9a07..c24d9b31a 100644 --- a/plugin/action/mask/mask.go +++ b/plugin/action/mask/mask.go @@ -26,6 +26,8 @@ pipelines: actions: - type: mask metric_subsystem_name: "some_name" + ignore_fields: + - trace_id masks: - mask: re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b" @@ -189,7 +191,9 @@ func compileMasks(masks []Mask, logger *zap.Logger) ([]Mask, *regexp.Regexp) { patterns := make([]string, 0, len(masks)) for i := range masks { compileMask(&masks[i], logger) - patterns = append(patterns, masks[i].Re) + if masks[i].Re != "" { + patterns = append(patterns, masks[i].Re) + } } combinedPattern := strings.Join(patterns, "|") @@ -440,14 +444,17 @@ func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult { p.valueNodes = getValueNodeList(root, p.valueNodes, p.ignoredFields) for _, v := range p.valueNodes { value := v.AsBytes() - if p.config.SkipMismatched && !p.matchRe.Match(value) { - continue - } + valueIsCommonMatched := p.matchRe.Match(value) p.sourceBuf = append(p.sourceBuf[:0], value...) p.maskBuf = append(p.maskBuf[:0], p.sourceBuf...) for i := range p.config.Masks { mask := &p.config.Masks[i] + if mask.Re != "" && p.config.SkipMismatched && !valueIsCommonMatched { + // skips messages not matched common regex + continue + } + p.maskBuf, locApplied = p.maskValue(mask, p.sourceBuf, p.maskBuf) p.sourceBuf = append(p.sourceBuf[:0], p.maskBuf...) if !locApplied { diff --git a/plugin/action/mask/mask_test.go b/plugin/action/mask/mask_test.go index 6003209cc..98f4da7ec 100644 --- a/plugin/action/mask/mask_test.go +++ b/plugin/action/mask/mask_test.go @@ -482,6 +482,7 @@ func TestPlugin(t *testing.T) { } config := test.NewConfig(&Config{ + SkipMismatched: true, Masks: []Mask{ { Re: `a(x*)b`, @@ -528,25 +529,76 @@ func TestPlugin(t *testing.T) { } } -func createConfig() Config { - config := Config{ +func TestWithEmptyRegex(t *testing.T) { + suits := []struct { + name string + input []string + expected []string + comment string + }{ + { + name: "ID&card", + input: []string{`{"field1":"Индивидуальный предприниматель Иванов Иван Иванович"}`}, + expected: []string{`{"field1":"Индивидуальный предприниматель Иванов Иван Иванович","access_token_leaked":"personal_data_leak"}`}, + comment: "Add field access_token_leaked", + }, + } + + config := test.NewConfig(&Config{ SkipMismatched: true, Masks: []Mask{ { - Re: `a(x*)b`, - Groups: []int{0}, + MatchRules: []matchrule.RuleSet{ + { + Rules: []matchrule.Rule{ + { + Values: []string{"Индивидуальный предприниматель"}, + Mode: matchrule.ModeContains, + CaseInsensitive: false, + }, + }, + }, + }, + AppliedField: "access_token_leaked", + AppliedValue: "personal_data_leak", + MetricName: "sec_dataleak_predprinimatel", + MetricLabels: []string{"service"}, }, { Re: kDefaultCardRegExp, Groups: []int{1, 2, 3, 4}, }, - { - Re: kDefaultIDRegExp, - Groups: []int{0}, - }, }, + }, nil) + + for _, s := range suits { + t.Run(s.name, func(t *testing.T) { + sut, input, output := test.NewPipelineMock( + test.NewActionPluginStaticInfo(factory, config, + pipeline.MatchModeAnd, + nil, + false)) + wg := sync.WaitGroup{} + wg.Add(len(s.input)) + + outEvents := make([]string, 0, len(s.expected)) + output.SetOutFn(func(e *pipeline.Event) { + outEvents = append(outEvents, e.Root.EncodeToString()) + wg.Done() + }) + + for _, in := range s.input { + input.In(0, "test.log", 0, []byte(in)) + } + + wg.Wait() + sut.Stop() + + for i := range s.expected { + assert.Equal(t, s.expected[i], outEvents[i], s.comment) + } + }) } - return config } //nolint:funlen @@ -707,7 +759,7 @@ func TestPluginWithComplexMasks(t *testing.T) { for _, s := range suits { t.Run(s.name, func(t *testing.T) { config := test.NewConfig(&Config{ - SkipMismatched: true, + SkipMismatched: true, Masks: s.masks, AppliedMetricName: s.metricName, AppliedMetricLabels: s.metricLabels,