[processor/redaction] Introduce blocked_key_patterns parameter (#37664

)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
- added `blocked_key_patterns` parameter to mask values of attributes,
which keys match at least one of the defined patterns
- refactored tests to make them easier to extend

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes #35830

#### Follow-up


#38161

---------

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>

.chloggen/redaction-blocked-patterns.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: processor/redaction
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Introduce 'blocked_key_patterns' parameter"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [35830]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

processor/redactionprocessor/README.md

@@ -73,6 +73,11 @@ processors:
     # Any keys in this list are allowed so they don't need to be in both lists.
     ignored_keys:
       - safe_attribute
+    # blocked_key_patterns is a list of blocked span attribute key patterns. Span attributes
+    # matching the regexes on the list are masked.
+    blocked_key_patterns:
+      - ".*token.*"
+      - ".*api_key.*"
     # blocked_values is a list of regular expressions for blocking values of
     # allowed span attributes. Values that match are masked
     blocked_values:
@@ -111,6 +116,9 @@ part of the value is not masked even if it matches the regular expression for a
 If the value matches the regular expression for a blocked value only, the matching
 part of the value is masked with a fixed length of asterisks.
 
+`blocked_key_patterns` applies to the values of the keys matching one of the patterns.
+The value is then masked according to the configuration.
+
 For example, if `notes` is on the list of allowed keys, then the `notes`
 attribute is retained. However, if there is a value such as a credit card
 number in the `notes` field that matched a regular expression on the list of

processor/redactionprocessor/config.go

@@ -14,6 +14,10 @@ type Config struct {
 	// allow all keys, you should explicitly set AllowAllKeys
 	AllowedKeys []string `mapstructure:"allowed_keys"`
 
+	// BlockedKeyPatterns is a list of blocked span attribute key patterns. Span attributes
+	// matching the regexes on the list are masked.
+	BlockedKeyPatterns []string `mapstructure:"blocked_key_patterns"`
+
 	// IgnoredKeys is a list of span attribute keys that are not redacted.
 	// Span attributes in this list are allowed to pass through the filter
 	// without being changed or removed.

processor/redactionprocessor/config_test.go

@@ -26,12 +26,13 @@ func TestLoadConfig(t *testing.T) {
 		{
 			id: component.NewIDWithName(metadata.Type, ""),
 			expected: &Config{
-				AllowAllKeys:  false,
-				AllowedKeys:   []string{"description", "group", "id", "name"},
-				IgnoredKeys:   []string{"safe_attribute"},
-				BlockedValues: []string{"4[0-9]{12}(?:[0-9]{3})?", "(5[1-5][0-9]{14})"},
-				AllowedValues: []string{".+@mycompany.com"},
-				Summary:       debug,
+				AllowAllKeys:       false,
+				AllowedKeys:        []string{"description", "group", "id", "name"},
+				IgnoredKeys:        []string{"safe_attribute"},
+				BlockedValues:      []string{"4[0-9]{12}(?:[0-9]{3})?", "(5[1-5][0-9]{14})"},
+				BlockedKeyPatterns: []string{".*token.*", ".*api_key.*"},
+				AllowedValues:      []string{".+@mycompany.com"},
+				Summary:            debug,
 			},
 		},
 		{

processor/redactionprocessor/factory_test.go

@@ -19,6 +19,7 @@ func TestDefaultConfiguration(t *testing.T) {
 	assert.Empty(t, c.AllowedKeys)
 	assert.Empty(t, c.BlockedValues)
 	assert.Empty(t, c.AllowedValues)
+	assert.Empty(t, c.BlockedKeyPatterns)
 }
 
 func TestCreateTestProcessor(t *testing.T) {

processor/redactionprocessor/processor.go

@@ -28,6 +28,8 @@ type redaction struct {
 	blockRegexList map[string]*regexp.Regexp
 	// Attribute values allowed in a span
 	allowRegexList map[string]*regexp.Regexp
+	// Attribute keys blocked in a span
+	blockKeyRegexList map[string]*regexp.Regexp
 	// Redaction processor configuration
 	config *Config
 	// Logger
@@ -43,6 +45,11 @@ func newRedaction(ctx context.Context, config *Config, logger *zap.Logger) (*red
 		// TODO: Placeholder for an error metric in the next PR
 		return nil, fmt.Errorf("failed to process block list: %w", err)
 	}
+	blockKeysRegexList, err := makeRegexList(ctx, config.BlockedKeyPatterns)
+	if err != nil {
+		// TODO: Placeholder for an error metric in the next PR
+		return nil, fmt.Errorf("failed to process block keys list: %w", err)
+	}
 
 	allowRegexList, err := makeRegexList(ctx, config.AllowedValues)
 	if err != nil {
@@ -51,12 +58,13 @@ func newRedaction(ctx context.Context, config *Config, logger *zap.Logger) (*red
 	}
 
 	return &redaction{
-		allowList:      allowList,
-		ignoreList:     ignoreList,
-		blockRegexList: blockRegexList,
-		allowRegexList: allowRegexList,
-		config:         config,
-		logger:         logger,
+		allowList:         allowList,
+		ignoreList:        ignoreList,
+		blockRegexList:    blockRegexList,
+		allowRegexList:    allowRegexList,
+		blockKeyRegexList: blockKeysRegexList,
+		config:            config,
+		logger:            logger,
 	}, nil
 }
 
@@ -197,7 +205,6 @@ func (s *redaction) processAttrs(_ context.Context, attributes pcommon.Map) {
 		}
 
 		strVal := value.Str()
-
 		// Allow any values matching the allowed list regex
 		for _, compiledRE := range s.allowRegexList {
 			if match := compiledRE.MatchString(strVal); match {
@@ -206,11 +213,20 @@ func (s *redaction) processAttrs(_ context.Context, attributes pcommon.Map) {
 			}
 		}
 
+		// Mask any blocked keys for the other attributes
+		for _, compiledRE := range s.blockKeyRegexList {
+			if match := compiledRE.MatchString(k); match {
+				toBlock = append(toBlock, k)
+				maskedValue := compiledRE.ReplaceAllString(strVal, "****")
+				value.SetStr(maskedValue)
+				return true
+			}
+		}
+
 		// Mask any blocked values for the other attributes
 		var matched bool
 		for _, compiledRE := range s.blockRegexList {
-			match := compiledRE.MatchString(strVal)
-			if match {
+			if match := compiledRE.MatchString(strVal); match {
 				if !matched {
 					matched = true
 					toBlock = append(toBlock, k)