Fix decoded line allowlist (#1788)

* fix

* B patch

* buh bye

.gitleaksignore

@@ -938,3 +938,4 @@ e56870aecf8ac2733affc82e1ac25e3be9092db3:cmd/generate/config/rules/curl.go:gener
 239e6323639b18ba8a1081f17788a658da075dd7:cmd/generate/config/rules/prefect.go:prefect-api-token:26
 239e6323639b18ba8a1081f17788a658da075dd7:cmd/generate/config/rules/databricks.go:databricks-api-token:25
 239e6323639b18ba8a1081f17788a658da075dd7:cmd/generate/config/rules/databricks.go:generic-api-key:22
+c24a3a654b18d689732ec3199f0a19a254505b04:detect/detect_test.go:generic-api-key:52

detect/decoder.go

@@ -279,3 +279,25 @@ func segmentWithDecodedOverlap(encodedSegments []EncodedSegment, start, end int)
 
 	return nil
 }
+
+func (s EncodedSegment) lineStartIndex(currentRaw string) int {
+	for i := s.decodedStart; i > -1; i-- {
+		c := currentRaw[i]
+		if c == '\n' {
+			return i
+		}
+	}
+
+	return 0
+}
+
+func (s EncodedSegment) lineEndIndex(currentRaw string, matchLen int) int {
+	for i := s.decodedStart; i < s.decodedStart+matchLen; i++ {
+		c := currentRaw[i]
+		if c == '\n' {
+			return i
+		}
+	}
+
+	return len(currentRaw) - 1
+}

detect/detect.go

@@ -384,6 +384,7 @@ MatchLoop:
 
 		// For any meta data from decoding
 		var metaTags []string
+		currentLine := ""
 
 		// Check if the decoded portions of the segment overlap with the match
 		// to see if its potentially a new match
@@ -391,6 +392,7 @@ MatchLoop:
 			if segment := segmentWithDecodedOverlap(encodedSegments, matchIndex[0], matchIndex[1]); segment != nil {
 				matchIndex = segment.adjustMatchIndex(matchIndex)
 				metaTags = append(metaTags, segment.tags()...)
+				currentLine = currentRaw[segment.lineStartIndex(currentRaw):segment.lineEndIndex(currentRaw, matchIndex[1]-matchIndex[0])]
 			} else {
 				// This item has already been added to a finding
 				continue
@@ -433,6 +435,10 @@ MatchLoop:
 			continue
 		}
 
+		if currentLine == "" {
+			currentLine = finding.Line
+		}
+
 		// Set the value of |secret|, if the pattern contains at least one capture group.
 		// (The first element is the full match, hence we check >= 2.)
 		groups := r.Regex.FindStringSubmatch(finding.Secret)
@@ -474,7 +480,7 @@ MatchLoop:
 		case "match":
 			globalAllowlistTarget = finding.Match
 		case "line":
-			globalAllowlistTarget = finding.Line
+			globalAllowlistTarget = currentLine
 		}
 		if d.Config.Allowlist.RegexAllowed(globalAllowlistTarget) {
 			logger.Trace().
@@ -496,7 +502,7 @@ MatchLoop:
 			case "match":
 				allowlistTarget = finding.Match
 			case "line":
-				allowlistTarget = finding.Line
+				allowlistTarget = currentLine
 			}
 
 			var (

detect/detect_test.go

@@ -47,6 +47,9 @@ c21hbGwtc2VjcmV0
 secret=ZGVjb2RlZC1zZWNyZXQtdmFsdWU=
 # The above encoded again
 c2VjcmV0PVpHVmpiMlJsWkMxelpXTnlaWFF0ZG1Gc2RXVT0=
+
+# Confirm you can ignore on the decoded value
+password="bFJxQkstejVrZjQtcGxlYXNlLWlnbm9yZS1tZS1YLVhJSk0yUGRkdw=="
 `
 
 func TestDetect(t *testing.T) {
@@ -469,6 +472,20 @@ func TestDetect(t *testing.T) {
 					EndColumn:   49,
 					Entropy:     3.3037016,
 				},
+				{ // This just confirms that with no allowlist the pattern is detected (i.e. the regex is good)
+					Description: "Make sure this would be detected with no allowlist",
+					Secret:      "lRqBK-z5kf4-please-ignore-me-X-XIJM2Pddw",
+					Match:       "password=\"lRqBK-z5kf4-please-ignore-me-X-XIJM2Pddw\"",
+					File:        "tmp.go",
+					Line:        "\npassword=\"bFJxQkstejVrZjQtcGxlYXNlLWlnbm9yZS1tZS1YLVhJSk0yUGRkdw==\"",
+					RuleID:      "decoded-password-dont-ignore",
+					Tags:        []string{"decode-ignore", "decoded:base64", "decode-depth:1"},
+					StartLine:   23,
+					EndLine:     23,
+					StartColumn: 2,
+					EndColumn:   68,
+					Entropy:     4.5841837,
+				},
 			},
 		},
 	}

testdata/config/base64_encoded.toml

@@ -73,3 +73,54 @@
   regex = '''secret=(decoded-secret-value)'''
   tags = ['overlapping']
   secretGroup = 1
+
+# -----BEGIN REGEX TARGET DECODED MATCH PATTERNS-----
+[[rules]]
+  id = 'decoded-password-dont-ignore'
+  description = 'Make sure this would be detected with no allowlist'
+  regex = '''password\s*=\s*\"([^\"]+please-ignore-me[^\"]+)\"'''
+  tags = ['decode-ignore']
+  secretGroup = 1
+
+[[rules]]
+  id = 'decoded-password-ignore-secret'
+  description = 'Test ignore on decoded secrets: regexTarget = "secret"'
+  regex = '''password\s*=\s*\"([^\"]+please-ignore-me[^\"]+)\"'''
+  tags = ['decode-ignore']
+  secretGroup = 1
+
+  [[rules.allowlists]]
+    regexTarget = 'secret'
+    regexes = [
+      # The decoded segment that we are testing against
+      'please-ignore-me',
+    ]
+
+[[rules]]
+  id = 'decoded-password-ignore-match'
+  description = 'Test ignore on decoded secrets: regexTarget = "match"'
+  regex = '''password\s*=\s*\"([^\"]+please-ignore-me[^\"]+)\"'''
+  tags = ['decode-ignore']
+  secretGroup = 1
+
+  [[rules.allowlists]]
+    regexTarget = 'match'
+    regexes = [
+      # The decoded segment that we are testing against
+      'please-ignore-me',
+    ]
+
+[[rules]]
+  id = 'decoded-password-ignore-line'
+  description = 'Test ignore on decoded secrets: regexTarget = "line"'
+  regex = '''password\s*=\s*\"([^\"]+please-ignore-me[^\"]+)\"'''
+  tags = ['decode-ignore']
+  secretGroup = 1
+
+  [[rules.allowlists]]
+    regexTarget = 'line'
+    regexes = [
+      # The decoded segment that we are testing against
+      'please-ignore-me',
+    ]
+# -----END REGEX TARGET DECODED MATCH PATTERNS-----