chore: refine generic-api-key fps + trace logging (#1720)

cmd/generate/config/rules/generic.go

@@ -52,11 +52,11 @@ func GenericCredential() *config.Rule {
 				Regexes: []*regexp.Regexp{
 					regexp.MustCompile(`(?i)(` +
 						// Access
-						`accessor` +
+						`access(ibility|or)` +
 						`|access[_.-]?id` +
 						`|random[_.-]?access` +
 						// API
-						`|api[_.-]?(version|id)` + // version/id -> not a secret
+						`|api[_.-]?(id|name|version)` + // id/name/version -> not a secret
 						`|rapid|capital` + // common words containing "api"
 						`|[a-z0-9-]*?api[a-z0-9-]*?:jar:` + // Maven META-INF dependencies that contain "api" in the name.
 						// Auth
@@ -84,12 +84,22 @@ func GenericCredential() *config.Rule {
 						`|(api|credentials|token)[_.-]?(endpoint|ur[il])` +
 						`|public[_.-]?token` +
 						`|(key|token)[_.-]?file` +
+						// Empty variables capturing the next line (e.g., .env files)
+						`|(?-i:([A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(\n|\z))` +
+						`|(?-i:([A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(\n|\z))` +
 						`)`),
 				},
 				StopWords: append(DefaultStopWords,
 					"6fe4476ee5a1832882e326b506d14126", // https://github.com/yarnpkg/berry/issues/6201
 				),
 			},
+			{
+				RegexTarget: "line",
+				Regexes: []*regexp.Regexp{
+					// Docker build secrets (https://docs.docker.com/build/building/secrets/#using-build-secrets).
+					regexp.MustCompile(`--mount=type=secret,`),
+				},
+			},
 		},
 	}
 
@@ -129,6 +139,7 @@ func GenericCredential() *config.Rule {
 		// Access
 		`"accessor":"rA1wk0Y45YCufyfq",`,
 		`report_access_id: e8e4df51-2054-49b0-ab1c-516ac95c691d`,
+		`accessibilityYesOptionId = "0736f5ef-7e88-499a-80cc-90c85d2a5180"`,
 		`_RandomAccessIterator>
 _LIBCPP_CONSTEXPR_AFTER_CXX11 `,
 
@@ -141,6 +152,7 @@ _LIBCPP_CONSTEXPR_AFTER_CXX11 `,
 		`[DEBUG]		org.slf4j.slf4j-api:jar:1.7.8.:compile (version managed from default)`,
 		`[DEBUG]		org.neo4j.neo4j-graphdb-api:jar:3.5.12:test`,
 		`apiUrl=apigee.corpint.com`,
+		`X-API-Name": "NRG0-Hermes-INTERNAL-API",`,
 		// TODO: Jetbrains IML files (requires line-level allowlist).
 		// `<orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="projcet" />`
 
@@ -173,7 +185,6 @@ _LIBCPP_CONSTEXPR_AFTER_CXX11 `,
 		`minisat-master-keying:x64-uwp=fail`,
 		`IceSSL.KeyFile=s_rsa1024_priv.pem`,
 		`"bucket_key": "SalesResults-1.2"`,
-		// `<TAR key="REF_ID_923.properties" value="/opts/config/alias/"/>`,
 		`<key tag="SecurityIdentifier" name="SecurityIdentifier" type="STRING" />`,
 		// `packageKey":` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
 		`schemaKey = 'DOC_Vector_5_32'`,
@@ -196,6 +207,8 @@ _LIBCPP_CONSTEXPR_AFTER_CXX11 `,
 		`BlindKeySignatures = Ed25519.BlindKeySignatures`,
 		`AVEncVideoMaxKeyframeDistance, "2987123a-ba93-4704-b489-ec1e5f25292c"`,
 		// `<add key="SchemaTable" value="G:\SchemaTable.xml" />`,
+		//`    { key: '9df21e95-3848-409d-8f94-c675cdfee839', value: 'Americas' },`,
+		// `<TAR key="REF_ID_923.properties" value="/opts/config/alias/"/>`,
 		//	`secret:
 		// secretName: app-decryption-secret
 		// items:
@@ -216,6 +229,9 @@ R5: Regulatory--21`,
 		`  <UserSecretsId>79a3edd0-2092-40a2-a04d-dcb46d5ca9ed</UserSecretsId>`,
 		`secret_length = X25519.secret_length`,
 		`secretSize must be >= XXH3_SECRET_SIZE_MIN`,
+		`# get build time secret for authentication
+#RUN --mount=type=secret,id=jfrog_secret \
+#    JFROG_SECRET = $(cat /run/secrets/jfrog_secret) && \`,
 
 		// Token
 		`    access_token_url='https://github.com/login/oauth/access_token',`,
@@ -226,8 +242,10 @@ R5: Regulatory--21`,
 
 		// General
 		`clientId = "73082700-1f09-405b-80d0-3131bfd6272d"`,
-		//		`GITHUB_API_KEY=
-		//DYNATRACE_API_KEY=`,
+		`GITHUB_API_KEY=
+DYNATRACE_API_KEY=`,
+		`snowflake.password=
+jdbc.snowflake.url=`,
 	}
 	return utils.Validate(r, tps, fps)
 }
@@ -243,7 +261,7 @@ func newPlausibleSecret(regex string) string {
 		if !regexp.MustCompile(`[1-9]`).MatchString(secret) {
 			continue
 		}
-		if allowList.ContainsStopWord(secret) {
+		if ok, _ := allowList.ContainsStopWord(secret); ok {
 			continue
 		}
 		return secret

config/allowlist.go

@@ -57,16 +57,17 @@ type Allowlist struct {
 }
 
 // CommitAllowed returns true if the commit is allowed to be ignored.
-func (a *Allowlist) CommitAllowed(c string) bool {
+func (a *Allowlist) CommitAllowed(c string) (bool, string) {
 	if c == "" {
-		return false
+		return false, ""
 	}
+
 	for _, commit := range a.Commits {
 		if commit == c {
-			return true
+			return true, c
 		}
 	}
-	return false
+	return false, ""
 }
 
 // PathAllowed returns true if the path is allowed to be ignored.
@@ -79,14 +80,14 @@ func (a *Allowlist) RegexAllowed(secret string) bool {
 	return anyRegexMatch(secret, a.Regexes)
 }
 
-func (a *Allowlist) ContainsStopWord(s string) bool {
+func (a *Allowlist) ContainsStopWord(s string) (bool, string) {
 	s = strings.ToLower(s)
 	for _, stopWord := range a.StopWords {
 		if strings.Contains(s, strings.ToLower(stopWord)) {
-			return true
+			return true, stopWord
 		}
 	}
-	return false
+	return false, ""
 }
 
 func (a *Allowlist) Validate() error {

config/allowlist_test.go

@@ -40,7 +40,8 @@ func TestCommitAllowed(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		assert.Equal(t, tt.commitAllowed, tt.allowlist.CommitAllowed(tt.commit))
+		isAllowed, _ := tt.allowlist.CommitAllowed(tt.commit)
+		assert.Equal(t, tt.commitAllowed, isAllowed)
 	}
 }
 

config/gitleaks.toml

@@ -585,7 +585,7 @@ regexes = [
 [[rules.allowlists]]
 regexTarget = "match"
 regexes = [
-    '''(?i)(accessor|access[_.-]?id|random[_.-]?access|api[_.-]?(version|id)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(credentials?[_.-]?id|withCredentials)|(bucket|foreign|hot|idx|natural|primary|pub(lic)?|schema|sequence)[_.-]?key|key[_.-]?(alias|board|code|frame|id|length|mesh|name|pair|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(id|name)|keyVaultToStoreSecrets|key(store|tab)[_.-]?(file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(secret)[_.-]?(length|name|size)|UserSecretsId|(io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(api|credentials|token)[_.-]?(endpoint|ur[il])|public[_.-]?token|(key|token)[_.-]?file)''',
+    '''(?i)(access(ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(credentials?[_.-]?id|withCredentials)|(bucket|foreign|hot|idx|natural|primary|pub(lic)?|schema|sequence)[_.-]?key|key[_.-]?(alias|board|code|frame|id|length|mesh|name|pair|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(id|name)|keyVaultToStoreSecrets|key(store|tab)[_.-]?(file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(secret)[_.-]?(length|name|size)|UserSecretsId|(io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(api|credentials|token)[_.-]?(endpoint|ur[il])|public[_.-]?token|(key|token)[_.-]?file|(?-i:([A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(\n|\z))|(?-i:([A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(\n|\z)))''',
 ]
 stopwords = [
     "000000",
@@ -2066,6 +2066,11 @@ stopwords = [
     "zsh_",
     "6fe4476ee5a1832882e326b506d14126",
 ]
+[[rules.allowlists]]
+regexTarget = "line"
+regexes = [
+    '''--mount=type=secret,''',
+]
 
 [[rules]]
 id = "github-app-token"

detect/detect.go

@@ -267,9 +267,9 @@ func (d *Detector) detectRule(fragment Fragment, currentRaw string, r config.Rul
 	// check if filepath or commit is allowed for this rule
 	for _, a := range r.Allowlists {
 		var (
-			isAllowed     bool
-			commitAllowed = a.CommitAllowed(fragment.CommitSHA)
-			pathAllowed   = a.PathAllowed(fragment.FilePath)
+			isAllowed             bool
+			commitAllowed, commit = a.CommitAllowed(fragment.CommitSHA)
+			pathAllowed           = a.PathAllowed(fragment.FilePath)
 		)
 		if a.MatchCondition == config.AllowlistMatchAnd {
 			// Determine applicable checks.
@@ -296,7 +296,7 @@ func (d *Detector) detectRule(fragment Fragment, currentRaw string, r config.Rul
 		if isAllowed {
 			event := logger.Trace().Str("condition", a.MatchCondition.String())
 			if commitAllowed {
-				event.Bool("allowed-commit", commitAllowed)
+				event.Str("allowed-commit", commit)
 			}
 			if pathAllowed {
 				event.Bool("allowed-path", pathAllowed)
@@ -452,9 +452,10 @@ MatchLoop:
 				Str("finding", globalAllowlistTarget).
 				Msg("skipping finding: global allowlist regex")
 			continue
-		} else if d.Config.Allowlist.ContainsStopWord(finding.Secret) {
+		} else if ok, word := d.Config.Allowlist.ContainsStopWord(finding.Secret); ok {
 			logger.Trace().
 				Str("finding", finding.Secret).
+				Str("allowed-stopword", word).
 				Msg("skipping finding: global allowlist stopword")
 			continue
 		}
@@ -470,18 +471,19 @@ MatchLoop:
 			}
 
 			var (
-				isAllowed        bool
-				commitAllowed    bool
-				pathAllowed      bool
-				regexAllowed     = a.RegexAllowed(allowlistTarget)
-				containsStopword = a.ContainsStopWord(finding.Secret)
+				isAllowed              bool
+				commitAllowed          bool
+				commit                 string
+				pathAllowed            bool
+				regexAllowed           = a.RegexAllowed(allowlistTarget)
+				containsStopword, word = a.ContainsStopWord(finding.Secret)
 			)
 			// check if the secret is in the list of stopwords
 			if a.MatchCondition == config.AllowlistMatchAnd {
 				// Determine applicable checks.
 				var allowlistChecks []bool
 				if len(a.Commits) > 0 {
-					commitAllowed = a.CommitAllowed(fragment.CommitSHA)
+					commitAllowed, commit = a.CommitAllowed(fragment.CommitSHA)
 					allowlistChecks = append(allowlistChecks, commitAllowed)
 				}
 				if len(a.Paths) > 0 {
@@ -506,7 +508,7 @@ MatchLoop:
 					Str("finding", finding.Secret).
 					Str("condition", a.MatchCondition.String())
 				if commitAllowed {
-					event.Bool("allowed-commit", commitAllowed)
+					event.Str("allowed-commit", commit)
 				}
 				if pathAllowed {
 					event.Bool("allowed-path", pathAllowed)
@@ -515,7 +517,7 @@ MatchLoop:
 					event.Bool("allowed-regex", regexAllowed)
 				}
 				if containsStopword {
-					event.Bool("allowed-stopword", containsStopword)
+					event.Str("allowed-stopword", word)
 				}
 				event.Msg("skipping finding: rule allowlist")
 				continue MatchLoop

detect/git.go

@@ -42,7 +42,8 @@ func (d *Detector) DetectGit(cmd *sources.GitCmd, remote *RemoteInfo) ([]report.
 			commitSHA := ""
 			if gitdiffFile.PatchHeader != nil {
 				commitSHA = gitdiffFile.PatchHeader.SHA
-				if d.Config.Allowlist.CommitAllowed(gitdiffFile.PatchHeader.SHA) {
+				if ok, c := d.Config.Allowlist.CommitAllowed(gitdiffFile.PatchHeader.SHA); ok {
+					logging.Trace().Str("allowed-commit", c).Msg("skipping commit: global allowlist")
 					continue
 				}
 			}