feat(generic-api-key): exclude keywords (#1587)

* fix(config.tmpl) missing \n for rules with regexes AND stopwords

patch provided by rgmx

fix(config.tmpl) omit `condition=OR`

* feat(generic-api-key): exclude keywords

prevent matching the rule generic-api-key on e.g.
- public_key
- api_version
- client_id
- client_name
- keyword
- monkey

feat(generic-api-key): review findings

- combine capture groups
- remove mapKey from allowList

fix(generic): make monkey, donkey, case-sensitive

avoids matching on e.g. pokemonKey=

* feat(generic) remove client, add credential

exclude more composite words with key

cmd/generate/config/rules/config.tmpl

@@ -52,20 +52,18 @@ tags = [
 {{- if eq (len $rule.Allowlists) 1 }}{{ with index $rule.Allowlists 0}}{{ if or .Regexes .Paths .Commits .StopWords }}
 # NOTE: Gitleaks >= v8.21.0 should use [[rules.allowlists] instead.
 [rules.allowlist]
-{{ with .MatchCondition }}condition = "{{ . }}"
-{{ end -}}
-{{- with .Commits }}commits = [
+{{- with .MatchCondition }}{{println}}condition = "{{ .MatchCondition.String }}"{{ end }}
+{{- with .Commits -}}{{println}}commits = [
     {{ range $j, $commit := . }}"{{ $commit }}",{{ end }}
 ]{{ end }}
-{{- with .Paths }}paths = [
+{{- with .Paths }}{{println}}paths = [
     {{ range $j, $path := . }}'''{{ $path }}''',{{ end }}
 ]{{ end }}
-{{- with .RegexTarget }}regexTarget = "{{ . }}"
-{{ end -}}
-{{- with .Regexes }}regexes = [{{ range $i, $regex := . }}
+{{- if and .RegexTarget .Regexes }}{{println}}regexTarget = "{{ .RegexTarget }}"{{ end -}}
+{{- with .Regexes }}{{println}}regexes = [{{ range $i, $regex := . }}
     '''{{ $regex }}''',{{ end }}
 ]{{ end }}
-{{- with .StopWords }}stopwords = [{{ range $j, $stopword := . }}
+{{- with .StopWords }}{{println}}stopwords = [{{ range $j, $stopword := . }}
     "{{ $stopword }}",{{ end }}
 ]{{ end }}{{ end }}{{ end }}
 {{ else }}
@@ -84,7 +82,7 @@ tags = [
 {{- with $allowlist.Regexes }}regexes = [{{ range $i, $regex := . }}
     '''{{ $regex }}''',{{ end }}
 ]{{ end }}
-{{- with $allowlist.StopWords }}stopwords = [{{ range $j, $stopword := . }}
+{{ with $allowlist.StopWords }}stopwords = [{{ range $j, $stopword := . }}
     "{{ $stopword }}",{{ end }}
 ]{{ end }}{{ end }}{{ end }}
 {{ end }}{{ end }}

cmd/generate/config/rules/generic.go

@@ -2,7 +2,9 @@ package rules
 
 import (
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
+	"regexp"
 )
 
 func GenericCredential() *config.Rule {
@@ -15,7 +17,8 @@ func GenericCredential() *config.Rule {
 			"api",
 			"token",
 			"secret",
-			"client",
+			"credential",
+			"creds",
 			"passwd",
 			"password",
 			"auth",
@@ -26,7 +29,8 @@ func GenericCredential() *config.Rule {
 			"api",
 			"token",
 			"secret",
-			"client",
+			"credential",
+			"creds",
 			"passwd",
 			"password",
 			"auth",
@@ -35,7 +39,22 @@ func GenericCredential() *config.Rule {
 		Entropy: 3.5,
 		Allowlists: []config.Allowlist{
 			{
-				StopWords: DefaultStopWords,
+				Description:    "Allowlist for Generic API Keys",
+				MatchCondition: config.AllowlistMatchOr,
+				Regexes: []*regexp.Regexp{
+					regexp.MustCompile(`(?i)(` +
+						`public[_.-]?(key|token)` + // public key -> not a secret
+						`|api[_.-]?(version|id)` + // version/id -> not a secret
+						`|(secret)[_.-]?name` + // name of e.g. env variable
+						`|issuerkeyhash` + // part of ssl cert
+						`|(?-i:[DdMm]onkey|[DM]ONKEY)|keying` + // common words containing "key"
+						`|(primary|foreign|natural|definition|hot)[_.-]?key` +
+						`|key[_.-]?(alias|board|code|stone|storetype|word|up|down|left|right)` +
+						`|rapid|capital` + // common words containing "api"
+						`)`),
+				},
+				RegexTarget: "match",
+				StopWords:   DefaultStopWords,
 			},
 		},
 	}
@@ -44,14 +63,54 @@ func GenericCredential() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("generic", "CLOJARS_34bf0e88955ff5a1c328d6a7491acc4f48e865a7b8dd4d70a70749037443"), //gitleaks:allow
 		utils.GenerateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"),
-		`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
+		`"credentials" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
 		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`,
+		`passwd = ` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
+		`private-key: ` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{100}`),
+		`mySecretString=` + newPlausibleSecret(`[a-zA-Z0-9]{30}`),
+		`some_api_token_123 = "` + newPlausibleSecret(`[a-zA-Z0-9]{60}`) + `"`,
+		`todo_secret_do_not_commit = ` + newPlausibleSecret(`[a-zA-Z0-9]{30}`),
+		`creds = ` + newPlausibleSecret(`[a-zA-Z0-9]{30}`),
 	}
 	fps := []string{
 		`client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`,
 		`password combination.
 
 R5: Regulatory--21`,
+		`public_key = "9Cnzj4p4WGeKLs1Pt8QuKUpRKfFLfRYC9AIKjbJTWit"`,
+		`publicToken = "9Cnzj4p4WGeKLs1Pt8QuKUpRKfFLfRYC9AIKjbJTWit"`,
+		`clientId = "73082700-1f09-405b-80d0-3131bfd6272d"`,
+		`MantleAPI_version=9a038989604e8da62ecddbe2094b16ce1b778be1`,
+		`COMMUNICATION_API_VERSION=rc0.13.0_20230412_0712-SNAPSHOT`,
+		`LLM_SECRET_NAME = "NEXUS-GPT4-API-KEY"`,
+		`keyword: "Befaehigung_P2"`,
+		`minisat-master-keying:x64-uwp=fail`,
+		`monkeys-audio:mx64-uwp=fail`,
+		`rapidstring:marm64-uwp=fail`,
+		`<entry key="jetbrains.mps.v8_elimination" value="executed" />`,
+		`event-bus-message-api:rc0.15.0_20231217_1420-SNAPSHOT'`,
+		`primaryKey=` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
+		`foreignKey=` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
+		`key_down_event=` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
+		`issuerKeyHash=` + newPlausibleSecret(`[a-zA-Z0-9\-_.=]{30}`),
 	}
 	return utils.Validate(r, tps, fps)
 }
+
+func newPlausibleSecret(regex string) string {
+	allowList := config.Allowlist{StopWords: DefaultStopWords}
+	// attempt to generate a random secret,
+	// retrying until it contains at least one digit and no stop words
+	// TODO: currently the DefaultStopWords list contains many short words,
+	//  so there is a significant chance of generating a secret that contains a stop word
+	for {
+		secret := secrets.NewSecret(regex)
+		if !regexp.MustCompile(`[1-9]`).MatchString(secret) {
+			continue
+		}
+		if allowList.ContainsStopWord(secret) {
+			continue
+		}
+		return secret
+	}
+}

cmd/generate/config/rules/stopwords.go

@@ -638,6 +638,7 @@ var DefaultStopWords = []string{
 	"jbos",
 	"jekyll",
 	"jenkin",
+	"jetbrains",
 	"job-",
 	"job.",
 	"job_",
@@ -1181,6 +1182,7 @@ var DefaultStopWords = []string{
 	"smart",
 	"smtp",
 	"snake",
+	"snapshot",
 	"snippet",
 	"soap",
 	"social",

config/gitleaks.toml

@@ -529,14 +529,15 @@ keywords = ["aiza"]
 [[rules]]
 id = "generic-api-key"
 description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations."
-regex = '''(?i)[\w.-]{0,10}?(?:key|api|token|secret|client|passwd|password|auth|access)(?:[ \t\w.-]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+regex = '''(?i)[\w.-]{0,10}?(?:key|api|token|secret|credential|creds|passwd|password|auth|access)(?:[ \t\w.-]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
 entropy = 3.5
 keywords = [
     "key",
     "api",
     "token",
     "secret",
-    "client",
+    "credential",
+    "creds",
     "passwd",
     "password",
     "auth",
@@ -544,6 +545,10 @@ keywords = [
 ]
 # NOTE: Gitleaks >= v8.21.0 should use [[rules.allowlists] instead.
 [rules.allowlist]
+regexTarget = "match"
+regexes = [
+    '''(?i)(public[_.-]?(key|token)|api[_.-]?(version|id)|(secret)[_.-]?name|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(primary|foreign|natural|definition|hot)[_.-]?key|key[_.-]?(alias|board|code|stone|storetype|word|up|down|left|right)|rapid|capital)''',
+]
 stopwords = [
     "000000",
     "aaaaaa",
@@ -1171,6 +1176,7 @@ stopwords = [
     "jbos",
     "jekyll",
     "jenkin",
+    "jetbrains",
     "job-",
     "job.",
     "job_",
@@ -1714,6 +1720,7 @@ stopwords = [
     "smart",
     "smtp",
     "snake",
+    "snapshot",
     "snippet",
     "soap",
     "social",