feat(config): add placeholder regexes to global allowlist (#1547)

cmd/generate/config/base/config.go

@@ -10,10 +10,30 @@ func CreateGlobalConfig() config.Config {
 		Title: "gitleaks config",
 		Allowlist: config.Allowlist{
 			Description: "global allow lists",
+			Regexes: []*regexp.Regexp{
+				// ----------- General placeholders -----------
+				regexp.MustCompile(`(?i)^true|false|null$`),
+				// ----------- Interpolated Variables -----------
+				// Ansible (https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html)
+				regexp.MustCompile(`^\{\{[ \t]*[\w ().|]+[ \t]*}}$`),
+				// GitHub Actions
+				// https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables
+				// https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions
+				regexp.MustCompile(`^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$`),
+				// NuGet (https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file#using-environment-variables)
+				regexp.MustCompile(`^%([A-Z_]+|[a-z_]+)%$`),
+				// Urban Code Deploy (https://www.ibm.com/support/pages/replace-token-step-replaces-replacement-values-windows-variables)
+				regexp.MustCompile(`^@([A-Z_]+|[a-z_]+)@$`),
+
+				// ----------- Environment Variables -----------
+				regexp.MustCompile(`^\$(\d+|{\d+})$`),
+				regexp.MustCompile(`^\$([A-Z_]+|[a-z_]+)$`),
+				regexp.MustCompile(`^\${([A-Z_]+|[a-z_]+)}$`),
+			},
 			Paths: []*regexp.Regexp{
 				regexp.MustCompile(`^gitleaks\.toml$`),
 
-				// Images
+				// ----------- Documents and media -----------
 				regexp.MustCompile(`(?i)\.(bmp|gif|jpe?g|svg|tiff?)$`),
 				regexp.MustCompile(`(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$`),
 

cmd/generate/config/base/config_test.go

@@ -0,0 +1,69 @@
+package base
+
+import (
+	"testing"
+)
+
+func TestConfigAllowlistRegexes(t *testing.T) {
+	tests := map[string]struct {
+		invalid []string
+		valid   []string
+	}{
+		"general placeholders": {
+			invalid: []string{
+				`true`, `True`, `false`, `False`, `null`, `NULL`,
+			},
+		},
+		"interpolated variables - ansible": {
+			invalid: []string{
+				`{{ x }}`, `{{ password }}`, `{{password}}`, `{{ data.proxy_password }}`,
+				`{{ dict1 | ansible.builtin.combine(dict2) }}`,
+			},
+		},
+		"interpolated variables - github actions": {
+			invalid: []string{
+				`${{ env.First_Name }}`,
+				`${{ env.DAY_OF_WEEK == 'Monday' }}`,
+				`${{env.JAVA_VERSION}}`,
+				`${{ github.event.issue.title }}`,
+				`${{ github.repository == "Gattocrucco/lsqfitgp" }}`,
+				`${{ github.event.pull_request.number || github.ref }}`,
+				`${{ github.event_name == 'pull_request' && github.event.action == 'unassigned' }}`,
+				`${{ secrets.SuperSecret }}`,
+				`${{ vars.JOB_NAME }}`,
+				`${{ vars.USE_VARIABLES == 'true' }}`,
+			},
+		},
+		"interpolated variables - nuget": {
+			invalid: []string{
+				`%MY_PASSWORD%`, `%password%`,
+			},
+		},
+		"interpolated variables - ucd": {
+			invalid: []string{`@password@`, `@LDAP_PASS@`},
+			valid:   []string{`@username@mastodon.example`},
+		},
+		"environment variables": {
+			invalid: []string{`$2`, `$GIT_PASSWORD`, `${GIT_PASSWORD}`, `$password`},
+			valid:   []string{`$yP@R.@=ibxI`, `$2a6WCust9aE`, `${not_complete1`},
+		},
+	}
+
+	cfg := CreateGlobalConfig()
+	allowlist := cfg.Allowlist
+	for name, cases := range tests {
+		t.Run(name, func(t *testing.T) {
+			for _, c := range cases.invalid {
+				if !allowlist.RegexAllowed(c) {
+					t.Errorf("invalid value not marked as allowed: %s", c)
+				}
+			}
+
+			for _, c := range cases.valid {
+				if allowlist.RegexAllowed(c) {
+					t.Errorf("valid value marked as allowed: %s", c)
+				}
+			}
+		})
+	}
+}

cmd/generate/config/rules/config.tmpl

@@ -15,8 +15,8 @@ description = "{{.Allowlist.Description}}"
 {{- with .Allowlist.RegexTarget }}
 regexTarget = "{{ . }}"{{ end -}}
 {{- with .Allowlist.Regexes }}
-regexes = [
-    {{ range $i, $regex := . }}'''{{ $regex }}''',{{ end }}
+regexes = [{{ range $i, $regex := . }}
+    '''{{ $regex }}''',{{ end }}
 ]{{ end }}
 {{- with .Allowlist.Paths }}
 paths = [{{ range $i, $path := . }}
@@ -45,20 +45,21 @@ tags = [
 ]{{ end }}
 {{ if or $rule.Allowlist.Regexes $rule.Allowlist.Paths $rule.Allowlist.Commits $rule.Allowlist.StopWords }}
 [rules.allowlist]
-{{ with $rule.Allowlist.RegexTarget }}
+{{- with $rule.Allowlist.RegexTarget }}
 regexTarget = "{{ . }}"{{ end -}}
 {{- with $rule.Allowlist.Regexes }}
-regexes = [
-    {{ range $j, $regex := . }}'''{{ $regex }}''',{{ end }}
+regexes = [{{ range $i, $regex := . }}
+    '''{{ $regex }}''',{{ end }}
 ]{{ end }}
 {{- with $rule.Allowlist.Paths }}paths = [
-    {{ range $j, $path := . }}"{{ $path }}",{{ end }}
+    {{ range $j, $path := . }}'''{{ $path }}''',{{ end }}
 ]{{ end }}
 {{- with $rule.Allowlist.Commits }}commits = [
     {{ range $j, $commit := . }}"{{ $commit }}",{{ end }}
 ]{{ end }}
-{{- with $rule.Allowlist.StopWords }}stopwords = [{{ range $j, $stopword := . }}
+{{- with $rule.Allowlist.StopWords }}
+stopwords = [{{ range $j, $stopword := . }}
     "{{ $stopword }}",{{ end }}
 ]{{ end }}
 {{ end }}
-{{ end }}
+{{ end -}}

config/gitleaks.toml

@@ -12,6 +12,16 @@ title = "gitleaks config"
 
 [allowlist]
 description = "global allow lists"
+regexes = [
+    '''(?i)^true|false|null$''',
+    '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''',
+    '''^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''',
+    '''^%([A-Z_]+|[a-z_]+)%$''',
+    '''^@([A-Z_]+|[a-z_]+)@$''',
+    '''^\$(\d+|{\d+})$''',
+    '''^\$([A-Z_]+|[a-z_]+)$''',
+    '''^\${([A-Z_]+|[a-z_]+)}$''',
+]
 paths = [
     '''^gitleaks\.toml$''',
     '''(?i)\.(bmp|gif|jpe?g|svg|tiff?)$''',
@@ -2149,7 +2159,6 @@ path = '''(?i)\.ya?ml$'''
 keywords = ["secret"]
 
 [rules.allowlist]
-
 regexes = [
     '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''',
 ]
@@ -2336,9 +2345,10 @@ entropy = 1
 keywords = ["<add key="]
 
 [rules.allowlist]
-
 regexes = [
-    '''33f!!lloppa''','''hal\+9ooo_da!sY''','''^\%\S.*\%$''',
+    '''33f!!lloppa''',
+    '''hal\+9ooo_da!sY''',
+    '''^\%\S.*\%$''',
 ]
 
 [[rules]]
@@ -2665,7 +2675,6 @@ entropy = 3
 keywords = ["sumo"]
 
 [rules.allowlist]
-
 regexTarget = "line"
 regexes = [
     '''sumOf''',
@@ -2755,7 +2764,6 @@ keywords = [
 ]
 
 [rules.allowlist]
-
 regexes = [
     '''s\.[A-Za-z]{24}''',
 ]
@@ -2784,4 +2792,3 @@ description = "Detected a Zendesk Secret Key, risking unauthorized access to cus
 regex = '''(?i)(?:zendesk)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
 keywords = ["zendesk"]
 
-