feat(generate): generate global (#1546)

cmd/generate/config/base/config.go

@@ -0,0 +1,69 @@
+package base
+
+import (
+	"github.com/zricethezav/gitleaks/v8/config"
+	"regexp"
+)
+
+func CreateGlobalConfig() config.Config {
+	return config.Config{
+		Title: "gitleaks config",
+		Allowlist: config.Allowlist{
+			Description: "global allow lists",
+			Paths: []*regexp.Regexp{
+				regexp.MustCompile(`^gitleaks\.toml$`),
+
+				// Images
+				regexp.MustCompile(`(?i)\.(bmp|gif|jpe?g|svg|tiff?)$`),
+				regexp.MustCompile(`(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$`),
+
+				// ----------- Golang files -----------
+				regexp.MustCompile(`go\.(mod|sum|work(\.sum)?)$`),
+				regexp.MustCompile(`(^|/)vendor/modules\.txt$`),
+				regexp.MustCompile(`(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)/.*$`),
+
+				// ----------- Java files -----------
+				// Gradle
+				regexp.MustCompile(`(^|/)gradlew(\.bat)?$`),
+				regexp.MustCompile(`(^|/)gradle\.lockfile$`),
+				regexp.MustCompile(`(^|/)mvnw(\.cmd)?$`),
+				regexp.MustCompile(`(^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$`),
+
+				// ----------- Node.js files -----------
+				// Dependencies and lock files.
+				regexp.MustCompile(`(^|/)node_modules/.*?$`),
+				regexp.MustCompile(`(^|/)package-lock\.json$`),
+				regexp.MustCompile(`(^|/)yarn\.lock$`),
+				regexp.MustCompile(`(^|/)pnpm-lock\.yaml$`),
+				regexp.MustCompile(`(^|/)npm-shrinkwrap\.json$`),
+				regexp.MustCompile(`(^|/)bower_components/.*?$`),
+
+				// ----------- Python files -----------
+				// Dependencies and lock files.
+				regexp.MustCompile(`(^|/)Pipfile\.lock$`),
+				regexp.MustCompile(`(^|/)poetry\.lock$`),
+				// Virtual environments
+				// env/lib/python3.7/site-packages/urllib3/util/url.py
+				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib/.+$`),
+				// /python/3.7.4/Lib/site-packages/dask/bytes/tests/test_bytes_utils.py
+				// python/3.7.4/Lib/site-packages/fsspec/utils.py
+				// python/2.7.16.32/Lib/bsddb/test/test_dbenv.py
+				regexp.MustCompile(`(?i)/?python/[23](\.\d{1,2})+/lib/.+$`),
+				// python/lib/python3.8/site-packages/boto3/data/ec2/2016-04-01/resources-1.json
+				// python/lib/python3.8/site-packages/botocore/data/alexaforbusiness/2017-11-09/service-2.json
+				regexp.MustCompile(`(?i)/?python/lib/python[23](\.\d{1,2})+/.+$`),
+				// dist-info directory (https://py-pkgs.org/04-package-structure.html#building-sdists-and-wheels)
+				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$`),
+
+				// ----------- Ruby files -----------
+				regexp.MustCompile(`(^|/)vendor/(bundle|ruby)/.*?$`),
+				regexp.MustCompile(`\.gem$`), // tar archive
+
+				// Misc
+				regexp.MustCompile(`verification-metadata.xml`),
+				regexp.MustCompile(`Database.refactorlog`),
+				//regexp.MustCompile(`vendor`),
+			},
+		},
+	}
+}

cmd/generate/config/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/base"
 	"os"
 	"text/template"
 
@@ -228,7 +229,9 @@ func main() {
 		log.Fatal().Err(err).Msg("Failed to create rules.toml")
 	}
 
-	if err = tmpl.Execute(f, config.Config{Rules: ruleLookUp}); err != nil {
+	cfg := base.CreateGlobalConfig()
+	cfg.Rules = ruleLookUp
+	if err = tmpl.Execute(f, cfg); err != nil {
 		log.Fatal().Err(err).Msg("could not execute template")
 	}
 

cmd/generate/config/rules/config.tmpl

@@ -8,24 +8,20 @@
 # Rules instruct gitleaks on what should be considered a secret.
 # Allowlists instruct gitleaks on what is allowed, i.e. not a secret.
 
-title = "gitleaks config"
+title = "{{.Title}}"
 
 [allowlist]
-description = "global allow lists"
-paths = [
-    '''gitleaks.toml''',
-    '''(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''',
-    '''(go.mod|go.sum|go.work|go.work.sum)$''',
-    '''gradle.lockfile''',
-    '''verification-metadata.xml''',
-    '''node_modules''',
-    '''package-lock.json''',
-    '''yarn.lock''',
-    '''pnpm-lock.yaml''',
-    '''poetry.lock''',
-    '''Database.refactorlog''',
-    '''vendor''',
-]
+description = "{{.Allowlist.Description}}"
+{{- with .Allowlist.RegexTarget }}
+regexTarget = "{{ . }}"{{ end -}}
+{{- with .Allowlist.Regexes }}
+regexes = [
+    {{ range $i, $regex := . }}'''{{ $regex }}''',{{ end }}
+]{{ end }}
+{{- with .Allowlist.Paths }}
+paths = [{{ range $i, $path := . }}
+    '''{{ $path }}''',{{ end }}
+]{{ end }}
 
 {{ range $i, $rule := .Rules }}[[rules]]
 id = "{{$rule.RuleID}}"
@@ -38,9 +34,11 @@ secretGroup = {{ . }}{{ end -}}
 {{- with $rule.Entropy }}
 entropy = {{ . }}{{ end -}}
 {{- with $rule.Keywords }}
-keywords = [
-    {{ range $j, $keyword := . }}"{{ $keyword }}",{{ end }}
-]{{ end }}
+{{- if gt (len .) 1}}
+keywords = [{{ range $j, $keyword := . }}
+    "{{ $keyword }}",{{ end }}
+]{{else}}
+keywords = [{{ range $j, $keyword := . }}"{{ $keyword }}"{{ end }}]{{end}}{{ end }}
 {{- with $rule.Tags }}
 tags = [
     {{ range $j, $tag := . }}"{{ $tag }}",{{ end }}

cmd/generate/config/utils/validate.go

@@ -6,6 +6,7 @@ package utils
 
 import (
 	"github.com/rs/zerolog/log"
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/base"
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/detect"
 	"strings"
@@ -82,8 +83,8 @@ func createSingleRuleDetector(r *config.Rule) *detect.Detector {
 	rules := map[string]config.Rule{
 		r.RuleID: *r,
 	}
-	return detect.NewDetector(config.Config{
-		Rules:    rules,
-		Keywords: uniqueKeywords,
-	})
+	cfg := base.CreateGlobalConfig()
+	cfg.Rules = rules
+	cfg.Keywords = uniqueKeywords
+	return detect.NewDetector(cfg)
 }

config/config.go

@@ -54,6 +54,7 @@ type ViperConfig struct {
 
 // Config is a configuration struct that contains rules and an allowlist if present.
 type Config struct {
+	Title       string
 	Extend      Extend
 	Path        string
 	Description string