refactor(config): keyword map (#1538)

cmd/generate/config/utils/validate.go

@@ -11,20 +11,9 @@ import (
 	"strings"
 )
 
-func Validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule {
-	// normalize keywords like in the config package
-	var keywords []string
-	for _, k := range r.Keywords {
-		keywords = append(keywords, strings.ToLower(k))
-	}
-	r.Keywords = keywords
-
-	rules := make(map[string]config.Rule)
-	rules[r.RuleID] = r
-	d := detect.NewDetector(config.Config{
-		Rules:    rules,
-		Keywords: keywords,
-	})
+func Validate(rule config.Rule, truePositives []string, falsePositives []string) *config.Rule {
+	r := &rule
+	d := createSingleRuleDetector(r)
 	for _, tp := range truePositives {
 		if len(d.DetectString(tp)) != 1 {
 			log.Fatal().
@@ -43,22 +32,12 @@ func Validate(r config.Rule, truePositives []string, falsePositives []string) *c
 				Msg("Failed to Validate. False positive was detected by regex.")
 		}
 	}
-	return &r
+	return r
 }
 
-func ValidateWithPaths(r config.Rule, truePositives map[string]string, falsePositives map[string]string) *config.Rule {
-	var keywords []string
-	for _, k := range r.Keywords {
-		keywords = append(keywords, strings.ToLower(k))
-	}
-	r.Keywords = keywords
-
-	rules := make(map[string]config.Rule)
-	rules[r.RuleID] = r
-	d := detect.NewDetector(config.Config{
-		Rules:    rules,
-		Keywords: keywords,
-	})
+func ValidateWithPaths(rule config.Rule, truePositives map[string]string, falsePositives map[string]string) *config.Rule {
+	r := &rule
+	d := createSingleRuleDetector(r)
 	for path, tp := range truePositives {
 		f := detect.Fragment{Raw: tp, FilePath: path}
 		if len(d.Detect(f)) != 1 {
@@ -81,5 +60,30 @@ func ValidateWithPaths(r config.Rule, truePositives map[string]string, falsePosi
 				Msg("Failed to Validate. False positive was detected by regex and/or path.")
 		}
 	}
-	return &r
+	return r
+}
+
+func createSingleRuleDetector(r *config.Rule) *detect.Detector {
+	// normalize keywords like in the config package
+	var (
+		uniqueKeywords = make(map[string]struct{})
+		keywords       []string
+	)
+	for _, keyword := range r.Keywords {
+		k := strings.ToLower(keyword)
+		if _, ok := uniqueKeywords[k]; ok {
+			continue
+		}
+		keywords = append(keywords, k)
+		uniqueKeywords[k] = struct{}{}
+	}
+	r.Keywords = keywords
+
+	rules := map[string]config.Rule{
+		r.RuleID: *r,
+	}
+	return detect.NewDetector(config.Config{
+		Rules:    rules,
+		Keywords: uniqueKeywords,
+	})
 }

config/config.go

@@ -59,7 +59,7 @@ type Config struct {
 	Description string
 	Rules       map[string]Rule
 	Allowlist   Allowlist
-	Keywords    []string
+	Keywords    map[string]struct{}
 
 	// used to keep sarif results consistent
 	OrderedRules []string
@@ -75,10 +75,10 @@ type Extend struct {
 
 func (vc *ViperConfig) Translate() (Config, error) {
 	var (
-		keywords     []string
+		keywords     = make(map[string]struct{})
 		orderedRules []string
+		rulesMap     = make(map[string]Rule)
 	)
-	rulesMap := make(map[string]Rule)
 
 	for _, r := range vc.Rules {
 		var allowlistRegexes []*regexp.Regexp
@@ -94,7 +94,7 @@ func (vc *ViperConfig) Translate() (Config, error) {
 			r.Keywords = []string{}
 		} else {
 			for _, k := range r.Keywords {
-				keywords = append(keywords, strings.ToLower(k))
+				keywords[strings.ToLower(k)] = struct{}{}
 			}
 		}
 
@@ -245,7 +245,9 @@ func (c *Config) extend(extensionConfig Config) {
 		if !ok {
 			// Rule doesn't exist, add it to the config.
 			c.Rules[ruleID] = baseRule
-			c.Keywords = append(c.Keywords, baseRule.Keywords...)
+			for _, k := range baseRule.Keywords {
+				c.Keywords[k] = struct{}{}
+			}
 			c.OrderedRules = append(c.OrderedRules, ruleID)
 		} else {
 			// Rule exists, merge our changes into the base.
@@ -255,10 +257,12 @@ func (c *Config) extend(extensionConfig Config) {
 			baseRule.Allowlist.RegexTarget = currentRule.Allowlist.RegexTarget
 			baseRule.Allowlist.StopWords = append(baseRule.Allowlist.StopWords, currentRule.Allowlist.StopWords...)
 			// The keywords from the base rule and the extended rule must be merged into the global keywords list
-			c.Keywords = append(c.Keywords, baseRule.Keywords...)
-			c.Keywords = append(c.Keywords, currentRule.Keywords...)
-
-			delete(c.Rules, ruleID)
+			for _, k := range baseRule.Keywords {
+				c.Keywords[k] = struct{}{}
+			}
+			for _, k := range currentRule.Keywords {
+				c.Keywords[k] = struct{}{}
+			}
 			c.Rules[ruleID] = baseRule
 		}
 	}

detect/detect.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"golang.org/x/exp/maps"
 )
 
 const (
@@ -108,7 +109,7 @@ func NewDetector(cfg config.Config) *Detector {
 		findingMutex:   &sync.Mutex{},
 		findings:       make([]report.Finding, 0),
 		Config:         cfg,
-		prefilter:      *ahocorasick.NewTrieBuilder().AddStrings(cfg.Keywords).Build(),
+		prefilter:      *ahocorasick.NewTrieBuilder().AddStrings(maps.Keys(cfg.Keywords)).Build(),
 		Sema:           semgroup.NewGroup(context.Background(), 40),
 	}
 }

go.mod

@@ -1,6 +1,8 @@
 module github.com/zricethezav/gitleaks/v8
 
-go 1.22
+go 1.22.0
+
+toolchain go1.22.5
 
 require (
 	github.com/BobuSumisu/aho-corasick v1.0.3
@@ -13,6 +15,7 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
 )
 
 require (
@@ -40,7 +43,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/text v0.3.8 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect

go.sum

@@ -309,6 +309,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -393,8 +395,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=