package config import ( _ "embed" "errors" "fmt" "sort" "strings" "github.com/spf13/viper" "github.com/zricethezav/gitleaks/v8/logging" "github.com/zricethezav/gitleaks/v8/regexp" ) var ( //go:embed gitleaks.toml DefaultConfig string // use to keep track of how many configs we can extend // yea I know, globals bad extendDepth int ) const maxExtendDepth = 2 // ViperConfig is the config struct used by the Viper config package // to parse the config file. This struct does not include regular expressions. // It is used as an intermediary to convert the Viper config to the Config struct. type ViperConfig struct { Title string Description string Extend Extend Rules []struct { ID string Description string Path string Regex string SecretGroup int Entropy float64 Keywords []string Tags []string // Deprecated: this is a shim for backwards-compatibility. // TODO: Remove this in 9.x. AllowList *viperRuleAllowlist Allowlists []*viperRuleAllowlist } // Deprecated: this is a shim for backwards-compatibility. // TODO: Remove this in 9.x. AllowList *viperGlobalAllowlist Allowlists []*viperGlobalAllowlist } type viperRuleAllowlist struct { Description string Condition string Commits []string Paths []string RegexTarget string Regexes []string StopWords []string } type viperGlobalAllowlist struct { TargetRules []string viperRuleAllowlist `mapstructure:",squash"` } // Config is a configuration struct that contains rules and an allowlist if present. type Config struct { Title string Extend Extend Path string Description string Rules map[string]Rule Keywords map[string]struct{} // used to keep sarif results consistent OrderedRules []string Allowlists []*Allowlist } // Extend is a struct that allows users to define how they want their // configuration extended by other configuration files. type Extend struct { Path string URL string UseDefault bool DisabledRules []string } func (vc *ViperConfig) Translate() (Config, error) { var ( keywords = make(map[string]struct{}) orderedRules []string rulesMap = make(map[string]Rule) ruleAllowlists = make(map[string][]*Allowlist) ) // Validate individual rules. for _, vr := range vc.Rules { var ( pathPat *regexp.Regexp regexPat *regexp.Regexp ) if vr.Path != "" { pathPat = regexp.MustCompile(vr.Path) } if vr.Regex != "" { regexPat = regexp.MustCompile(vr.Regex) } if vr.Keywords == nil { vr.Keywords = []string{} } else { for i, k := range vr.Keywords { keyword := strings.ToLower(k) keywords[keyword] = struct{}{} vr.Keywords[i] = keyword } } if vr.Tags == nil { vr.Tags = []string{} } cr := Rule{ RuleID: vr.ID, Description: vr.Description, Regex: regexPat, SecretGroup: vr.SecretGroup, Entropy: vr.Entropy, Path: pathPat, Keywords: vr.Keywords, Tags: vr.Tags, } // Parse the rule allowlists, including the older format for backwards compatibility. if vr.AllowList != nil { // TODO: Remove this in v9. if len(vr.Allowlists) > 0 { return Config{}, fmt.Errorf("%s: [rules.allowlist] is deprecated, it cannot be used alongside [[rules.allowlist]]", cr.RuleID) } vr.Allowlists = append(vr.Allowlists, vr.AllowList) } for _, a := range vr.Allowlists { allowlist, err := parseAllowlist(a) if err != nil { return Config{}, fmt.Errorf("%s: [[rules.allowlists]] %w", cr.RuleID, err) } cr.Allowlists = append(cr.Allowlists, allowlist) } orderedRules = append(orderedRules, cr.RuleID) rulesMap[cr.RuleID] = cr } // Assemble the config. c := Config{ Title: vc.Title, Description: vc.Description, Extend: vc.Extend, Rules: rulesMap, Keywords: keywords, OrderedRules: orderedRules, } // Parse the config allowlists, including the older format for backwards compatibility. if vc.AllowList != nil { // TODO: Remove this in v9. if len(vc.Allowlists) > 0 { return Config{}, errors.New("[allowlist] is deprecated, it cannot be used alongside [[allowlists]]") } vc.Allowlists = append(vc.Allowlists, vc.AllowList) } for _, a := range vc.Allowlists { allowlist, err := parseAllowlist(&a.viperRuleAllowlist) if err != nil { return Config{}, fmt.Errorf("[[allowlists]] %w", err) } // Allowlists with |targetRules| aren't added to the global list. if len(a.TargetRules) > 0 { for _, ruleID := range a.TargetRules { // It's not possible to validate |ruleID| until after extend. ruleAllowlists[ruleID] = append(ruleAllowlists[ruleID], allowlist) } } else { c.Allowlists = append(c.Allowlists, allowlist) } } if maxExtendDepth != extendDepth { // disallow both usedefault and path from being set if c.Extend.Path != "" && c.Extend.UseDefault { return Config{}, errors.New("unable to load config due to extend.path and extend.useDefault being set") } if c.Extend.UseDefault { if err := c.extendDefault(); err != nil { return Config{}, err } } else if c.Extend.Path != "" { if err := c.extendPath(); err != nil { return Config{}, err } } } // Validate the rules after everything has been assembled (including extended configs). if extendDepth == 0 { for _, rule := range c.Rules { if err := rule.Validate(); err != nil { return Config{}, err } } // Populate targeted configs. for ruleID, allowlists := range ruleAllowlists { rule, ok := c.Rules[ruleID] if !ok { return Config{}, fmt.Errorf("[[allowlists]] target rule ID '%s' does not exist", ruleID) } rule.Allowlists = append(rule.Allowlists, allowlists...) c.Rules[ruleID] = rule } } return c, nil } func parseAllowlist(a *viperRuleAllowlist) (*Allowlist, error) { var matchCondition AllowlistMatchCondition switch strings.ToUpper(a.Condition) { case "AND", "&&": matchCondition = AllowlistMatchAnd case "", "OR", "||": matchCondition = AllowlistMatchOr default: return nil, fmt.Errorf("unknown allowlist |condition| '%s' (expected 'and', 'or')", a.Condition) } // Validate the target. regexTarget := a.RegexTarget if regexTarget != "" { switch regexTarget { case "secret": regexTarget = "" case "match", "line": // do nothing default: return nil, fmt.Errorf("unknown allowlist |regexTarget| '%s' (expected 'match', 'line')", regexTarget) } } var allowlistRegexes []*regexp.Regexp for _, a := range a.Regexes { allowlistRegexes = append(allowlistRegexes, regexp.MustCompile(a)) } var allowlistPaths []*regexp.Regexp for _, a := range a.Paths { allowlistPaths = append(allowlistPaths, regexp.MustCompile(a)) } allowlist := &Allowlist{ Description: a.Description, MatchCondition: matchCondition, Commits: a.Commits, Paths: allowlistPaths, RegexTarget: regexTarget, Regexes: allowlistRegexes, StopWords: a.StopWords, } if err := allowlist.Validate(); err != nil { return nil, err } return allowlist, nil } func (c *Config) GetOrderedRules() []Rule { var orderedRules []Rule for _, id := range c.OrderedRules { if _, ok := c.Rules[id]; ok { orderedRules = append(orderedRules, c.Rules[id]) } } return orderedRules } func (c *Config) extendDefault() error { extendDepth++ viper.SetConfigType("toml") if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil { return fmt.Errorf("failed to load extended default config, err: %w", err) } defaultViperConfig := ViperConfig{} if err := viper.Unmarshal(&defaultViperConfig); err != nil { return fmt.Errorf("failed to load extended default config, err: %w", err) } cfg, err := defaultViperConfig.Translate() if err != nil { return fmt.Errorf("failed to load extended default config, err: %w", err) } logging.Debug().Msg("extending config with default config") c.extend(cfg) return nil } func (c *Config) extendPath() error { extendDepth++ viper.SetConfigFile(c.Extend.Path) if err := viper.ReadInConfig(); err != nil { return fmt.Errorf("failed to load extended config, err: %w", err) } extensionViperConfig := ViperConfig{} if err := viper.Unmarshal(&extensionViperConfig); err != nil { return fmt.Errorf("failed to load extended config, err: %w", err) } cfg, err := extensionViperConfig.Translate() if err != nil { return fmt.Errorf("failed to load extended config, err: %w", err) } logging.Debug().Msgf("extending config with %s", c.Extend.Path) c.extend(cfg) return nil } func (c *Config) extendURL() { // TODO } func (c *Config) extend(extensionConfig Config) { // Get config name for helpful log messages. var configName string if c.Extend.Path != "" { configName = c.Extend.Path } else { configName = "default" } // Convert |Config.DisabledRules| into a map for ease of access. disabledRuleIDs := map[string]struct{}{} for _, id := range c.Extend.DisabledRules { if _, ok := extensionConfig.Rules[id]; !ok { logging.Warn(). Str("rule-id", id). Str("config", configName). Msg("Disabled rule doesn't exist in extended config.") } disabledRuleIDs[id] = struct{}{} } for ruleID, baseRule := range extensionConfig.Rules { // Skip the rule. if _, ok := disabledRuleIDs[ruleID]; ok { logging.Debug(). Str("rule-id", ruleID). Str("config", configName). Msg("Ignoring rule from extended config.") continue } currentRule, ok := c.Rules[ruleID] if !ok { // Rule doesn't exist, add it to the config. c.Rules[ruleID] = baseRule for _, k := range baseRule.Keywords { c.Keywords[k] = struct{}{} } c.OrderedRules = append(c.OrderedRules, ruleID) } else { // Rule exists, merge our changes into the base. if currentRule.Description != "" { baseRule.Description = currentRule.Description } if currentRule.Entropy != 0 { baseRule.Entropy = currentRule.Entropy } if currentRule.SecretGroup != 0 { baseRule.SecretGroup = currentRule.SecretGroup } if currentRule.Regex != nil { baseRule.Regex = currentRule.Regex } if currentRule.Path != nil { baseRule.Path = currentRule.Path } baseRule.Tags = append(baseRule.Tags, currentRule.Tags...) baseRule.Keywords = append(baseRule.Keywords, currentRule.Keywords...) baseRule.Allowlists = append(baseRule.Allowlists, currentRule.Allowlists...) // The keywords from the base rule and the extended rule must be merged into the global keywords list for _, k := range baseRule.Keywords { c.Keywords[k] = struct{}{} } c.Rules[ruleID] = baseRule } } // append allowlists, not attempting to merge c.Allowlists = append(c.Allowlists, extensionConfig.Allowlists...) // sort to keep extended rules in order sort.Strings(c.OrderedRules) }