v7.x Update - Add ability to append repo configurations, and user supplied configurations to the specified or default configuration (#487)

* Update merge configuration changes for  Gitleaks 7.x

* Change merge to append, and did some clean up

config/config.go

@@ -2,7 +2,10 @@ package config
 
 import (
 	"fmt"
+	"io"
+	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 
@@ -249,13 +252,7 @@ func LoadRepoConfig(repo *git.Repository, repoConfig string) (Config, error) {
 		if err != nil {
 			return Config{}, err
 		}
-		var tomlLoader TomlLoader
-		_, err = toml.DecodeReader(r, &tomlLoader)
-		if err != nil {
-			return Config{}, err
-		}
-
-		return tomlLoader.Parse()
+		return parseTomlFile(r)
 	}
 
 	log.Debug("attempting to load repo config from bare worktree, this may use an old config")
@@ -274,15 +271,49 @@ func LoadRepoConfig(repo *git.Repository, repoConfig string) (Config, error) {
 		return Config{}, err
 	}
 
-	var tomlLoader TomlLoader
 	r, err := f.Reader()
+
 	if err != nil {
 		return Config{}, err
 	}
-	_, err = toml.DecodeReader(r, &tomlLoader)
+
+	return parseTomlFile(r)
+}
+
+// LoadAdditionalConfig Accepts a path to a gitleaks config and returns a Config struct
+func LoadAdditionalConfig(repoConfig string) (Config, error) {
+	file, err := os.Open(filepath.Clean(repoConfig))
 	if err != nil {
 		return Config{}, err
 	}
 
+	return parseTomlFile(file)
+}
+
+// AppendConfig Accepts a Config struct and will append those fields to this Config Struct's fields
+func (config *Config) AppendConfig(configToBeAppended Config) Config {
+	newAllowList := AllowList{
+		Description: "Appended Configuration",
+		Commits:     append(config.Allowlist.Commits, configToBeAppended.Allowlist.Commits...),
+		Files:       append(config.Allowlist.Files, configToBeAppended.Allowlist.Files...),
+		Paths:       append(config.Allowlist.Paths, configToBeAppended.Allowlist.Paths...),
+		Regexes:     append(config.Allowlist.Regexes, configToBeAppended.Allowlist.Regexes...),
+		Repos:       append(config.Allowlist.Repos, configToBeAppended.Allowlist.Repos...),
+	}
+
+	return Config{
+		Rules:     append(config.Rules, configToBeAppended.Rules...),
+		Allowlist: newAllowList,
+	}
+}
+
+// takes a File, makes sure it is a valid config, and parses it
+func parseTomlFile(f io.Reader) (Config, error) {
+	var tomlLoader TomlLoader
+	_, err := toml.DecodeReader(f, &tomlLoader)
+	if err != nil {
+		log.Errorf("Unable to read gitleaks config. Using defaults. Error: %s", err)
+		return Config{}, err
+	}
 	return tomlLoader.Parse()
 }

config/config_test.go

@@ -211,3 +211,73 @@ func writeTestConfig(toml string) (string, error) {
 
 	return tmpfile.Name(), nil
 }
+
+func TestAppendingConfiguration(t *testing.T) {
+	testRegexA, _ := regexp.Compile("a")
+	testRegexB, _ := regexp.Compile("b")
+
+	allowListA := AllowList{
+		Description: "Test Description",
+		Commits:     []string{"a"},
+		Files:       []*regexp.Regexp{testRegexA},
+		Paths:       []*regexp.Regexp{testRegexA},
+		Regexes:     []*regexp.Regexp{testRegexA},
+		Repos:       []*regexp.Regexp{testRegexA},
+	}
+
+	allowListB := AllowList{
+		Description: "Test Description",
+		Commits:     []string{"b"},
+		Files:       []*regexp.Regexp{testRegexB},
+		Paths:       []*regexp.Regexp{testRegexB},
+		Regexes:     []*regexp.Regexp{testRegexB},
+		Repos:       []*regexp.Regexp{testRegexB},
+	}
+
+	ruleA := Rule{Description: "a"}
+	ruleB := Rule{Description: "b"}
+
+	rulesA := []Rule{ruleA}
+	rulesB := []Rule{ruleB}
+
+	cfgA := Config{
+		Rules:     rulesA,
+		Allowlist: allowListA,
+	}
+
+	cfgB := Config{
+		Rules:     rulesB,
+		Allowlist: allowListB,
+	}
+
+	cfgAppended := cfgA.AppendConfig(cfgB)
+
+	if !(len(cfgAppended.Rules) == 2) {
+		t.Errorf("Length of Appended Rules = %d; want 2", len(cfgAppended.Rules))
+	}
+
+	if !(len(cfgAppended.Allowlist.Commits) == 2) {
+		t.Errorf("Length of Appended Allowed Commits = %d; want 2", len(cfgAppended.Allowlist.Commits))
+	}
+
+	if !(len(cfgAppended.Allowlist.Files) == 2) {
+		t.Errorf("Length of Appended Allowed Files = %d; want 2", len(cfgAppended.Allowlist.Files))
+	}
+
+	if !(len(cfgAppended.Allowlist.Paths) == 2) {
+		t.Errorf("Length of Appended Allowed Paths = %d; want 2", len(cfgAppended.Allowlist.Paths))
+	}
+
+	if !(len(cfgAppended.Allowlist.Regexes) == 2) {
+		t.Errorf("Length of Appended Allowed Regexes = %d; want 2", len(cfgAppended.Allowlist.Regexes))
+	}
+
+	if !(len(cfgAppended.Allowlist.Repos) == 2) {
+		t.Errorf("Length of Appended Allowed Repos = %d; want 2", len(cfgAppended.Allowlist.Repos))
+	}
+
+	if cfgAppended.Allowlist.Description != "Appended Configuration" {
+		t.Errorf("Allow List Description is = \"%s\"; want \"Appended Configuration\"", cfgAppended.Allowlist.Description)
+	}
+
+}

options/options.go

@@ -20,25 +20,27 @@ import (
 
 // Options stores values of command line options
 type Options struct {
-	Verbose        bool   `short:"v" long:"verbose" description:"Show verbose output from scan"`
-	Quiet          bool   `short:"q" long:"quiet" description:"Sets log level to error and only output leaks, one json object per line"`
-	RepoURL        string `short:"r" long:"repo-url" description:"Repository URL"`
-	Path           string `short:"p" long:"path" description:"Path to directory (repo if contains .git) or file"`
-	ConfigPath     string `short:"c" long:"config-path" description:"Path to config"`
-	RepoConfigPath string `long:"repo-config-path" description:"Path to gitleaks config relative to repo root"`
-	ClonePath      string `long:"clone-path" description:"Path to clone repo to disk"`
-	Version        bool   `long:"version" description:"Version number"`
-	Username       string `long:"username" description:"Username for git repo"`
-	Password       string `long:"password" description:"Password for git repo"`
-	AccessToken    string `long:"access-token" description:"Access token for git repo"`
-	Threads        int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	SSH            string `long:"ssh-key" description:"Path to ssh key used for auth"`
-	Unstaged       bool   `long:"unstaged" description:"Run gitleaks on unstaged code"`
-	Branch         string `long:"branch" description:"Branch to scan"`
-	Redact         bool   `long:"redact" description:"Redact secrets from log messages and leaks"`
-	Debug          bool   `long:"debug" description:"Log debug messages"`
-	NoGit          bool   `long:"no-git" description:"Treat git repos as plain directories and scan those files"`
-	CodeOnLeak     int    `long:"leaks-exit-code" default:"1" description:"Exit code when leaks have been encountered"`
+	Verbose          bool   `short:"v" long:"verbose" description:"Show verbose output from scan"`
+	Quiet            bool   `short:"q" long:"quiet" description:"Sets log level to error and only output leaks, one json object per line"`
+	RepoURL          string `short:"r" long:"repo-url" description:"Repository URL"`
+	Path             string `short:"p" long:"path" description:"Path to directory (repo if contains .git) or file"`
+	ConfigPath       string `short:"c" long:"config-path" description:"Path to config"`
+	RepoConfigPath   string `long:"repo-config-path" description:"Path to gitleaks config relative to repo root"`
+	ClonePath        string `long:"clone-path" description:"Path to clone repo to disk"`
+	Version          bool   `long:"version" description:"Version number"`
+	Username         string `long:"username" description:"Username for git repo"`
+	Password         string `long:"password" description:"Password for git repo"`
+	AccessToken      string `long:"access-token" description:"Access token for git repo"`
+	Threads          int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	SSH              string `long:"ssh-key" description:"Path to ssh key used for auth"`
+	Unstaged         bool   `long:"unstaged" description:"Run gitleaks on unstaged code"`
+	Branch           string `long:"branch" description:"Branch to scan"`
+	Redact           bool   `long:"redact" description:"Redact secrets from log messages and leaks"`
+	Debug            bool   `long:"debug" description:"Log debug messages"`
+	NoGit            bool   `long:"no-git" description:"Treat git repos as plain directories and scan those files"`
+	CodeOnLeak       int    `long:"leaks-exit-code" default:"1" description:"Exit code when leaks have been encountered"`
+	AppendRepoConfig bool   `long:"append-repo-config" description:"append the provided or default config with the repo config."`
+	AdditionalConfig string `long:"additional-config" description:"path to an additional gitleaks config to append with an existing config. Can be used with --append-repo-config to append up to three configurations"`
 
 	// Report Options
 	Report       string `short:"o" long:"report" description:"Report output path"`

scan/scan.go

@@ -45,6 +45,13 @@ func NewScanner(opts options.Options, cfg config.Config) (Scanner, error) {
 		return nil, err
 	}
 	if st == typeDirScanner {
+		if opts.AdditionalConfig != "" {
+			additionalCfg, err := config.LoadAdditionalConfig(opts.AdditionalConfig)
+			if err != nil {
+				return nil, err
+			}
+			cfg = cfg.AppendConfig(additionalCfg)
+		}
 		return NewParentScanner(opts, cfg), nil
 	}
 
@@ -56,12 +63,26 @@ func NewScanner(opts options.Options, cfg config.Config) (Scanner, error) {
 		}
 	}
 
-	// load up alternative config if possible, if not use manager's config
+	// load up alternative config if possible, if not use default/specified config. Will append if AppendRepoConfig is true
 	if opts.RepoConfigPath != "" && !opts.NoGit {
-		cfg, err = config.LoadRepoConfig(repo, opts.RepoConfigPath)
+		repoCfg, err := config.LoadRepoConfig(repo, opts.RepoConfigPath)
+		if err != nil {
+			return nil, err
+		}
+		if opts.AppendRepoConfig {
+			cfg = cfg.AppendConfig(repoCfg)
+		} else {
+			cfg = repoCfg
+		}
+	}
+
+	// append additional config with the rest of the config
+	if opts.AdditionalConfig != "" {
+		additionalCfg, err := config.LoadAdditionalConfig(opts.AdditionalConfig)
 		if err != nil {
 			return nil, err
 		}
+		cfg = cfg.AppendConfig(additionalCfg)
 	}
 
 	switch st {

scan/scan_test.go

@@ -441,6 +441,16 @@ func TestScan(t *testing.T) {
 			},
 			wantPath: "../test_data/test_allow_list_file.json",
 		},
+		{
+			description: "test allowlist files",
+			opts: options.Options{
+				Path:         "../test_data/test_repos/test_repo_10",
+				Report:       "../test_data/test_allow_list_file.json.got",
+				ReportFormat: "json",
+				ConfigPath:   "../test_data/test_configs/allowlist_files.toml",
+			},
+			wantPath: "../test_data/test_allow_list_file.json",
+		},
 		{
 			description: "test allowlist files no-git",
 			opts: options.Options{
@@ -464,27 +474,63 @@ func TestScan(t *testing.T) {
 			wantPath: "../test_data/test_allow_list_docx_no_git.json",
 		},
 		{
-			description: "test local repo two allowlist Commit config",
+			description: "test append repo config",
 			opts: options.Options{
-				Path:          "../test_data/test_repos/test_repo_2",
-				Report:        "../test_data/test_local_repo_two_allowlist_commits_files_at_commit.json.got",
-				ConfigPath:    "../test_data/test_configs/allowlist_commit.toml",
-				ReportFormat:  "json",
-				FilesAtCommit: "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+				Path:             "../test_data/test_repos/test_repo_10",
+				Report:           "../test_data/test_append_repo.json.got",
+				ReportFormat:     "json",
+				ConfigPath:       "../test_data/test_configs/nozips.toml",
+				RepoConfigPath:   ".gitleaks.toml",
+				AppendRepoConfig: true,
+			},
+			wantPath: "../test_data/test_append_repo.json",
+		},
+		{
+			description: "test additional config",
+			opts: options.Options{
+				Path:             "../test_data/test_repos/test_repo_10",
+				Report:           "../test_data/test_additional_config.json.got",
+				ReportFormat:     "json",
+				ConfigPath:       "../test_data/test_configs/nozips.toml",
+				AdditionalConfig: "../test_data/test_repos/test_repo_10/.gitleaks.toml",
+			},
+			wantPath: "../test_data/test_additional_config.json",
+		},
+		{
+			description: "test append repo config with additional config",
+			opts: options.Options{
+				Path:             "../test_data/test_repos/test_repo_10",
+				Report:           "../test_data/test_append_repo_additional_config.json.got",
+				ReportFormat:     "json",
+				ConfigPath:       "../test_data/test_configs/nozips.toml",
+				RepoConfigPath:   ".gitleaks.toml",
+				AppendRepoConfig: true,
+				AdditionalConfig: "../test_data/test_configs/allowlist_bad_docx_10.toml",
 			},
-			wantPath:  "../test_data/test_local_repo_two_allowlist_commits_files_at_commit.json",
 			wantEmpty: true,
 		},
 		{
-			description: "test local repo two global allowlist commit config",
+			description: "test append repo config with additional config",
 			opts: options.Options{
-				Path:          "../test_data/test_repos/test_repo_2",
-				Report:        "../test_data/test_local_repo_two_global_allowlist_files_at_commit.json.got",
-				ConfigPath:    "../test_data/test_configs/allowlist_global_files.toml",
-				ReportFormat:  "json",
-				FilesAtCommit: "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+				Path:             "../test_data/test_repos/test_repo_10",
+				Report:           "../test_data/test_append_repo_additional_config.json.got",
+				ReportFormat:     "json",
+				ConfigPath:       "../test_data/test_configs/nozips.toml",
+				RepoConfigPath:   ".gitleaks.toml",
+				AppendRepoConfig: true,
+				AdditionalConfig: "../test_data/test_configs/allowlist_bad_docx_10.toml",
+			},
+			wantEmpty: true,
+		},
+		{
+			description: "test file with no leak due to additional config no git",
+			opts: options.Options{
+				Path:             "../test_data/test_repos/test_dir_1",
+				Report:           "../test_data/fail.json.got",
+				ReportFormat:     "json",
+				NoGit:            false,
+				AdditionalConfig: "../test_data/test_configs/allowlist_allow_all_repo_1.toml",
 			},
-			wantPath:  "../test_data/test_local_repo_two_global_allowlist_files_at_commit.json",
 			wantEmpty: true,
 		},
 	}

test_data/test_additional_config.json

@@ -0,0 +1,18 @@
+[
+ {
+  "line": "",
+  "lineNumber": 1,
+  "offender": "Filename or path offender: tmp/bad.docx",
+  "commit": "b0f9b62dfe12e4e10de180359c6b9276472494f8",
+  "repo": "test_repo_10",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "Block dangerous filetypes",
+  "commitMessage": "Create bad.docx",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "tmp/bad.docx",
+  "date": "2020-12-09T11:02:10-05:00",
+  "tags": "key, extensions"
+ }
+]

test_data/test_append_repo.json

@@ -0,0 +1,18 @@
+[
+ {
+  "line": "",
+  "lineNumber": 1,
+  "offender": "Filename or path offender: tmp/bad.docx",
+  "commit": "b0f9b62dfe12e4e10de180359c6b9276472494f8",
+  "repo": "test_repo_10",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "Block dangerous filetypes",
+  "commitMessage": "Create bad.docx",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "tmp/bad.docx",
+  "date": "2020-12-09T11:02:10-05:00",
+  "tags": "key, extensions"
+ }
+]

test_data/test_configs/allowlist_allow_all_repo_1.toml

@@ -0,0 +1,3 @@
+[allowlist]
+	description = "Allowlisted files"
+	files = [ '''server.test.py''','''server.test2.py''']

test_data/test_configs/allowlist_bad_docx_10.toml

@@ -0,0 +1,3 @@
+[allowlist]
+	description = "Allowlisted files"
+	files = [ '''bad.docx''']

test_data/test_configs/nozips.toml

@@ -0,0 +1,4 @@
+
+[[rules]]
+	description = "No Zips"
+	file = '''.*\.zip'''