Add file regex support (#340)

* Modified Entropy to be able to apply it on regex match or specific capture group

* Config tests

* Small fixes on entropy ranges

* Fix toml configs
Fix tests

* Start adding support for rules with filename regex

Changed how config is parsed
Created new functions to inspect files and content

* File regex working

* Fixing tests and configs

* Implemented different regex for filename and filepath

* Fix comments

* Audit tests

* Config rename vars

* Fix audit tests

* Filename offender

* Indents and code styles

* Indents and code styles

* Removing bad test repo

* Adding test repo 6 for filename and filepath tests

* global whitelist path and fixes

* Fixes on tests and new test for global whitelisting paths

* Removing unused test

* Fix comment

audit/audit_test.go

@@ -253,6 +253,46 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_repo_five_commit.json",
 		},
+		{
+			description: "test local repo six filename",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_6",
+				Report:       "../test_data/test_local_repo_six_filename.json.got",
+				Config:       "../test_data/test_configs/regex_filename.toml",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_six_filename.json",
+		},
+		{
+			description: "test local repo six filepath",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_6",
+				Report:       "../test_data/test_local_repo_six_filepath.json.got",
+				Config:       "../test_data/test_configs/regex_filepath.toml",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_six_filepath.json",
+		},
+		{
+			description: "test local repo six filename and filepath",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_6",
+				Report:       "../test_data/test_local_repo_six_filepath_filename.json.got",
+				Config:       "../test_data/test_configs/regex_filepath_filename.toml",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_six_filepath_filename.json",
+		},
+		{
+			description: "test local repo six path globally whitelisted",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_6",
+				Report:       "../test_data/test_local_repo_six_path_globally_whitelisted.json.got",
+				Config:       "../test_data/test_configs/aws_key_global_whitelist_path.toml",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_six_path_globally_whitelisted.json",
+		},
 	}
 
 	for _, test := range tests {

audit/repo.go

@@ -171,35 +171,18 @@ func (repo *Repo) AuditUncommitted() error {
 				}
 			}
 
-			if fileMatched(filename, repo.config.Whitelist.File) {
-				log.Debugf("whitelisted file found, skipping audit of file: %s", filename)
-			} else if fileMatched(filename, repo.config.FileRegex) {
-				repo.Manager.SendLeaks(manager.Leak{
-					Line:     "N/A",
-					Offender: filename,
-					Commit:   c.Hash.String(),
-					Repo:     repo.Name,
-					Rule:     "file regex matched" + repo.config.FileRegex.String(),
-					Message:  c.Message,
-					Author:   c.Author.Name,
-					Email:    c.Author.Email,
-					Date:     c.Author.When,
-					File:     filename,
-				})
-			} else {
-				dmp := diffmatchpatch.New()
-				diffs := dmp.DiffMain(prevFileContents, currFileContents, false)
-				var diffContents string
-				for _, d := range diffs {
-					switch d.Type {
-					case diffmatchpatch.DiffInsert:
-						diffContents += fmt.Sprintf("%s\n", d.Text)
-					case diffmatchpatch.DiffDelete:
-						diffContents += fmt.Sprintf("%s\n", d.Text)
-					}
+			diffs := diffmatchpatch.New().DiffMain(prevFileContents, currFileContents, false)
+			var diffContents string
+			for _, d := range diffs {
+				switch d.Type {
+				case diffmatchpatch.DiffInsert:
+					diffContents += fmt.Sprintf("%s\n", d.Text)
+				case diffmatchpatch.DiffDelete:
+					diffContents += fmt.Sprintf("%s\n", d.Text)
 				}
-				InspectString(diffContents, c, repo, filename)
 			}
+
+			InspectFile(diffContents, filename, c, repo)
 		}
 	}
 

audit/util.go

@@ -3,6 +3,7 @@ package audit
 import (
 	"fmt"
 	"math"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
@@ -31,33 +32,16 @@ func inspectPatch(patch *object.Patch, c *object.Commit, repo *Repo) {
 		if f.IsBinary() {
 			continue
 		}
-		if fileMatched(getFileName(f), repo.config.Whitelist.File) {
-			log.Debugf("whitelisted file found, skipping audit of file: %s", getFileName(f))
-			continue
-		}
-		if fileMatched(getFileName(f), repo.config.FileRegex) {
-			repo.Manager.SendLeaks(manager.Leak{
-				Line:     "N/A",
-				Offender: getFileName(f),
-				Commit:   c.Hash.String(),
-				Repo:     repo.Name,
-				Rule:     "file regex matched" + repo.config.FileRegex.String(),
-				Author:   c.Author.Name,
-				Email:    c.Author.Email,
-				Date:     c.Author.When,
-				File:     getFileName(f),
-			})
-		}
 		for _, chunk := range f.Chunks() {
 			if chunk.Type() == fdiff.Delete || chunk.Type() == fdiff.Add {
-				InspectString(chunk.Content(), c, repo, getFileName(f))
+				InspectFile(chunk.Content(), getFileFullPath(f), c, repo)
 			}
 		}
 	}
 }
 
 // getFileName accepts a file patch and returns the filename
-func getFileName(f fdiff.FilePatch) string {
+func getFileFullPath(f fdiff.FilePatch) string {
 	fn := "???"
 	from, to := f.Files()
 	if from != nil {
@@ -69,6 +53,31 @@ func getFileName(f fdiff.FilePatch) string {
 	return fn
 }
 
+// getFileName accepts a string with full path and returns only path
+func getFilePath(fullpath string) string {
+	return filepath.Dir(fullpath)
+}
+
+// getFileName accepts a string with full path and returns only filename
+func getFileName(fullpath string) string {
+	return filepath.Base(fullpath)
+}
+
+// aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
+// trippedEntropy checks if a given capture group or offender falls in between entropy ranges
+// supplied by a custom gitleaks configuration. Gitleaks do not check entropy by default.
+func trippedEntropy(groups []string, rule config.Rule) bool {
+	for _, e := range rule.Entropies {
+		if len(groups) > e.Group {
+			entropy := shannonEntropy(groups[e.Group])
+			if entropy >= e.Min && entropy <= e.Max {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // getShannonEntropy https://en.wiktionary.org/wiki/Shannon_entropy
 func shannonEntropy(data string) (entropy float64) {
 	if data == "" {
@@ -89,21 +98,8 @@ func shannonEntropy(data string) (entropy float64) {
 	return entropy
 }
 
-// aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
-// trippedEntropy checks if a given groups or offender falls in between entropy ranges
-// supplied by a custom gitleaks configuration. Gitleaks do not check entropy by default.
-func trippedEntropy(groups []string, rule config.Rule) bool {
-	for _, e := range rule.Entropies {
-		if len(groups) > e.Group {
-			entropy := shannonEntropy(groups[e.Group])
-			if entropy >= e.Min && entropy <= e.Max {
-				return true
-			}
-		}
-	}
-	return false
-}
 
+// Checks if the given rule has a regex
 func ruleContainRegex(rule config.Rule) bool {
 	if rule.Regex == nil {
 		return false
@@ -114,103 +110,147 @@ func ruleContainRegex(rule config.Rule) bool {
 	return true
 }
 
-// InspectString accepts a string, commit object, repo, and filename. This function iterates over
-// all the rules set by the gitleaks config. If the rule contains entropy checks then entropy will be checked first.
-// Next, if the rule contains a regular expression then that will be checked.
-func InspectString(content string, c *object.Commit, repo *Repo, filename string) {
+// Checks if the given rule has a file name regex
+func ruleContainFileNameRegex(rule config.Rule) bool {
+	if rule.FileNameRegex == nil {
+		return false
+	}
+	if rule.FileNameRegex.String() == "" {
+		return false
+	}
+	return true
+}
+
+// Checks if the given rule has a file path regex
+func ruleContainFilePathRegex(rule config.Rule) bool {
+	if rule.FilePathRegex == nil {
+		return false
+	}
+	if rule.FilePathRegex.String() == "" {
+		return false
+	}
+	return true
+}
+
+func sendLeak(offender string, line string, filename string, rule config.Rule, c *object.Commit, repo *Repo) {
+	if repo.Manager.Opts.Redact {
+		line = strings.ReplaceAll(line, offender, "REDACTED")
+		offender = "REDACTED"
+	}
+
+	repo.Manager.SendLeaks(manager.Leak{
+		Line:     line,
+		Offender: offender,
+		Commit:   c.Hash.String(),
+		Repo:     repo.Name,
+		Message:  c.Message,
+		Rule:     rule.Description,
+		Author:   c.Author.Name,
+		Email:    c.Author.Email,
+		Date:     c.Author.When,
+		Tags:     strings.Join(rule.Tags, ", "),
+		File:     filename,
+	})
+}
+
+// InspectFile accepts a file content, fullpath of file, commit and repo. If the file is
+// binary OR if a file is matched on whitelisted files set in the configuration, then gitleaks
+// will skip auditing that file. It will check first if rules apply to this file comparing filename
+// and path to their respective rule regexes and inspect file content with inspectFileContents after.
+func InspectFile(content string, fullpath string, c *object.Commit, repo *Repo) {
+
+	filename := getFileName(fullpath)
+	path := getFilePath(fullpath)
+
+	// We want to check if there is a whitelist for this file
+	if len(repo.config.Whitelist.Files) != 0 {
+		for _, reFileName := range repo.config.Whitelist.Files {
+			if fileMatched(filename, reFileName) {
+				log.Debugf("whitelisted file found, skipping audit of file: %s", filename)
+				return
+			}
+		}
+	}
+
+	// We want to check if there is a whitelist for this path
+	if len(repo.config.Whitelist.Paths) != 0 {
+		for _, reFilePath := range repo.config.Whitelist.Paths {
+			if fileMatched(path, reFilePath) {
+				log.Debugf("file in whitelisted path found, skipping audit of file: %s", filename)
+				return
+			}
+		}
+	}
+
 	for _, rule := range repo.config.Rules {
-		if rule.Regex.String() == "" {
+		start := time.Now()
+
+		// For each rule we want to check filename whitelists
+		if isFileNameWhiteListed(filename, rule.Whitelist) || isFilePathWhiteListed(path, rule.Whitelist) {
+			continue
+		}
+    
+		// If it has fileNameRegex and it doesnt match we continue to next rule
+		if ruleContainFileNameRegex(rule) && !fileMatched(filename, rule.FileNameRegex) {
 			continue
 		}
 
-		if repo.timeoutReached() {
-			return
+		// If it has filePathRegex and it doesnt match we continue to next rule
+		if ruleContainFilePathRegex(rule) && !fileMatched(path, rule.FilePathRegex) {
+			continue
 		}
-		start := time.Now()
-		locs := rule.Regex.FindAllIndex([]byte(content), -1)
-		if len(locs) != 0 {
-			// check if any rules are whitelisting this leak
-			if len(rule.Whitelist) != 0 {
-				for _, wl := range rule.Whitelist {
-					if fileMatched(filename, wl.File) {
-						// if matched, go to next rule
-						goto NEXT
-					}
-				}
+
+		// If it doesnt contain a content regex then it is a filename regex match
+		if !ruleContainRegex(rule) {
+			sendLeak("Filename/path offender: "+filename, "N/A", fullpath, rule, c, repo)
+		} else {
+			//otherwise we check if it matches content regex
+			inspectFileContents(content, fullpath, rule, c, repo)
+		}
+
+		//	TODO should return filenameRegex if only file rule
+		repo.Manager.RecordTime(manager.RegexTime{
+			Time:  howLong(start),
+			Regex: rule.Regex.String(),
+		})
+	}
+}
+
+// InspectString accepts a string, commit object, repo, and filename. This function iterates over
+// all the rules set by the gitleaks config. If the rule contains entropy checks then entropy will be checked first.
+// Next, if the rule contains a regular expression then that will be checked.
+func inspectFileContents(content string, path string, rule config.Rule, c *object.Commit, repo *Repo) {
+	locs := rule.Regex.FindAllIndex([]byte(content), -1)
+	if len(locs) != 0 {
+		for _, loc := range locs {
+			start := loc[0]
+			end := loc[1]
+			for start != 0 && content[start] != '\n' {
+				start = start - 1
+			}
+			if start != 0 {
+				// skip newline
+				start = start + 1
 			}
-			for _, loc := range locs {
-				start := loc[0]
-				end := loc[1]
-				for start != 0 && content[start] != '\n' {
-					start = start - 1
-				}
-				if start != 0 {
-					// skip newline
-					start = start + 1
-				}
 
-				for end < len(content)-1 && content[end] != '\n' {
-					end = end + 1
-				}
+			for end < len(content)-1 && content[end] != '\n' {
+				end = end + 1
+			}
 
-				line := content[start:end]
-				offender := content[loc[0]:loc[1]]
-				groups := rule.Regex.FindStringSubmatch(offender)
+			line := content[start:end]
+			offender := content[loc[0]:loc[1]]
+			groups := rule.Regex.FindStringSubmatch(offender)
 
-				if len(rule.Whitelist) != 0 {
-					for _, wl := range rule.Whitelist {
-						if wl.Regex.FindString(offender) != "" {
-							goto NEXT
-						}
-					}
-				}
+			if isOffenderWhiteListed(offender, rule.Whitelist) {
+				continue
+			}
 
-				if len(rule.Entropies) != 0 {
-					if trippedEntropy(groups, rule) {
-						if repo.Manager.Opts.Redact {
-							line = strings.ReplaceAll(line, offender, "REDACTED")
-							offender = "REDACTED"
-						}
-						repo.Manager.SendLeaks(manager.Leak{
-							Line:     line,
-							Offender: offender,
-							Commit:   c.Hash.String(),
-							Repo:     repo.Name,
-							Message:  c.Message,
-							Rule:     rule.Description,
-							Author:   c.Author.Name,
-							Email:    c.Author.Email,
-							Date:     c.Author.When,
-							Tags:     strings.Join(rule.Tags, ", "),
-							File:     filename,
-						})
-					}
-				} else {
-					if repo.Manager.Opts.Redact {
-						line = strings.ReplaceAll(line, offender, "REDACTED")
-						offender = "REDACTED"
-					}
-					repo.Manager.SendLeaks(manager.Leak{
-						Line:     line,
-						Offender: offender,
-						Commit:   c.Hash.String(),
-						Message:  c.Message,
-						Repo:     repo.Name,
-						Rule:     rule.Description,
-						Author:   c.Author.Name,
-						Email:    c.Author.Email,
-						Date:     c.Author.When,
-						Tags:     strings.Join(rule.Tags, ", "),
-						File:     filename,
-					})
-				}
+			if len(rule.Entropies) != 0 && !trippedEntropy(groups, rule) {
+				continue
 			}
+
+			sendLeak(offender, line, path, rule, c, repo)
 		}
-		repo.Manager.RecordTime(manager.RegexTime{
-			Time:  time.Now().Sub(start).Nanoseconds(),
-			Regex: rule.Regex.String(),
-		})
-	NEXT:
 	}
 }
 
@@ -281,30 +321,13 @@ func inspectFilesAtCommit(c *object.Commit, repo *Repo) error {
 		} else if err != nil {
 			return err
 		}
-		if fileMatched(f, repo.config.Whitelist.File) {
-			log.Debugf("whitelisted file found, skipping audit of file: %s", f.Name)
-			return nil
-		}
 
-		if fileMatched(f.Name, repo.config.FileRegex) {
-			repo.Manager.SendLeaks(manager.Leak{
-				Line:     "N/A",
-				Offender: f.Name,
-				Commit:   c.Hash.String(),
-				Repo:     repo.Name,
-				Rule:     "file regex matched" + repo.config.FileRegex.String(),
-				Author:   c.Author.Name,
-				Email:    c.Author.Email,
-				Date:     c.Author.When,
-				File:     f.Name,
-			})
-		}
 		content, err := f.Contents()
 		if err != nil {
 			return err
 		}
 
-		InspectString(content, c, repo, f.Name)
+		InspectFile(content, f.Name, c, repo)
 
 		return nil
 	})
@@ -333,6 +356,39 @@ func isCommitWhiteListed(commitHash string, whitelistedCommits []string) bool {
 	return false
 }
 
+func isOffenderWhiteListed(offender string, whitelist []config.Whitelist) bool {
+	if len(whitelist) != 0 {
+		for _, wl := range whitelist {
+			if wl.Regex.FindString(offender) != "" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func isFileNameWhiteListed(filename string, whitelist []config.Whitelist) bool {
+	if len(whitelist) != 0 {
+		for _, wl := range whitelist {
+			if fileMatched(filename, wl.File) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func isFilePathWhiteListed(filepath string, whitelist []config.Whitelist) bool {
+	if len(whitelist) != 0 {
+		for _, wl := range whitelist {
+			if fileMatched(filepath, wl.Path) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func fileMatched(f interface{}, re *regexp.Regexp) bool {
 	if re == nil {
 		return false

config/config.go

@@ -16,9 +16,10 @@ type Whitelist struct {
 	Description string
 	Regex       *regexp.Regexp
 	File        *regexp.Regexp
+	Path        *regexp.Regexp
 }
 
-// entropy represents an entropy range
+// Entropy represents an entropy range
 type Entropy struct {
 	Min   float64
 	Max   float64
@@ -31,23 +32,24 @@ type Entropy struct {
 // that match is not whitelisted (globally or locally), then a leak will be appended
 // to the final audit report.
 type Rule struct {
-	Description string
-	Regex       *regexp.Regexp
-	Tags        []string
-	Whitelist   []Whitelist
-	Entropies   []Entropy
+	Description   string
+	Regex         *regexp.Regexp
+	FileNameRegex *regexp.Regexp
+	FilePathRegex *regexp.Regexp
+	Tags          []string
+	Whitelist     []Whitelist
+	Entropies     []Entropy
 }
 
 // Config is a composite struct of Rules and Whitelists
 // Each Rule contains a description, regular expression, tags, and whitelists if available
 type Config struct {
-	FileRegex *regexp.Regexp
-	Message   *regexp.Regexp
 	Rules     []Rule
 	Whitelist struct {
 		Description string
 		Commits     []string
-		File        *regexp.Regexp
+		Files       []*regexp.Regexp
+		Paths       []*regexp.Regexp
 	}
 }
 
@@ -55,20 +57,19 @@ type Config struct {
 // see the config in config/defaults.go for an example. TomlLoader is used
 // to generate Config values (compiling regexes, etc).
 type TomlLoader struct {
-	Global struct {
-		File    string
-		Message string
-	}
 	Whitelist struct {
 		Description string
 		Commits     []string
-		File        string
+		Files       []string
+		Paths       []string
 	}
 	Rules []struct {
-		Description string
-		Regex       string
-		Tags        []string
-		Entropies   []struct {
+		Description   string
+		Regex         string
+		FileNameRegex string
+		FilePathRegex string
+		Tags          []string
+		Entropies     []struct {
 			Min   string
 			Max   string
 			Group string
@@ -77,6 +78,7 @@ type TomlLoader struct {
 			Description string
 			Regex       string
 			File        string
+			Path        string
 		}
 	}
 }
@@ -116,22 +118,35 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 		if err != nil {
 			return cfg, fmt.Errorf("problem loading config: %v", err)
 		}
+		fileNameRe, err := regexp.Compile(rule.FileNameRegex)
+		if err != nil {
+			return cfg, fmt.Errorf("problem loading config: %v", err)
+		}
+		filePathRe, err := regexp.Compile(rule.FilePathRegex)
+		if err != nil {
+			return cfg, fmt.Errorf("problem loading config: %v", err)
+		}
 
 		// rule specific whitelists
 		var whitelists []Whitelist
 		for _, wl := range rule.Whitelist {
-			re, err := regexp.Compile(wl.Regex)
+			wlRe, err := regexp.Compile(wl.Regex)
+			if err != nil {
+				return cfg, fmt.Errorf("problem loading config: %v", err)
+			}
+			wlFileNameRe, err := regexp.Compile(wl.File)
 			if err != nil {
 				return cfg, fmt.Errorf("problem loading config: %v", err)
 			}
-			fileRe, err := regexp.Compile(wl.File)
+			wlFilePathRe, err := regexp.Compile(wl.Path)
 			if err != nil {
 				return cfg, fmt.Errorf("problem loading config: %v", err)
 			}
 			whitelists = append(whitelists, Whitelist{
 				Description: wl.Description,
-				File:        fileRe,
-				Regex:       re,
+				File:        wlFileNameRe,
+				Path:        wlFilePathRe,
+				Regex:       wlRe,
 			})
 		}
 
@@ -165,37 +180,32 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 		}
 
 		cfg.Rules = append(cfg.Rules, Rule{
-			Description: rule.Description,
-			Regex:       re,
-			Tags:        rule.Tags,
-			Whitelist:   whitelists,
-			Entropies:   entropies,
+			Description:   rule.Description,
+			Regex:         re,
+			FileNameRegex: fileNameRe,
+			FilePathRegex: filePathRe,
+			Tags:          rule.Tags,
+			Whitelist:     whitelists,
+			Entropies:     entropies,
 		})
 	}
 
-	// global leaks
-	if tomlLoader.Global.File != "" {
-		re, err := regexp.Compile(tomlLoader.Global.File)
-		if err != nil {
-			return cfg, fmt.Errorf("problem loading config: %v", err)
-		}
-		cfg.FileRegex = re
-	}
-	if tomlLoader.Global.Message != "" {
-		re, err := regexp.Compile(tomlLoader.Global.Message)
+	// global file name whitelists
+	for _, wlFileName := range tomlLoader.Whitelist.Files {
+		re, err := regexp.Compile(wlFileName)
 		if err != nil {
 			return cfg, fmt.Errorf("problem loading config: %v", err)
 		}
-		cfg.Message = re
+		cfg.Whitelist.Files = append(cfg.Whitelist.Files, re)
 	}
 
-	// global whitelists
-	if tomlLoader.Whitelist.File != "" {
-		re, err := regexp.Compile(tomlLoader.Whitelist.File)
+	// global file path whitelists
+	for _, wlFilePath := range tomlLoader.Whitelist.Paths {
+		re, err := regexp.Compile(wlFilePath)
 		if err != nil {
 			return cfg, fmt.Errorf("problem loading config: %v", err)
 		}
-		cfg.Whitelist.File = re
+		cfg.Whitelist.Paths = append(cfg.Whitelist.Paths, re)
 	}
 	cfg.Whitelist.Commits = tomlLoader.Whitelist.Commits
 	cfg.Whitelist.Description = tomlLoader.Whitelist.Description

config/config_test.go

@@ -54,13 +54,6 @@ func TestParse(t *testing.T) {
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
 		},
-		{
-			description: "test bad global message regex",
-			opts: options.Options{
-				Config: "../test_data/test_configs/bad_aws_key_message_regex.toml",
-			},
-			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
-		},
 		{
 			description: "test successful load big ol thing",
 			opts: options.Options{

examples/leaky-repo.toml

@@ -160,7 +160,8 @@ title = "gitleaks config"
 			Min = "5.2"
 			Max = "5.5"
 
-[Global]
+[[rules]]
+	description = "Files with keys and credentials"
     file = '''(?i)(id_rsa|passwd|id_rsa.pub|pgpass|pem|key|shadow)'''
 
 [whitelist]

examples/regex_and_file.toml

@@ -6,5 +6,7 @@
     regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
     tags = ["key", "AWS"]
 
-[Global]
+[[rules]]
+    description = "PEM Files"
     file = '''(.*)?pem$'''
+    tags = ["key", "PEM"]

hosts/github.go

@@ -161,7 +161,7 @@ func (g *Github) AuditPR() {
 				if f.Patch == nil {
 					continue
 				}
-				audit.InspectString(*f.Patch, &commitObj, repo, *f.Filename)
+				audit.InspectFile(*f.Patch, *f.Filename, &commitObj, repo)
 			}
 		}
 		page = resp.NextPage

test_data/test_configs/aws_key_file_regex.toml

@@ -8,6 +8,6 @@
     regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
     tags = ["key", "AWS"]
 
-[Global]
-    file = '''(.*)?py$'''
-
+[[rules]]
+	description = "Python files"
+	fileRegex = '''(.*)?py$'''

test_data/test_configs/aws_key_global_whitelist_file.toml

@@ -5,4 +5,6 @@
 
 [whitelist]
     description = "ignore md files"
-    file = '''(.*)?md$'''
+    files = [
+        '''(.*)?md$'''
+    ]

test_data/test_configs/aws_key_global_whitelist_path.toml

@@ -0,0 +1,10 @@
+[[rules]]
+    description = "AWS Manager ID"
+    regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+    tags = ["key", "AWS"]
+
+[whitelist]
+    description = "ignore config folders"
+    paths = [
+        '''config(uration)?'''
+    ]

test_data/test_configs/bad_aws_key_global_whitelist_file.toml

@@ -5,4 +5,4 @@
 
 [whitelist]
     description = "ignore md files"
-    file = '''???????'''
+    files = ['''???????''']

test_data/test_configs/bad_aws_key_message_regex.toml

@@ -1,13 +0,0 @@
-[[rules]]
-	description = "AWS Secret Key"
-	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
-	tags = ["key", "AWS"]
-
-[[rules]]
-    description = "AWS Manager ID"
-    regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
-    tags = ["key", "AWS"]
-
-[Global]
-    message = '''?????????????'''
-

test_data/test_configs/large.toml

@@ -135,11 +135,14 @@ title = "gitleaks config"
 			file = '''(.*)?Jenkinsfile$'''
 
 # Global rules. This instructs gitleaks to ignore all .pem files or if a message with
-[Global]
-    file = '''(.*?)(pem)'''
-    message = '''somethingbad'''
+[[rules]]
+	description = "Files with keys and credentials"
+    fileRegex = '''(.*?)(pem)'''
 
 [whitelist]
-	description = "image whitelists"
-	file = '''(.*?)(jpg|gif|doc|pdf|bin)$'''
+	description = "File whitelists"
+	files = [
+		'''(.*?)(jpg|gif)$''',
+		'''(.*?)(doc|pdf|bin)$'''
+		]
 

test_data/test_configs/regex_filename.toml

@@ -0,0 +1,6 @@
+
+[[rules]]
+	description = "Generic password"
+	regex = '''(?i)(password|token).{0,20}?\s*[:|=]\s*?(?-i)([0-9a-zA-Z\/+]{4,40})'''
+	fileNameRegex = '''.*\.ya?ml'''
+	tags = ["key", "Yml", "Yaml"]

test_data/test_configs/regex_filepath.toml

@@ -0,0 +1,6 @@
+
+[[rules]]
+    description = "AWS Manager ID"
+    regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+    filePathRegex = '''config(guration)?'''
+    tags = ["key", "AWS"]

test_data/test_configs/regex_filepath_filename.toml

@@ -0,0 +1,7 @@
+
+[[rules]]
+	description = "Generic password"
+	regex = '''(?i)(password|token).{0,20}?\s*[:|=]\s*?(?-i)([0-9a-zA-Z\/+]{4,40})'''
+	fileNameRegex = '''.*\.properties'''
+	filePathRegex = '''config(guration)?'''
+	tags = ["key", "Yml", "Yaml"]

test_data/test_local_owner_aws_leak.json

@@ -271,5 +271,18 @@
   "file": "secrets.py",
   "date": "2020-02-01T10:07:34-05:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_owner_aws_leak_depth_2.json

@@ -141,5 +141,18 @@
   "file": "secrets.py",
   "date": "2020-02-01T10:08:04-05:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_repo_one_aws_leak_and_file_leak.json

@@ -1,11 +1,11 @@
 [
  {
   "line": "N/A",
-  "offender": "server.test.py",
+  "offender": "Filename/path offender: server.test.py",
   "commit": "d274003914c707212cbe84e3e466a00013ccb639",
   "repo": "test_repo_1",
-  "rule": "file regex matched(.*)?py$",
-  "commitMessage": "",
+  "rule": "Python files",
+  "commitMessage": "comment\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "server.test.py",
@@ -13,29 +13,29 @@
   "tags": ""
  },
  {
-  "line": "N/A",
-  "offender": "server.test.py",
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
   "repo": "test_repo_1",
-  "rule": "file regex matched(.*)?py$",
-  "commitMessage": "",
+  "rule": "AWS Manager ID",
+  "commitMessage": "commit 1 with secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "server.test.py",
   "date": "2019-10-24T09:29:27-04:00",
-  "tags": ""
+  "tags": "key, AWS"
  },
  {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "line": "N/A",
+  "offender": "Filename/path offender: server.test.py",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
   "repo": "test_repo_1",
-  "rule": "AWS Manager ID",
+  "rule": "Python files",
   "commitMessage": "commit 1 with secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "server.test.py",
   "date": "2019-10-24T09:29:27-04:00",
-  "tags": "key, AWS"
+  "tags": ""
  }
 ]

test_data/test_local_repo_six.json

@@ -0,0 +1,44 @@
+[
+ {
+  "line": "api_token: exampleSecretPassword",
+  "lineNumber": 3,
+  "offender": "token: exampleSecretPassword",
+  "commit": "e7bf376f855ae9bc6e3bb27b5f9d65c57eba799d",
+  "repo": "test_repo_6",
+  "rule": "Generic password",
+  "commitMessage": "More secrets\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/config.yaml",
+  "date": "2020-02-19T04:08:26-05:00",
+  "tags": "key, Yml, Yaml"
+ },
+ {
+  "line": "db_password_token: exampleSecretPassword",
+  "lineNumber": 3,
+  "offender": "password_token: exampleSecretPassword",
+  "commit": "e7bf376f855ae9bc6e3bb27b5f9d65c57eba799d",
+  "repo": "test_repo_6",
+  "rule": "Generic password",
+  "commitMessage": "More secrets\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/config.yaml",
+  "date": "2020-02-19T04:08:26-05:00",
+  "tags": "key, Yml, Yaml"
+ },
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "lineNumber": 1,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "95cf0237f8cdfc0686a29df08260d8635f46c9b0",
+  "repo": "test_repo_6",
+  "rule": "AWS Manager ID",
+  "commitMessage": "AWS Key in properties\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-19T04:04:15-05:00",
+  "tags": "key, AWS"
+ }
+]

test_data/test_local_repo_six_filename.json

@@ -0,0 +1,28 @@
+[
+ {
+  "line": "api_token: exampleSecretPassword",
+  "offender": "token: exampleSecretPassword",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic password",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "application.yaml",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, Yml, Yaml"
+ },
+ {
+  "line": "db_password: verySecretProductionPassword",
+  "offender": "password: verySecretProductionPassword",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic password",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "application.yaml",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, Yml, Yaml"
+ }
+]

test_data/test_local_repo_six_filepath.json

@@ -0,0 +1,15 @@
+[
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, AWS"
+ }
+]

test_data/test_local_repo_six_filepath_filename.json

@@ -0,0 +1,15 @@
+[
+ {
+  "line": "apiToken=d41d8cd98f00b204e9800998ecf8427e",
+  "offender": "Token=d41d8cd98f00b204e9800998ecf8427e",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic password",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, Yml, Yaml"
+ }
+]

test_data/test_local_repo_six_path_globally_whitelisted.json

@@ -0,0 +1,15 @@
+[
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_6",
+  "rule": "AWS Manager ID",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, AWS"
+ }
+]

test_data/test_repos/test_repo_6/application.yaml

@@ -0,0 +1,3 @@
+appName: "Gitleaks-Testing-App"
+api_token: exampleSecretPassword
+db_password: verySecretProductionPassword

test_data/test_repos/test_repo_6/config/application.properties

@@ -0,0 +1,3 @@
+language=en_EN
+apiToken=d41d8cd98f00b204e9800998ecf8427e
+aws_access_key_id=AKIAIO5FODNN7EXAMPLE

test_data/test_repos/test_repo_6/dotGit/COMMIT_EDITMSG

@@ -0,0 +1 @@
+comment

test_data/test_repos/test_repo_6/dotGit/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

test_data/test_repos/test_repo_6/dotGit/config

@@ -0,0 +1,7 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true

test_data/test_repos/test_repo_6/dotGit/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

test_data/test_repos/test_repo_6/dotGit/info/exclude

@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~

test_data/test_repos/test_repo_6/dotGit/logs/HEAD

@@ -0,0 +1,4 @@
+0000000000000000000000000000000000000000 6557c92612d3b35979bd426d429255b3bf9fab74 zach rice <zricer@protonmail.com> 1571923767 -0400	commit (initial): commit 1 with secrets
+6557c92612d3b35979bd426d429255b3bf9fab74 d274003914c707212cbe84e3e466a00013ccb639 zach rice <zricer@protonmail.com> 1571925818 -0400	commit: comment
+d274003914c707212cbe84e3e466a00013ccb639 98b6c7cb3fb29a5993c4c95c56a2dc53050b9247 Noel Algora <noealgigu@gmail.com> 1582571595 -0500	commit: Adding some secrets in config folder
+98b6c7cb3fb29a5993c4c95c56a2dc53050b9247 98b6c7cb3fb29a5993c4c95c56a2dc53050b9247 Noel Algora <noealgigu@gmail.com> 1582571636 -0500	discard: [bc379e22c22a97ca5ffc871c1449af854f6f26c4]

test_data/test_repos/test_repo_6/dotGit/logs/refs/heads/master

@@ -0,0 +1,3 @@
+0000000000000000000000000000000000000000 6557c92612d3b35979bd426d429255b3bf9fab74 zach rice <zricer@protonmail.com> 1571923767 -0400	commit (initial): commit 1 with secrets
+6557c92612d3b35979bd426d429255b3bf9fab74 d274003914c707212cbe84e3e466a00013ccb639 zach rice <zricer@protonmail.com> 1571925818 -0400	commit: comment
+d274003914c707212cbe84e3e466a00013ccb639 98b6c7cb3fb29a5993c4c95c56a2dc53050b9247 Noel Algora <noealgigu@gmail.com> 1582571595 -0500	commit: Adding some secrets in config folder

test_data/test_repos/test_repo_6/dotGit/objects/9e/523225b31add24e72f2feb0b2645cfb36542dc

@@ -0,0 +1,3 @@
+x
MŽO‚@Ĺ;ď§č`‘%:XjE	˘<Éş;�¤»°ł~űĚ
+z§÷`~ďMVęFţ´Ó…P�ŇnÜHZ"�•-xIL”Eca™¶Ú÷>±Ç ‘Cľă¶®«,�¬)TNŔéżČm7
+]U¨$JŻ%ů“R.Ąw¬ÓBÎś`łÖ‡q|÷űIt
vÇmôÝy_6ďŮ?¨!ć‹0Šb´ťÔW\>ε9ć	…ęr:-†Aň+a}Ć^K§N$

test_data/test_repos/test_repo_6/dotGit/objects/a1/fd29ec14823d8bc4a8d1a2cfe35451580f5118

@@ -0,0 +1,3 @@
+x
uPMOÃ0åÚüŠGV1lc.LEB`†Ä¸!Miç*a]R5!þ;N‡øââÄö{ÏÏÎ+—ãìôt§³;Ì�z-:¢ƒKzQ«º"hç–ðEcê€à°RKÂÚ\‰zí5\Œ³~À¬™¦/š7Ì~Ú‘‡ªª_h­|PMÀÆ�næªXf]Rv�†ž¨ßTVÙh²Ì$ÈmSþÔ„ñÑׂ�D+dUÎþƒæF\âˆ%­Z}VJ{²n¨ßPAæ™ä@S"�è[Èôz2›ß=Üß̧w³Éôv~5}¸�IÁ¬HLv,’�nu"C¦F¢_ÅÏ?¼dáD’гª 9¬){üeÆŠ/,Ó$axZòu ƒgm
+‹(·jÏòc•~Üøé[+Ñ9ÈÞ%.öN"s<æ¸=àVéÅŒâ¯
+~L–îïbÔãþÂY¥eMª$

test_data/test_repos/test_repo_6/dotGit/objects/a5/196d1be8fb59edf8062bef36d3a602e0812139

@@ -0,0 +1 @@
+x
-�±jÃ0„;ç)<¤
µSJ§B¦@¦¾Aé Û¿lI¤_¸Yú쑃·ãø¾»Þ³ÇûÇÛSƒÉ)|n­ó’ѶÔYÒäoðe”Ö&†SW©£‹–Ç­Þ5ør±:EV“‹Óûf“C�¨¹«ä…	×Ä_�Yý
.âüZu�¥÷\\œ°°ø½Tz"GdQÐÖ…íW£*)f<—¸=¬
ÎâÆ‚ÅD…%Ë£ù¬ò¡û¦ùYÃÿîá›SC

test_data/test_repos/test_repo_6/dotGit/objects/cb/19a50e8cdeb7011eccdb13f3b739f00d775bab

@@ -0,0 +1,2 @@
+x
��Í
+1„½êS䨥MÚf+"¾J“­XØY‚OïŠ^¼íÀ00‡áéF
B\™…‚à=kÄ`±%!9Jë0ÌŽè½�\ã5	;x%½ÁT4Ãéõ‰érŸÆ:}*ÝAÇþÖ³�HöƳžÛ¾TØ–¡Ô’ºÝ~�…g©7xd�r}l3´Èó.Eë”
£E•Ü¸LÙ…�æÇ–T%P\ëÛüÁ~óP7oq,\é

test_data/test_repos/test_repo_6/dotGit/objects/ec/17ec1939b7c3e86b7cb6c0c4de6b0818a7e75e

@@ -0,0 +1,2 @@
+x
-ÍK‚0P×9E[�r§Kïà&0H
+Ȥ&ƒ…·7~–Ý]õÚ/âq¾\õ±ó!vyrµ«q‹à�Ö´0&‘¹×�&Hʉ”AHÔÏ< dÉÁD_E±e†<Y2l«‡)ÅœD-·_ù.àH¾À6…üÕOň´þ›1|6AUXk¶4�qÕ:Ç;÷xïj2k¹iBŽ
vÐBü

test_data/test_repos/test_repo_6/dotGit/refs/heads/master

@@ -0,0 +1 @@
+98b6c7cb3fb29a5993c4c95c56a2dc53050b9247

test_data/test_repos/test_repo_6/server.test.py

@@ -0,0 +1,10 @@
+# Do not hard code credentials
+client = boto3.client(
+    's3',
+    # Hard coded strings as credentials, not recommended.
+    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
+    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'
+)
+
+# adding another line
+