Merge pull request #196 from zricethezav/v2.0.0

V2.0.0

.gitignore

@@ -9,4 +9,4 @@
 *.test
 *.out
 
-tmp
+examples

CHANGELOG.md

@@ -1,5 +1,11 @@
 CHANGELOG
 =========
+2.0.0
+----
+- rules introduced in the gitleaks configurationn
+- removing `--entropy` option
+- removing `--single-search` option
+
 1.25.1
 ----
 - Fixing #188

Makefile

@@ -20,8 +20,3 @@ build-all:
 	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips"
 	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips"
 	env GOOS="darwin" GOARCH="amd64" go build -o "build/gitleaks-darwin-amd64"
-benchmark:
-	go test -run=Benchmark -bench=. -benchtime=5s
-benchmark-fast:
-	go test -bench=BenchmarkAuditLeakRepo -run=BenchmarkAuditLeakRepo$
-

README.md

@@ -77,12 +77,9 @@ Application Options:
       --owner-path=     Path to owner directory (repos discovered)
       --threads=        Maximum number of threads gitleaks spawns
       --disk            Clones repo(s) to disk
-      --single-search=  single regular expression to search for
       --config=         path to gitleaks config
       --ssh-key=        path to ssh key
       --exclude-forks   exclude forks for organization/user audits
-  -e, --entropy=        Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)
-      --noise-reduction Reduce the number of finds when entropy checks are enabled
       --repo-config     Load config from target repo. Config file must be ".gitleaks.toml"
       --branch=         Branch to audit
   -l, --log=            log level

gitleaks.toml

@@ -1,39 +1,92 @@
-title = "gitleaks config"
-[[regexes]]
-description = "AWS"
+title = "sample gitleaks config"
+
+# This is a sample config file for gitleaks. You can configure gitleaks what to search for and what to whitelist.
+# The output you are seeing here is the default gitleaks config. If GITLEAKS_CONFIG environment variable
+# is set, gitleaks will load configurations from that path. If option --config-path is set, gitleaks will load
+# configurations from that path. Gitleaks does not whitelist anything by default.
+[[rules]]
+description = "AWS Key"
 regex = '''AKIA[0-9A-Z]{16}'''
-[[regexes]]
-description = "RKCS8"
+tags = ["key", "AWS"]
+
+[[rules]]
+description = "PKCS8"
 regex = '''-----BEGIN PRIVATE KEY-----'''
-[[regexes]]
+tags = ["key", "PKCS8"]
+
+[[rules]]
 description = "RSA"
 regex = '''-----BEGIN RSA PRIVATE KEY-----'''
-[[regexes]]
-description = "Github"
-regex = '''(?i)github.*['\"][0-9a-zA-Z]{35,40}['\"]'''
-[[regexes]]
+tags = ["key", "RSA"]
+
+[[rules]]
 description = "SSH"
 regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
-[[regexes]]
-description = "Facebook"
-regex = '''(?i)facebook.*['\"][0-9a-f]{32}['\"]'''
-[[regexes]]
-description = "Twitter"
-regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]'''
-[[regexes]]
+tags = ["key", "SSH"]
+
+[[rules]]
 description = "PGP"
 regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
-[[regexes]]
-description = "Slack token"
-regex = '''xox[baprs]-.*'''
-[[regexes]]
-description = "Strip API Key"
-regex = '''(?i)(sk|pk)_(test|live)_[0-9a-zA-Z]{10,32}'''
+tags = ["key", "PGP"]
+
+[[rules]]
+description = "Facebook"
+regex = '''(?i)facebook(.{0,4})?['\"][0-9a-f]{32}['\"]'''
+tags = ["key", "Facebook"]
+
+[[rules]]
+description = "Twitter"
+regex = '''(?i)twitter(.{0,4})?['\"][0-9a-zA-Z]{35,44}['\"]'''
+tags = ["key", "Twitter"]
+
+[[rules]]
+description = "Github"
+regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
+tags = ["key", "Github"]
+
+[[rules]]
+description = "Slack"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+tags = ["key", "Slack"]
 
 [whitelist]
-regexes = [
-  # "AKIA.*EXAMPLE",
-]
 files = [
   "(.*?)(jpg|gif|doc|pdf|bin)$"
 ]
+#commits = [
+#  "whitelisted-commit1",
+#  "whitelisted-commit2",
+#]
+#repos = [
+#	"whitelisted-repo"
+#]
+
+# Additional Examples
+
+# [[rules]]
+# description = "Generic Key"
+# regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+# entropies = [
+#     "4.1-4.3",
+#     "5.5-6.3",
+# ]
+# entropyROI = "line"
+# filetypes = [".go", ".py", ".c"]
+# tags = ["key"]
+# severity = "8"
+#
+#
+# [[rules]]
+# description = "Generic Key"
+# regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+# entropies = ["4.1-4.3"]
+# filetypes = [".gee"]
+# entropyROI = "line"
+# tags = ["key"]
+# severity = "medium"
+
+# [[rules]]
+# description = "Any pem file"
+# filetypes = [".key"]
+# tags = ["pem"]
+# severity = "high"

src/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
 )
 
@@ -17,20 +18,27 @@ type entropyRange struct {
 	v2 float64
 }
 
-type Regex struct {
+// Rule instructs how gitleaks should audit each line of code
+type Rule struct {
 	description string
 	regex       *regexp.Regexp
+	severity    string
+	tags        []string
+	entropies   []*entropyRange
+	entropyROI  string
+	fileTypes   []*regexp.Regexp
 }
 
 // TomlConfig is used for loading gitleaks configs from a toml file
 type TomlConfig struct {
-	Regexes []struct {
+	Rules []struct {
 		Description string
 		Regex       string
-	}
-	Entropy struct {
-		LineRegexes []string
-		Ranges      []string
+		Entropies   []string
+		Tags        []string
+		Severity    string
+		EntropyROI  string
+		FileTypes   []string
 	}
 	Whitelist struct {
 		Files   []string
@@ -42,18 +50,15 @@ type TomlConfig struct {
 
 // Config contains gitleaks config
 type Config struct {
-	Regexes   []Regex
+	Rules     []*Rule
 	WhiteList struct {
 		regexes []*regexp.Regexp
 		files   []*regexp.Regexp
 		commits map[string]bool
 		repos   []*regexp.Regexp
 	}
-	Entropy struct {
-		entropyRanges []*entropyRange
-		regexes       []*regexp.Regexp
-	}
-	sshAuth *ssh.PublicKeys
+	FileRules []*Rule
+	sshAuth   *ssh.PublicKeys
 }
 
 // loadToml loads of the toml config containing regexes and whitelists.
@@ -104,31 +109,37 @@ func newConfig() (*Config, error) {
 
 // updateConfig will update a the global config values
 func (config *Config) update(tomlConfig TomlConfig) error {
-	if len(tomlConfig.Entropy.Ranges) != 0 {
-		err := config.updateEntropyRanges(tomlConfig.Entropy.Ranges)
+	for _, rule := range tomlConfig.Rules {
+		re := regexp.MustCompile(rule.Regex)
+		ranges, err := getEntropyRanges(rule.Entropies)
+		var fileTypes = []*regexp.Regexp{}
+		for _, regex := range rule.FileTypes {
+			fileTypes = append(fileTypes, regexp.MustCompile(regex))
+		}
+
 		if err != nil {
-			return err
+			log.Errorf("could not create entropy range for %s, skipping rule", rule.Description)
+			continue
 		}
-	}
 
-	for _, regex := range tomlConfig.Entropy.LineRegexes {
-		config.Entropy.regexes = append(config.Entropy.regexes, regexp.MustCompile(regex))
-	}
+		r := &Rule{
+			description: rule.Description,
+			regex:       re,
+			severity:    rule.Severity,
+			tags:        rule.Tags,
+			entropies:   ranges,
+			entropyROI:  rule.EntropyROI,
+			fileTypes:   fileTypes,
+		}
 
-	if singleSearchRegex != nil {
-		config.Regexes = append(config.Regexes, Regex{
-			description: "single search",
-			regex:       singleSearchRegex,
-		})
-	} else {
-		for _, regex := range tomlConfig.Regexes {
-			config.Regexes = append(config.Regexes, Regex{
-				description: regex.Description,
-				regex:       regexp.MustCompile(regex.Regex),
-			})
+		if len(rule.Entropies) == 0 && rule.Regex == "" && len(fileTypes) != 0 {
+			config.FileRules = append(config.FileRules, r)
 		}
+		config.Rules = append(config.Rules, r)
+
 	}
 
+	// set whitelists
 	config.WhiteList.commits = make(map[string]bool)
 	for _, commit := range tomlConfig.Whitelist.Commits {
 		config.WhiteList.commits[commit] = true
@@ -147,30 +158,31 @@ func (config *Config) update(tomlConfig TomlConfig) error {
 }
 
 // entropyRanges hydrates entropyRanges which allows for fine tuning entropy checking
-func (config *Config) updateEntropyRanges(entropyLimitStr []string) error {
+func getEntropyRanges(entropyLimitStr []string) ([]*entropyRange, error) {
+	var ranges []*entropyRange
 	for _, span := range entropyLimitStr {
 		split := strings.Split(span, "-")
 		v1, err := strconv.ParseFloat(split[0], 64)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		v2, err := strconv.ParseFloat(split[1], 64)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if v1 > v2 {
-			return fmt.Errorf("entropy range must be ascending")
+			return nil, fmt.Errorf("entropy range must be ascending")
 		}
 		r := &entropyRange{
 			v1: v1,
 			v2: v2,
 		}
 		if r.v1 > 8.0 || r.v1 < 0.0 || r.v2 > 8.0 || r.v2 < 0.0 {
-			return fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
+			return nil, fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
 		}
-		config.Entropy.entropyRanges = append(config.Entropy.entropyRanges, r)
+		ranges = append(ranges, r)
 	}
-	return nil
+	return ranges, nil
 }
 
 // externalConfig will attempt to load a pinned ".gitleaks.toml" configuration file

src/constants.go

@@ -1,77 +1,79 @@
 package gitleaks
 
-const version = "1.25.1"
+const version = "2.0.0"
 
 const defaultGithubURL = "https://api.github.com/"
 const defaultThreadNum = 1
+
+// ErrExit used to signal an error during gitleaks execution
 const ErrExit = 2
+
+// LeakExit used to signal leaks present in audit
 const LeakExit = 1
 
 const defaultConfig = `
 # This is a sample config file for gitleaks. You can configure gitleaks what to search for and what to whitelist.
 # The output you are seeing here is the default gitleaks config. If GITLEAKS_CONFIG environment variable
-# is set, gitleaks will load configurations from that path. If option --config-path is set, gitleaks will load
+# is set, gitleaks will load configurations from that path. If option --config is set, gitleaks will load
 # configurations from that path. Gitleaks does not whitelist anything by default.
 
 title = "gitleaks config"
-# add regexes to the regex table
-[[regexes]]
-description = "AWS"
+[[rules]]
+description = "AWS Key"
 regex = '''AKIA[0-9A-Z]{16}'''
-[[regexes]]
+tags = ["key", "AWS"]
+severity = "high"
+
+[[rules]]
 description = "PKCS8"
 regex = '''-----BEGIN PRIVATE KEY-----'''
-[[regexes]]
+tags = ["key", "PKCS8"]
+
+[[rules]]
 description = "RSA"
 regex = '''-----BEGIN RSA PRIVATE KEY-----'''
-[[regexes]]
+tags = ["key", "RSA"]
+
+[[rules]]
 description = "SSH"
 regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
-[[regexes]]
+tags = ["key", "SSH"]
+
+[[rules]]
 description = "PGP"
 regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
-[[regexes]]
+tags = ["key", "PGP"]
+
+[[rules]]
 description = "Facebook"
 regex = '''(?i)facebook(.{0,4})?['\"][0-9a-f]{32}['\"]'''
-[[regexes]]
+tags = ["key", "Facebook"]
+
+[[rules]]
 description = "Twitter"
 regex = '''(?i)twitter(.{0,4})?['\"][0-9a-zA-Z]{35,44}['\"]'''
-[[regexes]]
+tags = ["key", "Twitter"]
+
+[[rules]]
 description = "Github"
 regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
-[[regexes]]
+tags = ["key", "Github"]
+
+[[rules]]
 description = "Slack"
 regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
-
-[entropy]
-lineregexes = [
-	"api",
-	"key",
-	"signature",
-	"secret",
-	"password",
-	"pass",
-	"pwd",
-	"token",
-	"curl",
-	"wget",
-	"https?",
-]
+tags = ["key", "Slack"]
 
 [whitelist]
 files = [
   "(.*?)(jpg|gif|doc|pdf|bin)$"
 ]
+
 #commits = [
-#  "BADHA5H1",
-#  "BADHA5H2",
+#  "whitelisted-commit1",
+#  "whitelisted-commit2",
 #]
 #repos = [
-#	"mygoodrepo"
-#]
-[misc]
-#entropy = [
-#	"3.3-4.30"
-#	"6.0-8.0
+#	"whitelisted-repo"
 #]
 `

src/constants_test.go

@@ -0,0 +1,123 @@
+package gitleaks
+
+import (
+	"io/ioutil"
+	"path"
+)
+
+const testWhitelistCommit = `
+[[rules]]
+description = "AWS"
+regex = '''AKIA[0-9A-Z]{16}'''
+
+[whitelist]
+commits = [
+  "eaeffdc65b4c73ccb67e75d96bd8743be2c85973",
+]
+`
+const testWhitelistFile = `
+[[rules]]
+description = "AWS"
+regex = '''AKIA[0-9A-Z]{16}'''
+
+[whitelist]
+files = [
+  ".go",
+]
+`
+
+const testWhitelistRegex = `
+[[rules]]
+description = "AWS"
+regex = '''AKIA[0-9A-Z]{16}'''
+
+[whitelist]
+regexes= [
+  "AKIA",
+]
+`
+
+const testWhitelistRepo = `
+[[rules]]
+description = "AWS"
+regex = '''AKIA[0-9A-Z]{16}'''
+
+[whitelist]
+repos = [
+  "gronit",
+]
+`
+
+const testEntropyRange = `
+[[rules]]
+description = "Entropy ranges"
+entropies = [
+  "7.5-8.0",
+  "3.2-3.4",
+]
+`
+const testBadEntropyRange = `
+[[rules]]
+description = "Bad entropy ranges"
+entropies = [
+  "8.0-3.0",
+]
+`
+const testBadEntropyRange2 = `
+[[rules]]
+description = "Bad entropy ranges"
+entropies = [
+  "8.0-8.9",
+]
+`
+
+const testEntropyWordRegexRange = `
+[[rules]]
+description = "test entropy regex ranges"
+regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+entropies = [
+	"4.1-4.3",
+]
+entropyROI="word"
+`
+
+const testEntropyRegexRange = `
+[[rules]]
+description = "test entropy regex ranges"
+regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+entropies = [
+	"4.1-4.3",
+]
+`
+
+const testMDFileType = `
+[[rules]]
+description = "test only markdown"
+filetypes = [".md"]
+`
+
+const testEntropyRegexRangeGoFilter = `
+[[rules]]
+description = "test entropy regex ranges"
+regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+entropies = [
+	"4.1-4.3",
+]
+filetypes = [".go"]
+`
+
+func testTomlLoader() string {
+	tmpDir, _ := ioutil.TempDir("", "whiteListConfigs")
+	ioutil.WriteFile(path.Join(tmpDir, "regex"), []byte(testWhitelistRegex), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "entropyWordRegex"), []byte(testEntropyWordRegexRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "entropyRegex"), []byte(testEntropyRegexRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "commit"), []byte(testWhitelistCommit), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "file"), []byte(testWhitelistFile), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "repo"), []byte(testWhitelistRepo), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "entropy"), []byte(testEntropyRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "badEntropy"), []byte(testBadEntropyRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "badEntropy2"), []byte(testBadEntropyRange2), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "mdFiles"), []byte(testMDFileType), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "entropyRegexGo"), []byte(testEntropyRegexRangeGoFilter), 0644)
+	return tmpDir
+}

src/core.go

@@ -3,7 +3,6 @@ package gitleaks
 import (
 	"io/ioutil"
 	"os"
-	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -13,15 +12,14 @@ import (
 )
 
 var (
-	opts              *Options
-	config            *Config
-	singleSearchRegex *regexp.Regexp
-	dir               string
-	threads           int
-	totalCommits      int64
-	commitMap         = make(map[string]bool)
-	auditDone         bool
-	mutex             = &sync.Mutex{}
+	opts         *Options
+	config       *Config
+	dir          string
+	threads      int
+	totalCommits int64
+	commitMap    = make(map[string]bool)
+	auditDone    bool
+	mutex        = &sync.Mutex{}
 )
 
 func init() {
@@ -29,7 +27,8 @@ func init() {
 	threads = defaultThreadNum
 }
 
-// Report is
+// Report can be exported as a json or csv. Used for logging informationn
+// about the audit, (duration and # of commits)
 type Report struct {
 	Leaks    []Leak
 	Duration string

src/entropy.go

@@ -23,30 +23,3 @@ func getShannonEntropy(data string) (entropy float64) {
 
 	return entropy
 }
-
-func entropyIsHighEnough(entropy float64) bool {
-	if entropy >= opts.Entropy && len(config.Entropy.entropyRanges) == 0 {
-		return true
-	}
-
-	for _, eR := range config.Entropy.entropyRanges {
-		if entropy > eR.v1 && entropy < eR.v2 {
-			return true
-		}
-	}
-
-	return false
-}
-
-func highEntropyLineIsALeak(line string) bool {
-	if !opts.NoiseReduction {
-		return true
-	}
-	for _, re := range config.Entropy.regexes {
-		if re.FindString(line) != "" {
-			return true
-		}
-	}
-
-	return false
-}

src/github.go

@@ -66,7 +66,7 @@ func auditGithubPR() ([]Leak, error) {
 					continue
 				}
 
-				commit := commitInfo{
+				commit := &commitInfo{
 					sha:      c.GetSHA(),
 					content:  *f.Patch,
 					filePath: *f.Filename,

src/gitleaks_test.go

@@ -16,82 +16,6 @@ import (
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
-const testWhitelistCommit = `
-[[regexes]]
-description = "AWS"
-regex = '''AKIA[0-9A-Z]{16}'''
-
-[whitelist]
-commits = [
-  "eaeffdc65b4c73ccb67e75d96bd8743be2c85973",
-]
-`
-const testWhitelistFile = `
-[[regexes]]
-description = "AWS"
-regex = '''AKIA[0-9A-Z]{16}'''
-
-[whitelist]
-files = [
-  ".go",
-]
-`
-
-const testWhitelistRegex = `
-[[regexes]]
-description = "AWS"
-regex = '''AKIA[0-9A-Z]{16}'''
-
-[whitelist]
-regexes= [
-  "AKIA",
-]
-`
-
-const testWhitelistRepo = `
-[[regexes]]
-description = "AWS"
-regex = '''AKIA[0-9A-Z]{16}'''
-
-[whitelist]
-repos = [
-  "gronit",
-]
-`
-
-const testEntropyRange = `
-[entropy]
-ranges = [
-  "7.5-8.0",
-  "3.2-3.4",
-]
-lineregexes = [
-	"(?i)api",
-	"(?i)key",
-	"signature",
-	"secret",
-	"password",
-	"pass",
-	"pwd",
-	"token",
-	"curl",
-	"wget",
-	"https?",
-]
-`
-const testBadEntropyRange = `
-[entropy]
-ranges = [
-  "8.0-3.0",
-]
-`
-const testBadEntropyRange2 = `
-[entropy]
-ranges = [
-  "8.0-8.9",
-]
-`
-
 func TestGetRepo(t *testing.T) {
 	var err error
 	dir, err = ioutil.TempDir("", "gitleaksTestRepo")
@@ -373,7 +297,7 @@ func TestWriteReport(t *testing.T) {
 			Line:     "eat",
 			Commit:   "your",
 			Offender: "veggies",
-			Type:     "and",
+			Rule:     "and",
 			Message:  "get",
 			Author:   "some",
 			File:     "sleep",
@@ -447,18 +371,6 @@ func TestWriteReport(t *testing.T) {
 
 }
 
-func testTomlLoader() string {
-	tmpDir, _ := ioutil.TempDir("", "whiteListConfigs")
-	ioutil.WriteFile(path.Join(tmpDir, "regex"), []byte(testWhitelistRegex), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "commit"), []byte(testWhitelistCommit), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "file"), []byte(testWhitelistFile), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "repo"), []byte(testWhitelistRepo), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "entropy"), []byte(testEntropyRange), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "badEntropy"), []byte(testBadEntropyRange), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "badEntropy2"), []byte(testBadEntropyRange2), 0644)
-	return tmpDir
-}
-
 func TestAuditRepo(t *testing.T) {
 	var leaks []Leak
 	configsDir := testTomlLoader()
@@ -630,23 +542,6 @@ func TestAuditRepo(t *testing.T) {
 			testOpts:    &Options{},
 			configPath:  path.Join(configsDir, "repo"),
 		},
-		{
-			repo:        leaksRepo,
-			description: "leaks present with entropy",
-			testOpts: &Options{
-				Entropy: 4.7,
-			},
-			numLeaks: 6,
-		},
-		{
-			repo:        leaksRepo,
-			description: "leaks present with entropy",
-			testOpts: &Options{
-				Entropy:        4.7,
-				NoiseReduction: true,
-			},
-			numLeaks: 2,
-		},
 		{
 			repo:        leaksRepo,
 			description: "Audit until specific commit",
@@ -666,26 +561,24 @@ func TestAuditRepo(t *testing.T) {
 		{
 			repo:        leaksRepo,
 			description: "toml entropy range from opts",
-			numLeaks:    454,
+			numLeaks:    266,
 			testOpts: &Options{
 				ConfigPath: path.Join(configsDir, "entropy"),
 			},
 		},
 		{
 			repo:        leaksRepo,
-			description: "toml entropy range",
-			numLeaks:    454,
+			description: "toml entropy regex word range",
+			numLeaks:    0,
 			testOpts:    &Options{},
-			configPath:  path.Join(configsDir, "entropy"),
+			configPath:  path.Join(configsDir, "entropyWordRegex"),
 		},
 		{
-			repo: leaksRepo,
-			testOpts: &Options{
-				NoiseReduction: true,
-			},
-			description: "toml entropy noise reduction range",
-			numLeaks:    64,
-			configPath:  path.Join(configsDir, "entropy"),
+			repo:        leaksRepo,
+			description: "toml entropy regex range",
+			numLeaks:    2,
+			testOpts:    &Options{},
+			configPath:  path.Join(configsDir, "entropyRegex"),
 		},
 		{
 			repo:           leaksRepo,
@@ -703,6 +596,20 @@ func TestAuditRepo(t *testing.T) {
 			configPath:     path.Join(configsDir, "badEntropy2"),
 			expectedErrMsg: "invalid entropy ranges, must be within 0.0-8.0",
 		},
+		{
+			repo:        leaksRepo,
+			description: "toml md files",
+			numLeaks:    5,
+			testOpts:    &Options{},
+			configPath:  path.Join(configsDir, "mdFiles"),
+		},
+		{
+			repo:        leaksRepo,
+			description: "toml entropys line regex go",
+			numLeaks:    2,
+			testOpts:    &Options{},
+			configPath:  path.Join(configsDir, "entropyRegexGo"),
+		},
 	}
 	g := goblin.Goblin(t)
 	for _, test := range tests {
@@ -710,7 +617,6 @@ func TestAuditRepo(t *testing.T) {
 			g.It(test.description, func() {
 				auditDone = false
 				opts = test.testOpts
-				totalCommits = 0
 
 				config, err = newConfig()
 				// config paths
@@ -723,14 +629,10 @@ func TestAuditRepo(t *testing.T) {
 					}
 				}
 				leaks, err = test.repo.audit()
-				if test.testOpts.Depth != 0 {
-					g.Assert(totalCommits).Equal(test.testOpts.Depth)
-				} else {
-					if opts.Redact {
-						g.Assert(leaks[0].Offender).Equal("REDACTED")
-					}
-					g.Assert(len(leaks)).Equal(test.numLeaks)
+				if opts.Redact {
+					g.Assert(leaks[0].Offender).Equal("REDACTED")
 				}
+				g.Assert(len(leaks)).Equal(test.numLeaks)
 			next:
 				os.Setenv("GITLEAKS_CONFIG", "")
 			})
@@ -775,30 +677,6 @@ func TestOptionGuard(t *testing.T) {
 			description:    "local and remote target",
 			expectedErrMsg: "github user set and local owner path",
 		},
-		{
-			testOpts: &Options{
-				GithubUser:   "fakeUser",
-				SingleSearch: "*/./....",
-			},
-			description:         "single search invalid regex gaurd",
-			expectedErrMsgFuzzy: "unable to compile regex: */./...., ",
-		},
-		{
-			testOpts: &Options{
-				GithubUser:   "fakeUser",
-				SingleSearch: "mystring",
-			},
-			description:    "single search regex gaurd",
-			expectedErrMsg: "",
-		},
-		{
-			testOpts: &Options{
-				GithubOrg: "fakeOrg",
-				Entropy:   9,
-			},
-			description:    "Invalid entropy level guard",
-			expectedErrMsg: "The maximum level of entropy is 8",
-		},
 	}
 	g := goblin.Goblin(t)
 	for _, test := range tests {
@@ -880,11 +758,6 @@ func TestLoadToml(t *testing.T) {
 		g.Describe("TestLoadToml", func() {
 			g.It(test.description, func() {
 				opts = test.testOpts
-				if test.singleSearch {
-					singleSearchRegex = regexp.MustCompile("test")
-				} else {
-					singleSearchRegex = nil
-				}
 				if test.configPath != "" {
 					os.Setenv("GITLEAKS_CONFIG", test.configPath)
 				} else {

src/options.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 	"time"
@@ -36,16 +35,13 @@ type Options struct {
 	OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
 
 	// Process options
-	Threads        int     `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	Disk           bool    `long:"disk" description:"Clones repo(s) to disk"`
-	SingleSearch   string  `long:"single-search" description:"single regular expression to search for"`
-	ConfigPath     string  `long:"config" description:"path to gitleaks config"`
-	SSHKey         string  `long:"ssh-key" description:"path to ssh key"`
-	ExcludeForks   bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
-	Entropy        float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
-	NoiseReduction bool    `long:"noise-reduction" description:"Reduce the number of finds when entropy checks are enabled"`
-	RepoConfig     bool    `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\""`
-	Branch         string  `long:"branch" description:"Branch to audit"`
+	Threads      int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	Disk         bool   `long:"disk" description:"Clones repo(s) to disk"`
+	ConfigPath   string `long:"config" description:"path to gitleaks config"`
+	SSHKey       string `long:"ssh-key" description:"path to ssh key"`
+	ExcludeForks bool   `long:"exclude-forks" description:"exclude forks for organization/user audits"`
+	RepoConfig   bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\""`
+	Branch       string `long:"branch" description:"Branch to audit"`
 	// TODO: IncludeMessages  string `long:"messages" description:"include commit messages in audit"`
 
 	// Output options
@@ -64,13 +60,13 @@ func ParseOpts() *Options {
 	_, err := parser.Parse()
 
 	if err != nil {
-		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
-			os.Exit(0)
+		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type != flags.ErrHelp {
+			parser.WriteHelp(os.Stdout)
 		}
+		os.Exit(0)
 	}
 
 	if len(os.Args) == 1 {
-		// TODO: this will be a feature, check locally
 		parser.WriteHelp(os.Stdout)
 		os.Exit(0)
 	}
@@ -95,7 +91,6 @@ func ParseOpts() *Options {
 
 // optsGuard prevents invalid options
 func (opts *Options) guard() error {
-	var err error
 	if opts.GithubOrg != "" && opts.GithubUser != "" {
 		return fmt.Errorf("github user and organization set")
 	} else if opts.GithubOrg != "" && opts.OwnerPath != "" {
@@ -129,16 +124,6 @@ func (opts *Options) guard() error {
 		}
 	}
 
-	if opts.SingleSearch != "" {
-		singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
-		if err != nil {
-			return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
-		}
-	}
-
-	if opts.Entropy > 8 {
-		return fmt.Errorf("The maximum level of entropy is 8")
-	}
 	if opts.Report != "" {
 		if !strings.HasSuffix(opts.Report, ".json") && !strings.HasSuffix(opts.Report, ".csv") {
 			return fmt.Errorf("Report should be a .json or .csv file")

src/repo.go

@@ -23,13 +23,16 @@ type Leak struct {
 	Line     string    `json:"line"`
 	Commit   string    `json:"commit"`
 	Offender string    `json:"offender"`
-	Type     string    `json:"reason"`
+	Rule     string    `json:"rule"`
+	Info     string    `json:"info"`
 	Message  string    `json:"commitMsg"`
 	Author   string    `json:"author"`
 	Email    string    `json:"email"`
 	File     string    `json:"file"`
 	Repo     string    `json:"repo"`
 	Date     time.Time `json:"date"`
+	Tags     string    `json:"tags"`
+	Severity string    `json:"severity"`
 }
 
 // RepoInfo contains a src-d git repository and other data about the repo
@@ -82,6 +85,9 @@ func (repoInfo *RepoInfo) clone() error {
 	} else if repoInfo.path != "" {
 		log.Infof("opening %s", opts.RepoPath)
 		repo, err = git.PlainOpen(repoInfo.path)
+		if err != nil {
+			log.Errorf("unable to open %s", opts.RepoPath)
+		}
 	} else {
 		// cloning to memory
 		log.Infof("cloning %s", opts.Repo)
@@ -235,6 +241,27 @@ func (repoInfo *RepoInfo) audit() ([]Leak, error) {
 					} else if to != nil {
 						filePath = to.Path()
 					}
+
+					for _, fr := range config.FileRules {
+						for _, r := range fr.fileTypes {
+							if r.FindString(filePath) != "" {
+								commitInfo := &commitInfo{
+									repoName: repoInfo.name,
+									filePath: filePath,
+									sha:      c.Hash.String(),
+									author:   c.Author.Name,
+									email:    c.Author.Email,
+									message:  strings.Replace(c.Message, "\n", " ", -1),
+									date:     c.Author.When,
+								}
+								leak := *newLeak("N/A", fmt.Sprintf("filetype %s found", r.String()), r.String(), fr, commitInfo)
+								mutex.Lock()
+								leaks = append(leaks, leak)
+								mutex.Unlock()
+							}
+						}
+					}
+
 					for _, re := range config.WhiteList.files {
 						if re.FindString(filePath) != "" {
 							log.Debugf("skipping whitelisted file (matched regex '%s'): %s", re.String(), filePath)
@@ -248,7 +275,7 @@ func (repoInfo *RepoInfo) audit() ([]Leak, error) {
 					chunks := f.Chunks()
 					for _, chunk := range chunks {
 						if chunk.Type() == diffType.Add || chunk.Type() == diffType.Delete {
-							diff := commitInfo{
+							diff := &commitInfo{
 								repoName: repoInfo.name,
 								filePath: filePath,
 								content:  chunk.Content(),
@@ -300,7 +327,7 @@ func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []Leak {
 		if err != nil {
 			return nil
 		}
-		diff := commitInfo{
+		diff := &commitInfo{
 			repoName: repoInfo.name,
 			filePath: f.Name,
 			content:  content,

src/utils.go

@@ -41,9 +41,9 @@ func writeReport(leaks []Leak) error {
 		}
 		defer f.Close()
 		w := csv.NewWriter(f)
-		w.Write([]string{"repo", "line", "commit", "offender", "reason", "commitMsg", "author", "email", "file", "date"})
+		w.Write([]string{"repo", "line", "commit", "offender", "rule", "info", "tags", "severity", "commitMsg", "author", "email", "file", "date"})
 		for _, leak := range leaks {
-			w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Type, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
+			w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Rule, leak.Info, leak.Tags, leak.Severity, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
 		}
 		w.Flush()
 	} else {
@@ -82,48 +82,86 @@ func writeReport(leaks []Leak) error {
 	return nil
 }
 
-// inspect will parse each line of the git diff's content against a set of regexes or
-// a set of regexes set by the config (see gitleaks.toml for example). This function
-// will skip lines that include a whitelisted regex. A list of leaks is returned.
-// If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
-func inspect(commit commitInfo) []Leak {
+// check rule will inspect a single line and return a leak if it encounters one
+func (rule *Rule) check(line string, commit *commitInfo) (*Leak, error) {
 	var (
-		leaks    []Leak
-		skipLine bool
+		match       string
+		fileMatch   string
+		entropy     float64
+		entropyWord string
 	)
-	lines := strings.Split(commit.content, "\n")
 
-	for _, line := range lines {
-		skipLine = false
-		for _, re := range config.Regexes {
-			match := re.regex.FindString(line)
-			if match == "" {
-				continue
-			}
-			if skipLine = isLineWhitelisted(line); skipLine {
-				break
-			}
-			leaks = addLeak(leaks, line, match, re.description, commit)
+	for _, f := range rule.fileTypes {
+		fileMatch = f.FindString(commit.filePath)
+		if fileMatch != "" {
+			break
 		}
+	}
 
-		if !skipLine && (opts.Entropy > 0 || len(config.Entropy.entropyRanges) != 0) {
+	if fileMatch == "" && len(rule.fileTypes) != 0 {
+		return nil, nil
+	}
+
+	if rule.entropies != nil {
+		if rule.entropyROI == "word" {
 			words := strings.Fields(line)
 			for _, word := range words {
-				entropy := getShannonEntropy(word)
-				// Only check entropyRegexes and whiteListRegexes once per line, and only if an entropy leak type
-				// was found above, since regex checks are expensive.
-				if !entropyIsHighEnough(entropy) {
-					continue
+				_entropy := getShannonEntropy(word)
+				for _, e := range rule.entropies {
+					if _entropy > e.v1 && _entropy < e.v2 {
+						entropy = _entropy
+						entropyWord = word
+						goto postEntropy
+					}
 				}
-				// If either the line is whitelisted or the line fails the noiseReduction check (when enabled),
-				// then we can skip checking the rest of the line for high entropy words.
-				if skipLine = !highEntropyLineIsALeak(line) || isLineWhitelisted(line); skipLine {
-					break
+			}
+		} else {
+			_entropy := getShannonEntropy(line)
+			for _, e := range rule.entropies {
+				if _entropy > e.v1 && _entropy < e.v2 {
+					entropy = _entropy
+					entropyWord = line
+					goto postEntropy
 				}
-				leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), commit)
 			}
 		}
 	}
+
+postEntropy:
+	if rule.regex != nil {
+		match = rule.regex.FindString(line)
+	}
+
+	if match != "" && entropy != 0.0 {
+		return newLeak(line, fmt.Sprintf("%s regex match and entropy met at %.2f", rule.regex.String(), entropy), entropyWord, rule, commit), nil
+	} else if match != "" && rule.entropies == nil {
+		return newLeak(line, fmt.Sprintf("%s regex match", rule.regex.String()), match, rule, commit), nil
+	} else if entropy != 0.0 && rule.regex.String() == "" {
+		return newLeak(line, fmt.Sprintf("entropy met at %.2f", entropy), entropyWord, rule, commit), nil
+	}
+	return nil, nil
+}
+
+// inspect will parse each line of the git diff's content against a set of regexes or
+// a set of regexes set by the config (see gitleaks.toml for example). This function
+// will skip lines that include a whitelisted regex. A list of leaks is returned.
+// If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
+func inspect(commit *commitInfo) []Leak {
+	var leaks []Leak
+	lines := strings.Split(commit.content, "\n")
+
+	for _, line := range lines {
+		for _, rule := range config.Rules {
+			if isLineWhitelisted(line) {
+				break
+			}
+			leak, err := rule.check(line, commit)
+			if err != nil || leak == nil {
+				continue
+			}
+			leaks = append(leaks, *leak)
+		}
+	}
 	return leaks
 }
 
@@ -138,19 +176,21 @@ func isLineWhitelisted(line string) bool {
 	return false
 }
 
-// addLeak is helper for func inspect() to append leaks if found during a diff check.
-func addLeak(leaks []Leak, line string, offender string, leakType string, commit commitInfo) []Leak {
-	leak := Leak{
+func newLeak(line string, info string, offender string, rule *Rule, commit *commitInfo) *Leak {
+	leak := &Leak{
 		Line:     line,
 		Commit:   commit.sha,
 		Offender: offender,
-		Type:     leakType,
+		Rule:     rule.description,
+		Info:     info,
 		Author:   commit.author,
 		Email:    commit.email,
 		File:     commit.filePath,
 		Repo:     commit.repoName,
 		Message:  commit.message,
 		Date:     commit.date,
+		Tags:     strings.Join(rule.tags, ", "),
+		Severity: rule.severity,
 	}
 	if opts.Redact {
 		leak.Offender = "REDACTED"
@@ -160,9 +200,7 @@ func addLeak(leaks []Leak, line string, offender string, leakType string, commit
 	if opts.Verbose {
 		leak.log()
 	}
-
-	leaks = append(leaks, leak)
-	return leaks
+	return leak
 }
 
 // discoverRepos walks all the children of `path`. If a child directory