sprucing up the joint

checks.go

@@ -5,30 +5,27 @@ import (
 	"strings"
 )
 
-// check each line of a diff and see if there are any potential secrets
-// [1] https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf
-func checkRegex(diff string) []string {
+func doChecks(diff string) []string {
 	var match string
 	var results []string
 	lines := strings.Split(diff, "\n")
 	for _, line := range lines {
-		// doubtful a leak would be on a line > 120 characters
-		if len(line) == 0 || len(line) > 120 {
-			continue
-		}
 		for _, re := range regexes {
 			match = re.FindString(line)
-			if len(match) == 0 {
+			if len(match) == 0 ||
+				(opts.Strict && containsStopWords(line)) ||
+				(opts.Entropy && !checkShannonEntropy(line)) {
 				continue
 			}
 			results = append(results, line)
 		}
 	}
 	return results
+
 }
 
 // checkShannonEntropy checks entropy of target
-func checkShannonEntropy(target string, entropy64Cutoff int, entropyHexCutoff int) bool {
+func checkShannonEntropy(target string) bool {
 	var (
 		sum             float64
 		targetBase64Len int
@@ -38,7 +35,6 @@ func checkShannonEntropy(target string, entropy64Cutoff int, entropyHexCutoff in
 		bits            int
 	)
 
-	// get assignment value
 	index := assignRegex.FindStringIndex(target)
 	if len(index) == 0 {
 		return false
@@ -61,7 +57,7 @@ func checkShannonEntropy(target string, entropy64Cutoff int, entropyHexCutoff in
 	}
 
 	bits = int(math.Ceil(sum*-1)) * targetBase64Len
-	if bits > entropy64Cutoff {
+	if bits > opts.B64EntropyCutoff {
 		return true
 	}
 
@@ -78,7 +74,7 @@ func checkShannonEntropy(target string, entropy64Cutoff int, entropyHexCutoff in
 		sum += f * math.Log2(f)
 	}
 	bits = int(math.Ceil(sum*-1)) * targetHexLen
-	return bits > entropyHexCutoff
+	return bits > opts.HexEntropyCutoff
 }
 
 // containsStopWords checks if there are any stop words in target

checks_test.go

@@ -4,23 +4,24 @@ import (
 	"testing"
 )
 
+func init(){
+	opts = &Options{
+		Concurrency:      10,
+		B64EntropyCutoff: 70,
+		HexEntropyCutoff: 40,
+		Entropy: false,
+	}
+}
+
 func TestCheckRegex(t *testing.T) {
 	var results []string
 	checks := map[string]int{
-		"github.com":                                                                                     0,
-		"github.com/user/":                                                                               0,
-		"github.com/user -- Sys":                                                                         0,
-		"github_api_client = \"sample key\"\naws=afewafewafewafewaf":                                     2,
-		"aws=\"afewafewafewafewaf\"":                                                                     1,
-		"aws\"afewafewafewafewaf\"":                                                                      0,
-		"heroku := \"afewafewafewafewaf\"":                                                               1,
-		"heroku_client_secret := \"afewafewafewafewaf\"":                                                 1,
-		"reddit_api_secreit = \"Fwe4fa431FgklreF\"":                                                      1,
-		"+ * [Github Help: Managing Deploy Keys](https://help.github.com/articles/managing-deploy-keys)": 0,
+		"aws=\"AKIALALEMEL33243OLIAE": 1,
+		"aws\"afewafewafewafewaf\"": 0,
 	}
 
 	for k, v := range checks {
-		results = checkRegex(k)
+		results = doChecks(k)
 		if v != len(results) {
 			t.Errorf("regexCheck failed on string %s", k)
 		}
@@ -36,7 +37,7 @@ func TestEntropy(t *testing.T) {
 		"aws_secret= \"AKIAIMNOJVGFDXXFE4OA\"":           true,
 	}
 	for k, v := range checks {
-		enoughEntropy = checkShannonEntropy(k, 70, 40)
+		enoughEntropy = checkShannonEntropy(k)
 		if v != enoughEntropy {
 			t.Errorf("checkEntropy failed for %s. Expected %t, got %t", k, v, enoughEntropy)
 		}

config.yml

@@ -1,11 +1,16 @@
 regexes:
-    - '[g|G][i|I][t|T][h|H][u|U][b|B].*(=|:=|<-).*\w+.*'
-    - '[a|A][w|W][s|S].*(=|:=|:|<-).*\w+.*'
-    - '[h|H][e|E][r|R][o|O][k|K][u|U].*(=|:=|<-).*\w+.*'
-    - '[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*(=|:=|<-).*\w+.*'
-    - '[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*(=|:=|<-).*\w+.*'
-    - '[r|R][e|E][d|D][d|D][i|I][t|T].*(=|:=|<-).*\w+.*'
-    - '[t|T][w|W][i|I][l|L][i|I][o|O].*(=|:=|<-).*\w+.*'
+    - "-----BEGIN RSA PRIVATE KEY-----"
+    - "-----BEGIN OPENSSH PRIVATE KEY-----"
+    - "-----BEGIN PGP PRIVATE KEY BLOCK-----"
+    - "[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*['|\"][0-9a-f]{32}['|\"]"
+    - "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*['|\"][0-9a-zA-Z]{35,44}['|\"]"
+    - "[g|G][i|I][t|T][h|H][u|U][b|B].*[['|\"]0-9a-zA-Z]{35,40}['|\"]"
+    - "AKIA[0-9A-Z]{16}"
+    - "[r|R][e|E][d|D][d|D][i|I][t|T].*['|\"][0-9a-zA-Z]{14}['|\"]"
+    - "[h|H][e|E][r|R][o|O][k|K][u|U].*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"
+    # add your own regex
+
+
 
 stopwords:
     - 'setting'
@@ -20,4 +25,5 @@ stopwords:
     - 'environment'
     - 'Environment'
     - 'ENVIRONMENT'
+    # add your own stopwords
 

leaks.go

@@ -107,10 +107,9 @@ func getLeaks(repoName string, opts *Options) []LeakElem {
 		currCommit := string(currCommitB)
 
 		go func(currCommit string, repoName string, commitWG *sync.WaitGroup,
-			gitLeakReceiverWG *sync.WaitGroup, opts *Options) {
+			gitLeakReceiverWG *sync.WaitGroup) {
 
 			defer commitWG.Done()
-			var leakPrs bool
 
 			if currCommit == "" {
 				return
@@ -131,22 +130,16 @@ func getLeaks(repoName string, opts *Options) []LeakElem {
 				return
 			}
 
-			lines := checkRegex(string(out))
+			lines := doChecks(string(out))
 			if len(lines) == 0 {
 				return
 			}
-
 			for _, line := range lines {
-				leakPrs = checkShannonEntropy(line, opts.B64EntropyCutoff, opts.HexEntropyCutoff)
-				if leakPrs {
-					if opts.Strict && containsStopWords(line) {
-						continue
-					}
-					gitLeakReceiverWG.Add(1)
-					gitLeaks <- LeakElem{line, currCommit}
-				}
+				gitLeakReceiverWG.Add(1)
+				gitLeaks <- LeakElem{line, currCommit}
 			}
-		}(currCommit, repoName, &commitWG, &gitLeakReceiverWG, opts)
+
+		}(currCommit, repoName, &commitWG, &gitLeakReceiverWG)
 	}
 
 	commitWG.Wait()

main.go

@@ -16,9 +16,10 @@ var (
 	appRoot     string
 	regexes     []*regexp.Regexp
 	stopWords 	[]string
-	assignRegex *regexp.Regexp
 	base64Chars string
 	hexChars    string
+	opts *Options
+	assignRegex *regexp.Regexp
 )
 
 // config
@@ -55,7 +56,6 @@ func init() {
 		log.Fatalf("Unmarshal: %v", err)
 	}
 
-	// regex from config
 	stopWords = c.StopWords
 	for _, re := range c.Regexes {
 		regexes = append(regexes, regexp.MustCompile(re))
@@ -65,7 +65,7 @@ func init() {
 
 func main() {
 	args := os.Args[1:]
-	opts := parseOptions(args)
+	opts = parseOptions(args)
 	if opts.RepoURL != "" {
 		start(opts)
 	} else if opts.UserURL != "" || opts.OrgURL != "" {

options.go

@@ -9,13 +9,14 @@ import (
 const usage = `usage: gitleaks [options] <url>
 
 Options:
-	-c 			Concurrency factor 
+	-c 			Concurrency factor (default is 10)
 	-u --user 		Git user url
 	-r --repo 		Git repo url
 	-o --org 		Git organization url
 	-s --strict 		Strict mode uses stopwords in config.yml 
-	-e --b64Entropy 	Base64 entropy cutoff, default is 70
-	-x --hexEntropy  	Hex entropy cutoff, default is 40
+	-b --b64Entropy 	Base64 entropy cutoff (default is 70)
+	-x --hexEntropy  	Hex entropy cutoff (default is 40)
+	-e --entropy	Enable entropy		
 	-h --help 		Display this message
 `
 
@@ -28,6 +29,7 @@ type Options struct {
 	OrgURL           string
 	RepoURL          string
 	Strict           bool
+	Entropy bool
 }
 
 // help prints the usage string and exits
@@ -36,8 +38,7 @@ func help() {
 	os.Exit(1)
 }
 
-// optionsNextInt is a parseOptions helper that returns the value (int) of an option
-// if valid.
+// optionsNextInt is a parseOptions helper that returns the value (int) of an option if valid
 func optionsNextInt(args []string, i *int) int {
 	if len(args) > *i+1 {
 		*i++
@@ -52,8 +53,7 @@ func optionsNextInt(args []string, i *int) int {
 	return argInt
 }
 
-// optionsNextString is a parseOptions helper that returns the value (string) of an option
-// if valid.
+// optionsNextString is a parseOptions helper that returns the value (string) of an option if valid
 func optionsNextString(args []string, i *int) string {
 	if len(args) > *i+1 {
 		*i++
@@ -70,6 +70,7 @@ func parseOptions(args []string) *Options {
 		Concurrency:      10,
 		B64EntropyCutoff: 70,
 		HexEntropyCutoff: 40,
+		Entropy: false,
 	}
 
 	for i := 0; i < len(args); i++ {
@@ -77,10 +78,12 @@ func parseOptions(args []string) *Options {
 		switch arg {
 		case "-s", "--strict":
 			opts.Strict = true
-		case "-e", "--b64Entropy":
+		case "-b", "--b64Entropy":
 			opts.B64EntropyCutoff = optionsNextInt(args, &i)
 		case "-x", "--hexEntropy":
 			opts.HexEntropyCutoff = optionsNextInt(args, &i)
+		case "-e", "--entropy":
+			opts.Entropy = true
 		case "-c":
 			opts.Concurrency = optionsNextInt(args, &i)
 		case "-o", "--org":