shannon entropy for base64 and hex, more options

checks.go

@@ -28,31 +28,57 @@ func checkRegex(diff string) []string {
 }
 
 // checkShannonEntropy checks entropy of target
-func checkShannonEntropy(target string, entropyCutoff int) bool {
+func checkShannonEntropy(target string, entropy64Cutoff int, entropyHexCutoff int) bool {
+	var (
+		sum             float64
+		targetBase64Len int
+		targetHexLen    int
+		base64Freq      = make(map[rune]float64)
+		hexFreq         = make(map[rune]float64)
+		bits            int
+	)
+
+	// get assignment value
 	index := assignRegex.FindStringIndex(target)
 	if len(index) == 0 {
 		return false
 	}
-
 	target = strings.Trim(target[index[1]:], " ")
 	if len(target) > 100 {
 		return false
 	}
 
-	var sum float64
-	frq := make(map[rune]float64)
-
+	// base64Shannon
 	for _, i := range target {
-		frq[i]++
+		if strings.Contains(base64Chars, string(i)) {
+			base64Freq[i]++
+			targetBase64Len++
+		}
 	}
-
-	for _, v := range frq {
-		f := v / float64(len(target))
+	for _, v := range base64Freq {
+		f := v / float64(targetBase64Len)
 		sum += f * math.Log2(f)
 	}
 
-	bits := int(math.Ceil(sum*-1)) * len(target)
-	return bits > entropyCutoff
+	bits = int(math.Ceil(sum*-1)) * targetBase64Len
+	if bits > entropy64Cutoff {
+		return true
+	}
+
+	// hexShannon
+	sum = 0
+	for _, i := range target {
+		if strings.Contains(hexChars, string(i)) {
+			hexFreq[i]++
+			targetHexLen++
+		}
+	}
+	for _, v := range hexFreq {
+		f := v / float64(targetHexLen)
+		sum += f * math.Log2(f)
+	}
+	bits = int(math.Ceil(sum*-1)) * targetHexLen
+	return bits > entropyHexCutoff
 }
 
 // containsStopWords checks if there are any stop words in target

checks_test.go

@@ -7,15 +7,16 @@ import (
 func TestCheckRegex(t *testing.T) {
 	var results []string
 	checks := map[string]int{
-		"github.com":                                                 0,
-		"github.com/user/":                                           0,
-		"github.com/user -- Sys":                                     0,
-		"github_api_client = \"sample key\"\naws=afewafewafewafewaf": 2,
-		"aws=\"afewafewafewafewaf\"":                                 1,
-		"aws\"afewafewafewafewaf\"":                                  0,
-		"heroku := \"afewafewafewafewaf\"":                           1,
-		"heroku_client_secret := \"afewafewafewafewaf\"":             1,
-		"reddit_api_secreit = \"Fwe4fa431FgklreF\"":                  1,
+		"github.com":                                                                                     0,
+		"github.com/user/":                                                                               0,
+		"github.com/user -- Sys":                                                                         0,
+		"github_api_client = \"sample key\"\naws=afewafewafewafewaf":                                     2,
+		"aws=\"afewafewafewafewaf\"":                                                                     1,
+		"aws\"afewafewafewafewaf\"":                                                                      0,
+		"heroku := \"afewafewafewafewaf\"":                                                               1,
+		"heroku_client_secret := \"afewafewafewafewaf\"":                                                 1,
+		"reddit_api_secreit = \"Fwe4fa431FgklreF\"":                                                      1,
+		"+ * [Github Help: Managing Deploy Keys](https://help.github.com/articles/managing-deploy-keys)": 0,
 	}
 
 	for k, v := range checks {
@@ -29,12 +30,13 @@ func TestCheckRegex(t *testing.T) {
 func TestEntropy(t *testing.T) {
 	var enoughEntropy bool
 	checks := map[string]bool{
-		"heroku_client_secret = simple":            false,
-		"reddit_api_secret = \"Fwe4fa431FgklreF\"": true,
-		"aws_secret= \"AKIAIMNOJVGFDXXXE4OA\"":     true,
+		"reddit_api_secret = settings./.http}":           false,
+		"heroku_client_secret = simple":                  false,
+		"reddit_api_secret = \"4ok1WFf57-EMswEfAFGewa\"": true,
+		"aws_secret= \"AKIAIMNOJVGFDXXFE4OA\"":           true,
 	}
 	for k, v := range checks {
-		enoughEntropy = checkShannonEntropy(k, 70)
+		enoughEntropy = checkShannonEntropy(k, 70, 40)
 		if v != enoughEntropy {
 			t.Errorf("checkEntropy failed for %s. Expected %t, got %t", k, v, enoughEntropy)
 		}

leaks.go

@@ -85,14 +85,8 @@ func getLeaks(repoName string, opts *Options) []LeakElem {
 		gitLeakReceiverWG sync.WaitGroup
 		gitLeaks          = make(chan LeakElem)
 		report            []LeakElem
-		concurrency       = 10
 	)
-
-	if opts.Concurrency != 0 {
-		concurrency = opts.Concurrency
-	}
-
-	semaphoreChan := make(chan struct{}, concurrency)
+	semaphoreChan := make(chan struct{}, opts.Concurrency)
 
 	go func(commitWG *sync.WaitGroup, gitLeakReceiverWG *sync.WaitGroup) {
 		for gitLeak := range gitLeaks {
@@ -143,7 +137,7 @@ func getLeaks(repoName string, opts *Options) []LeakElem {
 			}
 
 			for _, line := range lines {
-				leakPrs = checkShannonEntropy(line, opts.EntropyCutoff)
+				leakPrs = checkShannonEntropy(line, opts.B64EntropyCutoff, opts.HexEntropyCutoff)
 				if leakPrs {
 					if opts.Strict && containsStopWords(line) {
 						continue

main.go

@@ -15,6 +15,8 @@ var (
 	appRoot     string
 	regexes     map[string]*regexp.Regexp
 	assignRegex *regexp.Regexp
+	base64Chars string
+	hexChars    string
 )
 
 func init() {
@@ -28,15 +30,17 @@ func init() {
 	// TODO ability to add/filter regex
 	// client("AKAI32fJ334...",
 	regexes = map[string]*regexp.Regexp{
-		"github":   regexp.MustCompile(`[g|G][i|I][t|T][h|H][u|U][b|B].*(=|:|:=|<-).*\w+.*`),
+		"github":   regexp.MustCompile(`[g|G][i|I][t|T][h|H][u|U][b|B].*(=|:=|<-).*\w+.*`),
 		"aws":      regexp.MustCompile(`[a|A][w|W][s|S].*(=|:=|:|<-).*\w+.*`),
-		"heroku":   regexp.MustCompile(`[h|H][e|E][r|R][o|O][k|K][u|U].*(=|:=|:|<-).*\w+.*`),
-		"facebook": regexp.MustCompile(`[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*(=|:=|:|<-).*\w+.*`),
-		"twitter":  regexp.MustCompile(`[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*(=|:=|:|<-).*\w+.*`),
-		"reddit":   regexp.MustCompile(`[r|R][e|E][d|D][d|D][i|I][t|T].*(=|:=|:|<-).*\w+.*`),
-		"twilio":   regexp.MustCompile(`[t|T][w|W][i|I][l|L][i|I][o|O].*(=|:=|:|<-).*\w+.*`),
+		"heroku":   regexp.MustCompile(`[h|H][e|E][r|R][o|O][k|K][u|U].*(=|:=|<-).*\w+.*`),
+		"facebook": regexp.MustCompile(`[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*(=|:=|<-).*\w+.*`),
+		"twitter":  regexp.MustCompile(`[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*(=|:=|<-).*\w+.*`),
+		"reddit":   regexp.MustCompile(`[r|R][e|E][d|D][d|D][i|I][t|T].*(=|:=|<-).*\w+.*`),
+		"twilio":   regexp.MustCompile(`[t|T][w|W][i|I][l|L][i|I][o|O].*(=|:=|<-).*\w+.*`),
 	}
 	assignRegex = regexp.MustCompile(`(=|:|:=|<-)`)
+	base64Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+	hexChars = "1234567890abcdefABCDEF"
 }
 
 func main() {

options.go

@@ -18,12 +18,13 @@ Options:
 
 // Options for gitleaks
 type Options struct {
-	Concurrency   int
-	EntropyCutoff int
-	UserURL       string
-	OrgURL        string
-	RepoURL       string
-	Strict        bool
+	Concurrency      int
+	B64EntropyCutoff int
+	HexEntropyCutoff int
+	UserURL          string
+	OrgURL           string
+	RepoURL          string
+	Strict           bool
 }
 
 // help prints the usage string and exits
@@ -62,7 +63,11 @@ func optionsNextString(args []string, i *int) string {
 
 // parseOptions
 func parseOptions(args []string) *Options {
-	opts := &Options{}
+	opts := &Options{
+		Concurrency:      10,
+		B64EntropyCutoff: 70,
+		HexEntropyCutoff: 40,
+	}
 
 	// default is repo if no additional options
 	if len(args) == 1 {
@@ -76,7 +81,9 @@ func parseOptions(args []string) *Options {
 		case "-s":
 			opts.Strict = true
 		case "-e":
-			opts.EntropyCutoff = optionsNextInt(args, &i)
+			opts.B64EntropyCutoff = optionsNextInt(args, &i)
+		case "-x":
+			opts.HexEntropyCutoff = optionsNextInt(args, &i)
 		case "-c":
 			opts.Concurrency = optionsNextInt(args, &i)
 		case "-o":