Merge pull request #122 from zricethezav/feature/entropy-range

entropy ranges

CHANGELOG.md

@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.14.0
+----
+- Entropy Range support in gitleaks config
+
 1.13.0
 ----
 - Github PR support

gitleaks_test.go

@@ -67,6 +67,26 @@ repos = [
 ]
 `
 
+const testEntropyRange = `
+[misc]
+entropy = [
+  "7.5-8.0",
+  "3.3-3.4",
+]
+`
+const testBadEntropyRange = `
+[misc]
+entropy = [
+  "8.0-3.0",
+]
+`
+const testBadEntropyRange2 = `
+[misc]
+entropy = [
+  "8.0-8.9",
+]
+`
+
 var benchmarkRepo *RepoDescriptor
 var benchmarkLeaksRepo *RepoDescriptor
 
@@ -427,6 +447,9 @@ func testTomlLoader() string {
 	ioutil.WriteFile(path.Join(tmpDir, "commit"), []byte(testWhitelistCommit), 0644)
 	ioutil.WriteFile(path.Join(tmpDir, "file"), []byte(testWhitelistFile), 0644)
 	ioutil.WriteFile(path.Join(tmpDir, "repo"), []byte(testWhitelistRepo), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "entropy"), []byte(testEntropyRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "badEntropy"), []byte(testBadEntropyRange), 0644)
+	ioutil.WriteFile(path.Join(tmpDir, "badEntropy2"), []byte(testBadEntropyRange2), 0644)
 	return tmpDir
 }
 
@@ -637,8 +660,27 @@ func TestAuditRepo(t *testing.T) {
 				Depth: 2,
 			},
 		},
+		{
+			repo:        leaksRepo,
+			description: "toml entropy range",
+			numLeaks:    422,
+			configPath:  path.Join(configsDir, "entropy"),
+		},
+		{
+			repo:           leaksRepo,
+			description:    "toml bad entropy range",
+			numLeaks:       0,
+			configPath:     path.Join(configsDir, "badEntropy"),
+			expectedErrMsg: "entropy range must be ascending",
+		},
+		{
+			repo:           leaksRepo,
+			description:    "toml bad entropy2 range",
+			numLeaks:       0,
+			configPath:     path.Join(configsDir, "badEntropy2"),
+			expectedErrMsg: "invalid entropy ranges, must be within 0.0-8.0",
+		},
 	}
-
 	whiteListCommits = make(map[string]bool)
 	g := goblin.Goblin(t)
 	for _, test := range tests {
@@ -671,19 +713,24 @@ func TestAuditRepo(t *testing.T) {
 				} else {
 					whiteListRepos = nil
 				}
-
+				skip := false
 				// config paths
 				if test.configPath != "" {
 					os.Setenv("GITLEAKS_CONFIG", test.configPath)
-					loadToml()
+					err := loadToml()
+					if err != nil {
+						g.Assert(err.Error()).Equal(test.expectedErrMsg)
+						skip = true
+					}
 				}
+				if !skip {
+					leaks, err = auditGitRepo(test.repo)
 
-				leaks, err = auditGitRepo(test.repo)
-
-				if opts.Redact {
-					g.Assert(leaks[0].Offender).Equal("REDACTED")
+					if opts.Redact {
+						g.Assert(leaks[0].Offender).Equal("REDACTED")
+					}
+					g.Assert(len(leaks)).Equal(test.numLeaks)
 				}
-				g.Assert(len(leaks)).Equal(test.numLeaks)
 			})
 		})
 	}

main.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -113,6 +114,9 @@ type Config struct {
 		Branches []string
 		Repos    []string
 	}
+	Misc struct {
+		Entropy []string
+	}
 }
 
 type gitDiff struct {
@@ -127,8 +131,13 @@ type gitDiff struct {
 	author       string
 }
 
+type entropyRange struct {
+	v1 float64
+	v2 float64
+}
+
 const defaultGithubURL = "https://api.github.com/"
-const version = "1.13.0"
+const version = "1.14.0"
 const errExit = 2
 const leakExit = 1
 const defaultConfig = `
@@ -177,11 +186,17 @@ regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]'''
 #]
 
 #branches = [
-#	"dev/STUPDIFKNFEATURE"
+#	"dev/goodrepo"
 #]
 
 #repos = [
-#	"someYugeRepoWeKnowIsCLEAR"
+#	"mygoodrepo"
+#]
+
+[misc]
+#entropy = [
+#	"3.3-4.30"
+#	"6.0-8.0
 #]
 `
 
@@ -194,6 +209,7 @@ var (
 	whiteListCommits  map[string]bool
 	whiteListBranches []string
 	whiteListRepos    []string
+	entropyRanges     []entropyRange
 	fileDiffRegex     *regexp.Regexp
 	sshAuth           *ssh.PublicKeys
 	dir               string
@@ -622,11 +638,24 @@ func inspect(diff gitDiff) []Leak {
 			leaks = addLeak(leaks, line, match, leakType, diff)
 		}
 
-		if opts.Entropy > 0 {
+		if opts.Entropy > 0 || len(entropyRanges) != 0 {
+			entropyLeak := false
 			words := strings.Fields(line)
 			for _, word := range words {
-				if getShannonEntropy(word) >= opts.Entropy {
-					leaks = addLeak(leaks, line, word, "High Entropy", diff)
+				entropy := getShannonEntropy(word)
+				if entropy >= opts.Entropy && len(entropyRanges) == 0 {
+					entropyLeak = true
+				}
+				if len(entropyRanges) != 0 {
+					for _, eR := range entropyRanges {
+						if entropy > eR.v1 && entropy < eR.v2 {
+							entropyLeak = true
+						}
+					}
+				}
+				if entropyLeak {
+					leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), diff)
+					entropyLeak = false
 				}
 			}
 		}
@@ -813,6 +842,13 @@ func loadToml() error {
 		}
 	}
 
+	if len(config.Misc.Entropy) != 0 {
+		err := entropyLimits(config.Misc.Entropy)
+		if err != nil {
+			return err
+		}
+	}
+
 	if singleSearchRegex != nil {
 		regexes["singleSearch"] = singleSearchRegex
 	} else {
@@ -836,6 +872,33 @@ func loadToml() error {
 	return nil
 }
 
+// entropyLimits hydrates entropyRanges which allows for fine tuning entropy checking
+func entropyLimits(entropyLimitStr []string) error {
+	for _, span := range entropyLimitStr {
+		split := strings.Split(span, "-")
+		v1, err := strconv.ParseFloat(split[0], 64)
+		if err != nil {
+			return err
+		}
+		v2, err := strconv.ParseFloat(split[1], 64)
+		if err != nil {
+			return err
+		}
+		if v1 > v2 {
+			return fmt.Errorf("entropy range must be ascending")
+		}
+		r := entropyRange{
+			v1: v1,
+			v2: v2,
+		}
+		if r.v1 > 8.0 || r.v1 < 0.0 || r.v2 > 8.0 || r.v2 < 0.0 {
+			return fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
+		}
+		entropyRanges = append(entropyRanges, r)
+	}
+	return nil
+}
+
 // getSSHAuth return an ssh auth use by go-git to clone repos behind authentication.
 // If --ssh-key is set then it will attempt to load the key from that path. If not,
 // gitleaks will use the default $HOME/.ssh/id_rsa key