Enhance entropy (#333)

* Modified Entropy to be able to apply it on regex match or specific capture group

* Config tests

* Small fixes on entropy ranges

* Fix toml configs
Fix tests

* Indents and code styles

audit/audit_test.go

@@ -173,16 +173,6 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak.json",
 		},
-		{
-			description: "test entropy",
-			opts: options.Options{
-				RepoPath:     "../test_data/test_repos/test_repo_1",
-				Report:       "../test_data/test_entropy.json.got",
-				Config:       "../test_data/test_configs/entropy.toml",
-				ReportFormat: "json",
-			},
-			wantPath: "../test_data/test_entropy.json",
-		},
 		{
 			description: "test entropy and regex",
 			opts: options.Options{
@@ -427,7 +417,7 @@ func fileCheck(wantPath, gotPath string) error {
 	if !reflect.DeepEqual(gotLeaks, wantLeaks) {
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(string(want), string(got), false)
-		return fmt.Errorf("does not equal: %s", dmp.DiffPrettyText(diffs))
+		return fmt.Errorf("%s does not equal %s: %s", wantPath, gotPath, dmp.DiffPrettyText(diffs))
 	}
 	if err := os.Remove(gotPath); err != nil {
 		return err

audit/util.go

@@ -90,13 +90,15 @@ func shannonEntropy(data string) (entropy float64) {
 }
 
 // aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
-// trippedEntropy checks if a given line falls in between entropy ranges supplied
-// by a custom gitleaks configuration. Gitleaks do not check entropy by default.
-func trippedEntropy(line string, rule config.Rule) bool {
-	for _, e := range rule.Entropy {
-		entropy := shannonEntropy(line)
-		if entropy > e.P1 && entropy < e.P2 {
-			return true
+// trippedEntropy checks if a given groups or offender falls in between entropy ranges
+// supplied by a custom gitleaks configuration. Gitleaks do not check entropy by default.
+func trippedEntropy(groups []string, rule config.Rule) bool {
+	for _, e := range rule.Entropies {
+		if len(groups) > e.Group {
+			entropy := shannonEntropy(groups[e.Group])
+			if entropy >= e.Min && entropy <= e.Max {
+				return true
+			}
 		}
 	}
 	return false
@@ -117,77 +119,10 @@ func ruleContainRegex(rule config.Rule) bool {
 // Next, if the rule contains a regular expression then that will be checked.
 func InspectString(content string, c *object.Commit, repo *Repo, filename string) {
 	for _, rule := range repo.config.Rules {
-		// check entropy
-		if len(rule.Entropy) != 0 {
-			// an optimization would be to switch the regex from FindAllIndex to FindString
-			// since we are iterating on the lines if entropy rules exist...
-			for _, line := range strings.Split(content, "\n") {
-				if repo.timeoutReached() {
-					return
-				}
-				entropyTripped := trippedEntropy(line, rule)
-				if entropyTripped && !ruleContainRegex(rule) {
-					repo.Manager.SendLeaks(manager.Leak{
-						Line:     line,
-						Offender: fmt.Sprintf("Entropy range %+v", rule.Entropy),
-						Commit:   c.Hash.String(),
-						Repo:     repo.Name,
-						Message:  c.Message,
-						Rule:     rule.Description,
-						Author:   c.Author.Name,
-						Email:    c.Author.Email,
-						Date:     c.Author.When,
-						Tags:     strings.Join(rule.Tags, ", "),
-						File:     filename,
-					})
-				} else if entropyTripped {
-					if repo.timeoutReached() {
-						return
-					}
-					// entropy has been tripped which means if there is a regex specified in the same
-					// rule, we need to inspect the line for a regex match. In otherwords, the current rule has
-					// both entropy and regex set which work in combination. This helps narrow down false positives
-					// on searches for generic passwords in code.
-					match := rule.Regex.FindString(line)
-
-					// check if any rules are whitelisting this leak
-					if len(rule.Whitelist) != 0 {
-						for _, wl := range rule.Whitelist {
-							if fileMatched(filename, wl.File) {
-								// if matched, go to next rule
-								goto NEXTLINE
-							}
-							if wl.Regex.FindString(line) != "" {
-								goto NEXTLINE
-							}
-						}
-					}
-
-					if match != "" {
-						// both the regex and entropy in this rule have been tripped which means this line
-						// contains a leak
-						repo.Manager.SendLeaks(manager.Leak{
-							Line:     line,
-							Offender: match,
-							Commit:   c.Hash.String(),
-							Message:  c.Message,
-							Repo:     repo.Name,
-							Rule:     rule.Description,
-							Author:   c.Author.Name,
-							Email:    c.Author.Email,
-							Date:     c.Author.When,
-							Tags:     strings.Join(rule.Tags, ", "),
-							File:     filename,
-						})
-					}
-				}
-			NEXTLINE:
-			}
-			return
-		}
 		if rule.Regex.String() == "" {
 			continue
 		}
+
 		if repo.timeoutReached() {
 			return
 		}
@@ -218,33 +153,57 @@ func InspectString(content string, c *object.Commit, repo *Repo, filename string
 					end = end + 1
 				}
 
-				offender := content[loc[0]:loc[1]]
 				line := content[start:end]
+				offender := content[loc[0]:loc[1]]
+				groups := rule.Regex.FindStringSubmatch(offender)
 
 				if len(rule.Whitelist) != 0 {
 					for _, wl := range rule.Whitelist {
-						if wl.Regex.FindString(line) != "" {
+						if wl.Regex.FindString(offender) != "" {
 							goto NEXT
 						}
 					}
 				}
-				if repo.Manager.Opts.Redact {
-					line = strings.ReplaceAll(line, offender, "REDACTED")
-					offender = "REDACTED"
+
+				if len(rule.Entropies) != 0 {
+					if trippedEntropy(groups, rule) {
+						if repo.Manager.Opts.Redact {
+							line = strings.ReplaceAll(line, offender, "REDACTED")
+							offender = "REDACTED"
+						}
+						repo.Manager.SendLeaks(manager.Leak{
+							Line:     line,
+							Offender: offender,
+							Commit:   c.Hash.String(),
+							Repo:     repo.Name,
+							Message:  c.Message,
+							Rule:     rule.Description,
+							Author:   c.Author.Name,
+							Email:    c.Author.Email,
+							Date:     c.Author.When,
+							Tags:     strings.Join(rule.Tags, ", "),
+							File:     filename,
+						})
+					}
+				} else {
+					if repo.Manager.Opts.Redact {
+						line = strings.ReplaceAll(line, offender, "REDACTED")
+						offender = "REDACTED"
+					}
+					repo.Manager.SendLeaks(manager.Leak{
+						Line:     line,
+						Offender: offender,
+						Commit:   c.Hash.String(),
+						Message:  c.Message,
+						Repo:     repo.Name,
+						Rule:     rule.Description,
+						Author:   c.Author.Name,
+						Email:    c.Author.Email,
+						Date:     c.Author.When,
+						Tags:     strings.Join(rule.Tags, ", "),
+						File:     filename,
+					})
 				}
-				repo.Manager.SendLeaks(manager.Leak{
-					Line:     line,
-					Offender: offender,
-					Commit:   c.Hash.String(),
-					Message:  c.Message,
-					Repo:     repo.Name,
-					Rule:     rule.Description,
-					Author:   c.Author.Name,
-					Email:    c.Author.Email,
-					Date:     c.Author.When,
-					Tags:     strings.Join(rule.Tags, ", "),
-					File:     filename,
-				})
 			}
 		}
 		repo.Manager.RecordTime(manager.RegexTime{

config/config.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
-	"strings"
 
 	"github.com/zricethezav/gitleaks/v3/options"
 
@@ -20,8 +19,10 @@ type Whitelist struct {
 }
 
 // entropy represents an entropy range
-type entropy struct {
-	P1, P2 float64
+type Entropy struct {
+	Min   float64
+	Max   float64
+	Group int
 }
 
 // Rule is a struct that contains information that is loaded from a gitleaks config.
@@ -34,7 +35,7 @@ type Rule struct {
 	Regex       *regexp.Regexp
 	Tags        []string
 	Whitelist   []Whitelist
-	Entropy     []entropy
+	Entropies   []Entropy
 }
 
 // Config is a composite struct of Rules and Whitelists
@@ -67,8 +68,12 @@ type TomlLoader struct {
 		Description string
 		Regex       string
 		Tags        []string
-		Entropies   []string
-		Whitelist   []struct {
+		Entropies   []struct {
+			Min   string
+			Max   string
+			Group string
+		}
+		Whitelist []struct {
 			Description string
 			Regex       string
 			File        string
@@ -130,9 +135,33 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 			})
 		}
 
-		entropies, err := getEntropy(rule.Entropies)
-		if err != nil {
-			return cfg, err
+		var entropies []Entropy
+		for _, e := range rule.Entropies {
+			min, err := strconv.ParseFloat(e.Min, 64)
+			if err != nil {
+				return cfg, err
+			}
+			max, err := strconv.ParseFloat(e.Max, 64)
+			if err != nil {
+				return cfg, err
+			}
+			if e.Group == "" {
+				e.Group = "0"
+			}
+			group, err := strconv.ParseInt(e.Group, 10, 64)
+			if err != nil {
+				return cfg, err
+			} else if int(group) >= len(re.SubexpNames()) {
+				return cfg, fmt.Errorf("problem loading config: group cannot be higher than number of groups in regexp")
+			} else if group < 0 {
+				return cfg, fmt.Errorf("problem loading config: group cannot be lower than 0")
+			} else if min > 8.0 || min < 0.0 || max > 8.0 || max < 0.0 {
+				return cfg, fmt.Errorf("problem loading config: invalid entropy ranges, must be within 0.0-8.0")
+			} else if min > max {
+				return cfg, fmt.Errorf("problem loading config: entropy Min value cannot be higher than Max value")
+			}
+
+			entropies = append(entropies, Entropy{Min: min, Max: max, Group: int(group)})
 		}
 
 		cfg.Rules = append(cfg.Rules, Rule{
@@ -140,7 +169,7 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 			Regex:       re,
 			Tags:        rule.Tags,
 			Whitelist:   whitelists,
-			Entropy:     entropies,
+			Entropies:   entropies,
 		})
 	}
 
@@ -173,28 +202,3 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 
 	return cfg, nil
 }
-
-// getEntropy
-func getEntropy(entropyStr []string) ([]entropy, error) {
-	var ranges []entropy
-	for _, span := range entropyStr {
-		split := strings.Split(span, "-")
-		v1, err := strconv.ParseFloat(split[0], 64)
-		if err != nil {
-			return nil, err
-		}
-		v2, err := strconv.ParseFloat(split[1], 64)
-		if err != nil {
-			return nil, err
-		}
-		if v1 > v2 {
-			return nil, fmt.Errorf("entropy range must be ascending")
-		}
-		r := entropy{P1: v1, P2: v2}
-		if r.P1 > 8.0 || r.P1 < 0.0 || r.P2 > 8.0 || r.P2 < 0.0 {
-			return nil, fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
-		}
-		ranges = append(ranges, r)
-	}
-	return ranges, nil
-}

config/config_test.go

@@ -78,28 +78,49 @@ func TestParse(t *testing.T) {
 			opts: options.Options{
 				Config: "../test_data/test_configs/bad_entropy_1.toml",
 			},
-			wantErr: fmt.Errorf("entropy range must be ascending"),
+			wantErr: fmt.Errorf("problem loading config: entropy Min value cannot be higher than Max value"),
 		},
 		{
-			description: "test entropy value p2",
+			description: "test entropy value max",
 			opts: options.Options{
 				Config: "../test_data/test_configs/bad_entropy_2.toml",
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
-			description: "test entropy value p1",
+			description: "test entropy value min",
 			opts: options.Options{
 				Config: "../test_data/test_configs/bad_entropy_3.toml",
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
-			description: "test entropy value p1",
+			description: "test entropy value group",
 			opts: options.Options{
 				Config: "../test_data/test_configs/bad_entropy_4.toml",
 			},
-			wantErr: fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0"),
+			wantErr: fmt.Errorf("strconv.ParseInt: parsing \"x\": invalid syntax"),
+		},
+		{
+			description: "test entropy value group",
+			opts: options.Options{
+				Config: "../test_data/test_configs/bad_entropy_5.toml",
+			},
+			wantErr: fmt.Errorf("problem loading config: group cannot be lower than 0"),
+		},
+		{
+			description: "test entropy value group",
+			opts: options.Options{
+				Config: "../test_data/test_configs/bad_entropy_6.toml",
+			},
+			wantErr: fmt.Errorf("problem loading config: group cannot be higher than number of groups in regexp"),
+		},
+		{
+			description: "test entropy range limits",
+			opts: options.Options{
+				Config: "../test_data/test_configs/bad_entropy_7.toml",
+			},
+			wantErr: fmt.Errorf("problem loading config: invalid entropy ranges, must be within 0.0-8.0"),
 		},
 	}
 

examples/leaky-repo.toml

@@ -149,16 +149,16 @@ title = "gitleaks config"
 [[rules]]
 	description = "Pure Entropy"
 	regex = '''['|"][0-9a-zA-Z-._{}$\/=]{40,120}['|"]'''
-	entropies = [
-        "5.0-5.6"
-	]
+		[[rules.Entropies]]
+			Min = "5.0"
+			Max = "5.6"
 
 [[rules]]
 	description = "Entropy plus Generic Credential"
 	regex = '''(?i)(api_key|apikey|secret|key|api|password|pw)'''
-	entropies = [
-        "5.2-5.5"
-	]
+		[[rules.Entropies]]
+			Min = "5.2"
+			Max = "5.5"
 
 [Global]
     file = '''(?i)(id_rsa|passwd|id_rsa.pub|pgpass|pem|key|shadow)'''

examples/regex_and_entropy_config.toml

@@ -8,8 +8,10 @@
 [[rules]]
 	description = "entropy and regex"
 	regex = '''(?i)key(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]'''
-    entropies = [
-        "4.5-4.7",
-        "5.5-6.3",
-    ]
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "4.5"
+			Max = "5.7"
+		[[rules.Entropies]]
+			Min = "5.5"
+			Max = "6.3"

hosts/github.go

@@ -81,7 +81,6 @@ func (g *Github) Audit() {
 			githubRepos = append(githubRepos, r)
 		}
 
-
 		if resp == nil {
 			break
 		}

test_data/test_configs/bad_entropy_1.toml

@@ -1,7 +1,8 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "4.3-4.1",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "4.3"
+			Max = "4.1"
 

test_data/test_configs/bad_entropy_2.toml

@@ -1,7 +1,8 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "4.3-x",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "4.3"
+			Max = "x"
 

test_data/test_configs/bad_entropy_3.toml

@@ -1,7 +1,8 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "x-4.3",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "x"
+			Max = "4.1"
 

test_data/test_configs/bad_entropy_4.toml

@@ -1,7 +1,9 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "1-8.9",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "1.0"
+			Max = "8"
+			Group = "x"
 

test_data/test_configs/bad_entropy_5.toml

@@ -0,0 +1,9 @@
+[[rules]]
+	description = "entropy"
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
+	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "1.0"
+			Max = "8.5"
+			Group = "-2"
+

test_data/test_configs/bad_entropy_6.toml

@@ -0,0 +1,9 @@
+[[rules]]
+	description = "entropy"
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
+	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "1.0"
+			Max = "8.5"
+			Group = "2"
+

test_data/test_configs/bad_entropy_7.toml

@@ -0,0 +1,9 @@
+[[rules]]
+	description = "entropy"
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
+	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "1.0"
+			Max = "8.5"
+			Group = "0"
+

test_data/test_configs/entropy.toml

@@ -1,8 +1,11 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "4.5-4.7",
-        "5.5-6.3",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{40,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "4.5"
+			Max = "4.7"
+		[[rules.Entropies]]
+			Min = "5.5"
+			Max = "6.3"
 

test_data/test_configs/regex_entropy.toml

@@ -1,9 +1,14 @@
 [[rules]]
 	description = "entropy and regex"
-	regex = '''(?i)key(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]'''
-    entropies = [
-        "4.5-4.7",
-        "5.5-6.3",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "4.5"
+			Max = "4.7"
+			Group = "1"
+		[[rules.Entropies]]
+			Min = "5.5"
+			Max = "6.3"
+			Group = "1"
+	
 

test_data/test_local_repo_four_alt_config_entropy.json

@@ -1,28 +1,54 @@
 [
  {
-  "line": "    Just moments after the Cessna's inquiry, a Twin Beech piped up on frequency, in a rather superior tone, asking for his ground speed. \"I have you at one hundred and twenty-five knots of ground spe...",
-  "offender": "Entropy range [{P1:4.5 P2:4.7}]",
-  "commit": "d8ac0b73aeeb45843319cdc5ce506516eb49bf7a",
+  "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "\"99432bfewaf823ec3294e231\"",
+  "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
   "repo": "test_repo_4",
   "rule": "entropy",
-  "commitMessage": "removing secret.pem\n",
+  "commitMessage": "rm secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
-  "file": "secret.pem",
-  "date": "2019-10-25T13:08:39-04:00",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:26-04:00",
   "tags": "entropy"
  },
  {
-  "line": "    Just moments after the Cessna's inquiry, a Twin Beech piped up on frequency, in a rather superior tone, asking for his ground speed. \"I have you at one hundred and twenty-five knots of ground spe...",
-  "offender": "Entropy range [{P1:4.5 P2:4.7}]",
-  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "\"99432bfewaf823ec3294e231\"",
+  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
   "repo": "test_repo_4",
   "rule": "entropy",
-  "commitMessage": "committing pem\n",
+  "commitMessage": "more secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
-  "file": "secret.pem",
-  "date": "2019-10-25T13:07:41-04:00",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:08-04:00",
+  "tags": "entropy"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "\"99432bfewaf823ec3294e231\"",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_4",
+  "rule": "entropy",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "entropy"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "\"99432bfewaf823ec3294e231\"",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_4",
+  "rule": "entropy",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
   "tags": "entropy"
  }
 ]

test_data/test_regex_entropy.json

@@ -1,7 +1,7 @@
 [
  {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "offender": "key_id='AKIAIO5FODNN7EXAMPLE'",
+  "line": "    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
+  "offender": "'ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
   "repo": "test_repo_1",
   "rule": "entropy and regex",

test_data/test_regex_whitelist.json.got

@@ -1,15 +0,0 @@
-[
- {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_1",
-  "rule": "AWS Manager ID",
-  "commitMessage": "commit 1 with secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "server.test.py",
-  "date": "2019-10-24T09:29:27-04:00",
-  "tags": "key, AWS"
- }
-]

test_data/test_repos/test_repo_4/gitleaks.toml

@@ -1,6 +1,8 @@
 [[rules]]
 	description = "entropy"
-    entropies = [
-        "4.5-4.7",
-    ]
+	regex = '''['|"]([0-9a-zA-Z-._{}$\/\+=]{20,120})['|"]'''
 	tags = ["entropy"]
+		[[rules.Entropies]]
+			Min = "3.3"
+			Max = "3.5"
+			Group = "1"