init stopword work

checks.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	//"fmt"
+	"math"
 	"strings"
-
-	"github.com/nbutton23/zxcvbn-go"
 )
 
 // check each line of a diff and see if there are any potential secrets
@@ -28,25 +28,45 @@ func checkRegex(diff string) []string {
 	return results
 }
 
-// checkEntropy determines whether target contains enough
-// entropy for a hash
-// TODO remove stop words:
-// setting(s), config(s), property(s), etc
-func checkEntropy(target string) bool {
-	index := assignRegex.FindStringIndex(target)
-	if len(index) == 0 {
-		return false
+func checkShannonEntropy(target string, entropyCutoff int) bool {
+	var sum float64
+	frq := make(map[rune]float64)
+
+	for _, i := range target {
+		frq[i]++
 	}
 
-	// TODO check for stop words here
-	target = strings.Trim(target[index[1]:], " ")
+	for _, v := range frq {
+		f := v / float64(len(target))
+		sum += f * math.Log2(f)
+	}
 
-	if len(target) > 70 {
-		return false
+	bits := int(math.Ceil(sum*-1)) * len(target)
+	return bits > entropyCutoff
+}
+
+func checkStopWords(target string) bool {
+	stopWords := []string{
+		"setting",
+		"Setting",
+		"SETTING",
+		"info",
+		"Info",
+		"INFO",
+		"env",
+		"Env",
+		"ENV",
+		"environment",
+		"Environment",
+		"ENVIRONMENT",
 	}
 
-	entropy := zxcvbn.PasswordStrength(target, nil).Entropy
+	for _, stopWord := range stopWords {
+		if strings.Contains(target, stopWord) {
+			// fmt.Println("FOUND STOP", stopWord)
+			return true
+		}
+	}
+	return false
 
-	// tune this/make option
-	return entropy > 70
 }

leaks.go

@@ -22,15 +22,16 @@ type LeakElem struct {
 
 // start clones and determines if there are any leaks
 func start(opts *Options) {
-	fmt.Printf("\nEvaluating \x1b[37;1m%s\x1b[0m...\n", opts.RepoURL)
 	c := make(chan os.Signal, 2)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 
+	fmt.Printf("Cloning \x1b[37;1m%s\x1b[0m...\n", opts.RepoURL)
 	err := exec.Command("git", "clone", opts.RepoURL).Run()
 	if err != nil {
 		log.Printf("failed to clone repo %v", err)
 		return
 	}
+	fmt.Printf("Evaluating \x1b[37;1m%s\x1b[0m...\n", opts.RepoURL)
 	repoName := getLocalRepoName(opts.RepoURL)
 	if err = os.Chdir(repoName); err != nil {
 		log.Fatal(err)
@@ -41,7 +42,7 @@ func start(opts *Options) {
 		os.Exit(1)
 	}()
 
-	report := getLeaks(repoName, opts.Concurrency)
+	report := getLeaks(repoName, opts)
 	if len(report) == 0 {
 		fmt.Printf("No Leaks detected for \x1b[35;2m%s\x1b[0m...\n\n", opts.RepoURL)
 	}
@@ -76,7 +77,7 @@ func cleanup(repoName string) {
 }
 
 // getLeaks will attempt to find gitleaks
-func getLeaks(repoName string, concurrency int) []LeakElem {
+func getLeaks(repoName string, opts *Options) []LeakElem {
 	var (
 		out               []byte
 		err               error
@@ -84,15 +85,18 @@ func getLeaks(repoName string, concurrency int) []LeakElem {
 		gitLeakReceiverWG sync.WaitGroup
 		gitLeaks          = make(chan LeakElem)
 		report            []LeakElem
+		concurrency       = 10
 	)
 
-	if concurrency == 0 {
-		concurrency = 10
+	if opts.Concurrency != 0 {
+		concurrency = opts.Concurrency
 	}
+
 	semaphoreChan := make(chan struct{}, concurrency)
 
 	go func(commitWG *sync.WaitGroup, gitLeakReceiverWG *sync.WaitGroup) {
 		for gitLeak := range gitLeaks {
+			//defer gitLeakReceiverWG.Done()
 			fmt.Println(gitLeak)
 			report = append(report, gitLeak)
 			gitLeakReceiverWG.Done()
@@ -108,7 +112,10 @@ func getLeaks(repoName string, concurrency int) []LeakElem {
 	commitWG.Add(len(commits))
 	for _, currCommitB := range commits {
 		currCommit := string(currCommitB)
-		go func(currCommit string, repoName string, commitWG *sync.WaitGroup, gitLeakReceiverWG *sync.WaitGroup) {
+
+		go func(currCommit string, repoName string, commitWG *sync.WaitGroup,
+			gitLeakReceiverWG *sync.WaitGroup, opts *Options) {
+
 			defer commitWG.Done()
 			var leakPrs bool
 
@@ -137,13 +144,16 @@ func getLeaks(repoName string, concurrency int) []LeakElem {
 			}
 
 			for _, line := range lines {
-				leakPrs = checkEntropy(line)
+				leakPrs = checkShannonEntropy(line, opts.EntropyCutoff)
 				if leakPrs {
+					if opts.Strict && checkStopWords(line) {
+						continue
+					}
 					gitLeakReceiverWG.Add(1)
 					gitLeaks <- LeakElem{line, currCommit}
 				}
 			}
-		}(currCommit, repoName, &commitWG, &gitLeakReceiverWG)
+		}(currCommit, repoName, &commitWG, &gitLeakReceiverWG, opts)
 	}
 
 	commitWG.Wait()

options.go

@@ -18,10 +18,12 @@ Options:
 
 // Options for gitleaks
 type Options struct {
-	Concurrency int
-	UserURL     string
-	OrgURL      string
-	RepoURL     string
+	Concurrency   int
+	EntropyCutoff int
+	UserURL       string
+	OrgURL        string
+	RepoURL       string
+	Strict        bool
 }
 
 // help prints the usage string and exits
@@ -71,6 +73,10 @@ func parseOptions(args []string) *Options {
 	for i := 0; i < len(args); i++ {
 		arg := args[i]
 		switch arg {
+		case "-s":
+			opts.Strict = true
+		case "-e":
+			opts.EntropyCutoff = optionsNextInt(args, &i)
 		case "-c":
 			opts.Concurrency = optionsNextInt(args, &i)
 		case "-o":