better naming

checks.go

@@ -1,51 +1,10 @@
 package main
 
 import (
-	"fmt"
 	"github.com/nbutton23/zxcvbn-go"
-	"log"
-	"os"
-	"os/exec"
 	"strings"
 )
 
-// checkDiff operates on a single diff between to chronological commits
-func checkDiff(commit1 string, commit2 string, repoName string) []string {
-	// var leakPrs bool
-	// var leaks []string
-	// _, seen := cache[commit1+commit2]
-	// if seen {
-	// 	fmt.Println("WE HAVE SEEN THIS")
-	// 	return []string{}
-	// }
-
-	if err := os.Chdir(fmt.Sprintf("%s/%s", appRoot, repoName)); err != nil {
-		log.Fatal(err)
-	}
-
-	cmd := exec.Command("git", "diff", commit1, commit2)
-	_, err := cmd.Output()
-	// fmt.Println(string(out))
-	if err != nil {
-		return []string{}
-	}
-	return []string{}
-
-	// cache[commit1+commit2] = true
-	// lines := checkRegex(string(out))
-	// if len(lines) == 0 {
-	// 	return []string{}
-	// }
-	//
-	// for _, line := range lines {
-	// 	leakPrs = checkEntropy(line)
-	// 	if leakPrs {
-	// 		leaks = append(leaks, line)
-	// 	}
-	// }
-	// return leaks
-}
-
 // check each line of a diff and see if there are any potential secrets
 // [1] https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf
 func checkRegex(diff string) []string {
@@ -54,7 +13,7 @@ func checkRegex(diff string) []string {
 	lines := strings.Split(diff, "\n")
 	for _, line := range lines {
 		// doubtful a leak would be on a line > 120 characters
-		if len(line) == 0 || len(line) > 80 {
+		if len(line) == 0 || len(line) > 120 {
 			continue
 		}
 		for _, re := range regexes {
@@ -85,7 +44,6 @@ func checkEntropy(target string) bool {
 		return false
 	}
 
-	// entropy := shannonEntropy(target)
 	entropy := zxcvbn.PasswordStrength(target, nil).Entropy
 
 	// tune this/make option

repo.go → leaks.go

@@ -8,8 +8,10 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"os/signal"
 	"strings"
 	"sync"
+	"syscall"
 )
 
 type ReportElem struct {
@@ -19,13 +21,7 @@ type ReportElem struct {
 	CommitB string   `json:"commitB"`
 }
 
-type Repo struct {
-	url  string
-	name string
-	path string
-}
-
-type MemoMessage struct {
+type GitLeak struct {
 	hash    string
 	leaks   []string
 	commitA string
@@ -33,10 +29,10 @@ type MemoMessage struct {
 	branch  string
 }
 
-var lock = sync.RWMutex{}
-var cmdLock = sync.Mutex{}
+func start(opts *Options, repoUrl string) {
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 
-func repoStart(repoUrl string) {
 	err := exec.Command("git", "clone", repoUrl).Run()
 	if err != nil {
 		log.Fatalf("failed to clone repo %v", err)
@@ -45,39 +41,45 @@ func repoStart(repoUrl string) {
 	if err := os.Chdir(repoName); err != nil {
 		log.Fatal(err)
 	}
+	go func() {
+		<-c
+		cleanup(repoName)
+		os.Exit(1)
+	}()
 
-	repo := Repo{repoUrl, repoName, ""}
-	report := repo.audit()
-	repo.cleanup()
+	report := getLeaks(repoName)
+	cleanup(repoName)
 
 	reportJson, _ := json.MarshalIndent(report, "", "\t")
-	err = ioutil.WriteFile(fmt.Sprintf("%s_leaks.json", repo.name), reportJson, 0644)
+	err = ioutil.WriteFile(fmt.Sprintf("%s_leaks.json", repoName), reportJson, 0644)
 }
 
 // cleanup changes to app root and recursive rms target repo
-func (repo Repo) cleanup() {
+func cleanup(repoName string) {
 	if err := os.Chdir(appRoot); err != nil {
 		log.Fatalf("failed cleaning up repo. Does the repo exist? %v", err)
 	}
-	err := exec.Command("rm", "-rf", repo.name).Run()
+	err := exec.Command("rm", "-rf", repoName).Run()
 	if err != nil {
 		log.Fatal(err)
 	}
 }
 
 // audit parses git branch --all
-func (repo Repo) audit() []ReportElem {
+func getLeaks(repoName string) []ReportElem {
 	var (
-		out     []byte
-		err     error
-		branch  string
-		commits [][]byte
-		// leaks   []string
+		out           []byte
+		err           error
+		branch        string
+		commits       [][]byte
 		wg            sync.WaitGroup
 		commitA       string
 		commitB       string
-		concurrent    = 10
+		concurrent    = 100
 		semaphoreChan = make(chan struct{}, concurrent)
+		gitLeaks      = make(chan GitLeak)
+		cmdLock       = sync.Mutex{}
+		cache         = make(map[string]bool)
 	)
 
 	out, err = exec.Command("git", "branch", "--all").Output()
@@ -88,16 +90,13 @@ func (repo Repo) audit() []ReportElem {
 	// iterate through branches, git rev-list <branch>
 	branches := bytes.Split(out, []byte("\n"))
 
-	messages := make(chan MemoMessage)
-
 	for i, branchB := range branches {
 		if i < 2 || i == len(branches)-1 {
 			continue
 		}
 		branch = string(bytes.Trim(branchB, " "))
 		cmdLock.Lock()
-		cmd := exec.Command("git", "rev-list", branch)
-		out, err := cmd.Output()
+		out, err := exec.Command("git", "rev-list", "--topo-order", branch).Output()
 		cmdLock.Unlock()
 		if err != nil {
 			fmt.Println("skipping branch", branch, err)
@@ -114,32 +113,26 @@ func (repo Repo) audit() []ReportElem {
 			commitB = string(currCommit)
 			_, seen := cache[commitA+commitB]
 			if seen {
-				fmt.Println("WE HAVE SEEN THIS")
 				continue
 			} else {
 				cache[commitA+commitB] = true
 			}
 
 			wg.Add(1)
-			go func(commitA string, commitB string,
-				j int, branch string) {
+			go func(commitA string, commitB string, branch string, repoName string) {
 				defer wg.Done()
 				var leakPrs bool
 				var leaks []string
-				fmt.Println(j, branch)
 
-				if err := os.Chdir(fmt.Sprintf("%s/%s", appRoot, repo.name)); err != nil {
+				if err := os.Chdir(fmt.Sprintf("%s/%s", appRoot, repoName)); err != nil {
 					log.Fatal(err)
 				}
 
 				semaphoreChan <- struct{}{}
-				fmt.Println("diffing", branch, j)
-				cmd := exec.Command("git", "diff", commitA, commitB)
-				out, err := cmd.Output()
+				out, err := exec.Command("git", "diff", commitA, commitB).Output()
 				<-semaphoreChan
 
 				if err != nil {
-					fmt.Println("no diff: ", err)
 					return
 				}
 				lines := checkRegex(string(out))
@@ -154,17 +147,17 @@ func (repo Repo) audit() []ReportElem {
 					}
 				}
 
-				messages <- MemoMessage{commitA + commitB, leaks, commitA, commitB, branch}
+				gitLeaks <- GitLeak{commitA + commitB, leaks, commitA, commitB, branch}
 
-			}(commitA, commitB, j, branch)
+			}(commitA, commitB, branch, repoName)
 		}
 	}
 	go func() {
-		for memoMsg := range messages {
-			fmt.Println(memoMsg)
-			if len(memoMsg.leaks) != 0 {
-				report = append(report, ReportElem{memoMsg.leaks, memoMsg.branch,
-					memoMsg.commitA, memoMsg.commitB})
+		for gitLeak := range gitLeaks {
+			if len(gitLeak.leaks) != 0 {
+				fmt.Println(gitLeak.branch)
+				report = append(report, ReportElem{gitLeak.leaks, gitLeak.branch,
+					gitLeak.commitA, gitLeak.commitB})
 			}
 		}
 	}()

main.go

@@ -5,9 +5,7 @@ import (
 	"regexp"
 )
 
-
 var (
-	cache       map[string]bool
 	appRoot     string
 	regexes     map[string]*regexp.Regexp
 	assignRegex *regexp.Regexp
@@ -16,9 +14,9 @@ var (
 
 func init() {
 	appRoot, _ = os.Getwd()
-	cache = make(map[string]bool)
 	// TODO update regex to look for things like:
-	// client("fewafewakwafejwkaf",
+	// TODO ability to add/filter regex
+	// client("AKAI32fJ334...",
 	regexes = map[string]*regexp.Regexp{
 		"github":   regexp.MustCompile(`[g|G][i|I][t|T][h|H][u|U][b|B].*(=|:|:=|<-).*\w+.*`),
 		"aws":      regexp.MustCompile(`[a|A][w|W][s|S].*(=|:=|:|<-).*\w+.*`),
@@ -32,14 +30,8 @@ func init() {
 }
 
 func main() {
-	args := os.Args[1:]
-	opts := parseOptions(args)
-	start(opts)
+	args := os.Args[2:]
+	repoUrl := os.Args[1]
+	opts := parseOptions(args, repoUrl)
+	start(opts, repoUrl)
 }
-
-func start(opts *Options) {
-	if opts.Repo != "" {
-		repoStart(opts.Repo)
-	}
-}
-

options.go

@@ -3,26 +3,21 @@ package main
 import (
 	"fmt"
 	"os"
+	"strconv"
 )
 
 // TODO regex on type.. user/organization can be treated as the same:
 // 	hittps://github.com/<user or org>
 // 	hittps://github.com/<user or org>/repo
-const usage = `usage: gogethunt [options]
+const usage = `usage: gitleaks [git link] [options]
 	
 Options:
-	-u --user		Target user
-	-r --repo 		Target repo
-	-o --org 		Target organization
-    -h --help 		Display this message
-	-e --entropy	Enable entropy detection
-	-r --regex 		Enable regex detection
+	-c 			Concurrency factor (potential number of git files open)
+	-h --help 		Display this message
 `
 
 type Options struct {
-	User string
-	Repo string
-	Org  string
+	Concurrency int
 }
 
 func help() {
@@ -30,6 +25,20 @@ func help() {
 	os.Exit(1)
 }
 
+func optionsNextInt(args []string, i *int) int {
+	if len(args) > *i+1 {
+		*i++
+	} else {
+		help()
+	}
+	argInt, err := strconv.Atoi(args[*i])
+	if err != nil {
+		fmt.Printf("Invalid %s option: %s\n", args[*i-1], args[*i])
+		help()
+	}
+	return argInt
+}
+
 func optionsNextString(args []string, i *int) string {
 	if len(args) > *i+1 {
 		*i++
@@ -39,17 +48,13 @@ func optionsNextString(args []string, i *int) string {
 	return args[*i]
 }
 
-func parseOptions(args []string) *Options {
+func parseOptions(args []string, repoUrl string) *Options {
 	opts := &Options{}
 	for i := 0; i < len(args); i++ {
 		arg := args[i]
 		switch arg {
-		case "-o", "--org":
-			opts.Org = optionsNextString(args, &i)
-		case "-r", "--repo":
-			opts.Repo = optionsNextString(args, &i)
-		case "-u", "--user":
-			opts.User = optionsNextString(args, &i)
+		case "-c":
+			opts.Concurrency = optionsNextInt(args, &i)
 		case "-h", "--help":
 			help()
 			return nil