Merge pull request #134 from zricethezav/fix/threads

Fix/threads

CHANGELOG.md

@@ -1,6 +1,13 @@
 CHANGELOG
 =========
 
+1.18.0
+----
+- fixing slack token
+- defaulting to single thread process
+- `--max-go=` --> `--threads=`
+- guarding `--threads=` > max threads available (you aint gettin any more performance)
+
 1.17.0
 ----
 - Default regex added to search: slack, telegram.

README.md

@@ -73,7 +73,7 @@ Application Options:
       --depth=         maximum commit depth
       --repo-path=     Path to repo
       --owner-path=    Path to owner directory (repos discovered)
-      --max-go=        Maximum number of concurrent go-routines gitleaks spawns
+      --threads=       Maximum number of threads gitleaks spawns
       --disk           Clones repo(s) to disk
       --all-refs       run audit on all refs
       --single-search= single regular expression to search for

gitleaks_test.go

@@ -504,7 +504,7 @@ func TestAuditRepo(t *testing.T) {
 			description: "two leaks present limit goroutines",
 			numLeaks:    2,
 			testOpts: Options{
-				MaxGoRoutines: 4,
+				Threads: 4,
 			},
 		},
 		{
@@ -898,7 +898,7 @@ func TestLoadToml(t *testing.T) {
 
 func BenchmarkAuditRepo1Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 1
+	opts.Threads = 1
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -907,7 +907,7 @@ func BenchmarkAuditRepo1Proc(b *testing.B) {
 
 func BenchmarkAuditRepo2Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 2
+	opts.Threads = 2
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -916,7 +916,7 @@ func BenchmarkAuditRepo2Proc(b *testing.B) {
 
 func BenchmarkAuditRepo4Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 4
+	opts.Threads = 4
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -925,7 +925,7 @@ func BenchmarkAuditRepo4Proc(b *testing.B) {
 
 func BenchmarkAuditRepo8Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 8
+	opts.Threads = 8
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -934,7 +934,7 @@ func BenchmarkAuditRepo8Proc(b *testing.B) {
 
 func BenchmarkAuditRepo10Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 10
+	opts.Threads = 10
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -943,7 +943,7 @@ func BenchmarkAuditRepo10Proc(b *testing.B) {
 
 func BenchmarkAuditRepo100Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 100
+	opts.Threads = 100
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -952,7 +952,7 @@ func BenchmarkAuditRepo100Proc(b *testing.B) {
 
 func BenchmarkAuditRepo1000Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 1000
+	opts.Threads = 1000
 	benchmarkRepo = getBenchmarkRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -960,7 +960,7 @@ func BenchmarkAuditRepo1000Proc(b *testing.B) {
 }
 func BenchmarkAuditLeakRepo1Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 1
+	opts.Threads = 1
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -969,7 +969,7 @@ func BenchmarkAuditLeakRepo1Proc(b *testing.B) {
 
 func BenchmarkAuditLeakRepo2Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 2
+	opts.Threads = 2
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -978,7 +978,7 @@ func BenchmarkAuditLeakRepo2Proc(b *testing.B) {
 
 func BenchmarkAuditLeakRepo4Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 4
+	opts.Threads = 4
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -987,7 +987,7 @@ func BenchmarkAuditLeakRepo4Proc(b *testing.B) {
 
 func BenchmarkAuditLeakRepo8Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 8
+	opts.Threads = 8
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -996,7 +996,7 @@ func BenchmarkAuditLeakRepo8Proc(b *testing.B) {
 
 func BenchmarkAuditLeakRepo10Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 10
+	opts.Threads = 10
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -1004,7 +1004,7 @@ func BenchmarkAuditLeakRepo10Proc(b *testing.B) {
 }
 func BenchmarkAuditLeakRepo100Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 100
+	opts.Threads = 100
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)
@@ -1012,7 +1012,7 @@ func BenchmarkAuditLeakRepo100Proc(b *testing.B) {
 }
 func BenchmarkAuditLeakRepo1000Proc(b *testing.B) {
 	loadToml()
-	opts.MaxGoRoutines = 1000
+	opts.Threads = 1000
 	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
 	for n := 0; n < b.N; n++ {
 		auditGitRepo(benchmarkRepo)

main.go

@@ -81,14 +81,14 @@ type Options struct {
 	OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
 
 	// Process options
-	MaxGoRoutines int     `long:"max-go" description:"Maximum number of concurrent go-routines gitleaks spawns"`
-	Disk          bool    `long:"disk" description:"Clones repo(s) to disk"`
-	AuditAllRefs  bool    `long:"all-refs" description:"run audit on all refs"`
-	SingleSearch  string  `long:"single-search" description:"single regular expression to search for"`
-	ConfigPath    string  `long:"config" description:"path to gitleaks config"`
-	SSHKey        string  `long:"ssh-key" description:"path to ssh key"`
-	ExcludeForks  bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
-	Entropy       float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
+	Threads      int     `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	Disk         bool    `long:"disk" description:"Clones repo(s) to disk"`
+	AuditAllRefs bool    `long:"all-refs" description:"run audit on all refs"`
+	SingleSearch string  `long:"single-search" description:"single regular expression to search for"`
+	ConfigPath   string  `long:"config" description:"path to gitleaks config"`
+	SSHKey       string  `long:"ssh-key" description:"path to ssh key"`
+	ExcludeForks bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
+	Entropy      float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
 	// TODO: IncludeMessages  string `long:"messages" description:"include commit messages in audit"`
 
 	// Output options
@@ -220,13 +220,14 @@ var (
 	fileDiffRegex     *regexp.Regexp
 	sshAuth           *ssh.PublicKeys
 	dir               string
-	maxGo             int
+	threads           int
 	totalCommits      int64
 )
 
 func init() {
 	log.SetOutput(os.Stdout)
-	maxGo = runtime.GOMAXPROCS(0) / 2
+	// threads = runtime.GOMAXPROCS(0) / 2
+	threads = 1
 	regexes = make(map[string]*regexp.Regexp)
 	whiteListCommits = make(map[string]bool)
 }
@@ -487,13 +488,13 @@ func auditGitReference(repo *RepoDescriptor, ref *plumbing.Reference) []Leak {
 		semaphore   chan bool
 	)
 	repoName = repo.name
-	if opts.MaxGoRoutines != 0 {
-		maxGo = opts.MaxGoRoutines
+	if opts.Threads != 0 {
+		threads = opts.Threads
 	}
 	if opts.RepoPath != "" {
-		maxGo = 1
+		threads = 1
 	}
-	semaphore = make(chan bool, maxGo)
+	semaphore = make(chan bool, threads)
 
 	cIter, err := repo.repository.Log(&git.LogOptions{From: ref.Hash()})
 	if err != nil {
@@ -740,6 +741,10 @@ func optsGuard() error {
 		return fmt.Errorf("github user set and local owner path")
 	}
 
+	if opts.Threads > runtime.GOMAXPROCS(0) {
+		return fmt.Errorf("%d available threads", runtime.GOMAXPROCS(0))
+	}
+
 	// do the URL Parse and error checking here, so we can skip it later
 	// empty string is OK, it will default to the public github URL.
 	if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {