tests, readme, bumping version

CHANGELOG.md

@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+1.19.0
+----
+- removed `--all-refs`. By default gitleaks now scans all branches, remote and local.
+- added commit memoizer to prevent duplicate commit audits
+- removed branch whitelist
+- removed branch from report as a commit is an object independent of branch
+- Better regexes for facebook, github, twitter (no more unbounded wildcards)
+
 1.18.0
 ----
 - fixing slack token

README.md

@@ -68,14 +68,12 @@ Application Options:
       --github-org=    Github organization to audit
       --github-url=    GitHub API Base URL, use for GitHub Enterprise. Example: https://github.example.com/api/v3/ (default: https://api.github.com/)
       --github-pr=     Github PR url to audit. This does not clone the repo. GITHUB_TOKEN must be set
-  -b, --branch=        branch name to audit (defaults to HEAD)
   -c, --commit=        sha of commit to stop at
       --depth=         maximum commit depth
       --repo-path=     Path to repo
       --owner-path=    Path to owner directory (repos discovered)
       --threads=       Maximum number of threads gitleaks spawns
       --disk           Clones repo(s) to disk
-      --all-refs       run audit on all refs
       --single-search= single regular expression to search for
       --config=        path to gitleaks config
       --ssh-key=       path to ssh key

gitleaks_test.go

@@ -35,16 +35,6 @@ files = [
   ".go",
 ]
 `
-const testWhitelistBranch = `
-[[regexes]]
-description = "AWS"
-regex = '''AKIA[0-9A-Z]{16}'''
-
-[whitelist]
-branches = [
-  "origin/master",
-]
-`
 
 const testWhitelistRegex = `
 [[regexes]]
@@ -88,35 +78,6 @@ entropy = [
 ]
 `
 
-var benchmarkRepo *RepoDescriptor
-var benchmarkLeaksRepo *RepoDescriptor
-
-func getBenchmarkLeaksRepo() *RepoDescriptor {
-	if benchmarkLeaksRepo != nil {
-		return benchmarkLeaksRepo
-	}
-	leaksR, _ := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-		URL: "https://github.com/gitleakstest/gronit.git",
-	})
-	benchmarkLeaksRepo = &RepoDescriptor{
-		repository: leaksR,
-	}
-	return benchmarkLeaksRepo
-}
-
-func getBenchmarkRepo() *RepoDescriptor {
-	if benchmarkRepo != nil {
-		return benchmarkRepo
-	}
-	bmRepo, _ := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-		URL: "https://github.com/apple/swift-package-manager.git",
-	})
-	benchmarkRepo = &RepoDescriptor{
-		repository: bmRepo,
-	}
-	return benchmarkRepo
-}
-
 func TestGetRepo(t *testing.T) {
 	var err error
 	dir, err = ioutil.TempDir("", "gitleaksTestRepo")
@@ -442,7 +403,6 @@ func TestWriteReport(t *testing.T) {
 func testTomlLoader() string {
 	tmpDir, _ := ioutil.TempDir("", "whiteListConfigs")
 	ioutil.WriteFile(path.Join(tmpDir, "regex"), []byte(testWhitelistRegex), 0644)
-	ioutil.WriteFile(path.Join(tmpDir, "branch"), []byte(testWhitelistBranch), 0644)
 	ioutil.WriteFile(path.Join(tmpDir, "commit"), []byte(testWhitelistCommit), 0644)
 	ioutil.WriteFile(path.Join(tmpDir, "file"), []byte(testWhitelistFile), 0644)
 	ioutil.WriteFile(path.Join(tmpDir, "repo"), []byte(testWhitelistRepo), 0644)
@@ -509,41 +469,6 @@ func TestAuditRepo(t *testing.T) {
 				Threads: 4,
 			},
 		},
-		{
-			repo:        leaksRepo,
-			description: "audit specific bad branch",
-			numLeaks:    2,
-			testOpts: Options{
-				Branch: "master",
-			},
-		},
-		{
-			repo:        leaksRepo,
-			description: "audit specific good branch",
-			numLeaks:    0,
-			testOpts: Options{
-				Branch: "dev",
-			},
-		},
-		{
-			repo:        leaksRepo,
-			description: "audit all branch",
-			numLeaks:    6,
-			testOpts: Options{
-				AuditAllRefs: true,
-			},
-		},
-		{
-			repo:        leaksRepo,
-			description: "audit all branch whitelist 1",
-			numLeaks:    4,
-			testOpts: Options{
-				AuditAllRefs: true,
-			},
-			whiteListBranches: []string{
-				"origin/master",
-			},
-		},
 		{
 			repo:        leaksRepo,
 			description: "two leaks present whitelist AWS.. no leaks",
@@ -592,15 +517,6 @@ func TestAuditRepo(t *testing.T) {
 			configPath:  path.Join(configsDir, "regex"),
 			numLeaks:    0,
 		},
-		{
-			repo:        leaksRepo,
-			description: "toml whitelist branch",
-			configPath:  path.Join(configsDir, "branch"),
-			testOpts: Options{
-				AuditAllRefs: true,
-			},
-			numLeaks: 4,
-		},
 		{
 			repo:        leaksRepo,
 			description: "toml whitelist file",
@@ -662,7 +578,7 @@ func TestAuditRepo(t *testing.T) {
 		{
 			repo:        leaksRepo,
 			description: "toml entropy range",
-			numLeaks:    283,
+			numLeaks:    284,
 			configPath:  path.Join(configsDir, "entropy"),
 		},
 		{
@@ -697,11 +613,6 @@ func TestAuditRepo(t *testing.T) {
 				} else {
 					whiteListCommits = nil
 				}
-				if test.whiteListBranches != nil {
-					whiteListBranches = test.whiteListBranches
-				} else {
-					whiteListBranches = nil
-				}
 				if test.whiteListRegexes != nil {
 					whiteListRegexes = test.whiteListRegexes
 				} else {
@@ -897,126 +808,3 @@ func TestLoadToml(t *testing.T) {
 		})
 	}
 }
-
-func BenchmarkAuditRepo1Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 1
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo2Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 2
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo4Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 4
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo8Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 8
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo10Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 10
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo100Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 100
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditRepo1000Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 1000
-	benchmarkRepo = getBenchmarkRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-func BenchmarkAuditLeakRepo1Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 1
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditLeakRepo2Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 2
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditLeakRepo4Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 4
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditLeakRepo8Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 8
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-
-func BenchmarkAuditLeakRepo10Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 10
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-func BenchmarkAuditLeakRepo100Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 100
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}
-func BenchmarkAuditLeakRepo1000Proc(b *testing.B) {
-	loadToml()
-	opts.Threads = 1000
-	benchmarkLeaksRepo = getBenchmarkLeaksRepo()
-	for n := 0; n < b.N; n++ {
-		auditGitRepo(benchmarkRepo)
-	}
-}

main.go

@@ -134,7 +134,7 @@ type entropyRange struct {
 }
 
 const defaultGithubURL = "https://api.github.com/"
-const version = "1.18.0"
+const version = "1.19.0"
 const errExit = 2
 const leakExit = 1
 const defaultConfig = `