Merge pull request #286 from zricethezav/v3.0.1

V3.0.1

Makefile

@@ -27,13 +27,13 @@ build:
 release-builds:
 	rm -rf build
 	mkdir build
-	env GOOS="windows" GOARCH="amd64" go build -o "build/gitleaks-windows-amd64.exe"
-	env GOOS="windows" GOARCH="386" go build -o "build/gitleaks-windows-386.exe"
-	env GOOS="linux" GOARCH="amd64" go build -o "build/gitleaks-linux-amd64"
-	env GOOS="linux" GOARCH="arm" go build -o "build/gitleaks-linux-arm"
-	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips"
-	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips"
-	env GOOS="darwin" GOARCH="amd64" go build -o "build/gitleaks-darwin-amd64"
+	env GOOS="windows" GOARCH="amd64" go build -o "build/gitleaks-windows-amd64.exe" $(LDFLAGS)
+	env GOOS="windows" GOARCH="386" go build -o "build/gitleaks-windows-386.exe" $(LDFLAGS)
+	env GOOS="linux" GOARCH="amd64" go build -o "build/gitleaks-linux-amd64" $(LDFLAGS)
+	env GOOS="linux" GOARCH="arm" go build -o "build/gitleaks-linux-arm" $(LDFLAGS)
+	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips" $(LDFLAGS)
+	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips" $(LDFLAGS)
+	env GOOS="darwin" GOARCH="amd64" go build -o "build/gitleaks-darwin-amd64" $(LDFLAGS)
 
 deploy:
 	@echo "$(DOCKER_PASSWORD)" | docker login -u "$(DOCKER_USERNAME)" --password-stdin

audit/audit_test.go

@@ -29,136 +29,152 @@ func TestAudit(t *testing.T) {
 		{
 			description: "test local repo one aws leak",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_local_repo_one_aws_leak.json.got",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_local_repo_one_aws_leak.json.got",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_one_aws_leak.json",
 		},
 		{
 			description: "test local repo one aws leak threaded",
 			opts: options.Options{
-				Threads:  runtime.GOMAXPROCS(0),
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_local_repo_one_aws_leak.json.got",
+				Threads:      runtime.GOMAXPROCS(0),
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_local_repo_one_aws_leak.json.got",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_one_aws_leak.json",
 		},
 		{
 			description: "test non existent repo",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/no_repo_here",
+				RepoPath:     "../test_data/test_repos/no_repo_here",
+				ReportFormat: "json",
 			},
 			emptyRepo: true,
 		},
 		{
 			description: "test local repo one aws leak whitelisted",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Config:   "../test_data/test_configs/aws_key_whitelist_python_files.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				ReportFormat: "json",
+				Config:       "../test_data/test_configs/aws_key_whitelist_python_files.toml",
 			},
 			wantEmpty: true,
 		},
 		{
 			description: "test local repo two leaks",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_2",
-				Report:   "../test_data/test_local_repo_two_leaks.json.got",
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks.json.got",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_two_leaks.json",
 		},
 		{
 			description: "test local repo two leaks globally whitelisted",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_2",
-				Config:   "../test_data/test_configs/aws_key_global_whitelist_file.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Config:       "../test_data/test_configs/aws_key_global_whitelist_file.toml",
+				ReportFormat: "json",
 			},
 			wantEmpty: true,
 		},
 		{
 			description: "test local repo two leaks whitelisted",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_2",
-				Config:   "../test_data/test_configs/aws_key_whitelist_files.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Config:       "../test_data/test_configs/aws_key_whitelist_files.toml",
+				ReportFormat: "json",
 			},
 			wantEmpty: true,
 		},
 		{
 			description: "test local repo three leaks dev branch",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_3",
-				Report:   "../test_data/test_local_repo_three_leaks.json.got",
-				Config:   "../test_data/test_configs/aws_key.toml",
-				Branch:   "dev",
+				RepoPath:     "../test_data/test_repos/test_repo_3",
+				Report:       "../test_data/test_local_repo_three_leaks.json.got",
+				Config:       "../test_data/test_configs/aws_key.toml",
+				Branch:       "dev",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_three_leaks.json",
 		},
 		{
 			description: "test local repo branch does not exist",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_3",
-				Branch:   "nobranch",
+				RepoPath:     "../test_data/test_repos/test_repo_3",
+				Branch:       "nobranch",
+				ReportFormat: "json",
 			},
 			wantEmpty: true,
 		},
 		{
 			description: "test local repo one aws leak single commit",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_local_repo_one_aws_leak_commit.json.got",
-				Commit:   "6557c92612d3b35979bd426d429255b3bf9fab74",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_local_repo_one_aws_leak_commit.json.got",
+				Commit:       "6557c92612d3b35979bd426d429255b3bf9fab74",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_one_aws_leak_commit.json",
 		},
 		{
 			description: "test local repo one aws leak AND leak on python files",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_local_repo_one_aws_leak_and_file_leak.json.got",
-				Config:   "../test_data/test_configs/aws_key_file_regex.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_local_repo_one_aws_leak_and_file_leak.json.got",
+				Config:       "../test_data/test_configs/aws_key_file_regex.toml",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_one_aws_leak_and_file_leak.json",
 		},
 		{
 			description: "test owner path",
 			opts: options.Options{
-				OwnerPath: "../test_data/test_repos/",
-				Report:    "../test_data/test_local_owner_aws_leak.json.got",
+				OwnerPath:    "../test_data/test_repos/",
+				Report:       "../test_data/test_local_owner_aws_leak.json.got",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak.json",
 		},
 		{
 			description: "test entropy",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_entropy.json.got",
-				Config:   "../test_data/test_configs/entropy.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_entropy.json.got",
+				Config:       "../test_data/test_configs/entropy.toml",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_entropy.json",
 		},
 		{
 			description: "test entropy and regex",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_regex_entropy.json.got",
-				Config:   "../test_data/test_configs/regex_entropy.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_regex_entropy.json.got",
+				Config:       "../test_data/test_configs/regex_entropy.toml",
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_regex_entropy.json",
 		},
 		{
 			description: "test local repo four entropy alternative config",
 			opts: options.Options{
-				RepoPath:   "../test_data/test_repos/test_repo_4",
-				Report:     "../test_data/test_local_repo_four_alt_config_entropy.json.got",
-				RepoConfig: true,
+				RepoPath:     "../test_data/test_repos/test_repo_4",
+				Report:       "../test_data/test_local_repo_four_alt_config_entropy.json.got",
+				RepoConfig:   true,
+				ReportFormat: "json",
 			},
 			wantPath: "../test_data/test_local_repo_four_alt_config_entropy.json",
 		},
 		{
 			description: "test local repo four entropy alternative config",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_1",
-				Report:   "../test_data/test_regex_whitelist.json.got",
-				Config:   "../test_data/test_configs/aws_key_aws_whitelisted.toml",
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_regex_whitelist.json.got",
+				Config:       "../test_data/test_configs/aws_key_aws_whitelisted.toml",
+				ReportFormat: "json",
 			},
 			wantEmpty: true,
 		},
@@ -222,9 +238,10 @@ func TestAuditUncommited(t *testing.T) {
 		{
 			description: "test audit local one leak",
 			opts: options.Options{
-				RepoPath:   "../test_data/test_repos/test_repo_1",
-				Report:     "../test_data/test_local_repo_one_aws_leak_uncommitted.json.got",
-				Uncommited: true,
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Report:       "../test_data/test_local_repo_one_aws_leak_uncommitted.json.got",
+				Uncommited:   true,
+				ReportFormat: "json",
 			},
 			wantPath:     "../test_data/test_local_repo_one_aws_leak_uncommitted.json",
 			fileToChange: "server.test.py",
@@ -233,8 +250,9 @@ func TestAuditUncommited(t *testing.T) {
 		{
 			description: "test audit local no leak",
 			opts: options.Options{
-				RepoPath:   "../test_data/test_repos/test_repo_1",
-				Uncommited: true,
+				RepoPath:     "../test_data/test_repos/test_repo_1",
+				Uncommited:   true,
+				ReportFormat: "json",
 			},
 			wantEmpty:    true,
 			fileToChange: "server.test.py",

audit/util.go

@@ -270,6 +270,20 @@ func inspectCommit(c *object.Commit, repo *Repo) error {
 			log.Debugf("whitelisted file found, skipping audit of file: %s", f.Name)
 			return nil
 		}
+
+		if fileMatched(f.Name, repo.config.FileRegex) {
+			repo.Manager.SendLeaks(manager.Leak{
+				Line:     "N/A",
+				Offender: f.Name,
+				Commit:   c.Hash.String(),
+				Repo:     repo.Name,
+				Rule:     "file regex matched" + repo.config.FileRegex.String(),
+				Author:   c.Author.Name,
+				Email:    c.Author.Email,
+				Date:     c.Author.When,
+				File:     f.Name,
+			})
+		}
 		content, err := f.Contents()
 		if err != nil {
 			return err

config/default.go

@@ -71,7 +71,6 @@ title = "gitleaks config"
 	regex = '''(?i)(api_key|apikey|secret)(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]'''
 	tags = ["key", "API", "generic"]
 
-
 [[rules]]
 	description = "Google API key"
 	regex = '''AIza[0-9A-Za-z\\-_]{35}'''

examples/leaky-repo.toml

@@ -0,0 +1,169 @@
+title = "gitleaks config"
+
+[[rules]]
+	description = "AWS Manager ID"
+	regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+	tags = ["key", "AWS"]
+
+[[rules]]
+	description = "AWS cred file info"
+	regex = '''(?i)(aws_access_key_id|aws_secret_access_key)(.{0,20})?=.[0-9a-zA-Z\/+]{20,40}'''
+	tags = ["AWS"]
+
+[[rules]]
+	description = "AWS Secret Key"
+	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
+	tags = ["key", "AWS"]
+
+[[rules]]
+	description = "AWS MWS key"
+	regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'''
+	tags = ["key", "AWS", "MWS"]
+
+[[rules]]
+	description = "Facebook Secret Key"
+	regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]'''
+	tags = ["key", "Facebook"]
+
+[[rules]]
+	description = "Facebook Client ID"
+	regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]'''
+	tags = ["key", "Facebook"]
+
+[[rules]]
+	description = "Twitter Secret Key"
+	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]'''
+	tags = ["key", "Twitter"]
+
+[[rules]]
+	description = "Twitter Client ID"
+	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]'''
+	tags = ["client", "Twitter"]
+
+[[rules]]
+	description = "Github"
+	regex = '''(?i)github(.{0,20})?(?-i)['\"][0-9a-zA-Z]{35,40}['\"]'''
+	tags = ["key", "Github"]
+
+[[rules]]
+	description = "LinkedIn Client ID"
+	regex = '''(?i)linkedin(.{0,20})?(?-i)['\"][0-9a-z]{12}['\"]'''
+	tags = ["client", "LinkedIn"]
+
+[[rules]]
+	description = "LinkedIn Secret Key"
+	regex = '''(?i)linkedin(.{0,20})?['\"][0-9a-z]{16}['\"]'''
+	tags = ["secret", "LinkedIn"]
+
+[[rules]]
+	description = "Slack"
+	regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+	tags = ["key", "Slack"]
+
+[[rules]]
+	description = "EC"
+	regex = '''-----BEGIN EC PRIVATE KEY-----'''
+	tags = ["key", "EC"]
+
+
+[[rules]]
+	description = "Google API key"
+	regex = '''AIza[0-9A-Za-z\\-_]{35}'''
+	tags = ["key", "Google"]
+
+
+[[rules]]
+	description = "Heroku API key"
+	regex = '''(?i)heroku(.{0,20})?['"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]'''
+	tags = ["key", "Heroku"]
+
+[[rules]]
+	description = "MailChimp API key"
+	regex = '''(?i)(mailchimp|mc)(.{0,20})?['"][0-9a-f]{32}-us[0-9]{1,2}['"]'''
+	tags = ["key", "Mailchimp"]
+
+[[rules]]
+	description = "Mailgun API key"
+	regex = '''(?i)(mailgun|mg)(.{0,20})?['"][0-9a-z]{32}['"]'''
+	tags = ["key", "Mailgun"]
+
+[[rules]]
+	description = "PayPal Braintree access token"
+	regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}'''
+	tags = ["key", "Paypal"]
+
+[[rules]]
+	description = "Picatic API key"
+	regex = '''sk_live_[0-9a-z]{32}'''
+	tags = ["key", "Picatic"]
+
+[[rules]]
+	description = "Slack Webhook"
+	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}'''
+	tags = ["key", "slack"]
+
+[[rules]]
+	description = "Stripe API key"
+	regex = '''(?i)stripe(.{0,20})?['\"][sk|rk]_live_[0-9a-zA-Z]{24}'''
+	tags = ["key", "Stripe"]
+
+[[rules]]
+	description = "Square access token"
+	regex = '''sq0atp-[0-9A-Za-z\-_]{22}'''
+	tags = ["key", "square"]
+
+[[rules]]
+	description = "Square OAuth secret"
+	regex = '''sq0csp-[0-9A-Za-z\\-_]{43}'''
+	tags = ["key", "square"]
+
+[[rules]]
+	description = "Twilio API key"
+	regex = '''(?i)twilio(.{0,20})?['\"][0-9a-f]{32}['\"]'''
+	tags = ["key", "twilio"]
+
+[[rules]]
+	description = "Env Var"
+	regex = '''(?i)(api_key|apikey|secret|key|api|password|pw|host)=[0-9a-zA-Z-_{}]{4,120}'''
+
+[[rules]]
+	description = "Port"
+	regex = '''(?i)port(.{0,4})?[0-9]{1,10}'''
+
+[[rules]]
+	description = "Email"
+	regex = '''[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}'''
+	tags = ["email"]
+
+[[rules]]
+	description = "Generic Credential"
+	regex = '''(?i)(dbpasswd|dbuser|dbname|dbhost|api_key|apikey|secret|key|api|password|user|guid|hostname|pw|auth)(.{0,20})?['|"][0-9a-zA-Z-_!{}/=]{4,120}['|"]'''
+	tags = ["key", "API", "generic"]
+
+[[rules]]
+	description = "WP-Config"
+	regex='''define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?['|"].{10,120}['|"]'''
+	tags = ["key", "API", "generic"]
+
+
+[[rules]]
+	description = "Pure Entropy"
+	regex = '''['|"][0-9a-zA-Z-._{}$\/=]{40,120}['|"]'''
+	entropies = [
+        "5.0-5.6"
+	]
+
+[[rules]]
+	description = "Entropy plus Generic Credential"
+	regex = '''(?i)(api_key|apikey|secret|key|api|password|pw)'''
+	entropies = [
+        "5.2-5.5"
+	]
+
+[Global]
+    file = '''(?i)(id_rsa|passwd|id_rsa.pub|pgpass|pem|key|shadow)'''
+
+[whitelist]
+	description = "image whitelists"
+	file = '''(.*?)(jpg|gif|doc|pdf|bin)$'''
+

manager/manager.go

@@ -1,6 +1,9 @@
 package manager
 
 import (
+	"crypto/sha1"
+	"encoding/csv"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"github.com/hako/durafmt"
@@ -26,9 +29,10 @@ type Manager struct {
 	CloneOptions *git.CloneOptions
 	CloneDir     string
 
-	leaks    []Leak
-	leakChan chan Leak
-	leakWG   *sync.WaitGroup
+	leaks     []Leak
+	leakChan  chan Leak
+	leakWG    *sync.WaitGroup
+	leakCache map[string]bool
 
 	stopChan chan os.Signal
 	metadata Metadata
@@ -38,17 +42,18 @@ type Manager struct {
 // Leak is a struct that contains information about some line of code that contains
 // sensitive information as determined by the rules set in a gitleaks config
 type Leak struct {
-	Line     string    `json:"line"`
-	Offender string    `json:"offender"`
-	Commit   string    `json:"commit"`
-	Repo     string    `json:"repo"`
-	Rule     string    `json:"rule"`
-	Message  string    `json:"commitMessage"`
-	Author   string    `json:"author"`
-	Email    string    `json:"email"`
-	File     string    `json:"file"`
-	Date     time.Time `json:"date"`
-	Tags     string    `json:"tags"`
+	Line       string    `json:"line"`
+	Offender   string    `json:"offender"`
+	Commit     string    `json:"commit"`
+	Repo       string    `json:"repo"`
+	Rule       string    `json:"rule"`
+	Message    string    `json:"commitMessage"`
+	Author     string    `json:"author"`
+	Email      string    `json:"email"`
+	File       string    `json:"file"`
+	Date       time.Time `json:"date"`
+	Tags       string    `json:"tags"`
+	lookupHash string
 }
 
 // AuditTime is a type used to determine total audit time
@@ -104,15 +109,30 @@ func (manager *Manager) GetLeaks() []Leak {
 // SendLeaks accepts a leak and is used by the audit pkg. This is the public function
 // that allows other packages to send leaks to the manager.
 func (manager *Manager) SendLeaks(l Leak) {
+	h := sha1.New()
+	h.Write([]byte(l.Commit + l.Offender + l.File))
+	l.lookupHash = hex.EncodeToString(h.Sum(nil))
 	manager.leakWG.Add(1)
 	manager.leakChan <- l
 }
 
+func (manager *Manager) alreadySeen(leak Leak) bool {
+	if _, ok := manager.leakCache[leak.lookupHash]; ok {
+		return true
+	}
+	manager.leakCache[leak.lookupHash] = true
+	return false
+}
+
 // receiveLeaks listens to leakChan for incoming leaks. If any are received, they are appended to the
 // manager's leaks for future reporting. If the -v/--verbose option is set the leaks will marshaled into
 // json and printed out.
 func (manager *Manager) receiveLeaks() {
 	for leak := range manager.leakChan {
+		if manager.alreadySeen(leak) {
+			manager.leakWG.Done()
+			continue
+		}
 		manager.leaks = append(manager.leaks, leak)
 		if manager.Opts.Verbose {
 			var b []byte
@@ -178,10 +198,11 @@ func NewManager(opts options.Options, cfg config.Config) (*Manager, error) {
 		Config:       cfg,
 		CloneOptions: cloneOpts,
 
-		stopChan: make(chan os.Signal, 1),
-		leakChan: make(chan Leak),
-		leakWG:   &sync.WaitGroup{},
-		metaWG:   &sync.WaitGroup{},
+		stopChan:  make(chan os.Signal, 1),
+		leakChan:  make(chan Leak),
+		leakWG:    &sync.WaitGroup{},
+		leakCache: make(map[string]bool),
+		metaWG:    &sync.WaitGroup{},
 		metadata: Metadata{
 			RegexTime: make(map[string]int64),
 			timings:   make(chan interface{}),
@@ -241,16 +262,23 @@ func (manager *Manager) Report() error {
 			return err
 		}
 
-		encoder := json.NewEncoder(file)
-		encoder.SetIndent("", " ")
-		err = encoder.Encode(manager.leaks)
-		if err != nil {
-			return err
-		}
-		err = file.Close()
-		if err != nil {
-			return err
+		if manager.Opts.ReportFormat == "json" {
+			encoder := json.NewEncoder(file)
+			encoder.SetIndent("", " ")
+			err = encoder.Encode(manager.leaks)
+			if err != nil {
+				return err
+			}
+		} else {
+			w := csv.NewWriter(file)
+			w.Write([]string{"repo", "line", "commit", "offender", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
+			for _, leak := range manager.GetLeaks() {
+				w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Rule, leak.Tags, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
+			}
+			w.Flush()
 		}
+		file.Close()
+
 		log.Infof("report written to %s", manager.Opts.Report)
 	}
 	return nil

manager/manager_test.go

@@ -1,8 +1,11 @@
 package manager
 
 import (
+	"crypto/rand"
+	"fmt"
 	"github.com/zricethezav/gitleaks/config"
 	"github.com/zricethezav/gitleaks/options"
+	"io"
 	"testing"
 )
 
@@ -28,7 +31,10 @@ func TestSendReceiveLeaks(t *testing.T) {
 		m, _ := NewManager(opts, cfg)
 
 		for i := 0; i < test.leaksToAdd; i++ {
-			m.SendLeaks(Leak{})
+			// we are testing the sync of sending/receiving leaks so we need
+			// the hash generation in sendLeaks to be unique for each iteration
+			// so I'm just setting the offender string as a uuid
+			m.SendLeaks(Leak{Offender: newUUID()})
 		}
 		got := m.GetLeaks()
 		if len(got) != test.leaksToAdd {
@@ -86,3 +92,15 @@ func TestSendReceiveMeta(t *testing.T) {
 		}
 	}
 }
+
+// newUUID generates a random UUID according to RFC 4122
+// Ripped from https://play.golang.org/p/4FkNSiUDMg
+func newUUID() string {
+	uuid := make([]byte, 16)
+	io.ReadFull(rand.Reader, uuid)
+	// variant bits; see section 4.1.1
+	uuid[8] = uuid[8]&^0xc0 | 0x80
+	// version 4 (pseudo-random); see section 4.1.3
+	uuid[6] = uuid[6]&^0xf0 | 0x40
+	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
+}

options/options.go

@@ -25,27 +25,28 @@ const (
 
 // Options stores values of command line options
 type Options struct {
-	Verbose     bool   `short:"v" long:"verbose" description:"Show verbose output from audit"`
-	Repo        string `short:"r" long:"repo" description:"Target repository"`
-	Config      string `long:"config" description:"config path"`
-	Disk        bool   `long:"disk" description:"Clones repo(s) to disk"`
-	Version     bool   `long:"version" description:"version number"`
-	Timeout     int    `long:"timeout" description:"Timeout (s)"`
-	Username    string `long:"username" description:"Username for git repo"`
-	Password    string `long:"password" description:"Password for git repo"`
-	AccessToken string `long:"access-token" description:"Access token for git repo"`
-	Commit      string `long:"commit" description:"sha of commit to audit"`
-	Threads     int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	SSH         string `long:"ssh-key" description:"path to ssh key used for auth"`
-	Uncommited  bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
-	RepoPath    string `long:"repo-path" description:"Path to repo"`
-	OwnerPath   string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
-	Branch      string `long:"branch" description:"Branch to audit"`
-	Report      string `long:"report" description:"path to write json leaks file"`
-	Redact      bool   `long:"redact" description:"redact secrets from log messages and leaks"`
-	Debug       bool   `long:"debug" description:"log debug messages"`
-	RepoConfig  bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
-	PrettyPrint bool   `long:"pretty" description:"Pretty print json if leaks are present"`
+	Verbose      bool   `short:"v" long:"verbose" description:"Show verbose output from audit"`
+	Repo         string `short:"r" long:"repo" description:"Target repository"`
+	Config       string `long:"config" description:"config path"`
+	Disk         bool   `long:"disk" description:"Clones repo(s) to disk"`
+	Version      bool   `long:"version" description:"version number"`
+	Timeout      int    `long:"timeout" description:"Timeout (s)"`
+	Username     string `long:"username" description:"Username for git repo"`
+	Password     string `long:"password" description:"Password for git repo"`
+	AccessToken  string `long:"access-token" description:"Access token for git repo"`
+	Commit       string `long:"commit" description:"sha of commit to audit"`
+	Threads      int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	SSH          string `long:"ssh-key" description:"path to ssh key used for auth"`
+	Uncommited   bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
+	RepoPath     string `long:"repo-path" description:"Path to repo"`
+	OwnerPath    string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
+	Branch       string `long:"branch" description:"Branch to audit"`
+	Report       string `long:"report" description:"path to write json leaks file"`
+	ReportFormat string `long:"report-format" default:"json" description:"json or csv"`
+	Redact       bool   `long:"redact" description:"redact secrets from log messages and leaks"`
+	Debug        bool   `long:"debug" description:"log debug messages"`
+	RepoConfig   bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
+	PrettyPrint  bool   `long:"pretty" description:"Pretty print json if leaks are present"`
 
 	// Hosts
 	Host         string `long:"host" description:"git hosting service like gitlab or github. Supported hosts include: Github, Gitlab"`

test_data/test_local_owner_aws_leak.json

@@ -116,19 +116,6 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
  },
- {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_2",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",
@@ -245,369 +232,5 @@
   "file": "secrets.md",
   "date": "2019-10-25T13:35:03-04:00",
   "tags": "key, API, generic"
- },
- {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "rm secrets again\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:32-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
-  "repo": "test_repo_3",
-  "rule": "Generic Credential",
-  "commitMessage": "rm secrets again\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:32-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "adding another one\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
-  "repo": "test_repo_3",
-  "rule": "Generic Credential",
-  "commitMessage": "adding another one\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "committing pem\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:07:41-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
-  "repo": "test_repo_3",
-  "rule": "Generic Credential",
-  "commitMessage": "committing pem\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:07:41-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_3",
-  "rule": "Generic Credential",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "adding aws key\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T12:58:39-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "rm secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:54:26-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "rm secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:54:26-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "more secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:54:08-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "more secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:54:08-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "64cfcee9aad1c84581631636bfc54f2050718d1a",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "rm secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:36:22-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "64cfcee9aad1c84581631636bfc54f2050718d1a",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "rm secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:36:22-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "deea550dd6c7acaf0e59432600593533984a2125",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "dev branch\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:35:03-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "deea550dd6c7acaf0e59432600593533984a2125",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "dev branch\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:35:03-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "rm secrets again\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:32-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "rm secrets again\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:32-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "adding another one\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "adding another one\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "committing pem\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:07:41-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "committing pem\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:07:41-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_4",
-  "rule": "Generic Credential",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, API, generic"
- },
- {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
- {
-  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
-  "repo": "test_repo_4",
-  "rule": "AWS Manager ID",
-  "commitMessage": "adding aws key\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T12:58:39-04:00",
-  "tags": "key, AWS"
  }
 ]

test_data/test_local_repo_one_aws_leak_and_file_leak.json

@@ -12,6 +12,19 @@
   "date": "2019-10-24T10:03:38-04:00",
   "tags": ""
  },
+ {
+  "line": "N/A",
+  "offender": "server.test.py",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "file regex matched(.*)?py$",
+  "commitMessage": "",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": ""
+ },
  {
   "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
   "offender": "AKIAIO5FODNN7EXAMPLE",

test_data/test_local_repo_three_leaks.json

@@ -77,19 +77,6 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
  },
- {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_3",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",

test_data/test_local_repo_two_leaks.json

@@ -103,19 +103,6 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
  },
- {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
-  "repo": "test_repo_2",
-  "rule": "AWS Manager ID",
-  "commitMessage": "wait this is actually adding an aws secret\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:01:27-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",