Go repo scan (#480)

* Real fast

* Addressing some golint stuff

go.mod

@@ -2,6 +2,8 @@ module github.com/zricethezav/gitleaks/v7
 
 go 1.15
 
+replace github.com/go-git/go-git/v5 => github.com/zricethezav/go-git/v5 v5.2.1
+
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/go-git/go-git/v5 v5.2.0

go.sum

@@ -66,6 +66,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+github.com/zricethezav/go-git/v5 v5.2.1 h1:snaoKv8ksDSz7NfBRXsBr9Yr8IKEKWRWf1zdhFmcFvI=
+github.com/zricethezav/go-git/v5 v5.2.1/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
@@ -78,6 +80,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -88,6 +91,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

scan/commit.go

@@ -20,7 +20,6 @@ type CommitScanner struct {
 	repo     *git.Repository
 	repoName string
 	commit   *object.Commit
-	patch    *object.Patch
 }
 
 // NewCommitScanner creates and returns a commit scanner
@@ -40,12 +39,6 @@ func (cs *CommitScanner) SetRepoName(repoName string) {
 	cs.repoName = repoName
 }
 
-// SetPatch sets the patch to be inspected by the commit scanner. This is used to avoid
-// a race condition when running a threaded repo scan
-func (cs *CommitScanner) SetPatch(patch *object.Patch) {
-	cs.patch = patch
-}
-
 // Scan kicks off a CommitScanner Scan
 func (cs *CommitScanner) Scan() (Report, error) {
 	var scannerReport Report
@@ -54,25 +47,23 @@ func (cs *CommitScanner) Scan() (Report, error) {
 		return facScanner.Scan()
 	}
 
-	if cs.patch == nil {
-		parent, err := cs.commit.Parent(0)
-		if err != nil {
-			return scannerReport, err
-		}
+	parent, err := cs.commit.Parent(0)
+	if err != nil {
+		return scannerReport, err
+	}
 
-		if parent == nil {
-			return scannerReport, nil
-		}
+	if parent == nil {
+		return scannerReport, nil
+	}
 
-		cs.patch, err = parent.Patch(cs.commit)
-		if err != nil {
-			return scannerReport, fmt.Errorf("could not generate Patch")
-		}
+	patch, err := parent.Patch(cs.commit)
+	if err != nil {
+		return scannerReport, fmt.Errorf("could not generate Patch")
 	}
 
-	patchContent := cs.patch.String()
+	patchContent := patch.String()
 
-	for _, f := range cs.patch.FilePatches() {
+	for _, f := range patch.FilePatches() {
 		if f.IsBinary() {
 			continue
 		}

scan/repo.go

@@ -1,17 +1,17 @@
 package scan
 
 import (
-	"fmt"
-	"sync"
+	"context"
+
+	"golang.org/x/sync/errgroup"
 
 	"github.com/zricethezav/gitleaks/v7/config"
 	"github.com/zricethezav/gitleaks/v7/options"
 
-	log "github.com/sirupsen/logrus"
-
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/go-git/go-git/v5/plumbing/storer"
+	log "github.com/sirupsen/logrus"
 )
 
 // RepoScanner is a repo scanner
@@ -19,35 +19,30 @@ type RepoScanner struct {
 	opts     options.Options
 	cfg      config.Config
 	repo     *git.Repository
+	throttle *Throttle
 	repoName string
-
-	leakChan  chan Leak
-	leakWG    *sync.WaitGroup
-	leakCache map[string]bool
-	leaks     []Leak
 }
 
 // NewRepoScanner returns a new repo scanner (go figure). This function also
 // sets up the leak listener for multi-threaded awesomeness.
 func NewRepoScanner(opts options.Options, cfg config.Config, repo *git.Repository) *RepoScanner {
 	rs := &RepoScanner{
-		opts:      opts,
-		cfg:       cfg,
-		repo:      repo,
-		leakChan:  make(chan Leak),
-		leakWG:    &sync.WaitGroup{},
-		leakCache: make(map[string]bool),
-		repoName:  getRepoName(opts),
+		opts:     opts,
+		cfg:      cfg,
+		repo:     repo,
+		throttle: NewThrottle(opts),
+		repoName: getRepoName(opts),
 	}
 
-	go rs.receiveLeaks()
-
 	return rs
 }
 
 // Scan kicks of a repo scan
 func (rs *RepoScanner) Scan() (Report, error) {
-	var scannerReport Report
+	var (
+		scannerReport Report
+		commits       chan *object.Commit
+	)
 	logOpts, err := logOptions(rs.repo, rs.opts)
 	if err != nil {
 		return scannerReport, err
@@ -56,80 +51,66 @@ func (rs *RepoScanner) Scan() (Report, error) {
 	if err != nil {
 		return scannerReport, err
 	}
-	semaphore := make(chan bool, howManyThreads(rs.opts.Threads))
-	wg := sync.WaitGroup{}
 
-	err = cIter.ForEach(func(c *object.Commit) error {
-		if c == nil || depthReached(scannerReport.Commits, rs.opts) {
-			return storer.ErrStop
-		}
+	g, _ := errgroup.WithContext(context.Background())
+	commits = make(chan *object.Commit)
+	leaks := make(chan Leak)
 
-		if rs.cfg.Allowlist.CommitAllowed(c.Hash.String()) {
-			return nil
-		}
-
-		// Check if at root
-		if len(c.ParentHashes) == 0 {
-			scannerReport.Commits++
-			facScanner := NewFilesAtCommitScanner(rs.opts, rs.cfg, rs.repo, c)
-			facScanner.repoName = rs.repoName
-			facReport, err := facScanner.Scan()
-			if err != nil {
-				return err
+	commitNum := 0
+	g.Go(func() error {
+		defer close(commits)
+		err = cIter.ForEach(func(c *object.Commit) error {
+			if c == nil || depthReached(commitNum, rs.opts) {
+				return storer.ErrStop
+			}
+
+			if rs.cfg.Allowlist.CommitAllowed(c.Hash.String()) {
+				return nil
+			}
+			commitNum++
+			commits <- c
+			if c.Hash.String() == rs.opts.CommitTo {
+				return storer.ErrStop
 			}
-			scannerReport.Leaks = append(scannerReport.Leaks, facReport.Leaks...)
-			return nil
-		}
 
-		// inspect first parent only as all other parents will be eventually reached
-		// (they exist as the tip of other branches, etc)
-		// See https://github.com/zricethezav/gitleaks/issues/413 for details
-		parent, err := c.Parent(0)
-		if err != nil || parent == nil {
 			return err
-		}
-		patch, err := parent.Patch(c)
-		if err != nil {
-			return fmt.Errorf("could not generate Patch")
-		}
-
-		scannerReport.Commits++
-		wg.Add(1)
-		semaphore <- true
-		go func(c *object.Commit, patch *object.Patch) {
-			defer func() {
-				<-semaphore
-				wg.Done()
-			}()
+		})
+		cIter.Close()
+		return nil
+	})
 
+	for commit := range commits {
+		c := commit
+		rs.throttle.Limit()
+		g.Go(func() error {
 			commitScanner := NewCommitScanner(rs.opts, rs.cfg, rs.repo, c)
 			commitScanner.SetRepoName(rs.repoName)
-			commitScanner.SetPatch(patch)
 			report, err := commitScanner.Scan()
+			rs.throttle.Release()
 			if err != nil {
 				log.Error(err)
 			}
 			for _, leak := range report.Leaks {
-				rs.leakWG.Add(1)
-				rs.leakChan <- leak
+				leaks <- leak
 			}
-		}(c, patch)
+			return nil
+		})
+	}
 
-		if c.Hash.String() == rs.opts.CommitTo {
-			return storer.ErrStop
-		}
-		return nil
-	})
+	go func() {
+		g.Wait()
+		close(leaks)
+	}()
 
-	wg.Wait()
-	rs.leakWG.Wait()
-	scannerReport.Leaks = append(scannerReport.Leaks, rs.leaks...)
-	return scannerReport, nil
+	for leak := range leaks {
+		scannerReport.Leaks = append(scannerReport.Leaks, leak)
+	}
+
+	scannerReport.Commits = commitNum
+	return scannerReport, g.Wait()
 }
 
-func (rs *RepoScanner) receiveLeaks() {
-	for leak := range rs.leakChan {
-		rs.leaks = append(rs.leaks, leak)
-		rs.leakWG.Done()
-	}
+// SetRepoName sets the repo name
+func (rs *RepoScanner) SetRepoName(repoName string) {
+	rs.repoName = repoName
 }

scan/scan_test.go

@@ -5,15 +5,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"reflect"
 	"runtime"
-	"sort"
 	"testing"
 
 	"github.com/zricethezav/gitleaks/v7/config"
 	"github.com/zricethezav/gitleaks/v7/options"
-
-	"github.com/sergi/go-diff/diffmatchpatch"
 )
 
 const testRepoBase = "../test_data/test_repos/"
@@ -202,6 +198,7 @@ func TestScan(t *testing.T) {
 				Path:         "../test_data/test_repos/",
 				Report:       "../test_data/test_local_owner_aws_leak.json.got",
 				ReportFormat: "json",
+				Threads:      runtime.GOMAXPROCS(0),
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak.json",
 		},
@@ -643,24 +640,68 @@ func fileCheck(wantPath, gotPath string) error {
 		return err
 	}
 
-	sort.Slice(gotLeaks, func(i, j int) bool {
-		return (gotLeaks)[i].Offender+(gotLeaks)[i].File < (gotLeaks)[j].Offender+(gotLeaks)[j].File
-	})
-	sort.Slice(wantLeaks, func(i, j int) bool {
-		return (wantLeaks)[i].Offender+(wantLeaks)[i].File < (wantLeaks)[j].Offender+(wantLeaks)[j].File
-	})
-
-	if !reflect.DeepEqual(gotLeaks, wantLeaks) {
-		dmp := diffmatchpatch.New()
-		diffs := dmp.DiffMain(string(want), string(got), false)
-		return fmt.Errorf("%s does not equal %s: %s", wantPath, gotPath, dmp.DiffPrettyText(diffs))
+	if len(wantLeaks) != len(gotLeaks) {
+		return fmt.Errorf("got %d leaks, want %d leaks", len(gotLeaks), len(wantLeaks))
+	}
+
+	for _, wantLeak := range wantLeaks {
+		found := false
+		for _, gotLeak := range gotLeaks {
+			if same(gotLeak, wantLeak) {
+				found = true
+			}
+		}
+		if !found {
+			return fmt.Errorf("unable to find %+v in got leaks", wantLeak)
+		}
 	}
+
 	if err := os.Remove(gotPath); err != nil {
 		return err
 	}
 	return nil
 }
 
+func same(l1, l2 Leak) bool {
+	if l1.Commit != l2.Commit {
+		return false
+	}
+
+	if l1.Offender != l2.Offender {
+		return false
+	}
+
+	if l1.Line != l2.Line {
+		return false
+	}
+
+	if l1.Tags != l2.Tags {
+		return false
+	}
+
+	if l1.LineNumber != l2.LineNumber {
+		return false
+	}
+
+	if l1.Author != l2.Author {
+		return false
+	}
+
+	if l1.LeakURL != l2.LeakURL {
+		return false
+	}
+
+	if l1.RepoURL != l2.RepoURL {
+		return false
+	}
+
+	if l1.Repo != l2.Repo {
+		return false
+	}
+	return true
+
+}
+
 func moveDotGit(from, to string) error {
 	repoDirs, err := ioutil.ReadDir("../test_data/test_repos")
 	if err != nil {

scan/throttle.go

@@ -0,0 +1,43 @@
+package scan
+
+import (
+	"runtime"
+
+	"github.com/zricethezav/gitleaks/v7/options"
+)
+
+const (
+	singleThreadCommitBuffer          = 1
+	multiThreadCommitBufferMultiplier = 10
+)
+
+// Throttle is a struct that limits the number of concurrent goroutines and sets the
+// number of threads available for gitleaks to use via GOMAXPROCS.
+type Throttle struct {
+	throttle chan bool
+}
+
+// NewThrottle accepts some options and returns a throttle for scanners to use
+func NewThrottle(opts options.Options) *Throttle {
+	t := Throttle{}
+	if opts.Threads <= 1 {
+		runtime.GOMAXPROCS(1)
+		t.throttle = make(chan bool, singleThreadCommitBuffer)
+		return &t
+	}
+
+	runtime.GOMAXPROCS(opts.Threads)
+	t.throttle = make(chan bool, multiThreadCommitBufferMultiplier*opts.Threads)
+	return &t
+
+}
+
+// Limit blocks new goroutines from spinning up if throttle is at capacity
+func (t *Throttle) Limit() {
+	t.throttle <- true
+}
+
+// Release releases the hold on the throttle, allowing more goroutines to be spun up
+func (t *Throttle) Release() {
+	<-t.throttle
+}

test_data/test_local_owner_aws_leak.json

@@ -4,7 +4,7 @@
   "lineNumber": 5,
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_1",
+  "repo": "test_repos",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
@@ -15,22 +15,6 @@
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
  },
- {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "lineNumber": 4,
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
-  "repo": "test_repo_2",
-  "repoURL": "",
-  "leakURL": "",
-  "rule": "AWS Access Key",
-  "commitMessage": "adding another one\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
   "lineNumber": 3,
@@ -48,35 +32,35 @@
   "tags": "key, AWS"
  },
  {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "lineNumber": 3,
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "lineNumber": 4,
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
   "repo": "test_repo_2",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "adding aws key\n",
+  "commitMessage": "adding another one\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "secrets.md",
-  "date": "2019-10-25T12:58:39-04:00",
+  "date": "2019-10-25T13:12:08-04:00",
   "tags": "key, AWS"
  },
  {
-  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
   "lineNumber": 3,
   "offender": "AKIALALEMEL33243OLIA",
-  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
-  "repo": "test_repo_3",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "more secrets\n",
+  "commitMessage": "adding aws key\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "secrets.md",
-  "date": "2019-10-25T13:54:08-04:00",
+  "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
  },
  {
@@ -96,19 +80,19 @@
   "tags": "key, AWS"
  },
  {
-  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
-  "lineNumber": 4,
-  "offender": "AKIALALEMEL33243OLIB",
-  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "lineNumber": 3,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
   "repo": "test_repo_3",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "adding another one\n",
+  "commitMessage": "more secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "secrets.md",
-  "date": "2019-10-25T13:12:08-04:00",
+  "date": "2019-10-25T13:54:08-04:00",
   "tags": "key, AWS"
  },
  {
@@ -143,6 +127,38 @@
   "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "lineNumber": 4,
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_3",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "AWS Access Key",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "lineNumber": 5,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "deea550dd6c7acaf0e59432600593533984a2125",
+  "repo": "test_repo_4",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "AWS Access Key",
+  "commitMessage": "dev branch\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:35:03-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
   "lineNumber": 3,
@@ -160,19 +176,19 @@
   "tags": "key, AWS"
  },
  {
-  "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "lineNumber": 5,
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "lineNumber": 3,
   "offender": "AKIALALEMEL33243OLIA",
-  "commit": "deea550dd6c7acaf0e59432600593533984a2125",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
   "repo": "test_repo_4",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "dev branch\n",
+  "commitMessage": "adding aws key\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "secrets.md",
-  "date": "2019-10-25T13:35:03-04:00",
+  "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
  },
  {
@@ -208,19 +224,19 @@
   "tags": "key, AWS"
  },
  {
-  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
-  "lineNumber": 3,
-  "offender": "AKIALALEMEL33243OLIA",
-  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
-  "repo": "test_repo_4",
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "lineNumber": 1,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "1f2a4abc47dabf991e6af6f9770867ce0ac1f360",
+  "repo": "test_repo_5",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "adding aws key\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "secrets.md",
-  "date": "2019-10-25T12:58:39-04:00",
+  "commitMessage": "introduce secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:07:34-05:00",
   "tags": "key, AWS"
  },
  {
@@ -239,28 +255,12 @@
   "date": "2020-02-01T10:08:04-05:00",
   "tags": "key, AWS"
  },
- {
-  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
-  "lineNumber": 1,
-  "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "1f2a4abc47dabf991e6af6f9770867ce0ac1f360",
-  "repo": "test_repo_5",
-  "repoURL": "",
-  "leakURL": "",
-  "rule": "AWS Access Key",
-  "commitMessage": "introduce secrets\n",
-  "author": "Zach Rice",
-  "email": "zrice@gitlab.com",
-  "file": "secrets.py",
-  "date": "2020-02-01T10:07:34-05:00",
-  "tags": "key, AWS"
- },
  {
   "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
   "lineNumber": 5,
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_6",
+  "repo": "test_repos",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
@@ -287,22 +287,6 @@
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS"
  },
- {
-  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "lineNumber": 6,
-  "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
-  "repo": "test_repo_8",
-  "repoURL": "",
-  "leakURL": "",
-  "rule": "AWS Access Key",
-  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
-  "author": "Zachary Rice",
-  "email": "zricer@protonmail.com",
-  "file": "dummy.txt",
-  "date": "2020-07-25T14:44:48-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
   "lineNumber": 2,
@@ -336,19 +320,19 @@
   "tags": "key, AWS"
  },
  {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "lineNumber": 5,
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
   "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_9",
+  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
+  "repo": "test_repo_8",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "commit 1 with secrets\n",
-  "author": "zach rice",
+  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
+  "author": "Zachary Rice",
   "email": "zricer@protonmail.com",
-  "file": "server.test.py",
-  "date": "2019-10-24T09:29:27-04:00",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:44:48-04:00",
   "tags": "key, AWS"
  },
  {
@@ -366,5 +350,21 @@
   "file": "server.test.py",
   "date": "2020-08-12T13:36:20-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 5,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repos",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "AWS Access Key",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_owner_aws_leak_allowlist_repo.json

@@ -239,22 +239,6 @@
   "date": "2020-02-01T10:07:34-05:00",
   "tags": "key, AWS"
  },
- {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
-  "lineNumber": 5,
-  "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_6",
-  "repoURL": "",
-  "leakURL": "",
-  "rule": "AWS Access Key",
-  "commitMessage": "commit 1 with secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
-  "file": "server.test.py",
-  "date": "2019-10-24T09:29:27-04:00",
-  "tags": "key, AWS"
- },
  {
   "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
   "lineNumber": 3,
@@ -271,6 +255,22 @@
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 5,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repos",
+  "repoURL": "",
+  "leakURL": "",
+  "rule": "AWS Access Key",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
   "lineNumber": 6,
@@ -320,35 +320,35 @@
   "tags": "key, AWS"
  },
  {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE', #gitleaks:allow",
   "lineNumber": 5,
   "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "commit": "8d1fb60d2d80f0590f191ed5ace1e45ef780909a",
   "repo": "test_repo_9",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "commit 1 with secrets\n",
-  "author": "zach rice",
-  "email": "zricer@protonmail.com",
+  "commitMessage": "gitleaks allow secret\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
   "file": "server.test.py",
-  "date": "2019-10-24T09:29:27-04:00",
+  "date": "2020-08-12T13:36:20-04:00",
   "tags": "key, AWS"
  },
  {
-  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE', #gitleaks:allow",
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
   "lineNumber": 5,
   "offender": "AKIAIO5FODNN7EXAMPLE",
-  "commit": "8d1fb60d2d80f0590f191ed5ace1e45ef780909a",
-  "repo": "test_repo_9",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repos",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",
-  "commitMessage": "gitleaks allow secret\n",
-  "author": "Zach Rice",
-  "email": "zrice@gitlab.com",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
   "file": "server.test.py",
-  "date": "2020-08-12T13:36:20-04:00",
+  "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
  }
 ]

test_data/test_local_owner_aws_leak_depth_2.json

@@ -4,7 +4,7 @@
   "lineNumber": 5,
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
-  "repo": "test_repo_1",
+  "repo": "test_repos",
   "repoURL": "",
   "leakURL": "",
   "rule": "AWS Access Key",

test_data/test_local_repo_eight.json

@@ -5,14 +5,15 @@
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
   "repo": "test_repo_8",
+  "repoURL": "",
+  "leakURL": "",
   "rule": "AWS Access Key",
   "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
   "author": "Zachary Rice",
   "email": "zricer@protonmail.com",
   "file": "dummy.txt",
   "date": "2020-07-25T14:44:48-04:00",
-  "tags": "key, AWS",
-  "operation": "addition"
+  "tags": "key, AWS"
  },
  {
   "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
@@ -20,14 +21,15 @@
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "ce7e8177bbf8a172c06b6a1e370a374d5c19f660",
   "repo": "test_repo_8",
+  "repoURL": "",
+  "leakURL": "",
   "rule": "AWS Access Key",
   "commitMessage": "dummy.txt w/ text",
   "author": "Zachary Rice",
   "email": "zricer@protonmail.com",
   "file": "dummy.txt",
   "date": "2020-07-25T14:39:11-04:00",
-  "tags": "key, AWS",
-  "operation": "addition"
+  "tags": "key, AWS"
  },
  {
   "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
@@ -35,13 +37,14 @@
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "9267bc86ec1497471cbc6f3308f3527f7ef34b9d",
   "repo": "test_repo_8",
+  "repoURL": "",
+  "leakURL": "",
   "rule": "AWS Access Key",
   "commitMessage": "Update dummy.txt",
   "author": "Zachary Rice",
   "email": "zricer@protonmail.com",
   "file": "dummy.txt",
   "date": "2020-07-25T14:41:11-04:00",
-  "tags": "key, AWS",
-  "operation": "addition"
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_repo_one_aws_leak.json

@@ -5,13 +5,14 @@
   "offender": "AKIAIO5FODNN7EXAMPLE",
   "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
   "repo": "test_repo_1",
+  "repoURL": "",
+  "leakURL": "",
   "rule": "AWS Access Key",
   "commitMessage": "commit 1 with secrets\n",
   "author": "zach rice",
   "email": "zricer@protonmail.com",
   "file": "server.test.py",
   "date": "2019-10-24T09:29:27-04:00",
-  "tags": "key, AWS",
-  "operation": "addition"
+  "tags": "key, AWS"
  }
 ]