Adding commit range feature via 'commit-to' and 'commit-from' options

audit/audit.go

@@ -40,7 +40,7 @@ func runHelper(r *Repo) error {
 		}
 
 		// Check if we are checking uncommitted files. This is the default behavior
-		// for a "$gitleaks" command with no options set
+		// for a "$ gitleaks" command with no options set
 		if r.Manager.Opts.CheckUncommitted() {
 			if err := r.AuditUncommitted(); err != nil {
 				return err

audit/audit_test.go

@@ -1,6 +1,7 @@
 package audit
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/sergi/go-diff/diffmatchpatch"
 	"github.com/zricethezav/gitleaks/config"
@@ -8,8 +9,9 @@ import (
 	"github.com/zricethezav/gitleaks/options"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"runtime"
-	"strings"
+	"sort"
 	"testing"
 )
 
@@ -71,6 +73,37 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_repo_two_leaks.json",
 		},
+		{
+			description: "test local repo two leaks from commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_from.json.got",
+				ReportFormat: "json",
+				CommitFrom: "996865bb912f3bc45898a370a13aadb315014b55",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_from.json",
+		},
+		{
+			description: "test local repo two leaks to commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_to.json.got",
+				ReportFormat: "json",
+				CommitTo: "996865bb912f3bc45898a370a13aadb315014b55",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_to.json",
+		},
+		{
+			description: "test local repo two leaks range commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_range.json.got",
+				ReportFormat: "json",
+				CommitFrom: "d8ac0b73aeeb45843319cdc5ce506516eb49bf7a",
+				CommitTo: "51f6dcf6b89b93f4075ba92c400b075631a6cc93",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_range.json",
+		},
 		{
 			description: "test local repo two leaks globally whitelisted",
 			opts: options.Options{
@@ -312,6 +345,8 @@ func TestAuditUncommited(t *testing.T) {
 }
 
 func fileCheck(wantPath, gotPath string) error {
+	var gotLeaks []manager.Leak
+	var wantLeaks []manager.Leak
 	want, err := ioutil.ReadFile(wantPath)
 	if err != nil {
 		return err
@@ -322,11 +357,29 @@ func fileCheck(wantPath, gotPath string) error {
 		return err
 	}
 
-	if strings.Trim(string(want), "\n") != strings.Trim(string(got), "\n") {
+	// TODO compare JSONs
+	err = json.Unmarshal(got, &gotLeaks)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(want, &wantLeaks)
+	if err != nil {
+		return nil
+	}
+
+	sort.Slice(gotLeaks, func(i, j int) bool { return (gotLeaks)[i].Commit < (gotLeaks)[j].Commit})
+	sort.Slice(wantLeaks, func(i, j int) bool { return (wantLeaks)[i].Commit < (wantLeaks)[j].Commit})
+
+
+	if !reflect.DeepEqual(gotLeaks, wantLeaks) {
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(string(want), string(got), false)
 		return fmt.Errorf("does not equal: %s", dmp.DiffPrettyText(diffs))
+		// return fmt.Errorf("does not equal: %s", cmp.Diff(gotLeaks, wantLeaks))
 	}
+	//if strings.Trim(string(want), "\n") != strings.Trim(string(got), "\n") {
+	//}
 	if err := os.Remove(gotPath); err != nil {
 		return err
 	}

audit/repo.go

@@ -247,7 +247,7 @@ func (repo *Repo) Audit() error {
 	semaphore := make(chan bool, howManyThreads(repo.Manager.Opts.Threads))
 	wg := sync.WaitGroup{}
 	err = cIter.ForEach(func(c *object.Commit) error {
-		if c == nil {
+		if c == nil || c.Hash.String() == repo.Manager.Opts.CommitTo {
 			return storer.ErrStop
 		}
 

audit/util.go

@@ -339,8 +339,11 @@ func fileMatched(f interface{}, re *regexp.Regexp) bool {
 // It is similar to `git log {branch}`. Default behavior is to log ALL branches so
 // gitleaks gets the full git history.
 func getLogOptions(repo *Repo) (*git.LogOptions, error) {
+	var logOpts git.LogOptions
+	if repo.Manager.Opts.CommitFrom != "" {
+		logOpts.From = plumbing.NewHash(repo.Manager.Opts.CommitFrom)
+	}
 	if repo.Manager.Opts.Branch != "" {
-		var logOpts git.LogOptions
 		refs, err := repo.Storer.IterReferences()
 		if err != nil {
 			return nil, err
@@ -368,6 +371,9 @@ func getLogOptions(repo *Repo) (*git.LogOptions, error) {
 		}
 		return &logOpts, nil
 	}
+	if !logOpts.From.IsZero() {
+		return &logOpts, nil
+	}
 	return &git.LogOptions{All: true}, nil
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/google/go-cmp v0.4.0 // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4
 	github.com/jessevdk/go-flags v1.4.0

go.sum

@@ -21,6 +21,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -101,6 +103,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

manager/manager.go

@@ -248,9 +248,9 @@ func (manager *Manager) DebugOutput() {
 	log.Debugf("| Individual Regex Times |\n")
 	log.Debugf("--------------------------\n")
 	for k, v := range manager.metadata.RegexTime {
-		fmt.Fprintf(w, "%s\t%s\n", k, durafmt.Parse(time.Duration(v)*time.Nanosecond))
+		_, _ = fmt.Fprintf(w, "%s\t%s\n", k, durafmt.Parse(time.Duration(v)*time.Nanosecond))
 	}
-	w.Flush()
+	_ = w.Flush()
 
 }
 
@@ -282,13 +282,13 @@ func (manager *Manager) Report() error {
 			}
 		} else {
 			w := csv.NewWriter(file)
-			w.Write([]string{"repo", "line", "commit", "offender", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
+			_ = w.Write([]string{"repo", "line", "commit", "offender", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
 			for _, leak := range manager.GetLeaks() {
 				w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Rule, leak.Tags, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
 			}
 			w.Flush()
 		}
-		file.Close()
+		_ = file.Close()
 
 		log.Infof("report written to %s", manager.Opts.Report)
 	}

options/options.go

@@ -49,6 +49,8 @@ type Options struct {
 	Debug        bool   `long:"debug" description:"log debug messages"`
 	RepoConfig   bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
 	PrettyPrint  bool   `long:"pretty" description:"Pretty print json if leaks are present"`
+	CommitFrom   string `long:"commit-from" description:"Commit to start audit from"`
+	CommitTo     string `long:"commit-to" description:"Commit to stop audit"`
 
 	// Hosts
 	Host         string `long:"host" description:"git hosting service like gitlab or github. Supported hosts include: Github, Gitlab"`

test_data/test_local_repo_two_leaks_commit_from.json

@@ -0,0 +1,67 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, AWS"
+ }
+]

test_data/test_local_repo_two_leaks_commit_range.json

@@ -0,0 +1,54 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ }
+]

test_data/test_local_repo_two_leaks_commit_to.json

@@ -0,0 +1,54 @@
+[
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ }
+]