Merge pull request #317 from zricethezav/feature/commit-range

Adding commit range feature via 'commit-to' and 'commit-from' options

README.md

@@ -55,34 +55,39 @@ Usage:
   gitleaks [OPTIONS]
 
 Application Options:
-  -v, --verbose       Show verbose output from audit
-  -r, --repo=         Target repository
-      --config=       config path
-      --disk          Clones repo(s) to disk
-      --version       version number
-      --timeout=      Timeout (s)
-      --username=     Username for git repo
-      --password=     Password for git repo
-      --access-token= Access token for git repo
-      --commit=       sha of commit to audit
-      --threads=      Maximum number of threads gitleaks spawns
-      --ssh-key=      path to ssh key used for auth
-      --uncommitted   run gitleaks on uncommitted code
-      --repo-path=    Path to repo
-      --owner-path=   Path to owner directory (repos discovered)
-      --branch=       Branch to audit
-      --report=       path to write json leaks file
-      --redact        redact secrets from log messages and leaks
-      --debug         log debug messages
-      --repo-config   Load config from target repo. Config file must be ".gitleaks.toml" or "gitleaks.toml"
-      --pretty        Pretty print json if leaks are present
-      --host=         git hosting service like gitlab or github. Supported hosts include: Github, Gitlab
-      --org=          organization to audit
-      --user=         user to audit
-      --pr=           pull/merge request url
+  -v, --verbose        Show verbose output from audit
+  -r, --repo=          Target repository
+      --config=        config path
+      --disk           Clones repo(s) to disk
+      --version        version number
+      --timeout=       Timeout (s)
+      --username=      Username for git repo
+      --password=      Password for git repo
+      --access-token=  Access token for git repo
+      --commit=        sha of commit to audit
+      --threads=       Maximum number of threads gitleaks spawns
+      --ssh-key=       path to ssh key used for auth
+      --uncommitted    run gitleaks on uncommitted code
+      --repo-path=     Path to repo
+      --owner-path=    Path to owner directory (repos discovered)
+      --branch=        Branch to audit
+      --report=        path to write json leaks file
+      --report-format= json or csv (default: json)
+      --redact         redact secrets from log messages and leaks
+      --debug          log debug messages
+      --repo-config    Load config from target repo. Config file must be ".gitleaks.toml" or "gitleaks.toml"
+      --pretty         Pretty print json if leaks are present
+      --commit-from=   Commit to start audit from
+      --commit-to=     Commit to stop audit
+      --host=          git hosting service like gitlab or github. Supported hosts include: Github, Gitlab
+      --baseurl=       Base URL for API requests. Defaults to the public GitLab or GitHub API, but can be set to a domain endpoint to use with a self hosted server.
+      --org=           organization to audit
+      --user=          user to audit
+      --pr=            pull/merge request url
 
 Help Options:
-  -h, --help          Show this help message
+  -h, --help           Show this help message
+
 ```
 
 ### Docker usage examples

audit/audit.go

@@ -40,7 +40,7 @@ func runHelper(r *Repo) error {
 		}
 
 		// Check if we are checking uncommitted files. This is the default behavior
-		// for a "$gitleaks" command with no options set
+		// for a "$ gitleaks" command with no options set
 		if r.Manager.Opts.CheckUncommitted() {
 			if err := r.AuditUncommitted(); err != nil {
 				return err

audit/audit_test.go

@@ -1,16 +1,20 @@
 package audit
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/sergi/go-diff/diffmatchpatch"
-	"github.com/zricethezav/gitleaks/config"
-	"github.com/zricethezav/gitleaks/manager"
-	"github.com/zricethezav/gitleaks/options"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"runtime"
-	"strings"
+	"sort"
 	"testing"
+
+	"github.com/zricethezav/gitleaks/config"
+	"github.com/zricethezav/gitleaks/manager"
+	"github.com/zricethezav/gitleaks/options"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
 )
 
 const testRepoBase = "../test_data/test_repos/"
@@ -71,6 +75,37 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_repo_two_leaks.json",
 		},
+		{
+			description: "test local repo two leaks from commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_from.json.got",
+				ReportFormat: "json",
+				CommitFrom:   "996865bb912f3bc45898a370a13aadb315014b55",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_from.json",
+		},
+		{
+			description: "test local repo two leaks to commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_to.json.got",
+				ReportFormat: "json",
+				CommitTo:     "996865bb912f3bc45898a370a13aadb315014b55",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_to.json",
+		},
+		{
+			description: "test local repo two leaks range commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_range.json.got",
+				ReportFormat: "json",
+				CommitFrom:   "d8ac0b73aeeb45843319cdc5ce506516eb49bf7a",
+				CommitTo:     "51f6dcf6b89b93f4075ba92c400b075631a6cc93",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_range.json",
+		},
 		{
 			description: "test local repo two leaks globally whitelisted",
 			opts: options.Options{
@@ -312,6 +347,10 @@ func TestAuditUncommited(t *testing.T) {
 }
 
 func fileCheck(wantPath, gotPath string) error {
+	var (
+		gotLeaks  []manager.Leak
+		wantLeaks []manager.Leak
+	)
 	want, err := ioutil.ReadFile(wantPath)
 	if err != nil {
 		return err
@@ -322,7 +361,20 @@ func fileCheck(wantPath, gotPath string) error {
 		return err
 	}
 
-	if strings.Trim(string(want), "\n") != strings.Trim(string(got), "\n") {
+	err = json.Unmarshal(got, &gotLeaks)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(want, &wantLeaks)
+	if err != nil {
+		return nil
+	}
+
+	sort.Slice(gotLeaks, func(i, j int) bool { return (gotLeaks)[i].Commit < (gotLeaks)[j].Commit })
+	sort.Slice(wantLeaks, func(i, j int) bool { return (wantLeaks)[i].Commit < (wantLeaks)[j].Commit })
+
+	if !reflect.DeepEqual(gotLeaks, wantLeaks) {
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(string(want), string(got), false)
 		return fmt.Errorf("does not equal: %s", dmp.DiffPrettyText(diffs))

audit/repo.go

@@ -247,7 +247,7 @@ func (repo *Repo) Audit() error {
 	semaphore := make(chan bool, howManyThreads(repo.Manager.Opts.Threads))
 	wg := sync.WaitGroup{}
 	err = cIter.ForEach(func(c *object.Commit) error {
-		if c == nil {
+		if c == nil || c.Hash.String() == repo.Manager.Opts.CommitTo {
 			return storer.ErrStop
 		}
 

audit/util.go

@@ -18,12 +18,10 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
 
-const maxLineLen = 200
-
 // Inspect patch accepts a patch, commit, and repo. If the patches contains files that are
 // binary, then gitleaks will skip auditing that file OR if a file is matched on
 // whitelisted files set in the configuration. If a global rule for files is defined and a filename
-// matches said global rule, then a laek is sent to the manager.
+// matches said global rule, then a leak is sent to the manager.
 // After that, file chunks are created which are then inspected by InspectString()
 func inspectPatch(patch *object.Patch, c *object.Commit, repo *Repo) {
 	for _, f := range patch.FilePatches() {
@@ -339,8 +337,11 @@ func fileMatched(f interface{}, re *regexp.Regexp) bool {
 // It is similar to `git log {branch}`. Default behavior is to log ALL branches so
 // gitleaks gets the full git history.
 func getLogOptions(repo *Repo) (*git.LogOptions, error) {
+	var logOpts git.LogOptions
+	if repo.Manager.Opts.CommitFrom != "" {
+		logOpts.From = plumbing.NewHash(repo.Manager.Opts.CommitFrom)
+	}
 	if repo.Manager.Opts.Branch != "" {
-		var logOpts git.LogOptions
 		refs, err := repo.Storer.IterReferences()
 		if err != nil {
 			return nil, err
@@ -368,6 +369,9 @@ func getLogOptions(repo *Repo) (*git.LogOptions, error) {
 		}
 		return &logOpts, nil
 	}
+	if !logOpts.From.IsZero() {
+		return &logOpts, nil
+	}
 	return &git.LogOptions{All: true}, nil
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/google/go-cmp v0.4.0 // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4
 	github.com/jessevdk/go-flags v1.4.0

go.sum

@@ -21,6 +21,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -101,6 +103,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

manager/manager.go

@@ -248,9 +248,9 @@ func (manager *Manager) DebugOutput() {
 	log.Debugf("| Individual Regex Times |\n")
 	log.Debugf("--------------------------\n")
 	for k, v := range manager.metadata.RegexTime {
-		fmt.Fprintf(w, "%s\t%s\n", k, durafmt.Parse(time.Duration(v)*time.Nanosecond))
+		_, _ = fmt.Fprintf(w, "%s\t%s\n", k, durafmt.Parse(time.Duration(v)*time.Nanosecond))
 	}
-	w.Flush()
+	_ = w.Flush()
 
 }
 
@@ -282,13 +282,13 @@ func (manager *Manager) Report() error {
 			}
 		} else {
 			w := csv.NewWriter(file)
-			w.Write([]string{"repo", "line", "commit", "offender", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
+			_ = w.Write([]string{"repo", "line", "commit", "offender", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
 			for _, leak := range manager.GetLeaks() {
 				w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Rule, leak.Tags, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
 			}
 			w.Flush()
 		}
-		file.Close()
+		_ = file.Close()
 
 		log.Infof("report written to %s", manager.Opts.Report)
 	}

options/options.go

@@ -49,6 +49,8 @@ type Options struct {
 	Debug        bool   `long:"debug" description:"log debug messages"`
 	RepoConfig   bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
 	PrettyPrint  bool   `long:"pretty" description:"Pretty print json if leaks are present"`
+	CommitFrom   string `long:"commit-from" description:"Commit to start audit from"`
+	CommitTo     string `long:"commit-to" description:"Commit to stop audit"`
 
 	// Hosts
 	Host         string `long:"host" description:"git hosting service like gitlab or github. Supported hosts include: Github, Gitlab"`

test_data/test_local_repo_two_leaks_commit_from.json

@@ -0,0 +1,67 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, AWS"
+ }
+]

test_data/test_local_repo_two_leaks_commit_range.json

@@ -0,0 +1,54 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "996865bb912f3bc45898a370a13aadb315014b55",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "committing pem\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:07:41-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ }
+]

test_data/test_local_repo_two_leaks_commit_to.json

@@ -0,0 +1,54 @@
+[
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "AKIALALEMEL33243OLIB",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ }
+]