Merge pull request #329 from zricethezav/bug/commit-scan

Bug/commit scan

audit/audit_test.go

@@ -243,6 +243,26 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak_depth_2.json",
 		},
+		{
+			description: "test local repo five files at commit",
+			opts: options.Options{
+				RepoPath:      "../test_data/test_repos/test_repo_5",
+				Report:        "../test_data/test_local_repo_five_files_at_commit.json.got",
+				FilesAtCommit: "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+				ReportFormat:  "json",
+			},
+			wantPath: "../test_data/test_local_repo_five_files_at_commit.json",
+		},
+		{
+			description: "test local repo five at commit",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_5",
+				Report:       "../test_data/test_local_repo_five_commit.json.got",
+				Commit:       "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_five_commit.json",
+		},
 	}
 
 	for _, test := range tests {

audit/repo.go

@@ -236,19 +236,12 @@ func (repo *Repo) Audit() error {
 
 	auditTimeStart := time.Now()
 
-	// audit single Commit
+	// audit commit patches OR all files at commit. See https://github.com/zricethezav/gitleaks/issues/326
+	// TODO having --commit= and --files-at-commit= set should probably be guarded against
 	if repo.Manager.Opts.Commit != "" {
-		h := plumbing.NewHash(repo.Manager.Opts.Commit)
-		c, err := repo.CommitObject(h)
-		if err != nil {
-			return err
-		}
-
-		err = inspectCommit(c, repo)
-		if err != nil {
-			return err
-		}
-		return nil
+		return inspectCommit(repo.Manager.Opts.Commit, repo, inspectCommitPatches)
+	} else if repo.Manager.Opts.FilesAtCommit != "" {
+		return inspectCommit(repo.Manager.Opts.FilesAtCommit, repo, inspectFilesAtCommit)
 	}
 
 	logOpts, err := getLogOptions(repo)
@@ -271,7 +264,7 @@ func (repo *Repo) Audit() error {
 
 		if len(c.ParentHashes) == 0 {
 			cc++
-			err = inspectCommit(c, repo)
+			err = inspectFilesAtCommit(c, repo)
 			if err != nil {
 				return err
 			}

audit/util.go

@@ -255,11 +255,57 @@ func InspectString(content string, c *object.Commit, repo *Repo, filename string
 	}
 }
 
-// inspectCommit accepts a commit object and a repo. This function is only called when the --commit=
+type commitInspector func(c *object.Commit, repo *Repo) error
+
+func inspectCommit(hash string, repo *Repo, f commitInspector) error {
+	h := plumbing.NewHash(hash)
+	c, err := repo.CommitObject(h)
+	if err != nil {
+		return err
+	}
+	return f(c, repo)
+}
+
+// inspectCommitPatches accepts a commit object and a repo. This function is only called when the --commit=
+// option has been set. That option tells gitleaks to look only at a single commit and check the contents
+// of said commit. Similar to inspectPatch(), if the files contained in the commit are a binaries or if they are
+// whitelisted then those files will be skipped.
+func inspectCommitPatches(c *object.Commit, repo *Repo) error {
+	if len(c.ParentHashes) == 0 {
+		err := inspectFilesAtCommit(c, repo)
+		if err != nil {
+			return err
+		}
+	}
+
+	return c.Parents().ForEach(func(parent *object.Commit) error {
+		defer func() {
+			if err := recover(); err != nil {
+				// sometimes the patch generation will fail due to a known bug in
+				// sergi's go-diff: https://github.com/sergi/go-diff/issues/89.
+				// Once a fix has been merged I will remove this recover.
+				return
+			}
+		}()
+		if repo.timeoutReached() {
+			return nil
+		}
+		start := time.Now()
+		patch, err := c.Patch(parent)
+		if err != nil {
+			return fmt.Errorf("could not generate patch")
+		}
+		repo.Manager.RecordTime(manager.PatchTime(howLong(start)))
+		inspectPatch(patch, c, repo)
+		return nil
+	})
+}
+
+// inspectFilesAtCommit accepts a commit object and a repo. This function is only called when the --commit=
 // option has been set. That option tells gitleaks to look only at a single commit and check the contents
 // of said commit. Similar to inspectPatch(), if the files contained in the commit are a binaries or if they are
 // whitelisted then those files will be skipped.
-func inspectCommit(c *object.Commit, repo *Repo) error {
+func inspectFilesAtCommit(c *object.Commit, repo *Repo) error {
 	fIter, err := c.Files()
 	if err != nil {
 		return err

options/options.go

@@ -27,31 +27,32 @@ const (
 
 // Options stores values of command line options
 type Options struct {
-	Verbose      bool   `short:"v" long:"verbose" description:"Show verbose output from audit"`
-	Repo         string `short:"r" long:"repo" description:"Target repository"`
-	Config       string `long:"config" description:"config path"`
-	Disk         bool   `long:"disk" description:"Clones repo(s) to disk"`
-	Version      bool   `long:"version" description:"version number"`
-	Username     string `long:"username" description:"Username for git repo"`
-	Password     string `long:"password" description:"Password for git repo"`
-	AccessToken  string `long:"access-token" description:"Access token for git repo"`
-	Commit       string `long:"commit" description:"sha of commit to audit"`
-	Threads      int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	SSH          string `long:"ssh-key" description:"path to ssh key used for auth"`
-	Uncommited   bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
-	RepoPath     string `long:"repo-path" description:"Path to repo"`
-	OwnerPath    string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
-	Branch       string `long:"branch" description:"Branch to audit"`
-	Report       string `long:"report" description:"path to write json leaks file"`
-	ReportFormat string `long:"report-format" default:"json" description:"json or csv"`
-	Redact       bool   `long:"redact" description:"redact secrets from log messages and leaks"`
-	Debug        bool   `long:"debug" description:"log debug messages"`
-	RepoConfig   bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
-	PrettyPrint  bool   `long:"pretty" description:"Pretty print json if leaks are present"`
-	CommitFrom   string `long:"commit-from" description:"Commit to start audit from"`
-	CommitTo     string `long:"commit-to" description:"Commit to stop audit"`
-	Timeout      string `long:"timeout" description:"Time allowed per audit. Ex: 10us, 30s, 1m, 1h10m1s"`
-	Depth        int    `long:"depth" description:"Number of commits to audit"`
+	Verbose       bool   `short:"v" long:"verbose" description:"Show verbose output from audit"`
+	Repo          string `short:"r" long:"repo" description:"Target repository"`
+	Config        string `long:"config" description:"config path"`
+	Disk          bool   `long:"disk" description:"Clones repo(s) to disk"`
+	Version       bool   `long:"version" description:"version number"`
+	Username      string `long:"username" description:"Username for git repo"`
+	Password      string `long:"password" description:"Password for git repo"`
+	AccessToken   string `long:"access-token" description:"Access token for git repo"`
+	Commit        string `long:"commit" description:"sha of commit to audit"`
+	FilesAtCommit string `long:"files-at-commit" description:"sha of commit to audit all files at commit"`
+	Threads       int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	SSH           string `long:"ssh-key" description:"path to ssh key used for auth"`
+	Uncommited    bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
+	RepoPath      string `long:"repo-path" description:"Path to repo"`
+	OwnerPath     string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
+	Branch        string `long:"branch" description:"Branch to audit"`
+	Report        string `long:"report" description:"path to write json leaks file"`
+	ReportFormat  string `long:"report-format" default:"json" description:"json or csv"`
+	Redact        bool   `long:"redact" description:"redact secrets from log messages and leaks"`
+	Debug         bool   `long:"debug" description:"log debug messages"`
+	RepoConfig    bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
+	PrettyPrint   bool   `long:"pretty" description:"Pretty print json if leaks are present"`
+	CommitFrom    string `long:"commit-from" description:"Commit to start audit from"`
+	CommitTo      string `long:"commit-to" description:"Commit to stop audit"`
+	Timeout       string `long:"timeout" description:"Time allowed per audit. Ex: 10us, 30s, 1m, 1h10m1s"`
+	Depth         int    `long:"depth" description:"Number of commits to audit"`
 
 	// Hosts
 	Host         string `long:"host" description:"git hosting service like gitlab or github. Supported hosts include: Github, Gitlab"`

test_data/test_local_owner_aws_leak.json

@@ -232,5 +232,44 @@
   "file": "secrets.md",
   "date": "2019-10-25T13:35:03-04:00",
   "tags": "key, API, generic"
+ },
+ {
+  "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "\naws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "AKIAIO5FODNN7EXAMPL2",
+  "commit": "ca71fcdeda15f25f0cc661d90e8785c255925c27",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "introduce more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:08:04-05:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "1f2a4abc47dabf991e6af6f9770867ce0ac1f360",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "introduce secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:07:34-05:00",
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_owner_aws_leak_depth_2.json

@@ -115,5 +115,31 @@
   "file": "secrets.md",
   "date": "2019-10-25T13:54:08-04:00",
   "tags": "key, API, generic"
+ },
+ {
+  "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "\naws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "AKIAIO5FODNN7EXAMPL2",
+  "commit": "ca71fcdeda15f25f0cc661d90e8785c255925c27",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "introduce more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:08:04-05:00",
+  "tags": "key, AWS"
  }
 ]

test_data/test_local_repo_five_commit.json

@@ -0,0 +1,15 @@
+[
+ {
+  "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ }
+]

test_data/test_local_repo_five_files_at_commit.json

@@ -0,0 +1,41 @@
+[
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "AKIAIO5FODNN7EXAMPL2",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "more_secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ }
+]

test_data/test_repos/test_repo_5/dotGit/COMMIT_EDITMSG

@@ -0,0 +1 @@
+even more secrets

test_data/test_repos/test_repo_5/dotGit/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

test_data/test_repos/test_repo_5/dotGit/config

@@ -0,0 +1,7 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true

test_data/test_repos/test_repo_5/dotGit/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

test_data/test_repos/test_repo_5/dotGit/info/exclude

@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~

test_data/test_repos/test_repo_5/dotGit/logs/HEAD

@@ -0,0 +1,4 @@
+0000000000000000000000000000000000000000 547bc0caa26ce3f20bea9ad9bb0f7e3e9dc749ec Zach Rice <zrice@gitlab.com> 1580490269 -0500	commit (initial): init
+547bc0caa26ce3f20bea9ad9bb0f7e3e9dc749ec 1f2a4abc47dabf991e6af6f9770867ce0ac1f360 Zach Rice <zrice@gitlab.com> 1580569654 -0500	commit: introduce secrets
+1f2a4abc47dabf991e6af6f9770867ce0ac1f360 ca71fcdeda15f25f0cc661d90e8785c255925c27 Zach Rice <zrice@gitlab.com> 1580569684 -0500	commit: introduce more secrets
+ca71fcdeda15f25f0cc661d90e8785c255925c27 a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0 Zach Rice <zrice@gitlab.com> 1580571022 -0500	commit: even more secrets

test_data/test_repos/test_repo_5/dotGit/logs/refs/heads/master

@@ -0,0 +1,4 @@
+0000000000000000000000000000000000000000 547bc0caa26ce3f20bea9ad9bb0f7e3e9dc749ec Zach Rice <zrice@gitlab.com> 1580490269 -0500	commit (initial): init
+547bc0caa26ce3f20bea9ad9bb0f7e3e9dc749ec 1f2a4abc47dabf991e6af6f9770867ce0ac1f360 Zach Rice <zrice@gitlab.com> 1580569654 -0500	commit: introduce secrets
+1f2a4abc47dabf991e6af6f9770867ce0ac1f360 ca71fcdeda15f25f0cc661d90e8785c255925c27 Zach Rice <zrice@gitlab.com> 1580569684 -0500	commit: introduce more secrets
+ca71fcdeda15f25f0cc661d90e8785c255925c27 a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0 Zach Rice <zrice@gitlab.com> 1580571022 -0500	commit: even more secrets

test_data/test_repos/test_repo_5/dotGit/objects/1f/2a4abc47dabf991e6af6f9770867ce0ac1f360

@@ -0,0 +1,2 @@
+x
•ŽM
+Â0F]繀’ß&
ĎŕŇÝĚdb¶•4ÝxzsW<xĽŹ¶e©]cN˝1˨�±™°”	MA«ĽĆ’µˇ�0;ŚĹŻ]z�
�‰ŘŁ�!AN�޶ś2—�}Ţš|ÍňQ‰ĺőŰĆÜ_µż
/´-7©}T~J“wň¬ĽRbĐń®óżž¨ko[>FfgjÜwń;ěH�

test_data/test_repos/test_repo_5/dotGit/objects/54/7bc0caa26ce3f20bea9ad9bb0f7e3e9dc749ec

@@ -0,0 +1,4 @@
+x
•�A
+‚!F[{Š¹@1ú«)Dt†–ítû…L�iÓéó
+­>xðÞG£÷& ½>Èd*¶š’3ç@hü‘μéZB$olEçL.dTúÈ>&<ípoÄpùÎ5·g“WÊ'ý
+Ú´qe"Ñ!ªE×›ð¿žjï&êÚQ3–

test_data/test_repos/test_repo_5/dotGit/objects/a4/c9fb737d5552fd96fce5cc7eedb23353ba9ed0

@@ -0,0 +1,2 @@
+x
•ŽA
+Â0E]çs
e2Í$-ˆx—îÒéÄl#itáéÍ\=xŸ_òº.ˆèP‹*$%.29¥ë1«O®
ê58ï�Ìæ‹n$›dÒ)ZNÄ	E¼·Ó€Ú‡ž¥Å5ßuÎîQf¸-¢pþ–†ëc©Ï8ž$¯°Ü#‹DpDF4ͶwUÿíŒ~tƒ5…]¥hÝÍ\ïES

test_data/test_repos/test_repo_5/dotGit/objects/ca/71fcdeda15f25f0cc661d90e8785c255925c27

@@ -0,0 +1 @@
+x
•ŽAj1E»ö)t�yì‘l(¥gȲ;Y–“�N\g“Óg®ÐÕ‡�÷µïû6aYøc3Xk)![ñ
)�ÊZ#'Zjð>ÇbèþdØ}‚o‹D)¹Ji9{#iÔ23&b5õ-:yÎ[ð#zƒË¦Ÿ¯qÌ÷u›¿RÎÚ÷/ðk•2¥'\ÝA�wÓþë¹í>G¯Ï#³÷að06î
ÿGó

test_data/test_repos/test_repo_5/dotGit/refs/heads/master

@@ -0,0 +1 @@
+a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0

test_data/test_repos/test_repo_5/notes.md

@@ -0,0 +1 @@
+Init

test_data/test_repos/test_repo_5/secrets.py

@@ -0,0 +1,9 @@
+aws_access_key_id='AKIAIO5FODNN7EXAMPLE'
+print(aws_access_key_id)
+
+aws_access_key_id='AKIAIO5FODNN7EXAMPL2'
+print(aws_access_key_id)
+
+more_secrets = '99432bfewaf823ec3294e231'
+print(more_secrets)
+