Add commits and commits-file option (#433)

options/options.go

@@ -28,35 +28,40 @@ const (
 
 // Options stores values of command line options
 type Options struct {
-	Verbose       bool   `short:"v" long:"verbose" description:"Show verbose output from scan"`
-	Repo          string `short:"r" long:"repo" description:"Target repository"`
-	Config        string `long:"config" description:"config path"`
-	Disk          bool   `long:"disk" description:"Clones repo(s) to disk"`
-	Version       bool   `long:"version" description:"version number"`
-	Username      string `long:"username" description:"Username for git repo"`
-	Password      string `long:"password" description:"Password for git repo"`
-	AccessToken   string `long:"access-token" description:"Access token for git repo"`
-	Commit        string `long:"commit" description:"sha of commit to scan or \"latest\" to scan the last commit of the repository"`
-	FilesAtCommit string `long:"files-at-commit" description:"sha of commit to scan all files at commit"`
-	Threads       int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	SSH           string `long:"ssh-key" description:"path to ssh key used for auth"`
-	Uncommited    bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
-	RepoPath      string `long:"repo-path" description:"Path to repo"`
-	OwnerPath     string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
-	Branch        string `long:"branch" description:"Branch to scan"`
-	Report        string `long:"report" description:"path to write json leaks file"`
-	ReportFormat  string `long:"report-format" default:"json" description:"json, csv, sarif"`
-	Redact        bool   `long:"redact" description:"redact secrets from log messages and leaks"`
-	Debug         bool   `long:"debug" description:"log debug messages"`
-	RepoConfig    bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
-	PrettyPrint   bool   `long:"pretty" description:"Pretty print json if leaks are present"`
-	CommitFrom    string `long:"commit-from" description:"Commit to start scan from"`
-	CommitTo      string `long:"commit-to" description:"Commit to stop scan"`
-	CommitSince   string `long:"commit-since" description:"Scan commits more recent than a specific date. Ex: '2006-01-02' or '2006-01-02T15:04:05-0700' format."`
-	CommitUntil   string `long:"commit-until" description:"Scan commits older than a specific date. Ex: '2006-01-02' or '2006-01-02T15:04:05-0700' format."`
-	Timeout       string `long:"timeout" description:"Time allowed per scan. Ex: 10us, 30s, 1m, 1h10m1s"`
-	Depth         int    `long:"depth" description:"Number of commits to scan"`
-	Deletion      bool   `long:"include-deletion" description:"Scan for patch deletions in addition to patch additions"`
+	Verbose         bool   `short:"v" long:"verbose" description:"Show verbose output from scan"`
+	Repo            string `short:"r" long:"repo" description:"Target repository"`
+	Config          string `long:"config" description:"config path"`
+	Disk            bool   `long:"disk" description:"Clones repo(s) to disk"`
+	Version         bool   `long:"version" description:"version number"`
+	Username        string `long:"username" description:"Username for git repo"`
+	Password        string `long:"password" description:"Password for git repo"`
+	AccessToken     string `long:"access-token" description:"Access token for git repo"`
+	FilesAtCommit   string `long:"files-at-commit" description:"sha of commit to scan all files at commit"`
+	Threads         int    `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	SSH             string `long:"ssh-key" description:"path to ssh key used for auth"`
+	Uncommited      bool   `long:"uncommitted" description:"run gitleaks on uncommitted code"`
+	RepoPath        string `long:"repo-path" description:"Path to repo"`
+	OwnerPath       string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
+	Branch          string `long:"branch" description:"Branch to scan"`
+	Report          string `long:"report" description:"path to write json leaks file"`
+	ReportFormat    string `long:"report-format" default:"json" description:"json, csv, sarif"`
+	Redact          bool   `long:"redact" description:"redact secrets from log messages and leaks"`
+	Debug           bool   `long:"debug" description:"log debug messages"`
+	RepoConfig      bool   `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\" or \"gitleaks.toml\""`
+	PrettyPrint     bool   `long:"pretty" description:"Pretty print json if leaks are present"`
+
+	// Commit Options
+	Commit      string `long:"commit" description:"sha of commit to scan or \"latest\" to scan the last commit of the repository"`
+	Commits     string `long:"commits" description:"comma separated list of a commits to scan"`
+	CommitsFile string `long:"commits-file" description:"file of new line separated list of a commits to scan"`
+	CommitFrom  string `long:"commit-from" description:"Commit to start scan from"`
+	CommitTo    string `long:"commit-to" description:"Commit to stop scan"`
+	CommitSince string `long:"commit-since" description:"Scan commits more recent than a specific date. Ex: '2006-01-02' or '2006-01-02T15:04:05-0700' format."`
+	CommitUntil string `long:"commit-until" description:"Scan commits older than a specific date. Ex: '2006-01-02' or '2006-01-02T15:04:05-0700' format."`
+
+	Timeout         string `long:"timeout" description:"Time allowed per scan. Ex: 10us, 30s, 1m, 1h10m1s"`
+	Depth           int    `long:"depth" description:"Number of commits to scan"`
+	Deletion        bool   `long:"include-deletion" description:"Scan for patch deletions in addition to patch additions"`
 
 	// Hosts
 	Host         string `long:"host" description:"git hosting service like gitlab or github. Supported hosts include: Github, Gitlab"`

scan/scan.go

@@ -1,9 +1,12 @@
 package scan
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
+	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -71,11 +74,36 @@ func (repo *Repo) Scan() error {
 
 	scanTimeStart := time.Now()
 
-	// scan Commit patches OR all files at Commit. See https://github.com/zricethezav/gitleaks/issues/326
+	// See https://github.com/zricethezav/gitleaks/issues/326
+	// Scan commit patches, all files at a commit, or a range of commits
 	if repo.Manager.Opts.Commit != "" {
 		return scanCommit(repo.Manager.Opts.Commit, repo, scanCommitPatches)
 	} else if repo.Manager.Opts.FilesAtCommit != "" {
 		return scanCommit(repo.Manager.Opts.FilesAtCommit, repo, scanFilesAtCommit)
+	} else if repo.Manager.Opts.Commits != "" {
+		commits := strings.Split(repo.Manager.Opts.Commits, ",")
+		for _, c := range commits {
+			err := scanCommit(c, repo, scanCommitPatches)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	} else if repo.Manager.Opts.CommitsFile != "" {
+		file, err := os.Open(repo.Manager.Opts.CommitsFile)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			err := scanCommit(scanner.Text(), repo, scanCommitPatches)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
 	}
 
 	logOpts, err := getLogOptions(repo)

scan/scan_test.go

@@ -96,16 +96,36 @@ func TestScan(t *testing.T) {
 			wantPath: "../test_data/test_local_repo_two_leaks_commit_to.json",
 		},
 		{
-			description: "test local repo two leaks range Commit",
+			description: "test local repo two leaks to from Commit",
 			opts: options.Options{
 				RepoPath:     "../test_data/test_repos/test_repo_2",
-				Report:       "../test_data/test_local_repo_two_leaks_commit_range.json.got",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_to_from.json.got",
 				ReportFormat: "json",
 				CommitFrom:   "d8ac0b73aeeb45843319cdc5ce506516eb49bf7a",
 				CommitTo:     "51f6dcf6b89b93f4075ba92c400b075631a6cc93",
 			},
+			wantPath: "../test_data/test_local_repo_two_leaks_commit_to_from.json",
+		},
+		{
+			description: "test local repo two leaks list Commits",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_commit_range.json.got",
+				ReportFormat: "json",
+				Commits:      "d8ac0b73aeeb45843319cdc5ce506516eb49bf7a,996865bb912f3bc45898a370a13aadb315014b55,17471a5fda722a9e423f1a0d3f0d267ea009d41c,51f6dcf6b89b93f4075ba92c400b075631a6cc93,b10b3e2cb320a8c211fda94c4567299d37de7776",
+			},
 			wantPath: "../test_data/test_local_repo_two_leaks_commit_range.json",
 		},
+		{
+			description: "test local repo two leaks file list commits",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_2",
+				Report:       "../test_data/test_local_repo_two_leaks_file_commit_range.json.got",
+				ReportFormat: "json",
+				CommitsFile:  "../test_data/test_options/test_local_repo_commits.txt",
+			},
+			wantPath: "../test_data/test_local_repo_two_leaks_file_commit_range.json",
+		},
 		{
 			description: "test local repo two leaks globally allowlisted",
 			opts: options.Options{

test_data/test_local_repo_two_leaks_commit_range.json

@@ -13,5 +13,20 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS",
   "operation": "addition"
+ },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "lineNumber": 3,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
  }
 ]

test_data/test_local_repo_two_leaks_commit_to_from.json

@@ -0,0 +1,17 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "lineNumber": 5,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ }
+]

test_data/test_local_repo_two_leaks_file_commit_range.json

@@ -0,0 +1,32 @@
+[
+ {
+  "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
+  "lineNumber": 5,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "lineNumber": 3,
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ }
+]

test_data/test_options/test_local_repo_commits.txt

@@ -0,0 +1,5 @@
+d8ac0b73aeeb45843319cdc5ce506516eb49bf7a
+996865bb912f3bc45898a370a13aadb315014b55
+17471a5fda722a9e423f1a0d3f0d267ea009d41c
+51f6dcf6b89b93f4075ba92c400b075631a6cc93
+b10b3e2cb320a8c211fda94c4567299d37de7776