linting, more testing, cleaning

checks.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	_ "fmt"
 	"math"
 	"strings"
 )

main.go

@@ -9,9 +9,14 @@ import (
 	_ "time"
 )
 
-const EXIT_CLEAN = 0
-const EXIT_FAILURE = 1
-const EXIT_LEAKS = 2
+// ExitClean : no leaks have been found
+const ExitClean = 0
+
+// ExitFailure : gitleaks has encountered an error or SIGINT
+const ExitFailure = 1
+
+// ExitLeaks : leaks are present in scanned repos
+const ExitLeaks = 2
 
 // package globals
 var (
@@ -55,5 +60,5 @@ func main() {
 
 func failF(format string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, format, args...)
-	os.Exit(EXIT_FAILURE)
+	os.Exit(ExitFailure)
 }

options.go

@@ -193,7 +193,7 @@ func (opts *Options) parseOptions(args []string) error {
 			opts.LogLevel = opts.nextInt(args, &i)
 		case "-h", "--help":
 			help()
-			os.Exit(EXIT_CLEAN)
+			os.Exit(ExitClean)
 		default:
 			if match, value := opts.optString(arg, "--token="); match {
 				opts.Token = value
@@ -219,7 +219,7 @@ func (opts *Options) parseOptions(args []string) error {
 					} else {
 						fmt.Printf("Unknown option %s\n\n", arg)
 						help()
-						os.Exit(EXIT_CLEAN)
+						os.Exit(ExitClean)
 					}
 				}
 			} else {
@@ -242,7 +242,7 @@ func (opts *Options) parseOptions(args []string) error {
 
 		if _, err := os.Stat(dotGitPath); os.IsNotExist(err) {
 			fmt.Printf("gitleaks has no target")
-			os.Exit(EXIT_CLEAN)
+			os.Exit(ExitClean)
 		} else {
 			opts.LocalMode = true
 			opts.RepoPath = pwd
@@ -258,7 +258,7 @@ func (opts *Options) parseOptions(args []string) error {
 func (opts *Options) failF(format string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, format, args...)
 	help()
-	os.Exit(EXIT_FAILURE)
+	os.Exit(ExitFailure)
 }
 
 // guards will prevent gitleaks from continuing if any invalid options

owner.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	_ "fmt"
 	"github.com/google/go-github/github"
 	"golang.org/x/oauth2"
 	"io/ioutil"
@@ -15,6 +14,7 @@ import (
 	"strings"
 )
 
+// Owner blah blah
 type Owner struct {
 	name        string
 	url         string
@@ -24,9 +24,14 @@ type Owner struct {
 	repos       []Repo
 }
 
+// ownerPath is used by newOwner and is responsible for returning a path parsed from
+// opts.ClonePath, PWD, or a temporary directory. If a user provides --clone-path=$Home/Desktop/audits
+// then the owner path with be $HOME/Desktop/audits. If the user does not provide a --clone-path= argument
+// then ownerPath will return the current working directory. If the user sets the temporary option, then
+// ownerPath will be $TMPDIR/ownerName. For example running gitleaks on github.com/mozilla, ownerPath would
+// return $TMPDIR/mozilla
 func ownerPath(ownerName string) (string, error) {
 	if opts.Tmp {
-		fmt.Println("creating tmp")
 		dir, err := ioutil.TempDir("", ownerName)
 		return dir, err
 	} else if opts.ClonePath != "" {
@@ -37,11 +42,14 @@ func ownerPath(ownerName string) (string, error) {
 	} else {
 		return os.Getwd()
 	}
-
 }
 
-// newOwner instantiates an owner and creates any necessary resources for said owner.
-// newOwner returns a Owner struct pointer
+// newOwner is the entry point for gitleaks after all the options have been parsed and
+// is responsible for returning an Owner pointer. If running in localmode then the Owner
+// that gets created will create a single repo specified in opts.RepoPath. Otherwise
+// newOwner will go out to github and fetch all the repos associated with the owner if
+// gitleaks is running in owner mode. If gitleaks is running in a non-local repo mode, then
+// newOwner will skip hitting the github api and go directly to cloning.
 func newOwner() *Owner {
 	name := ownerName()
 	ownerPath, err := ownerPath(name)
@@ -73,11 +81,6 @@ func newOwner() *Owner {
 		return owner
 	}
 
-	/*
-		err := owner.setupDir()
-		if err != nil {
-			owner.failF("%v", err)
-		}*/
 	err = owner.fetchRepos()
 	if err != nil {
 		owner.failF("%v", err)
@@ -107,13 +110,13 @@ func (owner *Owner) fetchRepos() error {
 			orgOpt := &github.RepositoryListByOrgOptions{
 				ListOptions: github.ListOptions{PerPage: 10},
 			}
-			err = owner.fetchOrgRepos(orgOpt, gitClient, ctx)
+			err = owner.fetchOrgRepos(ctx, orgOpt, gitClient)
 		} else {
 			// user account type
 			userOpt := &github.RepositoryListOptions{
 				ListOptions: github.ListOptions{PerPage: 10},
 			}
-			err = owner.fetchUserRepos(userOpt, gitClient, ctx)
+			err = owner.fetchUserRepos(ctx, userOpt, gitClient)
 		}
 	}
 	return err
@@ -122,8 +125,8 @@ func (owner *Owner) fetchRepos() error {
 // fetchOrgRepos used by fetchRepos is responsible for parsing github's org repo response. If no
 // github token is available then fetchOrgRepos might run into a rate limit in which case owner will
 // log an error and gitleaks will exit. The rate limit for no token is 50 req/hour... not much.
-func (owner *Owner) fetchOrgRepos(orgOpts *github.RepositoryListByOrgOptions, gitClient *github.Client,
-	ctx context.Context) error {
+func (owner *Owner) fetchOrgRepos(ctx context.Context, orgOpts *github.RepositoryListByOrgOptions,
+	gitClient *github.Client) error {
 	var (
 		githubRepos []*github.Repository
 		resp        *github.Response
@@ -135,9 +138,10 @@ func (owner *Owner) fetchOrgRepos(orgOpts *github.RepositoryListByOrgOptions, gi
 			ctx, owner.name, orgOpts)
 		owner.addRepos(githubRepos)
 		if _, ok := err.(*github.RateLimitError); ok {
-			fmt.Println("Hit rate limit")
+			log.Printf("hit rate limit retreiving %s, continuing with partial audit\n",
+				owner.name)
 		} else if err != nil {
-			return fmt.Errorf("failed fetching org repos, bad request")
+			return fmt.Errorf("failed obtaining %s repos from githuib api, bad request", owner.name)
 		} else if resp.NextPage == 0 {
 			break
 		}
@@ -150,8 +154,8 @@ func (owner *Owner) fetchOrgRepos(orgOpts *github.RepositoryListByOrgOptions, gi
 // github token is available then fetchUserRepos might run into a rate limit in which case owner will
 // log an error and gitleaks will exit. The rate limit for no token is 50 req/hour... not much.
 // sorry for the redundancy
-func (owner *Owner) fetchUserRepos(userOpts *github.RepositoryListOptions, gitClient *github.Client,
-	ctx context.Context) error {
+func (owner *Owner) fetchUserRepos(ctx context.Context, userOpts *github.RepositoryListOptions,
+	gitClient *github.Client) error {
 	var (
 		githubRepos []*github.Repository
 		resp        *github.Response
@@ -162,10 +166,11 @@ func (owner *Owner) fetchUserRepos(userOpts *github.RepositoryListOptions, gitCl
 			ctx, owner.name, userOpts)
 		owner.addRepos(githubRepos)
 		if _, ok := err.(*github.RateLimitError); ok {
-			fmt.Println("Hit rate limit")
+			log.Printf("hit rate limit retreiving %s, continuing with partial audit\n",
+				owner.name)
 			break
 		} else if err != nil {
-			return fmt.Errorf("failed fetching user repos, bad request")
+			return fmt.Errorf("failed obtaining %s repos from github api, bad request", owner.name)
 		} else if resp.NextPage == 0 {
 			break
 		}
@@ -182,19 +187,20 @@ func (owner *Owner) addRepos(githubRepos []*github.Repository) {
 	}
 }
 
-// auditRepos
+// auditRepos is responsible for auditing all the owner's
+// repos. auditRepos is used by main and will return the following exit codes
+// 0: The audit succeeded with no findings
+// 1: The audit failed, or wasn't attempted due to an execution failure.
+// 2: The audit succeeded, and secrets / patterns were found.
 func (owner *Owner) auditRepos() int {
-	exitCode := EXIT_CLEAN
+	exitCode := ExitClean
 	for _, repo := range owner.repos {
 		leaksPst, err := repo.audit()
 		if err != nil {
-			failF("%v\n", err)
+			failF("%v", err)
 		}
 		if leaksPst {
-			log.Printf("\x1b[31;2mLEAKS DETECTED for %s\x1b[0m!\n", repo.name)
-			exitCode = EXIT_LEAKS
-		} else {
-			log.Printf("No Leaks detected for \x1b[32;2m%s\x1b[0m\n", repo.name)
+			exitCode = ExitLeaks
 		}
 	}
 	return exitCode
@@ -204,14 +210,15 @@ func (owner *Owner) auditRepos() int {
 // and exits with a exit code 2
 func (owner *Owner) failF(format string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, format, args...)
-	os.Exit(EXIT_FAILURE)
+	os.Exit(ExitFailure)
 }
 
-// rmTmp removes the temporary repo
+// rmTmp removes the owner's temporary repo. rmTmp will only get called if temporary
+// mode is set. rmTmp is called on a SIGINT and after the audits have finished
 func (owner *Owner) rmTmp() {
 	log.Printf("removing tmp gitleaks repo for %s\n", owner.name)
 	os.RemoveAll(owner.path)
-	os.Exit(EXIT_FAILURE)
+	os.Exit(ExitFailure)
 }
 
 // ownerType returns the owner type extracted from opts.
@@ -244,8 +251,6 @@ func ownerName() string {
 // githubTokenClient creates an oauth client from your github access token.
 // Gitleaks will attempt to retrieve your github access token from a cli argument
 // or an env var - "GITHUB_TOKEN".
-// Might be good to eventually parse the token from a Config or creds file in
-// $GITLEAKS_HOME
 func githubTokenClient() *http.Client {
 	var token string
 	if opts.Token != "" {

repo.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 )
 
+// Repo is
 type Repo struct {
 	name       string
 	url        string
@@ -22,6 +23,7 @@ type Repo struct {
 	reportPath string
 }
 
+// Leak is
 type Leak struct {
 	Line     string `json:"line"`
 	Commit   string `json:"commit"`
@@ -34,6 +36,7 @@ type Leak struct {
 	RepoURL  string `json:"repoURL"`
 }
 
+// Commit is
 type Commit struct {
 	Hash   string
 	Author string
@@ -41,7 +44,7 @@ type Commit struct {
 	Msg    string
 }
 
-// running gitleaks on local repo
+// newLocalRepo will such and such
 func newLocalRepo(repoPath string) *Repo {
 	_, name := path.Split(repoPath)
 	repo := &Repo{
@@ -53,17 +56,23 @@ func newLocalRepo(repoPath string) *Repo {
 
 }
 
+// newRepo
 func newRepo(name string, url string, path string) *Repo {
 	repo := &Repo{
-		name: name,
-		url:  url,
-		// TODO handle existing one
+		name:       name,
+		url:        url,
 		path:       path,
 		reportPath: opts.ReportPath,
 	}
 	return repo
 }
 
+// rmTmp
+func (repo *Repo) rmTmp() {
+	log.Printf("removing tmp gitleaks repo %s\n", repo.path)
+	os.Remove(repo.path)
+}
+
 // Audit operates on a single repo and searches the full or partial history of the repo.
 // A semaphore is declared for every repo to bind concurrency. If unbounded, the system will throw a
 // `too many open files` error. Eventually, gitleaks should use src-d/go-git to avoid shelling out
@@ -82,6 +91,10 @@ func (repo *Repo) audit() (bool, error) {
 		leaksPst          bool
 	)
 
+	if opts.Tmp {
+		defer repo.rmTmp()
+	}
+
 	dotGitPath := filepath.Join(repo.path, ".git")
 
 	// Navigate to proper location to being audit. Clone repo
@@ -91,13 +104,13 @@ func (repo *Repo) audit() (bool, error) {
 			return false, fmt.Errorf("%s does not exist", repo.path)
 		}
 		// no repo present, clone it
-		log.Printf("Cloning \x1b[37;1m%s\x1b[0m into %s...\n", repo.url, repo.path)
+		log.Printf("cloning \x1b[37;1m%s\x1b[0m into %s...\n", repo.url, repo.path)
 		err = exec.Command("git", "clone", repo.url, repo.path).Run()
 		if err != nil {
 			return false, fmt.Errorf("cannot clone %s into %s", repo.url, repo.path)
 		}
 	} else {
-		log.Printf("Fetching \x1b[37;1m%s\x1b[0m from %s ...\n", repo.name, repo.path)
+		log.Printf("fetching \x1b[37;1m%s\x1b[0m from %s ...\n", repo.name, repo.path)
 		err = exec.Command("git", "fetch").Run()
 		if err != nil {
 			return false, fmt.Errorf("cannot fetch %s from %s", repo.url, repo.path)
@@ -138,10 +151,13 @@ func (repo *Repo) audit() (bool, error) {
 	gitLeakReceiverWG.Wait()
 	if len(leaks) != 0 {
 		leaksPst = true
+		log.Printf("\x1b[31;2mLEAKS DETECTED for %s\x1b[0m!\n", repo.name)
+	} else {
+		log.Printf("No Leaks detected for \x1b[32;2m%s\x1b[0m\n", repo.name)
 	}
 
 	if (opts.ReportPath != "" || opts.ReportOut) && len(leaks) != 0 {
-		err = repo.writeReport()
+		err = repo.writeReport(leaks)
 		if err != nil {
 			return leaksPst, fmt.Errorf("could not write report to %s", opts.ReportPath)
 		}
@@ -150,12 +166,13 @@ func (repo *Repo) audit() (bool, error) {
 }
 
 // Used by audit, writeReport will generate a report and write it out to
-// $GITLEAKS_HOME/report/<owner>/<repo>. No report will be generated if
-// no leaks have been found
-func (repo *Repo) writeReport() error {
-	reportJSON, _ := json.MarshalIndent(repo.leaks, "", "\t")
+// --report-path=<path> if specified, otherwise a report will be generated to
+// $PWD/<repo_name>_leaks.json. No report will be generated if
+// no leaks have been found or --report-out is not set.
+func (repo *Repo) writeReport(leaks []Leak) error {
+	reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
 	if _, err := os.Stat(opts.ReportPath); os.IsNotExist(err) {
-		os.Mkdir(opts.ReportPath, os.ModePerm)
+		os.MkdirAll(opts.ReportPath, os.ModePerm)
 	}
 	reportFileName := fmt.Sprintf("%s_leaks.json", repo.name)
 	reportFile := filepath.Join(repo.reportPath, reportFileName)
@@ -163,7 +180,7 @@ func (repo *Repo) writeReport() error {
 	if err != nil {
 		return err
 	}
-	log.Printf("report for %s written to %s", repo.name, reportFileName)
+	log.Printf("report for %s written to %s", repo.name, reportFile)
 	return nil
 }
 
@@ -218,7 +235,7 @@ func auditDiff(currCommit Commit, repo *Repo, commitWG *sync.WaitGroup,
 
 	if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil {
 		// TODO handle this better
-		os.Exit(EXIT_FAILURE)
+		os.Exit(ExitFailure)
 	}
 
 	commitCmp := fmt.Sprintf("%s^!", currCommit.Hash)
@@ -227,7 +244,7 @@ func auditDiff(currCommit Commit, repo *Repo, commitWG *sync.WaitGroup,
 	<-semaphoreChan
 
 	if err != nil {
-		os.Exit(EXIT_FAILURE)
+		os.Exit(ExitFailure)
 	}
 
 	leaks := doChecks(string(out), currCommit, repo)

repo_test.go

@@ -1,17 +1,20 @@
 package main
 
 import (
-	"testing"
+	_ "fmt"
 	"os"
-	"fmt"
+	"testing"
 )
 
-func TestNewRepo(t *testing.T) {
-	// TODO
-}
-
 func TestNewLocalRepo(t *testing.T) {
-	// TODO
+	r := newLocalRepo("")
+	if r.path != "" {
+		t.Error()
+	}
+	r = newLocalRepo("some/path")
+	if r.name != "path" || r.path != "some/path" {
+		t.Error()
+	}
 }
 
 func TestWriteReport(t *testing.T) {
@@ -32,20 +35,32 @@ func TestAudit(t *testing.T) {
 	opts.Tmp = true
 	opts.URL = "https://github.com/zricethezav/gronit"
 	owner := newOwner()
-	r := newRepo("gronit", "https://github.com/zricethezav/gronit", owner.path)
-	r.audit()
+	r := newRepo("gronit", opts.URL, owner.path)
+	leaksPst, _ := r.audit()
+	if !leaksPst {
+		// TODO setup actual test repo
+		t.Error()
+	}
 
+	// new owner
+	opts.URL = "https://github.com/kelseyhightower/nocode"
+	owner = newOwner()
+	r = newRepo("nocode", opts.URL, owner.path)
+	leaksPst, _ = r.audit()
+	if leaksPst {
+		t.Error()
+	}
 }
 
 func sampleLeak() *Leak {
 	return &Leak{
-		Line: "yoo",
-		Commit: "mycommit",
+		Line:     "yoo",
+		Commit:   "mycommit",
 		Offender: "oh boy",
-		Reason: "hello",
-		Msg: "msg",
-		Time: "time",
-		Author: "lol",
-		RepoURL: "yooo",
+		Reason:   "hello",
+		Msg:      "msg",
+		Time:     "time",
+		Author:   "lol",
+		RepoURL:  "yooo",
 	}
 }