structured logs

main.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"fmt"
 	_ "fmt"
 	"go.uber.org/zap"
 	_ "io/ioutil"
 	"os"
 	"regexp"
 	_ "time"
-	"fmt"
 )
 
 const EXIT_CLEAN = 0
@@ -23,7 +23,7 @@ var (
 	assignRegex   *regexp.Regexp
 	fileDiffRegex *regexp.Regexp
 	logger        *zap.Logger
-	opts 		  *Options
+	opts          *Options
 )
 
 func init() {
@@ -58,5 +58,3 @@ func failF(format string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, format, args...)
 	os.Exit(EXIT_FAILURE)
 }
-
-

options.go

@@ -2,14 +2,14 @@ package main
 
 import (
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
-	"regexp"
 	"github.com/mitchellh/go-homedir"
-	"path/filepath"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
 )
 
 const DEBUG = 0
@@ -43,42 +43,42 @@ Other
  -h --help 		Display this message
  --token=<STR>    	Github API token
  --stopwords  		Enables stopwords
+ --pretty 		Enables pretty printing for humans, otherwise you'll get logs'
 
 `
 
-
 // Options for gitleaks. need to support remote repo/owner
 // and local repo/owner mode
 type Options struct {
-	URL 			 string
-	RepoPath 	     string
+	URL      string
+	RepoPath string
 
-	ClonePath 		 string
-	ReportPath 	     string
+	ClonePath  string
+	ReportPath string
 
 	Concurrency      int
 	B64EntropyCutoff int
 	HexEntropyCutoff int
 
 	// MODES
-	UserMode         bool
-	OrgMode          bool
-	RepoMode       	 bool
-	LocalMode 		 bool
+	UserMode  bool
+	OrgMode   bool
+	RepoMode  bool
+	LocalMode bool
 
 	// OPTS
-	Strict           bool
-	Entropy          bool
-	SinceCommit      string
-	Persist          bool
-	IncludeForks     bool
-	Tmp              bool
-	ReportOut 		 bool
-	Token            string
+	Strict       bool
+	Entropy      bool
+	SinceCommit  string
+	Persist      bool
+	IncludeForks bool
+	Tmp          bool
+	ReportOut    bool
+	Token        string
 
 	// LOGS/REPORT
-	Verbose          bool
-	LogLevel 		 int
+	LogLevel    int
+	PrettyPrint bool
 }
 
 // help prints the usage string and exits
@@ -135,13 +135,13 @@ func (opts *Options) optInt(arg string, prefixes ...string) (bool, int) {
 }
 
 // newOpts generates opts and parses arguments
-func newOpts(args []string) (*Options) {
+func newOpts(args []string) *Options {
 	opts, err := defaultOptions()
-	if err != nil{
+	if err != nil {
 		opts.failF("%v", err)
 	}
 	err = opts.parseOptions(args)
-	if err != nil{
+	if err != nil {
 		opts.failF("%v", err)
 	}
 	opts.setupLogger()
@@ -171,9 +171,9 @@ func defaultOptions() (*Options, error) {
 		Concurrency:      10,
 		B64EntropyCutoff: 70,
 		HexEntropyCutoff: 40,
-		LogLevel: INFO,
-		ClonePath: filepath.Join(gitleaksHome, "clone"),
-		ReportPath: filepath.Join(gitleaksHome, "report"),
+		LogLevel:         INFO,
+		ClonePath:        filepath.Join(gitleaksHome, "clone"),
+		ReportPath:       filepath.Join(gitleaksHome, "report"),
 	}, nil
 }
 
@@ -211,6 +211,8 @@ func (opts *Options) parseOptions(args []string) error {
 
 		case "--report-out":
 			opts.ReportOut = true
+		case "--pretty":
+			opts.PrettyPrint = true
 
 		case "-t", "--temp":
 			opts.Tmp = true
@@ -222,7 +224,6 @@ func (opts *Options) parseOptions(args []string) error {
 		default:
 			// TARGETS
 			if i == len(args)-1 {
-				fmt.Println(arg[i])
 				if opts.LocalMode {
 					opts.RepoPath = args[i]
 				} else {
@@ -250,7 +251,7 @@ func (opts *Options) parseOptions(args []string) error {
 		}
 	}
 	err := opts.guards()
-	if err != nil{
+	if err != nil {
 		fmt.Printf("%v", err)
 	}
 	return err
@@ -275,7 +276,7 @@ func (opts *Options) guards() error {
 		return fmt.Errorf("Cannot run Gitleaks on more than one mode\n")
 	} else if (opts.OrgMode || opts.UserMode) && opts.RepoMode {
 		return fmt.Errorf("Cannot run Gitleaks on more than one mode\n")
-	} else if (opts.OrgMode || opts.RepoMode) && opts.UserMode{
+	} else if (opts.OrgMode || opts.RepoMode) && opts.UserMode {
 		return fmt.Errorf("Cannot run Gitleaks on more than one mode\n")
 	} else if opts.LocalMode && opts.Tmp {
 		return fmt.Errorf("Cannot run Gitleaks with temp settings and local mode\n")
@@ -307,6 +308,11 @@ func (opts *Options) setupLogger() {
 	case ERROR:
 		atom.SetLevel(zap.ErrorLevel)
 	}
+
+	// set to ErrorLevel if pretty printing
+	if opts.PrettyPrint{
+		atom.SetLevel(zap.ErrorLevel)
+	}
 }
 
 // isGithubTarget checks if url is a valid github target

owner.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	_ "fmt"
 	"github.com/google/go-github/github"
 	"golang.org/x/oauth2"
@@ -11,7 +12,6 @@ import (
 	"os/signal"
 	"path"
 	"strings"
-	"fmt"
 )
 
 type Owner struct {
@@ -25,8 +25,8 @@ type Owner struct {
 
 // newOwner instantiates an owner and creates any necessary resources for said owner.
 // newOwner returns a Owner struct pointer
-func newOwner() (*Owner)  {
-	name  := ownerName()
+func newOwner() *Owner {
+	name := ownerName()
 	owner := &Owner{
 		name:        name,
 		url:         opts.URL,
@@ -44,7 +44,6 @@ func newOwner() (*Owner)  {
 		owner.rmTmp()
 	}()
 
-
 	// if running on local repo, just go right to it.
 	if opts.LocalMode {
 		repo := newLocalRepo(opts.RepoPath)
@@ -162,14 +161,14 @@ func (owner *Owner) addRepos(githubRepos []*github.Repository) {
 }
 
 // auditRepos
-func (owner *Owner) auditRepos() (int) {
+func (owner *Owner) auditRepos() int {
 	exitCode := EXIT_CLEAN
 	for _, repo := range owner.repos {
 		leaksPst, err := repo.audit(owner)
 		if err != nil {
 			failF("%v\n", err)
 		}
-		if leaksPst{
+		if leaksPst {
 			exitCode = EXIT_LEAKS
 		}
 	}
@@ -190,8 +189,7 @@ func (owner *Owner) setupDir() error {
 	if opts.Tmp {
 		dir, err := ioutil.TempDir("", owner.name)
 		if err != nil {
-			return err
-			owner.failF("Unabled to create temp directories for cloning")
+			fmt.Errorf("unable to create temp directories for cloning")
 		}
 		owner.path = dir
 	} else {
@@ -200,9 +198,6 @@ func (owner *Owner) setupDir() error {
 		}
 	}
 	return nil
-
-	// TODO could be handled via option
-	// owner.reportPath = filepath.Join(gitLeaksPath, "report", owner.name)
 }
 
 // rmTmp removes the temporary repo
@@ -226,11 +221,11 @@ func ownerType() string {
 // ownerName returns the owner name extracted from the urls provided in opts.
 // If no RepoURL, OrgURL, or UserURL is provided, then owner will log an error
 // and gitleaks will exit.
-func ownerName() (string) {
+func ownerName() string {
 	if opts.RepoMode {
 		splitSlashes := strings.Split(opts.URL, "/")
 		return splitSlashes[len(splitSlashes)-2]
-	} else if opts.UserMode|| opts.OrgMode {
+	} else if opts.UserMode || opts.OrgMode {
 		_, ownerName := path.Split(opts.URL)
 		return ownerName
 	}

repo.go

@@ -8,9 +8,9 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"sync"
-	"path"
 )
 
 type Repo struct {
@@ -53,22 +53,28 @@ func newLocalRepo(repoPath string) *Repo {
 
 func newRepo(name string, url string) *Repo {
 	repo := &Repo{
-		name:  name,
-		url:   url,
+		name: name,
+		url:  url,
 		// TODO handle existing one
-		path:  opts.ClonePath + "/" + name,
+		path: opts.ClonePath + "/" + name,
 	}
 	return repo
 }
 
-func (repo *Repo) rLog(msg string) {
+func (repo *Repo) Info(msg string) {
 	// logger should have these infos: msg, repo, owner, time
-	logger.Debug("Beginning audit",
+	logger.Info(msg,
 		zap.String("repo", repo.name),
 		zap.String("repo_path", repo.path),
 	)
 }
 
+func (repo *Repo) PrettyPrintF(format string, args ...interface{}) {
+	if opts.PrettyPrint {
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+}
+
 // Audit operates on a single repo and searches the full or partial history of the repo.
 // A semaphore is declared for every repo to bind concurrency. If unbounded, the system will throw a
 // `too many open files` error. Eventually, gitleaks should use src-d/go-git to avoid shelling out
@@ -84,7 +90,7 @@ func (repo *Repo) audit(owner *Owner) (bool, error) {
 		gitLeaksChan      = make(chan Leak)
 		leaks             []Leak
 		semaphoreChan     = make(chan struct{}, opts.Concurrency)
-		leaksPst 		  bool
+		leaksPst          bool
 	)
 
 	dotGitPath := filepath.Join(repo.path, ".git")
@@ -96,13 +102,15 @@ func (repo *Repo) audit(owner *Owner) (bool, error) {
 			return false, fmt.Errorf("%s does not exist", repo.path)
 		}
 		// no repo present, clone it
-		fmt.Printf("Cloning \x1b[37;1m%s\x1b[0m into %s...\n", repo.url, repo.path)
+		repo.Info("cloning")
+		repo.PrettyPrintF("Cloning \x1b[37;1m%s\x1b[0m...\n", repo.url)
 		err = exec.Command("git", "clone", repo.url, repo.path).Run()
 		if err != nil {
 			return false, fmt.Errorf("cannot clone %s into %s", repo.url, repo.path)
 		}
 	} else {
-		fmt.Printf("Checking \x1b[37;1m%s\x1b[0m from %s...\n", repo.url, repo.path)
+		repo.Info("fetching")
+		repo.PrettyPrintF("Fetching \x1b[37;1m%s\x1b[0m...\n", repo.url)
 		err = exec.Command("git", "fetch").Run()
 		if err != nil {
 			return false, fmt.Errorf("cannot fetch %s from %s", repo.url, repo.path)
@@ -141,7 +149,7 @@ func (repo *Repo) audit(owner *Owner) (bool, error) {
 	go reportAggregator(&gitLeakReceiverWG, gitLeaksChan, &leaks)
 	commitWG.Wait()
 	gitLeakReceiverWG.Wait()
-	if len(leaks) != 0{
+	if len(leaks) != 0 {
 		leaksPst = true
 	}
 
@@ -170,7 +178,7 @@ func (repo *Repo) writeReport() error {
 	if err != nil {
 		return err
 	}
-	fmt.Printf("Report written to %s\n", reportFile)
+	repo.Info(fmt.Sprintf("Report written to %s\n", reportFile))
 	return nil
 }
 
@@ -201,13 +209,26 @@ func parseRevList(revList [][]byte) []Commit {
 // reportAggregator is a go func responsible for ...
 func reportAggregator(gitLeakReceiverWG *sync.WaitGroup, gitLeaks chan Leak, leaks *[]Leak) {
 	for gitLeak := range gitLeaks {
-		b, err := json.MarshalIndent(gitLeak, "", "   ")
-		if err != nil {
-			// make waitgroup errArray
-			fmt.Println("failed to output leak:", err)
-		}
-		fmt.Println(string(b))
+		logger.Info("leak",
+			zap.String("line", gitLeak.Line),
+			zap.String("commit", gitLeak.Commit),
+			zap.String("offender", gitLeak.Offender),
+			zap.String("Reason", gitLeak.Reason),
+			zap.String("author", gitLeak.Author),
+			zap.String("file", gitLeak.File),
+			zap.String("repoURL", gitLeak.RepoURL),
+			zap.String("timeOfCommit", gitLeak.Time),
+		)
 		*leaks = append(*leaks, gitLeak)
+		if opts.PrettyPrint {
+			b, err := json.MarshalIndent(gitLeak, "", "   ")
+			if err != nil {
+				// handle this?
+				fmt.Println("failed to output leak:", err)
+			}
+			fmt.Println(string(b))
+		}
+
 		gitLeakReceiverWG.Done()
 	}
 }