Merge pull request #57 from zricethezav/feature/external-regex

external repo file support

README.md

@@ -1,4 +1,10 @@
-![Alt Text](https://github.com/zricethezav/gifs/blob/master/gitleaks1.png) [![Build Status](https://travis-ci.org/zricethezav/gitleaks.svg?branch=master)](https://travis-ci.org/zricethezav/gitleaks)
+<p align="center">
+  <img alt="gitleaks" src="https://raw.githubusercontent.com/zricethezav/gifs/master/gitleaks4.png" height="140" />
+  <p align="center">
+      <a href="https://travis-ci.org/zricethezav/gitleaks"><img alt="Travis" src="https://img.shields.io/travis/zricethezav/gitleaks/master.svg?style=flat-square"></a>
+  </p>
+</p>
+
 ## Audit git repos for secrets and keys
 
 #### Installing
@@ -31,6 +37,7 @@ Options:
  --report-path=<STR>    Save report to path, gitleaks default behavior is to save report to pwd
  --clone-path=<STR>     Gitleaks will clone repos here, default pwd
  --concurrency=<INT>    Upper bound on concurrent diffs
+ --regex-file=<STR>     Path to regex file for external regex matching
  --since=<STR>          Commit to stop at
  --b64Entropy=<INT>     Base64 entropy cutoff (default is 70)
  --hexEntropy=<INT>     Hex entropy cutoff (default is 40)
@@ -97,13 +104,6 @@ docker build -t gitleaks .
 docker run --rm --name=gitleaks gitleaks https://github.com/zricethezav/gitleaks
 ```
 
-### cypherphunky
+##### Support
 BTC: 1H2rSXDJZxWcTk2Ugr5P9r9m93m2NhL4xj
 
-BCH: qp4mdaef04g5d0xpgecx78fmruk6vgl4pgqtetrl9h
-
-ETH: 0xe48b4Fce6A1C1a9C780376032895b06b1709AddF
-
-LTC: LRhDzMyGos5CtZMoSTEx5rdLksPUwSrtuz
-
-s/o to @jlakowski for the gimp skillz

checks.go

@@ -22,7 +22,6 @@ func doChecks(diff string, commit Commit, repo *Repo) []Leak {
 				file = line[idx[1]:]
 			}
 		}
-
 		for leakType, re := range regexes {
 			match = re.FindString(line)
 			if len(match) == 0 ||
@@ -44,6 +43,31 @@ func doChecks(diff string, commit Commit, repo *Repo) []Leak {
 			}
 			leaks = append(leaks, leak)
 		}
+
+		// Check for external regex matches
+		if externalRegex != nil {
+			for _, re := range externalRegex {
+				match = re.FindString(line)
+				if len(match) == 0 ||
+					(opts.Strict && containsStopWords(line)) ||
+					(opts.Entropy && !checkShannonEntropy(line, opts)) {
+					continue
+				}
+
+				leak = Leak{
+					Line:     line,
+					Commit:   commit.Hash,
+					Offender: match,
+					Reason:   "match: " + re.String(),
+					Msg:      commit.Msg,
+					Time:     commit.Time,
+					Author:   commit.Author,
+					File:     file,
+					RepoURL:  repo.url,
+				}
+				leaks = append(leaks, leak)
+			}
+		}
 	}
 	return leaks
 

checks_test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"testing"
+	"os"
+	"bufio"
+	"fmt"
 )
 
 func TestCheckRegex(t *testing.T) {
@@ -29,6 +32,32 @@ func TestCheckRegex(t *testing.T) {
 	}
 }
 
+func TestExternalRegex(t *testing.T) {
+	opts, err := defaultOptions()
+	if err != nil {
+		t.Error()
+	}
+	file, err := os.Create("testregex.txt")
+	if err != nil {
+		t.Error()
+	}
+	defer file.Close()
+
+	w := bufio.NewWriter(file)
+	fmt.Fprintln(w, "AKIA[0-9A-Z]{16}")
+	w.Flush()
+
+	opts.RegexFile = "testregex.txt"
+	opts.loadExternalRegex()
+	leaks := doChecks("aws=\"AKIALALEMEL33243OLIAE",
+		Commit{}, &Repo{url:"someurl"})
+	if len(leaks) != 2 {
+		// leak from default regex, leak from external
+		t.Error()
+	}
+	os.Remove("testregex.txt")
+}
+
 func TestEntropy(t *testing.T) {
 	var enoughEntropy bool
 	opts := &Options{

main.go

@@ -18,6 +18,7 @@ const ExitLeaks = 2
 // package globals
 var (
 	regexes       map[string]*regexp.Regexp
+	externalRegex []*regexp.Regexp
 	stopWords     []string
 	base64Chars   string
 	hexChars      string
@@ -38,7 +39,7 @@ func init() {
 	regexes = map[string]*regexp.Regexp{
 		"PKCS8":    regexp.MustCompile("-----BEGIN PRIVATE KEY-----"),
 		"RSA":      regexp.MustCompile("-----BEGIN RSA PRIVATE KEY-----"),
-                "DSA":      regexp.MustCompile("-----BEGIN DSA PRIVATE KEY-----"),
+		"DSA":      regexp.MustCompile("-----BEGIN DSA PRIVATE KEY-----"),
 		"SSH":      regexp.MustCompile("-----BEGIN OPENSSH PRIVATE KEY-----"),
 		"Facebook": regexp.MustCompile("(?i)facebook.*['\"][0-9a-f]{32}['\"]"),
 		"Twitter":  regexp.MustCompile("(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]"),

options.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"bufio"
 )
 
 const usage = `
@@ -52,6 +53,7 @@ type Options struct {
 	Tmp          bool
 	Token        string
 	Verbose  bool
+	RegexFile string
 }
 
 // help prints the usage string and exits
@@ -174,6 +176,8 @@ func (opts *Options) parseOptions(args []string) error {
 				opts.HexEntropyCutoff = value
 			} else if match, value := opts.optInt(arg, "--concurrency="); match {
 				opts.Concurrency = value
+			} else if match, value := opts.optString(arg, "--regex-file="); match {
+				opts.RegexFile = value
 			} else if i == len(args)-1 {
 				if opts.LocalMode {
 					opts.RepoPath = filepath.Clean(args[i])
@@ -192,7 +196,14 @@ func (opts *Options) parseOptions(args []string) error {
 		}
 	}
 
-	// TODO cleanup this logic
+	if opts.RegexFile != "" {
+		err := opts.loadExternalRegex()
+		if err != nil {
+			return fmt.Errorf("unable to load regex from file %s: %v",
+				opts.RegexFile, err)
+		}
+	}
+
 	if !opts.RepoMode && !opts.UserMode && !opts.OrgMode && !opts.LocalMode {
 		if opts.URL != "" {
 			opts.RepoMode = true
@@ -223,6 +234,22 @@ func (opts *Options) parseOptions(args []string) error {
 	return err
 }
 
+// loadExternalRegex
+func (opts *Options) loadExternalRegex() error {
+	file, err := os.Open(opts.RegexFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		externalRegex = append(externalRegex, regexp.MustCompile(scanner.Text()))
+	}
+
+	return nil
+}
+
 // failF prints a failure message out to stderr, displays help
 // and exits with a exit code 2
 func (opts *Options) failF(format string, args ...interface{}) {
@@ -251,7 +278,6 @@ func (opts *Options) guards() error {
 	} else if opts.ClonePath != "" && opts.Tmp {
 		return fmt.Errorf("Cannot run Gitleaks with --clone-path set and temporary repo\n")
 	}
-
 	return nil
 }
 

owner_test.go

@@ -15,11 +15,12 @@ func TestOwnerPath(t *testing.T) {
 	if pwd != p {
 		t.Error()
 	}
-	opts.ClonePath = "test"
+	opts.ClonePath = "gitleaksTestClonePath"
 	p, err = ownerPath("nameToIgnore")
-	if p != "test" {
+	if p != "gitleaksTestClonePath" {
 		t.Error()
 	}
+	os.Remove("gitleaksTestClonePath")
 }
 
 func TestNewOwner(t *testing.T) {