workin on new filters

src/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
 )
 
@@ -17,20 +18,23 @@ type entropyRange struct {
 	v2 float64
 }
 
-type Regex struct {
+// Rule instructs how gitleaks should audit each line of code
+type Rule struct {
 	description string
 	regex       *regexp.Regexp
+	severity    string
+	tags        []string
+	entropies   []*entropyRange
 }
 
 // TomlConfig is used for loading gitleaks configs from a toml file
 type TomlConfig struct {
-	Regexes []struct {
+	Rules []struct {
 		Description string
 		Regex       string
-	}
-	Entropy struct {
-		LineRegexes []string
-		Ranges      []string
+		Entropies   []string
+		Tags        []string
+		Severity    string
 	}
 	Whitelist struct {
 		Files   []string
@@ -42,17 +46,13 @@ type TomlConfig struct {
 
 // Config contains gitleaks config
 type Config struct {
-	Regexes   []Regex
+	Rules     []*Rule
 	WhiteList struct {
 		regexes []*regexp.Regexp
 		files   []*regexp.Regexp
 		commits map[string]bool
 		repos   []*regexp.Regexp
 	}
-	Entropy struct {
-		entropyRanges []*entropyRange
-		regexes       []*regexp.Regexp
-	}
 	sshAuth *ssh.PublicKeys
 }
 
@@ -104,31 +104,39 @@ func newConfig() (*Config, error) {
 
 // updateConfig will update a the global config values
 func (config *Config) update(tomlConfig TomlConfig) error {
-	if len(tomlConfig.Entropy.Ranges) != 0 {
-		err := config.updateEntropyRanges(tomlConfig.Entropy.Ranges)
+	for _, rule := range tomlConfig.Rules {
+		re := regexp.MustCompile(rule.Regex)
+		ranges, err := getEntropyRanges(rule.Entropies)
 		if err != nil {
-			return err
+			log.Errorf("could not create entropy range for %s, skipping rule", rule.Description)
+			continue
 		}
+		r := &Rule{
+			description: rule.Description,
+			regex:       re,
+			severity:    rule.Severity,
+			tags:        rule.Tags,
+			entropies:   ranges,
+		}
+		config.Rules = append(config.Rules, r)
 	}
 
-	for _, regex := range tomlConfig.Entropy.LineRegexes {
-		config.Entropy.regexes = append(config.Entropy.regexes, regexp.MustCompile(regex))
-	}
-
-	if singleSearchRegex != nil {
-		config.Regexes = append(config.Regexes, Regex{
-			description: "single search",
-			regex:       singleSearchRegex,
-		})
-	} else {
-		for _, regex := range tomlConfig.Regexes {
-			config.Regexes = append(config.Regexes, Regex{
-				description: regex.Description,
-				regex:       regexp.MustCompile(regex.Regex),
-			})
+	// set stand alone rules from opts
+	if opts.Entropy != 0.0 {
+		ranges, err := getEntropyRanges([]string{fmt.Sprintf("0.0-%s", opts.Entropy)})
+		if err != nil {
+			log.Fatalf("could not create entropy range for %s", opts.Entropy)
 		}
+		r := &Rule{
+			description: "Entropy ",
+			severity:    "5",
+			tags:        []string{"entropy"},
+			entropies:   ranges,
+		}
+		config.Rules = append(config.Rules, r)
 	}
 
+	// set whitelists
 	config.WhiteList.commits = make(map[string]bool)
 	for _, commit := range tomlConfig.Whitelist.Commits {
 		config.WhiteList.commits[commit] = true
@@ -147,30 +155,31 @@ func (config *Config) update(tomlConfig TomlConfig) error {
 }
 
 // entropyRanges hydrates entropyRanges which allows for fine tuning entropy checking
-func (config *Config) updateEntropyRanges(entropyLimitStr []string) error {
+func getEntropyRanges(entropyLimitStr []string) ([]*entropyRange, error) {
+	var ranges []*entropyRange
 	for _, span := range entropyLimitStr {
 		split := strings.Split(span, "-")
 		v1, err := strconv.ParseFloat(split[0], 64)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		v2, err := strconv.ParseFloat(split[1], 64)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if v1 > v2 {
-			return fmt.Errorf("entropy range must be ascending")
+			return nil, fmt.Errorf("entropy range must be ascending")
 		}
 		r := &entropyRange{
 			v1: v1,
 			v2: v2,
 		}
 		if r.v1 > 8.0 || r.v1 < 0.0 || r.v2 > 8.0 || r.v2 < 0.0 {
-			return fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
+			return nil, fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
 		}
-		config.Entropy.entropyRanges = append(config.Entropy.entropyRanges, r)
+		ranges = append(ranges, r)
 	}
-	return nil
+	return ranges, nil
 }
 
 // externalConfig will attempt to load a pinned ".gitleaks.toml" configuration file

src/constants.go

@@ -1,6 +1,6 @@
 package gitleaks
 
-const version = "1.25.1"
+const version = "2.0.0"
 
 const defaultGithubURL = "https://api.github.com/"
 const defaultThreadNum = 1
@@ -14,49 +14,14 @@ const defaultConfig = `
 # configurations from that path. Gitleaks does not whitelist anything by default.
 
 title = "gitleaks config"
-# add regexes to the regex table
-[[regexes]]
+# add rules to the rule table
+[[rules]]
 description = "AWS"
 regex = '''AKIA[0-9A-Z]{16}'''
-[[regexes]]
-description = "PKCS8"
-regex = '''-----BEGIN PRIVATE KEY-----'''
-[[regexes]]
-description = "RSA"
-regex = '''-----BEGIN RSA PRIVATE KEY-----'''
-[[regexes]]
-description = "SSH"
-regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
-[[regexes]]
-description = "PGP"
-regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
-[[regexes]]
-description = "Facebook"
-regex = '''(?i)facebook(.{0,4})?['\"][0-9a-f]{32}['\"]'''
-[[regexes]]
-description = "Twitter"
-regex = '''(?i)twitter(.{0,4})?['\"][0-9a-zA-Z]{35,44}['\"]'''
-[[regexes]]
-description = "Github"
-regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
-[[regexes]]
-description = "Slack"
-regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
-
-[entropy]
-lineregexes = [
-	"api",
-	"key",
-	"signature",
-	"secret",
-	"password",
-	"pass",
-	"pwd",
-	"token",
-	"curl",
-	"wget",
-	"https?",
-]
+tags = ["key", "AWS"]
+entropies = [ "3.3-4.30" ]
+filetypes = [ "*.go" ]
+severity = "5"
 
 [whitelist]
 files = [

src/core.go

@@ -3,7 +3,6 @@ package gitleaks
 import (
 	"io/ioutil"
 	"os"
-	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -13,15 +12,14 @@ import (
 )
 
 var (
-	opts              *Options
-	config            *Config
-	singleSearchRegex *regexp.Regexp
-	dir               string
-	threads           int
-	totalCommits      int64
-	commitMap         = make(map[string]bool)
-	auditDone         bool
-	mutex             = &sync.Mutex{}
+	opts         *Options
+	config       *Config
+	dir          string
+	threads      int
+	totalCommits int64
+	commitMap    = make(map[string]bool)
+	auditDone    bool
+	mutex        = &sync.Mutex{}
 )
 
 func init() {
@@ -31,7 +29,7 @@ func init() {
 
 // Report is
 type Report struct {
-	Leaks    []Leak
+	Leaks    []*Leak
 	Duration string
 	Commits  int64
 }
@@ -39,7 +37,7 @@ type Report struct {
 // Run is the entry point for gitleaks
 func Run(optsL *Options) (*Report, error) {
 	var (
-		leaks []Leak
+		leaks []*Leak
 		err   error
 	)
 

src/entropy.go

@@ -23,30 +23,3 @@ func getShannonEntropy(data string) (entropy float64) {
 
 	return entropy
 }
-
-func entropyIsHighEnough(entropy float64) bool {
-	if entropy >= opts.Entropy && len(config.Entropy.entropyRanges) == 0 {
-		return true
-	}
-
-	for _, eR := range config.Entropy.entropyRanges {
-		if entropy > eR.v1 && entropy < eR.v2 {
-			return true
-		}
-	}
-
-	return false
-}
-
-func highEntropyLineIsALeak(line string) bool {
-	if !opts.NoiseReduction {
-		return true
-	}
-	for _, re := range config.Entropy.regexes {
-		if re.FindString(line) != "" {
-			return true
-		}
-	}
-
-	return false
-}

src/github.go

@@ -21,8 +21,8 @@ import (
 var githubPages = 100
 
 // auditPR audits a single github PR
-func auditGithubPR() ([]Leak, error) {
-	var leaks []Leak
+func auditGithubPR() ([]*Leak, error) {
+	var leaks []*Leak
 	ctx := context.Background()
 	githubClient := github.NewClient(githubToken())
 	splits := strings.Split(opts.GithubPR, "/")
@@ -66,7 +66,7 @@ func auditGithubPR() ([]Leak, error) {
 					continue
 				}
 
-				commit := commitInfo{
+				commit := &commitInfo{
 					sha:      c.GetSHA(),
 					content:  *f.Patch,
 					filePath: *f.Filename,
@@ -91,7 +91,7 @@ func auditGithubPR() ([]Leak, error) {
 // First, we gather all the github repositories from the github api (this doesnt actually clone the repo).
 // After all the repos have been pulled from github's api we proceed to audit the repos by calling auditGithubRepo.
 // If an error occurs during an audit of a repo, that error is logged but won't break the execution cycle.
-func auditGithubRepos() ([]Leak, error) {
+func auditGithubRepos() ([]*Leak, error) {
 	var (
 		err              error
 		githubRepos      []*github.Repository
@@ -100,7 +100,7 @@ func auditGithubRepos() ([]Leak, error) {
 		githubOrgOptions *github.RepositoryListByOrgOptions
 		githubOptions    *github.RepositoryListOptions
 		done             bool
-		leaks            []Leak
+		leaks            []*Leak
 		ownerDir         string
 	)
 	ctx := context.Background()

src/gitlab.go

@@ -17,11 +17,11 @@ const gitlabPages = 100
 // auditGitlabRepos kicks off audits if --gitlab-user or --gitlab-org options are set.
 // Getting all repositories from the GitLab API and run audit. If an error occurs during an audit of a repo,
 // that error is logged.
-func auditGitlabRepos() ([]Leak, error) {
+func auditGitlabRepos() ([]*Leak, error) {
 	var (
 		ps      []*gitlab.Project
 		resp    *gitlab.Response
-		leaks   []Leak
+		leaks   []*Leak
 		tempDir string
 		err     error
 	)

src/options.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 	"time"
@@ -36,16 +35,14 @@ type Options struct {
 	OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
 
 	// Process options
-	Threads        int     `long:"threads" description:"Maximum number of threads gitleaks spawns"`
-	Disk           bool    `long:"disk" description:"Clones repo(s) to disk"`
-	SingleSearch   string  `long:"single-search" description:"single regular expression to search for"`
-	ConfigPath     string  `long:"config" description:"path to gitleaks config"`
-	SSHKey         string  `long:"ssh-key" description:"path to ssh key"`
-	ExcludeForks   bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
-	Entropy        float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
-	NoiseReduction bool    `long:"noise-reduction" description:"Reduce the number of finds when entropy checks are enabled"`
-	RepoConfig     bool    `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\""`
-	Branch         string  `long:"branch" description:"Branch to audit"`
+	Threads      int     `long:"threads" description:"Maximum number of threads gitleaks spawns"`
+	Disk         bool    `long:"disk" description:"Clones repo(s) to disk"`
+	ConfigPath   string  `long:"config" description:"path to gitleaks config"`
+	SSHKey       string  `long:"ssh-key" description:"path to ssh key"`
+	ExcludeForks bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
+	Entropy      float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
+	RepoConfig   bool    `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\""`
+	Branch       string  `long:"branch" description:"Branch to audit"`
 	// TODO: IncludeMessages  string `long:"messages" description:"include commit messages in audit"`
 
 	// Output options
@@ -95,7 +92,6 @@ func ParseOpts() *Options {
 
 // optsGuard prevents invalid options
 func (opts *Options) guard() error {
-	var err error
 	if opts.GithubOrg != "" && opts.GithubUser != "" {
 		return fmt.Errorf("github user and organization set")
 	} else if opts.GithubOrg != "" && opts.OwnerPath != "" {
@@ -129,13 +125,6 @@ func (opts *Options) guard() error {
 		}
 	}
 
-	if opts.SingleSearch != "" {
-		singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
-		if err != nil {
-			return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
-		}
-	}
-
 	if opts.Entropy > 8 {
 		return fmt.Errorf("The maximum level of entropy is 8")
 	}

src/repo.go

@@ -30,6 +30,8 @@ type Leak struct {
 	File     string    `json:"file"`
 	Repo     string    `json:"repo"`
 	Date     time.Time `json:"date"`
+	Tags     string    `json:"tags"`
+	Severity string    `json:"severity"`
 }
 
 // RepoInfo contains a src-d git repository and other data about the repo
@@ -104,10 +106,10 @@ func (repoInfo *RepoInfo) clone() error {
 }
 
 // audit performs an audit
-func (repoInfo *RepoInfo) audit() ([]Leak, error) {
+func (repoInfo *RepoInfo) audit() ([]*Leak, error) {
 	var (
 		err         error
-		leaks       []Leak
+		leaks       []*Leak
 		commitCount int64
 		commitWg    sync.WaitGroup
 		semaphore   chan bool
@@ -248,7 +250,7 @@ func (repoInfo *RepoInfo) audit() ([]Leak, error) {
 					chunks := f.Chunks()
 					for _, chunk := range chunks {
 						if chunk.Type() == diffType.Add || chunk.Type() == diffType.Delete {
-							diff := commitInfo{
+							diff := &commitInfo{
 								repoName: repoInfo.name,
 								filePath: filePath,
 								content:  chunk.Content(),
@@ -279,8 +281,8 @@ func (repoInfo *RepoInfo) audit() ([]Leak, error) {
 	return leaks, nil
 }
 
-func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []Leak {
-	var leaks []Leak
+func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []*Leak {
+	var leaks []*Leak
 	fIter, err := c.Files()
 	if err != nil {
 		return nil
@@ -300,7 +302,7 @@ func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []Leak {
 		if err != nil {
 			return nil
 		}
-		diff := commitInfo{
+		diff := &commitInfo{
 			repoName: repoInfo.name,
 			filePath: f.Name,
 			content:  content,

src/utils.go

@@ -28,7 +28,7 @@ type commitInfo struct {
 
 // writeReport writes a report to a file specified in the --report= option.
 // Default format for report is JSON. You can use the --csv option to write the report as a csv
-func writeReport(leaks []Leak) error {
+func writeReport(leaks []*Leak) error {
 	if len(leaks) == 0 {
 		return nil
 	}
@@ -82,46 +82,59 @@ func writeReport(leaks []Leak) error {
 	return nil
 }
 
+// check rule will inspect a single line and return a leak if it encounters one
+func (rule *Rule) check(line string, commit *commitInfo) (*Leak, error) {
+	var (
+		match       string
+		entropy     float64
+		entropyWord string
+	)
+
+	if rule.entropies != nil {
+		words := strings.Fields(line)
+		for _, word := range words {
+			_entropy := getShannonEntropy(word)
+			for _, e := range rule.entropies {
+				if _entropy > e.v1 && _entropy < e.v2 {
+					entropy = _entropy
+					entropyWord = word
+				}
+			}
+		}
+	}
+
+	if rule.regex != nil {
+		match = rule.regex.FindString(line)
+	}
+
+	if match != "" && entropy != 0.0 {
+		return newLeak(line, entropyWord, rule, commit), nil
+	} else if match != "" && rule.entropies == nil {
+		return newLeak(line, match, rule, commit), nil
+	} else if entropy != 0.0 && rule.regex == nil {
+		return newLeak(line, entropyWord, rule, commit), nil
+	}
+	return nil, nil
+}
+
 // inspect will parse each line of the git diff's content against a set of regexes or
 // a set of regexes set by the config (see gitleaks.toml for example). This function
 // will skip lines that include a whitelisted regex. A list of leaks is returned.
 // If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
-func inspect(commit commitInfo) []Leak {
-	var (
-		leaks    []Leak
-		skipLine bool
-	)
+func inspect(commit *commitInfo) []*Leak {
+	var leaks []*Leak
 	lines := strings.Split(commit.content, "\n")
 
 	for _, line := range lines {
-		skipLine = false
-		for _, re := range config.Regexes {
-			match := re.regex.FindString(line)
-			if match == "" {
-				continue
-			}
-			if skipLine = isLineWhitelisted(line); skipLine {
+		for _, rule := range config.Rules {
+			if isLineWhitelisted(line) {
 				break
 			}
-			leaks = addLeak(leaks, line, match, re.description, commit)
-		}
-
-		if !skipLine && (opts.Entropy > 0 || len(config.Entropy.entropyRanges) != 0) {
-			words := strings.Fields(line)
-			for _, word := range words {
-				entropy := getShannonEntropy(word)
-				// Only check entropyRegexes and whiteListRegexes once per line, and only if an entropy leak type
-				// was found above, since regex checks are expensive.
-				if !entropyIsHighEnough(entropy) {
-					continue
-				}
-				// If either the line is whitelisted or the line fails the noiseReduction check (when enabled),
-				// then we can skip checking the rest of the line for high entropy words.
-				if skipLine = !highEntropyLineIsALeak(line) || isLineWhitelisted(line); skipLine {
-					break
-				}
-				leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), commit)
+			leak, err := rule.check(line, commit)
+			if err != nil {
+				continue
 			}
+			leaks = append(leaks, leak)
 		}
 	}
 	return leaks
@@ -138,31 +151,30 @@ func isLineWhitelisted(line string) bool {
 	return false
 }
 
-// addLeak is helper for func inspect() to append leaks if found during a diff check.
-func addLeak(leaks []Leak, line string, offender string, leakType string, commit commitInfo) []Leak {
-	leak := Leak{
+func newLeak(line string, msg string, rule *Rule, commit *commitInfo) *Leak {
+	leak := &Leak{
 		Line:     line,
 		Commit:   commit.sha,
-		Offender: offender,
-		Type:     leakType,
+		Offender: msg,
+		Type:     rule.description,
 		Author:   commit.author,
 		Email:    commit.email,
 		File:     commit.filePath,
 		Repo:     commit.repoName,
 		Message:  commit.message,
 		Date:     commit.date,
+		Tags:     strings.Join(rule.tags, ", "),
+		Severity: rule.severity,
 	}
 	if opts.Redact {
 		leak.Offender = "REDACTED"
-		leak.Line = strings.Replace(line, offender, "REDACTED", -1)
+		leak.Line = strings.Replace(line, msg, "REDACTED", -1)
 	}
 
 	if opts.Verbose {
 		leak.log()
 	}
-
-	leaks = append(leaks, leak)
-	return leaks
+	return leak
 }
 
 // discoverRepos walks all the children of `path`. If a child directory