zach rice 6 лет назад
Родитель
Сommit
7bd55e33b5
10 измененных файлов с 144 добавлено и 141 удалено
  1. BIN
      src/.gitleaks_test.go.swp
  2. 18 14
      src/config.go
  3. 48 4
      src/constants.go
  4. 8 3
      src/core.go
  5. 4 4
      src/github.go
  6. 2 2
      src/gitlab.go
  7. 16 81
      src/gitleaks_test.go
  8. 4 4
      src/options.go
  9. 6 5
      src/repo.go
  10. 38 24
      src/utils.go

BIN
src/.gitleaks_test.go.swp


+ 18 - 14
src/config.go

@@ -25,6 +25,7 @@ type Rule struct {
 	severity    string
 	tags        []string
 	entropies   []*entropyRange
+    entropyROI  string
 }
 
 // TomlConfig is used for loading gitleaks configs from a toml file
@@ -35,6 +36,7 @@ type TomlConfig struct {
 		Entropies   []string
 		Tags        []string
 		Severity    string
+        EntropyROI  string
 	}
 	Whitelist struct {
 		Files   []string
@@ -117,24 +119,26 @@ func (config *Config) update(tomlConfig TomlConfig) error {
 			severity:    rule.Severity,
 			tags:        rule.Tags,
 			entropies:   ranges,
+            entropyROI:  rule.EntropyROI,
 		}
 		config.Rules = append(config.Rules, r)
 	}
 
-	// set stand alone rules from opts
-	if opts.Entropy != 0.0 {
-		ranges, err := getEntropyRanges([]string{fmt.Sprintf("0.0-%s", opts.Entropy)})
-		if err != nil {
-			log.Fatalf("could not create entropy range for %s", opts.Entropy)
-		}
-		r := &Rule{
-			description: "Entropy ",
-			severity:    "5",
-			tags:        []string{"entropy"},
-			entropies:   ranges,
-		}
-		config.Rules = append(config.Rules, r)
-	}
+	// // set stand alone rules from opts
+	// if opts.Entropy != 0.0 {
+	// 	ranges, err := getEntropyRanges([]string{fmt.Sprintf("0.0-%s", opts.Entropy)})
+	// 	if err != nil {
+	// 		log.Fatalf("could not create entropy range for %s", opts.Entropy)
+	// 	}
+	// 	r := &Rule{
+	// 		description: "Entropy ",
+	// 		severity:    "5",
+	// 		tags:        []string{"entropy"},
+	// 		entropies:   ranges,
+    //         entropyROI:
+	// 	}
+	// 	config.Rules = append(config.Rules, r)
+	// }
 
 	// set whitelists
 	config.WhiteList.commits = make(map[string]bool)

+ 48 - 4
src/constants.go

@@ -15,13 +15,57 @@ const defaultConfig = `
 
 title = "gitleaks config"
 # add rules to the rule table
+# [[rules]]
+# description = "Generic Key"
+# regex = '''(?i)key(.{0,6})?(:|=|=>|:=)'''
+# entropies = ["4.1-4.3"]
+# entropyROI = "line"
+# tags = ["key"]
+
 [[rules]]
-description = "AWS"
+description = "AWS Key"
 regex = '''AKIA[0-9A-Z]{16}'''
 tags = ["key", "AWS"]
-entropies = [ "3.3-4.30" ]
-filetypes = [ "*.go" ]
-severity = "5"
+
+[[rules]]
+description = "PKCS8"
+regex = '''-----BEGIN PRIVATE KEY-----'''
+tags = ["key", "PKCS8"]
+
+[[rules]]
+description = "RSA"
+regex = '''-----BEGIN RSA PRIVATE KEY-----'''
+tags = ["key", "RSA"]
+
+[[rules]]
+description = "SSH"
+regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
+tags = ["key", "SSH"]
+
+[[rules]]
+description = "PGP"
+regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
+tags = ["key", "PGP"]
+
+[[rules]]
+description = "Facebook"
+regex = '''(?i)facebook(.{0,4})?['\"][0-9a-f]{32}['\"]'''
+tags = ["key", "Facebook"]
+
+[[rules]]
+description = "Twitter"
+regex = '''(?i)twitter(.{0,4})?['\"][0-9a-zA-Z]{35,44}['\"]'''
+tags = ["key", "Twitter"]
+
+[[rules]]
+description = "Github"
+regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
+tags = ["key", "Github"]
+
+[[rules]]
+description = "Slack"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+tags = ["key", "Slack"]
 
 [whitelist]
 files = [

+ 8 - 3
src/core.go

@@ -27,9 +27,10 @@ func init() {
 	threads = defaultThreadNum
 }
 
-// Report is
+// Report can be exported as a json or csv. Used for logging informationn
+// about the audit, (duration and # of commits)
 type Report struct {
-	Leaks    []*Leak
+	Leaks    []Leak
 	Duration string
 	Commits  int64
 }
@@ -37,7 +38,7 @@ type Report struct {
 // Run is the entry point for gitleaks
 func Run(optsL *Options) (*Report, error) {
 	var (
-		leaks []*Leak
+		leaks []Leak
 		err   error
 	)
 
@@ -47,6 +48,10 @@ func Run(optsL *Options) (*Report, error) {
 	if err != nil {
 		return nil, err
 	}
+    if opts.ValidateConfig {
+        log.Info("valid gitleaks config")
+        return nil, nil
+    }
 
 	if opts.Disk {
 		// temporary directory where all the gitleaks plain clones will reside

+ 4 - 4
src/github.go

@@ -21,8 +21,8 @@ import (
 var githubPages = 100
 
 // auditPR audits a single github PR
-func auditGithubPR() ([]*Leak, error) {
-	var leaks []*Leak
+func auditGithubPR() ([]Leak, error) {
+	var leaks []Leak
 	ctx := context.Background()
 	githubClient := github.NewClient(githubToken())
 	splits := strings.Split(opts.GithubPR, "/")
@@ -91,7 +91,7 @@ func auditGithubPR() ([]*Leak, error) {
 // First, we gather all the github repositories from the github api (this doesnt actually clone the repo).
 // After all the repos have been pulled from github's api we proceed to audit the repos by calling auditGithubRepo.
 // If an error occurs during an audit of a repo, that error is logged but won't break the execution cycle.
-func auditGithubRepos() ([]*Leak, error) {
+func auditGithubRepos() ([]Leak, error) {
 	var (
 		err              error
 		githubRepos      []*github.Repository
@@ -100,7 +100,7 @@ func auditGithubRepos() ([]*Leak, error) {
 		githubOrgOptions *github.RepositoryListByOrgOptions
 		githubOptions    *github.RepositoryListOptions
 		done             bool
-		leaks            []*Leak
+		leaks            []Leak
 		ownerDir         string
 	)
 	ctx := context.Background()

+ 2 - 2
src/gitlab.go

@@ -17,11 +17,11 @@ const gitlabPages = 100
 // auditGitlabRepos kicks off audits if --gitlab-user or --gitlab-org options are set.
 // Getting all repositories from the GitLab API and run audit. If an error occurs during an audit of a repo,
 // that error is logged.
-func auditGitlabRepos() ([]*Leak, error) {
+func auditGitlabRepos() ([]Leak, error) {
 	var (
 		ps      []*gitlab.Project
 		resp    *gitlab.Response
-		leaks   []*Leak
+		leaks   []Leak
 		tempDir string
 		err     error
 	)

+ 16 - 81
src/gitleaks_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 const testWhitelistCommit = `
-[[regexes]]
+[[rules]]
 description = "AWS"
 regex = '''AKIA[0-9A-Z]{16}'''
 
@@ -27,7 +27,7 @@ commits = [
 ]
 `
 const testWhitelistFile = `
-[[regexes]]
+[[rules]]
 description = "AWS"
 regex = '''AKIA[0-9A-Z]{16}'''
 
@@ -38,7 +38,7 @@ files = [
 `
 
 const testWhitelistRegex = `
-[[regexes]]
+[[rules]]
 description = "AWS"
 regex = '''AKIA[0-9A-Z]{16}'''
 
@@ -49,7 +49,7 @@ regexes= [
 `
 
 const testWhitelistRepo = `
-[[regexes]]
+[[rules]]
 description = "AWS"
 regex = '''AKIA[0-9A-Z]{16}'''
 
@@ -60,34 +60,24 @@ repos = [
 `
 
 const testEntropyRange = `
-[entropy]
-ranges = [
+[[rules]]
+description = "Entropy ranges"
+entropies = [
   "7.5-8.0",
   "3.2-3.4",
 ]
-lineregexes = [
-	"(?i)api",
-	"(?i)key",
-	"signature",
-	"secret",
-	"password",
-	"pass",
-	"pwd",
-	"token",
-	"curl",
-	"wget",
-	"https?",
-]
 `
 const testBadEntropyRange = `
-[entropy]
-ranges = [
+[[rules]]
+description = "Bad entropy ranges"
+entropies = [
   "8.0-3.0",
 ]
 `
 const testBadEntropyRange2 = `
-[entropy]
-ranges = [
+[[rules]]
+description = "Bad entropy ranges"
+entropies = [
   "8.0-8.9",
 ]
 `
@@ -373,7 +363,7 @@ func TestWriteReport(t *testing.T) {
 			Line:     "eat",
 			Commit:   "your",
 			Offender: "veggies",
-			Type:     "and",
+			Rule:     "and",
 			Message:  "get",
 			Author:   "some",
 			File:     "sleep",
@@ -630,23 +620,6 @@ func TestAuditRepo(t *testing.T) {
 			testOpts:    &Options{},
 			configPath:  path.Join(configsDir, "repo"),
 		},
-		{
-			repo:        leaksRepo,
-			description: "leaks present with entropy",
-			testOpts: &Options{
-				Entropy: 4.7,
-			},
-			numLeaks: 6,
-		},
-		{
-			repo:        leaksRepo,
-			description: "leaks present with entropy",
-			testOpts: &Options{
-				Entropy:        4.7,
-				NoiseReduction: true,
-			},
-			numLeaks: 2,
-		},
 		{
 			repo:        leaksRepo,
 			description: "Audit until specific commit",
@@ -666,7 +639,7 @@ func TestAuditRepo(t *testing.T) {
 		{
 			repo:        leaksRepo,
 			description: "toml entropy range from opts",
-			numLeaks:    454,
+			numLeaks:    426,
 			testOpts: &Options{
 				ConfigPath: path.Join(configsDir, "entropy"),
 			},
@@ -674,19 +647,10 @@ func TestAuditRepo(t *testing.T) {
 		{
 			repo:        leaksRepo,
 			description: "toml entropy range",
-			numLeaks:    454,
+			numLeaks:    426,
 			testOpts:    &Options{},
 			configPath:  path.Join(configsDir, "entropy"),
 		},
-		{
-			repo: leaksRepo,
-			testOpts: &Options{
-				NoiseReduction: true,
-			},
-			description: "toml entropy noise reduction range",
-			numLeaks:    64,
-			configPath:  path.Join(configsDir, "entropy"),
-		},
 		{
 			repo:           leaksRepo,
 			description:    "toml bad entropy range",
@@ -775,30 +739,6 @@ func TestOptionGuard(t *testing.T) {
 			description:    "local and remote target",
 			expectedErrMsg: "github user set and local owner path",
 		},
-		{
-			testOpts: &Options{
-				GithubUser:   "fakeUser",
-				SingleSearch: "*/./....",
-			},
-			description:         "single search invalid regex gaurd",
-			expectedErrMsgFuzzy: "unable to compile regex: */./...., ",
-		},
-		{
-			testOpts: &Options{
-				GithubUser:   "fakeUser",
-				SingleSearch: "mystring",
-			},
-			description:    "single search regex gaurd",
-			expectedErrMsg: "",
-		},
-		{
-			testOpts: &Options{
-				GithubOrg: "fakeOrg",
-				Entropy:   9,
-			},
-			description:    "Invalid entropy level guard",
-			expectedErrMsg: "The maximum level of entropy is 8",
-		},
 	}
 	g := goblin.Goblin(t)
 	for _, test := range tests {
@@ -880,11 +820,6 @@ func TestLoadToml(t *testing.T) {
 		g.Describe("TestLoadToml", func() {
 			g.It(test.description, func() {
 				opts = test.testOpts
-				if test.singleSearch {
-					singleSearchRegex = regexp.MustCompile("test")
-				} else {
-					singleSearchRegex = nil
-				}
 				if test.configPath != "" {
 					os.Setenv("GITLEAKS_CONFIG", test.configPath)
 				} else {

+ 4 - 4
src/options.go

@@ -40,7 +40,6 @@ type Options struct {
 	ConfigPath   string  `long:"config" description:"path to gitleaks config"`
 	SSHKey       string  `long:"ssh-key" description:"path to ssh key"`
 	ExcludeForks bool    `long:"exclude-forks" description:"exclude forks for organization/user audits"`
-	Entropy      float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
 	RepoConfig   bool    `long:"repo-config" description:"Load config from target repo. Config file must be \".gitleaks.toml\""`
 	Branch       string  `long:"branch" description:"Branch to audit"`
 	// TODO: IncludeMessages  string `long:"messages" description:"include commit messages in audit"`
@@ -52,6 +51,7 @@ type Options struct {
 	Redact       bool   `long:"redact" description:"redact secrets from log messages and report"`
 	Version      bool   `long:"version" description:"version number"`
 	SampleConfig bool   `long:"sample-config" description:"prints a sample config file"`
+    ValidateConfig bool `long:"validate-config" description:"validate gitleaks config"`
 }
 
 // ParseOpts parses the options
@@ -125,9 +125,9 @@ func (opts *Options) guard() error {
 		}
 	}
 
-	if opts.Entropy > 8 {
-		return fmt.Errorf("The maximum level of entropy is 8")
-	}
+	// if opts.Entropy > 8 {
+	// 	return fmt.Errorf("The maximum level of entropy is 8")
+	// }
 	if opts.Report != "" {
 		if !strings.HasSuffix(opts.Report, ".json") && !strings.HasSuffix(opts.Report, ".csv") {
 			return fmt.Errorf("Report should be a .json or .csv file")

+ 6 - 5
src/repo.go

@@ -23,7 +23,8 @@ type Leak struct {
 	Line     string    `json:"line"`
 	Commit   string    `json:"commit"`
 	Offender string    `json:"offender"`
-	Type     string    `json:"reason"`
+	Rule string    `json:"rule"`
+	Info string    `json:"info"`
 	Message  string    `json:"commitMsg"`
 	Author   string    `json:"author"`
 	Email    string    `json:"email"`
@@ -106,10 +107,10 @@ func (repoInfo *RepoInfo) clone() error {
 }
 
 // audit performs an audit
-func (repoInfo *RepoInfo) audit() ([]*Leak, error) {
+func (repoInfo *RepoInfo) audit() ([]Leak, error) {
 	var (
 		err         error
-		leaks       []*Leak
+		leaks       []Leak
 		commitCount int64
 		commitWg    sync.WaitGroup
 		semaphore   chan bool
@@ -281,8 +282,8 @@ func (repoInfo *RepoInfo) audit() ([]*Leak, error) {
 	return leaks, nil
 }
 
-func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []*Leak {
-	var leaks []*Leak
+func (repoInfo *RepoInfo) auditSingleCommit(c *object.Commit) []Leak {
+	var leaks []Leak
 	fIter, err := c.Files()
 	if err != nil {
 		return nil

+ 38 - 24
src/utils.go

@@ -28,7 +28,7 @@ type commitInfo struct {
 
 // writeReport writes a report to a file specified in the --report= option.
 // Default format for report is JSON. You can use the --csv option to write the report as a csv
-func writeReport(leaks []*Leak) error {
+func writeReport(leaks []Leak) error {
 	if len(leaks) == 0 {
 		return nil
 	}
@@ -43,7 +43,7 @@ func writeReport(leaks []*Leak) error {
 		w := csv.NewWriter(f)
 		w.Write([]string{"repo", "line", "commit", "offender", "reason", "commitMsg", "author", "email", "file", "date"})
 		for _, leak := range leaks {
-			w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Type, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
+			w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Rule, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
 		}
 		w.Flush()
 	} else {
@@ -91,28 +91,41 @@ func (rule *Rule) check(line string, commit *commitInfo) (*Leak, error) {
 	)
 
 	if rule.entropies != nil {
-		words := strings.Fields(line)
-		for _, word := range words {
-			_entropy := getShannonEntropy(word)
-			for _, e := range rule.entropies {
-				if _entropy > e.v1 && _entropy < e.v2 {
-					entropy = _entropy
-					entropyWord = word
-				}
-			}
-		}
+        if rule.entropyROI == "line" {
+            _entropy := getShannonEntropy(line)
+            for _, e := range rule.entropies {
+                if _entropy > e.v1 && _entropy < e.v2 {
+                    entropy = _entropy
+                    entropyWord = line
+                    goto postEntropy
+                }
+            }
+        } else {
+            words := strings.Fields(line)
+            for _, word := range words {
+                _entropy := getShannonEntropy(word)
+                for _, e := range rule.entropies {
+                    if _entropy > e.v1 && _entropy < e.v2 {
+                        entropy = _entropy
+                        entropyWord = word
+                        goto postEntropy
+                    }
+                }
+            }
+        }
 	}
 
+postEntropy:
 	if rule.regex != nil {
 		match = rule.regex.FindString(line)
 	}
 
 	if match != "" && entropy != 0.0 {
-		return newLeak(line, entropyWord, rule, commit), nil
+		return newLeak(line, fmt.Sprintf("%s regex match and entropy met at %.2f", rule.regex.String(), entropy), entropyWord, rule, commit), nil
 	} else if match != "" && rule.entropies == nil {
-		return newLeak(line, match, rule, commit), nil
-	} else if entropy != 0.0 && rule.regex == nil {
-		return newLeak(line, entropyWord, rule, commit), nil
+		return newLeak(line, fmt.Sprintf("%s regex match", rule.regex.String()), match, rule, commit), nil
+	} else if entropy != 0.0 && rule.regex.String() == "" {
+		return newLeak(line, fmt.Sprintf("entropy met at %.2f", entropy), entropyWord, rule, commit), nil
 	}
 	return nil, nil
 }
@@ -121,8 +134,8 @@ func (rule *Rule) check(line string, commit *commitInfo) (*Leak, error) {
 // a set of regexes set by the config (see gitleaks.toml for example). This function
 // will skip lines that include a whitelisted regex. A list of leaks is returned.
 // If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
-func inspect(commit *commitInfo) []*Leak {
-	var leaks []*Leak
+func inspect(commit *commitInfo) []Leak {
+	var leaks []Leak
 	lines := strings.Split(commit.content, "\n")
 
 	for _, line := range lines {
@@ -131,10 +144,10 @@ func inspect(commit *commitInfo) []*Leak {
 				break
 			}
 			leak, err := rule.check(line, commit)
-			if err != nil {
+			if err != nil || leak == nil {
 				continue
 			}
-			leaks = append(leaks, leak)
+			leaks = append(leaks, *leak)
 		}
 	}
 	return leaks
@@ -151,12 +164,13 @@ func isLineWhitelisted(line string) bool {
 	return false
 }
 
-func newLeak(line string, msg string, rule *Rule, commit *commitInfo) *Leak {
+func newLeak(line string, info string, offender string, rule *Rule, commit *commitInfo) *Leak {
 	leak := &Leak{
 		Line:     line,
 		Commit:   commit.sha,
-		Offender: msg,
-		Type:     rule.description,
+		Offender: offender,
+		Rule:     rule.description,
+        Info: info,
 		Author:   commit.author,
 		Email:    commit.email,
 		File:     commit.filePath,
@@ -168,7 +182,7 @@ func newLeak(line string, msg string, rule *Rule, commit *commitInfo) *Leak {
 	}
 	if opts.Redact {
 		leak.Offender = "REDACTED"
-		leak.Line = strings.Replace(line, msg, "REDACTED", -1)
+		leak.Line = strings.Replace(line, offender, "REDACTED", -1)
 	}
 
 	if opts.Verbose {