|
|
@@ -77,13 +77,14 @@ type Options struct {
|
|
|
OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
|
|
|
|
|
|
// Process options
|
|
|
- Threads int `long:"threads" description:"Maximum number of threads gitleaks spawns"`
|
|
|
- Disk bool `long:"disk" description:"Clones repo(s) to disk"`
|
|
|
- SingleSearch string `long:"single-search" description:"single regular expression to search for"`
|
|
|
- ConfigPath string `long:"config" description:"path to gitleaks config"`
|
|
|
- SSHKey string `long:"ssh-key" description:"path to ssh key"`
|
|
|
- ExcludeForks bool `long:"exclude-forks" description:"exclude forks for organization/user audits"`
|
|
|
- Entropy float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
|
|
|
+ Threads int `long:"threads" description:"Maximum number of threads gitleaks spawns"`
|
|
|
+ Disk bool `long:"disk" description:"Clones repo(s) to disk"`
|
|
|
+ SingleSearch string `long:"single-search" description:"single regular expression to search for"`
|
|
|
+ ConfigPath string `long:"config" description:"path to gitleaks config"`
|
|
|
+ SSHKey string `long:"ssh-key" description:"path to ssh key"`
|
|
|
+ ExcludeForks bool `long:"exclude-forks" description:"exclude forks for organization/user audits"`
|
|
|
+ Entropy float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
|
|
|
+ NoiseReduction bool `long:"noise-reduction" description:"Reduce the number of finds when entropy checks are enabled"`
|
|
|
// TODO: IncludeMessages string `long:"messages" description:"include commit messages in audit"`
|
|
|
|
|
|
// Output options
|
|
|
@@ -101,6 +102,9 @@ type Config struct {
|
|
|
Description string
|
|
|
Regex string
|
|
|
}
|
|
|
+ Entropy struct {
|
|
|
+ LineRegexes []string
|
|
|
+ }
|
|
|
Whitelist struct {
|
|
|
Files []string
|
|
|
Regexes []string
|
|
|
@@ -130,7 +134,7 @@ type entropyRange struct {
|
|
|
}
|
|
|
|
|
|
const defaultGithubURL = "https://api.github.com/"
|
|
|
-const version = "1.20.0"
|
|
|
+const version = "1.22.0"
|
|
|
const errExit = 2
|
|
|
const leakExit = 1
|
|
|
const defaultConfig = `
|
|
|
@@ -169,9 +173,24 @@ regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
|
|
|
description = "Slack"
|
|
|
regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
|
|
|
|
|
|
+[entropy]
|
|
|
+lineregexes = [
|
|
|
+ "api",
|
|
|
+ "key",
|
|
|
+ "signature",
|
|
|
+ "secret",
|
|
|
+ "password",
|
|
|
+ "pass",
|
|
|
+ "pwd",
|
|
|
+ "token",
|
|
|
+ "curl",
|
|
|
+ "wget",
|
|
|
+ "https?",
|
|
|
+]
|
|
|
+
|
|
|
[whitelist]
|
|
|
files = [
|
|
|
- "(.*?)(jpg|gif|doc|pdf|bin)$"
|
|
|
+ "(.*?)(jpg|gif|doc|pdf|bin)$"
|
|
|
]
|
|
|
#commits = [
|
|
|
# "BADHA5H1",
|
|
|
@@ -196,6 +215,7 @@ var (
|
|
|
whiteListCommits map[string]bool
|
|
|
whiteListRepos []*regexp.Regexp
|
|
|
entropyRanges []entropyRange
|
|
|
+ entropyRegexes []*regexp.Regexp
|
|
|
fileDiffRegex *regexp.Regexp
|
|
|
sshAuth *ssh.PublicKeys
|
|
|
dir string
|
|
|
@@ -325,6 +345,10 @@ func run() ([]Leak, error) {
|
|
|
// writeReport writes a report to a file specified in the --report= option.
|
|
|
// Default format for report is JSON. You can use the --csv option to write the report as a csv
|
|
|
func writeReport(leaks []Leak) error {
|
|
|
+ if len(leaks) == 0 {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
var err error
|
|
|
log.Infof("writing report to %s", opts.Report)
|
|
|
if strings.HasSuffix(opts.Report, ".csv") {
|
|
|
@@ -488,11 +512,6 @@ func auditGitReference(repo *RepoDescriptor, ref *plumbing.Reference) []Leak {
|
|
|
if bin || err != nil {
|
|
|
return nil
|
|
|
}
|
|
|
- for _, re := range whiteListFiles {
|
|
|
- if re.FindString(f.Name) != "" {
|
|
|
- return nil
|
|
|
- }
|
|
|
- }
|
|
|
content, err := f.Contents()
|
|
|
if err != nil {
|
|
|
return nil
|
|
|
@@ -560,12 +579,6 @@ func auditGitReference(repo *RepoDescriptor, ref *plumbing.Reference) []Leak {
|
|
|
} else if to != nil {
|
|
|
filePath = to.Path()
|
|
|
}
|
|
|
- for _, re := range whiteListFiles {
|
|
|
- if re.FindString(filePath) != "" {
|
|
|
- skipFile = true
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
if skipFile {
|
|
|
continue
|
|
|
}
|
|
|
@@ -604,12 +617,19 @@ func auditGitReference(repo *RepoDescriptor, ref *plumbing.Reference) []Leak {
|
|
|
// will skip lines that include a whitelisted regex. A list of leaks is returned.
|
|
|
// If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
|
|
|
func inspect(diff gitDiff) []Leak {
|
|
|
- lines := strings.Split(diff.content, "\n")
|
|
|
var (
|
|
|
leaks []Leak
|
|
|
skipLine bool
|
|
|
)
|
|
|
|
|
|
+ for _, re := range whiteListFiles {
|
|
|
+ if re.FindString(diff.filePath) != "" {
|
|
|
+ return leaks
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ lines := strings.Split(diff.content, "\n")
|
|
|
+
|
|
|
for _, line := range lines {
|
|
|
skipLine = false
|
|
|
for leakType, re := range regexes {
|
|
|
@@ -634,23 +654,12 @@ func inspect(diff gitDiff) []Leak {
|
|
|
}
|
|
|
|
|
|
if opts.Entropy > 0 || len(entropyRanges) != 0 {
|
|
|
- entropyLeak := false
|
|
|
words := strings.Fields(line)
|
|
|
for _, word := range words {
|
|
|
entropy := getShannonEntropy(word)
|
|
|
- if entropy >= opts.Entropy && len(entropyRanges) == 0 {
|
|
|
- entropyLeak = true
|
|
|
- }
|
|
|
- if len(entropyRanges) != 0 {
|
|
|
- for _, eR := range entropyRanges {
|
|
|
- if entropy > eR.v1 && entropy < eR.v2 {
|
|
|
- entropyLeak = true
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if entropyLeak {
|
|
|
+
|
|
|
+ if entropyIsHighEnough(entropy) && highEntropyLineIsALeak(line) {
|
|
|
leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), diff)
|
|
|
- entropyLeak = false
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -673,7 +682,7 @@ func addLeak(leaks []Leak, line string, offender string, leakType string, diff g
|
|
|
}
|
|
|
if opts.Redact {
|
|
|
leak.Offender = "REDACTED"
|
|
|
- leak.Line = "REDACTED"
|
|
|
+ leak.Line = strings.Replace(line, offender, "REDACTED", -1)
|
|
|
}
|
|
|
|
|
|
if opts.Verbose {
|
|
|
@@ -704,6 +713,36 @@ func getShannonEntropy(data string) (entropy float64) {
|
|
|
return entropy
|
|
|
}
|
|
|
|
|
|
+func entropyIsHighEnough(entropy float64) bool {
|
|
|
+ if entropy >= opts.Entropy && len(entropyRanges) == 0 {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(entropyRanges) != 0 {
|
|
|
+ for _, eR := range entropyRanges {
|
|
|
+ if entropy > eR.v1 && entropy < eR.v2 {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
+func highEntropyLineIsALeak(line string) bool {
|
|
|
+ if !opts.NoiseReduction {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, re := range entropyRegexes {
|
|
|
+ if re.FindString(line) != "" {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
// discoverRepos walks all the children of `path`. If a child directory
|
|
|
// contain a .git file then that repo will be added to the list of repos returned
|
|
|
func discoverRepos(ownerPath string) ([]*RepoDescriptor, error) {
|
|
|
@@ -846,6 +885,10 @@ func loadToml() error {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ for _, regex := range config.Entropy.LineRegexes {
|
|
|
+ entropyRegexes = append(entropyRegexes, regexp.MustCompile(regex))
|
|
|
+ }
|
|
|
+
|
|
|
if singleSearchRegex != nil {
|
|
|
regexes["singleSearch"] = singleSearchRegex
|
|
|
} else {
|
Adam Kobi