main.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878
  1. package main
  2. import (
  3. "crypto/md5"
  4. "encoding/csv"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "io/ioutil"
  9. "math"
  10. "net"
  11. "net/url"
  12. "os"
  13. "os/user"
  14. "path"
  15. "path/filepath"
  16. "regexp"
  17. "runtime"
  18. "strconv"
  19. "strings"
  20. "sync"
  21. "time"
  22. "gopkg.in/src-d/go-git.v4/plumbing"
  23. "gopkg.in/src-d/go-git.v4/plumbing/object"
  24. "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
  25. "gopkg.in/src-d/go-git.v4/storage/memory"
  26. "github.com/BurntSushi/toml"
  27. "github.com/google/go-github/github"
  28. "github.com/hako/durafmt"
  29. flags "github.com/jessevdk/go-flags"
  30. log "github.com/sirupsen/logrus"
  31. git "gopkg.in/src-d/go-git.v4"
  32. )
  33. // Leak represents a leaked secret or regex match.
  34. type Leak struct {
  35. Line string `json:"line"`
  36. Commit string `json:"commit"`
  37. Offender string `json:"offender"`
  38. Type string `json:"reason"`
  39. Message string `json:"commitMsg"`
  40. Author string `json:"author"`
  41. File string `json:"file"`
  42. Repo string `json:"repo"`
  43. Date time.Time `json:"date"`
  44. }
  45. // RepoDescriptor contains a src-d git repository and other data about the repo
  46. type RepoDescriptor struct {
  47. path string
  48. url string
  49. name string
  50. repository *git.Repository
  51. err error
  52. }
  53. // Options for gitleaks
  54. type Options struct {
  55. // remote target options
  56. Repo string `short:"r" long:"repo" description:"Repo url to audit"`
  57. GithubUser string `long:"github-user" description:"Github user to audit"`
  58. GithubOrg string `long:"github-org" description:"Github organization to audit"`
  59. GithubURL string `long:"github-url" default:"https://api.github.com/" description:"GitHub API Base URL, use for GitHub Enterprise. Example: https://github.example.com/api/v3/"`
  60. GithubPR string `long:"github-pr" description:"Github PR url to audit. This does not clone the repo. GITHUB_TOKEN must be set"`
  61. GitLabUser string `long:"gitlab-user" description:"GitLab user ID to audit"`
  62. GitLabOrg string `long:"gitlab-org" description:"GitLab group ID to audit"`
  63. Commit string `short:"c" long:"commit" description:"sha of commit to stop at"`
  64. Depth int `long:"depth" description:"maximum commit depth"`
  65. // local target option
  66. RepoPath string `long:"repo-path" description:"Path to repo"`
  67. OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
  68. // Process options
  69. Threads int `long:"threads" description:"Maximum number of threads gitleaks spawns"`
  70. Disk bool `long:"disk" description:"Clones repo(s) to disk"`
  71. SingleSearch string `long:"single-search" description:"single regular expression to search for"`
  72. ConfigPath string `long:"config" description:"path to gitleaks config"`
  73. SSHKey string `long:"ssh-key" description:"path to ssh key"`
  74. ExcludeForks bool `long:"exclude-forks" description:"exclude forks for organization/user audits"`
  75. Entropy float64 `long:"entropy" short:"e" description:"Include entropy checks during audit. Entropy scale: 0.0(no entropy) - 8.0(max entropy)"`
  76. // TODO: IncludeMessages string `long:"messages" description:"include commit messages in audit"`
  77. // Output options
  78. Log string `short:"l" long:"log" description:"log level"`
  79. Verbose bool `short:"v" long:"verbose" description:"Show verbose output from gitleaks audit"`
  80. Report string `long:"report" description:"path to write report file. Needs to be csv or json"`
  81. Redact bool `long:"redact" description:"redact secrets from log messages and report"`
  82. Version bool `long:"version" description:"version number"`
  83. SampleConfig bool `long:"sample-config" description:"prints a sample config file"`
  84. }
  85. // Config struct for regexes matching and whitelisting
  86. type Config struct {
  87. Regexes []struct {
  88. Description string
  89. Regex string
  90. }
  91. Whitelist struct {
  92. Files []string
  93. Regexes []string
  94. Commits []string
  95. Repos []string
  96. }
  97. Misc struct {
  98. Entropy []string
  99. }
  100. }
  101. type gitDiff struct {
  102. content string
  103. commit *object.Commit
  104. filePath string
  105. repoName string
  106. githubCommit *github.RepositoryCommit
  107. sha string
  108. message string
  109. author string
  110. date time.Time
  111. }
  112. type entropyRange struct {
  113. v1 float64
  114. v2 float64
  115. }
  116. const defaultGithubURL = "https://api.github.com/"
  117. const version = "1.19.1"
  118. const errExit = 2
  119. const leakExit = 1
  120. const defaultConfig = `
  121. # This is a sample config file for gitleaks. You can configure gitleaks what to search for and what to whitelist.
  122. # The output you are seeing here is the default gitleaks config. If GITLEAKS_CONFIG environment variable
  123. # is set, gitleaks will load configurations from that path. If option --config-path is set, gitleaks will load
  124. # configurations from that path. Gitleaks does not whitelist anything by default.
  125. title = "gitleaks config"
  126. # add regexes to the regex table
  127. [[regexes]]
  128. description = "AWS"
  129. regex = '''AKIA[0-9A-Z]{16}'''
  130. [[regexes]]
  131. description = "PKCS8"
  132. regex = '''-----BEGIN PRIVATE KEY-----'''
  133. [[regexes]]
  134. description = "RSA"
  135. regex = '''-----BEGIN RSA PRIVATE KEY-----'''
  136. [[regexes]]
  137. description = "SSH"
  138. regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
  139. [[regexes]]
  140. description = "PGP"
  141. regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
  142. [[regexes]]
  143. description = "Facebook"
  144. regex = '''(?i)facebook(.{0,4})?['\"][0-9a-f]{32}['\"]'''
  145. [[regexes]]
  146. description = "Twitter"
  147. regex = '''(?i)twitter(.{0,4})?['\"][0-9a-zA-Z]{35,44}['\"]'''
  148. [[regexes]]
  149. description = "Github"
  150. regex = '''(?i)github(.{0,4})?['\"][0-9a-zA-Z]{35,40}['\"]'''
  151. [[regexes]]
  152. description = "Slack"
  153. regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
  154. [whitelist]
  155. files = [
  156. "(.*?)(jpg|gif|doc|pdf|bin)$"
  157. ]
  158. #commits = [
  159. # "BADHA5H1",
  160. # "BADHA5H2",
  161. #]
  162. #repos = [
  163. # "mygoodrepo"
  164. #]
  165. [misc]
  166. #entropy = [
  167. # "3.3-4.30"
  168. # "6.0-8.0
  169. #]
  170. `
  171. var (
  172. opts Options
  173. regexes map[string]*regexp.Regexp
  174. singleSearchRegex *regexp.Regexp
  175. whiteListRegexes []*regexp.Regexp
  176. whiteListFiles []*regexp.Regexp
  177. whiteListCommits map[string]bool
  178. whiteListRepos []*regexp.Regexp
  179. entropyRanges []entropyRange
  180. fileDiffRegex *regexp.Regexp
  181. sshAuth *ssh.PublicKeys
  182. dir string
  183. threads int
  184. totalCommits int64
  185. commitMap = make(map[string]bool)
  186. cMutex = &sync.Mutex{}
  187. )
  188. func init() {
  189. log.SetOutput(os.Stdout)
  190. // threads = runtime.GOMAXPROCS(0) / 2
  191. threads = 1
  192. regexes = make(map[string]*regexp.Regexp)
  193. whiteListCommits = make(map[string]bool)
  194. }
  195. func main() {
  196. parser := flags.NewParser(&opts, flags.Default)
  197. _, err := parser.Parse()
  198. if err != nil {
  199. if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
  200. os.Exit(0)
  201. }
  202. }
  203. if len(os.Args) == 1 {
  204. parser.WriteHelp(os.Stdout)
  205. os.Exit(0)
  206. }
  207. if opts.Version {
  208. fmt.Println(version)
  209. os.Exit(0)
  210. }
  211. if opts.SampleConfig {
  212. fmt.Println(defaultConfig)
  213. os.Exit(0)
  214. }
  215. now := time.Now()
  216. leaks, err := run()
  217. if err != nil {
  218. if strings.Contains(err.Error(), "whitelisted") {
  219. log.Info(err.Error())
  220. os.Exit(0)
  221. }
  222. log.Error(err)
  223. os.Exit(errExit)
  224. }
  225. if opts.Report != "" {
  226. writeReport(leaks)
  227. }
  228. if len(leaks) != 0 {
  229. log.Warnf("%d leaks detected. %d commits inspected in %s", len(leaks), totalCommits, durafmt.Parse(time.Now().Sub(now)).String())
  230. os.Exit(leakExit)
  231. } else {
  232. log.Infof("%d leaks detected. %d commits inspected in %s", len(leaks), totalCommits, durafmt.Parse(time.Now().Sub(now)).String())
  233. }
  234. }
  235. // run parses options and kicks off the audit
  236. func run() ([]Leak, error) {
  237. var leaks []Leak
  238. setLogs()
  239. err := optsGuard()
  240. if err != nil {
  241. return nil, err
  242. }
  243. err = loadToml()
  244. if err != nil {
  245. return nil, err
  246. }
  247. sshAuth, err = getSSHAuth()
  248. if err != nil {
  249. return leaks, err
  250. }
  251. if opts.Disk {
  252. // temporary directory where all the gitleaks plain clones will reside
  253. dir, err = ioutil.TempDir("", "gitleaks")
  254. defer os.RemoveAll(dir)
  255. if err != nil {
  256. return nil, err
  257. }
  258. }
  259. // start audits
  260. if opts.Repo != "" || opts.RepoPath != "" {
  261. // Audit a single remote repo or a local repo.
  262. repo, err := cloneRepo()
  263. if err != nil {
  264. return leaks, err
  265. }
  266. return auditGitRepo(repo)
  267. } else if opts.OwnerPath != "" {
  268. // Audit local repos. Gitleaks will look for all child directories of OwnerPath for
  269. // git repos and perform an audit on said repos.
  270. repos, err := discoverRepos(opts.OwnerPath)
  271. if err != nil {
  272. return leaks, err
  273. }
  274. for _, repo := range repos {
  275. leaksFromRepo, err := auditGitRepo(repo)
  276. if err != nil {
  277. return leaks, err
  278. }
  279. leaks = append(leaksFromRepo, leaks...)
  280. }
  281. } else if opts.GithubOrg != "" || opts.GithubUser != "" {
  282. // Audit a github owner -- a user or organization.
  283. leaks, err = auditGithubRepos()
  284. if err != nil {
  285. return leaks, err
  286. }
  287. } else if opts.GitLabOrg != "" || opts.GitLabUser != "" {
  288. leaks, err = auditGitlabRepos()
  289. if err != nil {
  290. return leaks, err
  291. }
  292. } else if opts.GithubPR != "" {
  293. return auditGithubPR()
  294. }
  295. return leaks, nil
  296. }
  297. // writeReport writes a report to a file specified in the --report= option.
  298. // Default format for report is JSON. You can use the --csv option to write the report as a csv
  299. func writeReport(leaks []Leak) error {
  300. var err error
  301. log.Infof("writing report to %s", opts.Report)
  302. if strings.HasSuffix(opts.Report, ".csv") {
  303. f, err := os.Create(opts.Report)
  304. if err != nil {
  305. return err
  306. }
  307. defer f.Close()
  308. w := csv.NewWriter(f)
  309. w.Write([]string{"repo", "line", "commit", "offender", "reason", "commitMsg", "author", "file", "date"})
  310. for _, leak := range leaks {
  311. w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Type, leak.Message, leak.Author, leak.File, leak.Date.Format(time.RFC3339)})
  312. }
  313. w.Flush()
  314. } else {
  315. reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
  316. err = ioutil.WriteFile(opts.Report, reportJSON, 0644)
  317. }
  318. return err
  319. }
  320. // cloneRepo clones a repo to memory(default) or to disk if the --disk option is set.
  321. func cloneRepo() (*RepoDescriptor, error) {
  322. var (
  323. err error
  324. repo *git.Repository
  325. )
  326. // check if whitelist
  327. for _, re := range whiteListRepos {
  328. if re.FindString(opts.Repo) != "" {
  329. return nil, fmt.Errorf("skipping %s, whitelisted", opts.Repo)
  330. }
  331. }
  332. if opts.Disk {
  333. log.Infof("cloning %s", opts.Repo)
  334. cloneTarget := fmt.Sprintf("%s/%x", dir, md5.Sum([]byte(fmt.Sprintf("%s%s", opts.GithubUser, opts.Repo))))
  335. if strings.HasPrefix(opts.Repo, "git") {
  336. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  337. URL: opts.Repo,
  338. Progress: os.Stdout,
  339. Auth: sshAuth,
  340. })
  341. } else {
  342. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  343. URL: opts.Repo,
  344. Progress: os.Stdout,
  345. })
  346. }
  347. } else if opts.RepoPath != "" {
  348. log.Infof("opening %s", opts.RepoPath)
  349. repo, err = git.PlainOpen(opts.RepoPath)
  350. } else {
  351. log.Infof("cloning %s", opts.Repo)
  352. if strings.HasPrefix(opts.Repo, "git") {
  353. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  354. URL: opts.Repo,
  355. Progress: os.Stdout,
  356. Auth: sshAuth,
  357. })
  358. } else {
  359. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  360. URL: opts.Repo,
  361. Progress: os.Stdout,
  362. })
  363. }
  364. }
  365. return &RepoDescriptor{
  366. repository: repo,
  367. path: opts.RepoPath,
  368. url: opts.Repo,
  369. name: filepath.Base(opts.Repo),
  370. err: err,
  371. }, nil
  372. }
  373. // auditGitRepo beings an audit on a git repository
  374. func auditGitRepo(repo *RepoDescriptor) ([]Leak, error) {
  375. var (
  376. err error
  377. leaks []Leak
  378. )
  379. for _, re := range whiteListRepos {
  380. if re.FindString(repo.name) != "" {
  381. return leaks, fmt.Errorf("skipping %s, whitelisted", repo.name)
  382. }
  383. }
  384. // clear commit cache
  385. commitMap = make(map[string]bool)
  386. refs, err := repo.repository.Storer.IterReferences()
  387. if err != nil {
  388. return leaks, err
  389. }
  390. err = refs.ForEach(func(ref *plumbing.Reference) error {
  391. if ref.Name().IsTag() {
  392. return nil
  393. }
  394. branchLeaks := auditGitReference(repo, ref)
  395. for _, leak := range branchLeaks {
  396. leaks = append(leaks, leak)
  397. }
  398. return nil
  399. })
  400. return leaks, err
  401. }
  402. // auditGitReference beings the audit for a git reference. This function will
  403. // traverse the git reference and audit each line of each diff.
  404. func auditGitReference(repo *RepoDescriptor, ref *plumbing.Reference) []Leak {
  405. var (
  406. err error
  407. repoName string
  408. leaks []Leak
  409. commitCount int
  410. commitWg sync.WaitGroup
  411. mutex = &sync.Mutex{}
  412. semaphore chan bool
  413. )
  414. repoName = repo.name
  415. if opts.Threads != 0 {
  416. threads = opts.Threads
  417. }
  418. if opts.RepoPath != "" {
  419. threads = 1
  420. }
  421. semaphore = make(chan bool, threads)
  422. cIter, err := repo.repository.Log(&git.LogOptions{From: ref.Hash()})
  423. if err != nil {
  424. return nil
  425. }
  426. err = cIter.ForEach(func(c *object.Commit) error {
  427. if commitMap[c.Hash.String()] {
  428. return nil
  429. }
  430. cMutex.Lock()
  431. commitMap[c.Hash.String()] = true
  432. cMutex.Unlock()
  433. if c == nil || c.Hash.String() == opts.Commit || (opts.Depth != 0 && commitCount == opts.Depth) {
  434. cIter.Close()
  435. return errors.New("ErrStop")
  436. }
  437. commitCount = commitCount + 1
  438. totalCommits = totalCommits + 1
  439. if whiteListCommits[c.Hash.String()] {
  440. log.Infof("skipping commit: %s\n", c.Hash.String())
  441. return nil
  442. }
  443. err = c.Parents().ForEach(func(parent *object.Commit) error {
  444. commitWg.Add(1)
  445. semaphore <- true
  446. go func(c *object.Commit, parent *object.Commit) {
  447. var (
  448. filePath string
  449. skipFile bool
  450. )
  451. defer func() {
  452. commitWg.Done()
  453. <-semaphore
  454. if r := recover(); r != nil {
  455. log.Warnf("recoverying from panic on commit %s, likely large diff causing panic", c.Hash.String())
  456. }
  457. }()
  458. patch, err := c.Patch(parent)
  459. if err != nil {
  460. log.Warnf("problem generating patch for commit: %s\n", c.Hash.String())
  461. return
  462. }
  463. for _, f := range patch.FilePatches() {
  464. if f.IsBinary() {
  465. continue
  466. }
  467. skipFile = false
  468. from, to := f.Files()
  469. filePath = "???"
  470. if from != nil {
  471. filePath = from.Path()
  472. } else if to != nil {
  473. filePath = to.Path()
  474. }
  475. for _, re := range whiteListFiles {
  476. if re.FindString(filePath) != "" {
  477. skipFile = true
  478. break
  479. }
  480. }
  481. if skipFile {
  482. continue
  483. }
  484. chunks := f.Chunks()
  485. for _, chunk := range chunks {
  486. if chunk.Type() == 1 || chunk.Type() == 2 {
  487. diff := gitDiff{
  488. repoName: repoName,
  489. filePath: filePath,
  490. content: chunk.Content(),
  491. sha: c.Hash.String(),
  492. author: c.Author.String(),
  493. message: c.Message,
  494. date: c.Author.When,
  495. }
  496. chunkLeaks := inspect(diff)
  497. for _, leak := range chunkLeaks {
  498. mutex.Lock()
  499. leaks = append(leaks, leak)
  500. mutex.Unlock()
  501. }
  502. }
  503. }
  504. }
  505. }(c, parent)
  506. return nil
  507. })
  508. return nil
  509. })
  510. commitWg.Wait()
  511. return leaks
  512. }
  513. // inspect will parse each line of the git diff's content against a set of regexes or
  514. // a set of regexes set by the config (see gitleaks.toml for example). This function
  515. // will skip lines that include a whitelisted regex. A list of leaks is returned.
  516. // If verbose mode (-v/--verbose) is set, then checkDiff will log leaks as they are discovered.
  517. func inspect(diff gitDiff) []Leak {
  518. lines := strings.Split(diff.content, "\n")
  519. var (
  520. leaks []Leak
  521. skipLine bool
  522. )
  523. for _, line := range lines {
  524. skipLine = false
  525. for leakType, re := range regexes {
  526. match := re.FindString(line)
  527. if match == "" {
  528. continue
  529. }
  530. // if offender matches whitelist regex, ignore it
  531. for _, wRe := range whiteListRegexes {
  532. whitelistMatch := wRe.FindString(line)
  533. if whitelistMatch != "" {
  534. skipLine = true
  535. break
  536. }
  537. }
  538. if skipLine {
  539. break
  540. }
  541. leaks = addLeak(leaks, line, match, leakType, diff)
  542. }
  543. if opts.Entropy > 0 || len(entropyRanges) != 0 {
  544. entropyLeak := false
  545. words := strings.Fields(line)
  546. for _, word := range words {
  547. entropy := getShannonEntropy(word)
  548. if entropy >= opts.Entropy && len(entropyRanges) == 0 {
  549. entropyLeak = true
  550. }
  551. if len(entropyRanges) != 0 {
  552. for _, eR := range entropyRanges {
  553. if entropy > eR.v1 && entropy < eR.v2 {
  554. entropyLeak = true
  555. }
  556. }
  557. }
  558. if entropyLeak {
  559. leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), diff)
  560. entropyLeak = false
  561. }
  562. }
  563. }
  564. }
  565. return leaks
  566. }
  567. // addLeak is helper for func inspect() to append leaks if found during a diff check.
  568. func addLeak(leaks []Leak, line string, offender string, leakType string, diff gitDiff) []Leak {
  569. leak := Leak{
  570. Line: line,
  571. Commit: diff.sha,
  572. Offender: offender,
  573. Type: leakType,
  574. Author: diff.author,
  575. File: diff.filePath,
  576. Repo: diff.repoName,
  577. Message: diff.message,
  578. Date: diff.date,
  579. }
  580. if opts.Redact {
  581. leak.Offender = "REDACTED"
  582. leak.Line = "REDACTED"
  583. }
  584. if opts.Verbose {
  585. leak.log()
  586. }
  587. leaks = append(leaks, leak)
  588. return leaks
  589. }
  590. // getShannonEntropy https://en.wiktionary.org/wiki/Shannon_entropy
  591. func getShannonEntropy(data string) (entropy float64) {
  592. if data == "" {
  593. return 0
  594. }
  595. charCounts := make(map[rune]int)
  596. for _, char := range data {
  597. charCounts[char]++
  598. }
  599. invLength := 1.0 / float64(len(data))
  600. for _, count := range charCounts {
  601. freq := float64(count) * invLength
  602. entropy -= freq * math.Log2(freq)
  603. }
  604. return entropy
  605. }
  606. // discoverRepos walks all the children of `path`. If a child directory
  607. // contain a .git file then that repo will be added to the list of repos returned
  608. func discoverRepos(ownerPath string) ([]*RepoDescriptor, error) {
  609. var (
  610. err error
  611. repos []*RepoDescriptor
  612. )
  613. files, err := ioutil.ReadDir(ownerPath)
  614. if err != nil {
  615. return repos, err
  616. }
  617. for _, f := range files {
  618. if f.IsDir() {
  619. repoPath := path.Join(ownerPath, f.Name())
  620. r, err := git.PlainOpen(repoPath)
  621. if err != nil {
  622. continue
  623. }
  624. repos = append(repos, &RepoDescriptor{
  625. repository: r,
  626. name: f.Name(),
  627. path: repoPath,
  628. })
  629. }
  630. }
  631. return repos, err
  632. }
  633. // setLogLevel sets log level for gitleaks. Default is Warning
  634. func setLogs() {
  635. switch opts.Log {
  636. case "info":
  637. log.SetLevel(log.InfoLevel)
  638. case "debug":
  639. log.SetLevel(log.DebugLevel)
  640. case "warn":
  641. log.SetLevel(log.WarnLevel)
  642. default:
  643. log.SetLevel(log.InfoLevel)
  644. }
  645. log.SetFormatter(&log.TextFormatter{
  646. FullTimestamp: true,
  647. })
  648. }
  649. // optsGuard prevents invalid options
  650. func optsGuard() error {
  651. var err error
  652. if opts.GithubOrg != "" && opts.GithubUser != "" {
  653. return fmt.Errorf("github user and organization set")
  654. } else if opts.GithubOrg != "" && opts.OwnerPath != "" {
  655. return fmt.Errorf("github organization set and local owner path")
  656. } else if opts.GithubUser != "" && opts.OwnerPath != "" {
  657. return fmt.Errorf("github user set and local owner path")
  658. }
  659. if opts.Threads > runtime.GOMAXPROCS(0) {
  660. return fmt.Errorf("%d available threads", runtime.GOMAXPROCS(0))
  661. }
  662. // do the URL Parse and error checking here, so we can skip it later
  663. // empty string is OK, it will default to the public github URL.
  664. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  665. if !strings.HasSuffix(opts.GithubURL, "/") {
  666. opts.GithubURL += "/"
  667. }
  668. ghURL, err := url.Parse(opts.GithubURL)
  669. if err != nil {
  670. return err
  671. }
  672. tcpPort := "443"
  673. if ghURL.Scheme == "http" {
  674. tcpPort = "80"
  675. }
  676. timeout := time.Duration(1 * time.Second)
  677. _, err = net.DialTimeout("tcp", ghURL.Host+":"+tcpPort, timeout)
  678. if err != nil {
  679. return fmt.Errorf("%s unreachable, error: %s", ghURL.Host, err)
  680. }
  681. }
  682. if opts.SingleSearch != "" {
  683. singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
  684. if err != nil {
  685. return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
  686. }
  687. }
  688. if opts.Entropy > 8 {
  689. return fmt.Errorf("The maximum level of entropy is 8")
  690. }
  691. if opts.Report != "" {
  692. if !strings.HasSuffix(opts.Report, ".json") && !strings.HasSuffix(opts.Report, ".csv") {
  693. return fmt.Errorf("Report should be a .json or .csv file")
  694. }
  695. dirPath := filepath.Dir(opts.Report)
  696. if _, err := os.Stat(dirPath); os.IsNotExist(err) {
  697. return fmt.Errorf("%s does not exist", dirPath)
  698. }
  699. }
  700. return nil
  701. }
  702. // loadToml loads of the toml config containing regexes and whitelists.
  703. // This function will first look if the configPath is set and load the config
  704. // from that file. Otherwise will then look for the path set by the GITHLEAKS_CONIFG
  705. // env var. If that is not set, then gitleaks will continue with the default configs
  706. // specified by the const var at the top `defaultConfig`
  707. func loadToml() error {
  708. var (
  709. config Config
  710. configPath string
  711. )
  712. if opts.ConfigPath != "" {
  713. configPath = opts.ConfigPath
  714. _, err := os.Stat(configPath)
  715. if err != nil {
  716. return fmt.Errorf("no gitleaks config at %s", configPath)
  717. }
  718. } else {
  719. configPath = os.Getenv("GITLEAKS_CONFIG")
  720. }
  721. if configPath != "" {
  722. if _, err := toml.DecodeFile(configPath, &config); err != nil {
  723. return fmt.Errorf("problem loading config: %v", err)
  724. }
  725. } else {
  726. _, err := toml.Decode(defaultConfig, &config)
  727. if err != nil {
  728. return fmt.Errorf("problem loading default config: %v", err)
  729. }
  730. }
  731. if len(config.Misc.Entropy) != 0 {
  732. err := entropyLimits(config.Misc.Entropy)
  733. if err != nil {
  734. return err
  735. }
  736. }
  737. if singleSearchRegex != nil {
  738. regexes["singleSearch"] = singleSearchRegex
  739. } else {
  740. for _, regex := range config.Regexes {
  741. regexes[regex.Description] = regexp.MustCompile(regex.Regex)
  742. }
  743. }
  744. whiteListCommits = make(map[string]bool)
  745. for _, commit := range config.Whitelist.Commits {
  746. whiteListCommits[commit] = true
  747. }
  748. for _, regex := range config.Whitelist.Files {
  749. whiteListFiles = append(whiteListFiles, regexp.MustCompile(regex))
  750. }
  751. for _, regex := range config.Whitelist.Regexes {
  752. whiteListRegexes = append(whiteListRegexes, regexp.MustCompile(regex))
  753. }
  754. for _, regex := range config.Whitelist.Repos {
  755. whiteListRepos = append(whiteListRepos, regexp.MustCompile(regex))
  756. }
  757. return nil
  758. }
  759. // entropyLimits hydrates entropyRanges which allows for fine tuning entropy checking
  760. func entropyLimits(entropyLimitStr []string) error {
  761. for _, span := range entropyLimitStr {
  762. split := strings.Split(span, "-")
  763. v1, err := strconv.ParseFloat(split[0], 64)
  764. if err != nil {
  765. return err
  766. }
  767. v2, err := strconv.ParseFloat(split[1], 64)
  768. if err != nil {
  769. return err
  770. }
  771. if v1 > v2 {
  772. return fmt.Errorf("entropy range must be ascending")
  773. }
  774. r := entropyRange{
  775. v1: v1,
  776. v2: v2,
  777. }
  778. if r.v1 > 8.0 || r.v1 < 0.0 || r.v2 > 8.0 || r.v2 < 0.0 {
  779. return fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0")
  780. }
  781. entropyRanges = append(entropyRanges, r)
  782. }
  783. return nil
  784. }
  785. // getSSHAuth return an ssh auth use by go-git to clone repos behind authentication.
  786. // If --ssh-key is set then it will attempt to load the key from that path. If not,
  787. // gitleaks will use the default $HOME/.ssh/id_rsa key
  788. func getSSHAuth() (*ssh.PublicKeys, error) {
  789. var (
  790. sshKeyPath string
  791. )
  792. if opts.SSHKey != "" {
  793. sshKeyPath = opts.SSHKey
  794. } else {
  795. // try grabbing default
  796. c, err := user.Current()
  797. if err != nil {
  798. return nil, nil
  799. }
  800. sshKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", c.HomeDir)
  801. }
  802. sshAuth, err := ssh.NewPublicKeysFromFile("git", sshKeyPath, "")
  803. if err != nil {
  804. if strings.HasPrefix(opts.Repo, "git") {
  805. // if you are attempting to clone a git repo via ssh and supply a bad ssh key,
  806. // the clone will fail.
  807. return nil, fmt.Errorf("unable to generate ssh key: %v", err)
  808. }
  809. }
  810. return sshAuth, nil
  811. }
  812. func (leak Leak) log() {
  813. b, _ := json.MarshalIndent(leak, "", " ")
  814. fmt.Println(string(b))
  815. }