main.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884
  1. package main
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/csv"
  6. "encoding/json"
  7. "fmt"
  8. "io/ioutil"
  9. "net"
  10. "net/http"
  11. "net/url"
  12. "os"
  13. "os/user"
  14. "path"
  15. "path/filepath"
  16. "regexp"
  17. "strings"
  18. "sync"
  19. "time"
  20. "gopkg.in/src-d/go-git.v4/plumbing"
  21. "golang.org/x/oauth2"
  22. "gopkg.in/src-d/go-git.v4/plumbing/object"
  23. "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
  24. "gopkg.in/src-d/go-git.v4/storage/memory"
  25. "github.com/BurntSushi/toml"
  26. "github.com/google/go-github/github"
  27. flags "github.com/jessevdk/go-flags"
  28. log "github.com/sirupsen/logrus"
  29. git "gopkg.in/src-d/go-git.v4"
  30. )
  31. // Leak represents a leaked secret or regex match. This will be output to stdout and/or the report
  32. type Leak struct {
  33. Line string `json:"line"`
  34. Commit string `json:"commit"`
  35. Offender string `json:"offender"`
  36. Type string `json:"reason"`
  37. Message string `json:"commitMsg"`
  38. Author string `json:"author"`
  39. File string `json:"file"`
  40. Branch string `json:"branch"`
  41. Repo string `json:"repo"`
  42. }
  43. // Repo contains the actual git repository and meta data about the repo
  44. type Repo struct {
  45. path string
  46. url string
  47. name string
  48. leaks []Leak
  49. repository *git.Repository
  50. err error
  51. }
  52. // Owner contains a collection of repos. This could represent an org or user.
  53. type Owner struct {
  54. path string
  55. url string
  56. repos []Repo
  57. }
  58. // Options for gitleaks
  59. type Options struct {
  60. // remote target options
  61. Repo string `short:"r" long:"repo" description:"Repo url to audit"`
  62. GithubUser string `long:"github-user" description:"Github user to audit"`
  63. GithubOrg string `long:"github-org" description:"Github organization to audit"`
  64. GithubURL string `long:"github-url" default:"https://api.github.com/" description:"GitHub API Base URL, use for GitHub Enterprise. Example: https://github.example.com/api/v3/"`
  65. IncludePrivate bool `short:"p" long:"private" description:"Include private repos in audit"`
  66. /*
  67. TODO:
  68. GitLabUser string `long:"gitlab-user" description:"User url to audit"`
  69. GitLabOrg string `long:"gitlab-org" description:"Organization url to audit"`
  70. */
  71. Branch string `short:"b" long:"branch" description:"branch name to audit (defaults to HEAD)"`
  72. Commit string `short:"c" long:"commit" description:"sha of commit to stop at"`
  73. // local target option
  74. RepoPath string `long:"repo-path" description:"Path to repo"`
  75. OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
  76. // Process options
  77. MaxGoRoutines int `long:"max-go" description:"Maximum number of concurrent go-routines gitleaks spawns"`
  78. Disk bool `long:"disk" description:"Clones repo(s) to disk"`
  79. AuditAllRefs bool `long:"all-refs" description:"run audit on all refs"`
  80. SingleSearch string `long:"single-search" description:"single regular expression to search for"`
  81. ConfigPath string `long:"config" description:"path to gitleaks config"`
  82. SSHKey string `long:"ssh-key" description:"path to ssh key"`
  83. // TODO: IncludeMessages string `long:"messages" description:"include commit messages in audit"`
  84. // Output options
  85. Log string `short:"l" long:"log" description:"log level"`
  86. Verbose bool `short:"v" long:"verbose" description:"Show verbose output from gitleaks audit"`
  87. Report string `long:"report" description:"path to write report file"`
  88. CSV bool `long:"csv" description:"report output to csv"`
  89. Redact bool `long:"redact" description:"redact secrets from log messages and report"`
  90. Version bool `long:"version" description:"version number"`
  91. }
  92. // Config struct for regexes matching and whitelisting
  93. type Config struct {
  94. Regexes []struct {
  95. Description string
  96. Regex string
  97. }
  98. Whitelist struct {
  99. Files []string
  100. Regexes []string
  101. Commits []string
  102. Branches []string
  103. }
  104. }
  105. const defaultGithubURL = "https://api.github.com/"
  106. const version = "1.5.0"
  107. const defaultConfig = `
  108. title = "gitleaks config"
  109. # add regexes to the regex table
  110. [[regexes]]
  111. description = "AWS"
  112. regex = '''AKIA[0-9A-Z]{16}'''
  113. [[regexes]]
  114. description = "RKCS8"
  115. regex = '''-----BEGIN PRIVATE KEY-----'''
  116. [[regexes]]
  117. description = "RSA"
  118. regex = '''-----BEGIN RSA PRIVATE KEY-----'''
  119. [[regexes]]
  120. description = "Github"
  121. regex = '''(?i)github.*['\"][0-9a-zA-Z]{35,40}['\"]'''
  122. [[regexes]]
  123. description = "SSH"
  124. regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
  125. [[regexes]]
  126. description = "Facebook"
  127. regex = '''(?i)facebook.*['\"][0-9a-f]{32}['\"]'''
  128. [[regexes]]
  129. description = "Twitter"
  130. regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]'''
  131. [whitelist]
  132. #regexes = [
  133. # "AKAIMYFAKEAWKKEY",
  134. #]
  135. #files = [
  136. # "(.*?)(jpg|gif|doc|pdf|bin)$"
  137. #]
  138. #commits = [
  139. # "BADHA5H1",
  140. # "BADHA5H2",
  141. #]
  142. #branches = [
  143. # "dev/STUPDIFKNFEATURE"
  144. #]
  145. `
  146. var (
  147. opts Options
  148. regexes map[string]*regexp.Regexp
  149. singleSearchRegex *regexp.Regexp
  150. whiteListRegexes []*regexp.Regexp
  151. whiteListFiles []*regexp.Regexp
  152. whiteListCommits map[string]bool
  153. whiteListBranches []string
  154. fileDiffRegex *regexp.Regexp
  155. sshAuth *ssh.PublicKeys
  156. dir string
  157. )
  158. func init() {
  159. log.SetOutput(os.Stdout)
  160. regexes = make(map[string]*regexp.Regexp)
  161. whiteListCommits = make(map[string]bool)
  162. }
  163. func main() {
  164. var (
  165. leaks []Leak
  166. )
  167. _, err := flags.Parse(&opts)
  168. if opts.Version {
  169. fmt.Println(version)
  170. os.Exit(0)
  171. }
  172. if err != nil {
  173. os.Exit(1)
  174. }
  175. setLogs()
  176. err = optsGuard()
  177. if err != nil {
  178. log.Fatal(err)
  179. }
  180. err = loadToml()
  181. if err != nil {
  182. log.Fatal(err)
  183. }
  184. if opts.IncludePrivate {
  185. // if including private repos use ssh as authentication
  186. sshAuth, err = getSSHAuth()
  187. if err != nil {
  188. log.Fatal(err)
  189. }
  190. }
  191. if opts.Disk {
  192. // temporary directory where all the gitleaks plain clones will reside
  193. dir, err = ioutil.TempDir("", "gitleaks")
  194. defer os.RemoveAll(dir)
  195. if err != nil {
  196. log.Fatal(err)
  197. }
  198. }
  199. leaks, err = startAudits()
  200. if err != nil {
  201. log.Fatal(err)
  202. }
  203. if opts.Report != "" {
  204. writeReport(leaks)
  205. }
  206. if len(leaks) != 0 {
  207. log.Errorf("leaks detected")
  208. os.Exit(1)
  209. }
  210. }
  211. func startAudits() ([]Leak, error) {
  212. var leaks []Leak
  213. // start audits
  214. if opts.Repo != "" || opts.RepoPath != "" {
  215. repo, err := getRepo()
  216. if err != nil {
  217. return leaks, err
  218. }
  219. return auditRepo(repo)
  220. } else if opts.OwnerPath != "" {
  221. repos, err := discoverRepos(opts.OwnerPath)
  222. if err != nil {
  223. return leaks, err
  224. }
  225. for _, repo := range repos {
  226. leaksFromRepo, err := auditRepo(repo)
  227. if err != nil {
  228. return leaks, err
  229. }
  230. leaks = append(leaksFromRepo, leaks...)
  231. }
  232. } else if opts.GithubOrg != "" || opts.GithubUser != "" {
  233. githubRepos, err := getGithubRepos()
  234. if err != nil {
  235. return leaks, err
  236. }
  237. for _, githubRepo := range githubRepos {
  238. leaksFromRepo, err := auditGithubRepo(githubRepo)
  239. if len(leaksFromRepo) == 0 {
  240. log.Infof("no leaks found for repo %s", *githubRepo.Name)
  241. } else {
  242. log.Warnf("leaks found for repo %s", *githubRepo.Name)
  243. }
  244. if err != nil {
  245. return leaks, err
  246. }
  247. leaks = append(leaks, leaksFromRepo...)
  248. }
  249. }
  250. return leaks, nil
  251. }
  252. // writeReport writes a report to opts.Report in JSON.
  253. func writeReport(leaks []Leak) error {
  254. var err error
  255. log.Infof("writing report to %s", opts.Report)
  256. if opts.CSV {
  257. f, err := os.Create(opts.Report)
  258. if err != nil {
  259. return err
  260. }
  261. defer f.Close()
  262. w := csv.NewWriter(f)
  263. w.Write([]string{"repo", "line", "commit", "offender", "reason", "commitMsg", "author", "file", "branch"})
  264. for _, leak := range leaks {
  265. w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Type, leak.Message, leak.Author, leak.File, leak.Branch})
  266. }
  267. w.Flush()
  268. } else {
  269. reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
  270. err = ioutil.WriteFile(opts.Report, reportJSON, 0644)
  271. }
  272. return err
  273. }
  274. // getRepo is responsible for cloning a repo in-memory or to disk, or opening a local repo and creating
  275. // a repo object
  276. func getRepo() (Repo, error) {
  277. var (
  278. err error
  279. repo *git.Repository
  280. )
  281. if opts.Disk {
  282. log.Infof("cloning %s", opts.Repo)
  283. cloneTarget := fmt.Sprintf("%s/%x", dir, md5.Sum([]byte(fmt.Sprintf("%s%s", opts.GithubUser, opts.Repo))))
  284. if opts.IncludePrivate {
  285. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  286. URL: opts.Repo,
  287. Progress: os.Stdout,
  288. Auth: sshAuth,
  289. })
  290. } else {
  291. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  292. URL: opts.Repo,
  293. Progress: os.Stdout,
  294. })
  295. }
  296. } else if opts.RepoPath != "" {
  297. // use existing repo
  298. log.Infof("opening %s", opts.Repo)
  299. repo, err = git.PlainOpen(opts.RepoPath)
  300. } else {
  301. log.Infof("cloning %s", opts.Repo)
  302. if opts.IncludePrivate {
  303. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  304. URL: opts.Repo,
  305. Progress: os.Stdout,
  306. Auth: sshAuth,
  307. })
  308. } else {
  309. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  310. URL: opts.Repo,
  311. Progress: os.Stdout,
  312. })
  313. }
  314. }
  315. return Repo{
  316. repository: repo,
  317. path: opts.RepoPath,
  318. url: opts.Repo,
  319. name: filepath.Base(opts.Repo),
  320. err: err,
  321. }, nil
  322. }
  323. // auditRef audits a git reference
  324. // TODO: need to add a layer of parallelism here and a cache for parent+child commits so we don't
  325. // double dip
  326. func auditRef(repo Repo, ref *plumbing.Reference, commitWg *sync.WaitGroup, commitChan chan []Leak) error {
  327. var (
  328. err error
  329. prevCommit *object.Commit
  330. limitGoRoutines bool
  331. semaphore chan bool
  332. repoName string
  333. )
  334. repoName = repo.name
  335. // goroutine limiting
  336. if opts.MaxGoRoutines != 0 {
  337. semaphore = make(chan bool, opts.MaxGoRoutines)
  338. limitGoRoutines = true
  339. }
  340. cIter, err := repo.repository.Log(&git.LogOptions{From: ref.Hash()})
  341. if err != nil {
  342. return err
  343. }
  344. err = cIter.ForEach(func(c *object.Commit) error {
  345. if c.Hash.String() == opts.Commit {
  346. cIter.Close()
  347. }
  348. if whiteListCommits[c.Hash.String()] {
  349. log.Infof("skipping commit: %s\n", c.Hash.String())
  350. return nil
  351. }
  352. if limitGoRoutines {
  353. semaphore <- true
  354. }
  355. commitWg.Add(1)
  356. go func(c *object.Commit, prevCommit *object.Commit) {
  357. var (
  358. leaks []Leak
  359. filePath string
  360. skipFile bool
  361. )
  362. if prevCommit == nil {
  363. t, _ := c.Tree()
  364. files := t.Files()
  365. err := files.ForEach(func(file *object.File) error {
  366. content, err := file.Contents()
  367. if err != nil {
  368. return err
  369. }
  370. leaks = append(leaks, checkDiff(content, c, file.Name, string(ref.Name()), repoName)...)
  371. return nil
  372. })
  373. if err != nil {
  374. log.Warnf("problem generating diff for commit: %s\n", c.Hash.String())
  375. if limitGoRoutines {
  376. <-semaphore
  377. }
  378. commitChan <- leaks
  379. return
  380. }
  381. } else {
  382. patch, err := c.Patch(prevCommit)
  383. if err != nil {
  384. log.Warnf("problem generating patch for commit: %s\n", c.Hash.String())
  385. if limitGoRoutines {
  386. <-semaphore
  387. }
  388. commitChan <- leaks
  389. return
  390. }
  391. for _, f := range patch.FilePatches() {
  392. skipFile = false
  393. from, to := f.Files()
  394. filePath = "???"
  395. if from != nil {
  396. filePath = from.Path()
  397. } else if to != nil {
  398. filePath = to.Path()
  399. }
  400. for _, re := range whiteListFiles {
  401. if re.FindString(filePath) != "" {
  402. skipFile = true
  403. break
  404. }
  405. }
  406. if skipFile {
  407. continue
  408. }
  409. chunks := f.Chunks()
  410. for _, chunk := range chunks {
  411. if chunk.Type() == 1 || chunk.Type() == 2 {
  412. // only check if adding or removing
  413. leaks = append(leaks, checkDiff(chunk.Content(), prevCommit, filePath, string(ref.Name()), repoName)...)
  414. }
  415. }
  416. }
  417. }
  418. if limitGoRoutines {
  419. <-semaphore
  420. }
  421. commitChan <- leaks
  422. }(c, prevCommit)
  423. prevCommit = c
  424. return nil
  425. })
  426. return nil
  427. }
  428. // auditRepo performs an audit on a repository checking for regex matching and ignoring
  429. // files and regexes that are whitelisted
  430. func auditRepo(repo Repo) ([]Leak, error) {
  431. var (
  432. err error
  433. leaks []Leak
  434. commitWg sync.WaitGroup
  435. )
  436. ref, err := repo.repository.Head()
  437. if err != nil {
  438. return leaks, err
  439. }
  440. // leak messaging
  441. commitChan := make(chan []Leak, 1)
  442. if opts.AuditAllRefs {
  443. skipBranch := false
  444. refs, err := repo.repository.Storer.IterReferences()
  445. if err != nil {
  446. return leaks, err
  447. }
  448. err = refs.ForEach(func(ref *plumbing.Reference) error {
  449. for _, b := range whiteListBranches {
  450. if strings.HasSuffix(string(ref.Name()), b) {
  451. skipBranch = true
  452. }
  453. }
  454. if skipBranch {
  455. skipBranch = false
  456. return nil
  457. }
  458. auditRef(repo, ref, &commitWg, commitChan)
  459. return nil
  460. })
  461. } else {
  462. if opts.Branch != "" {
  463. foundBranch := false
  464. refs, _ := repo.repository.Storer.IterReferences()
  465. branch := strings.Split(opts.Branch, "/")[len(strings.Split(opts.Branch, "/"))-1]
  466. err = refs.ForEach(func(refBranch *plumbing.Reference) error {
  467. if strings.Split(refBranch.Name().String(), "/")[len(strings.Split(refBranch.Name().String(), "/"))-1] == branch {
  468. foundBranch = true
  469. ref = refBranch
  470. }
  471. return nil
  472. })
  473. if foundBranch == false {
  474. log.Fatalf("No branch with name", opts.Branch)
  475. return nil, nil
  476. }
  477. }
  478. auditRef(repo, ref, &commitWg, commitChan)
  479. }
  480. go func() {
  481. for commitLeaks := range commitChan {
  482. if commitLeaks != nil {
  483. for _, leak := range commitLeaks {
  484. leaks = append(leaks, leak)
  485. }
  486. }
  487. commitWg.Done()
  488. }
  489. }()
  490. commitWg.Wait()
  491. return leaks, err
  492. }
  493. // checkDiff accepts a string diff and commit object then performs a
  494. // regex check
  495. func checkDiff(diff string, commit *object.Commit, filePath string, branch string, repo string) []Leak {
  496. lines := strings.Split(diff, "\n")
  497. var (
  498. leaks []Leak
  499. skipLine bool
  500. )
  501. for _, line := range lines {
  502. skipLine = false
  503. for leakType, re := range regexes {
  504. match := re.FindString(line)
  505. if match == "" {
  506. continue
  507. }
  508. // if offender matches whitelist regex, ignore it
  509. for _, wRe := range whiteListRegexes {
  510. whitelistMatch := wRe.FindString(line)
  511. if whitelistMatch != "" {
  512. skipLine = true
  513. break
  514. }
  515. }
  516. if skipLine {
  517. break
  518. }
  519. leak := Leak{
  520. Line: line,
  521. Commit: commit.Hash.String(),
  522. Offender: match,
  523. Type: leakType,
  524. Message: commit.Message,
  525. Author: commit.Author.String(),
  526. File: filePath,
  527. Branch: branch,
  528. Repo: repo,
  529. }
  530. if opts.Redact {
  531. leak.Offender = "REDACTED"
  532. leak.Line = "REDACTED"
  533. }
  534. if opts.Verbose {
  535. leak.log()
  536. }
  537. leaks = append(leaks, leak)
  538. }
  539. }
  540. return leaks
  541. }
  542. // auditOwner audits all of the owner's(user or org) repos
  543. func getGithubRepos() ([]*github.Repository, error) {
  544. var (
  545. err error
  546. githubRepos []*github.Repository
  547. rs []*github.Repository
  548. resp *github.Response
  549. githubClient *github.Client
  550. githubOrgOptions *github.RepositoryListByOrgOptions
  551. githubOptions *github.RepositoryListOptions
  552. )
  553. ctx := context.Background()
  554. if opts.GithubOrg != "" {
  555. githubClient = github.NewClient(githubToken())
  556. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  557. ghURL, _ := url.Parse(opts.GithubURL)
  558. githubClient.BaseURL = ghURL
  559. }
  560. githubOrgOptions = &github.RepositoryListByOrgOptions{
  561. ListOptions: github.ListOptions{PerPage: 10},
  562. }
  563. } else if opts.GithubUser != "" {
  564. githubClient = github.NewClient(githubToken())
  565. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  566. ghURL, _ := url.Parse(opts.GithubURL)
  567. githubClient.BaseURL = ghURL
  568. }
  569. githubOptions = &github.RepositoryListOptions{
  570. Affiliation: "owner",
  571. ListOptions: github.ListOptions{
  572. PerPage: 10,
  573. },
  574. }
  575. }
  576. for {
  577. if opts.GithubUser != "" {
  578. if opts.IncludePrivate {
  579. rs, resp, err = githubClient.Repositories.List(ctx, "", githubOptions)
  580. } else {
  581. rs, resp, err = githubClient.Repositories.List(ctx, opts.GithubUser, githubOptions)
  582. }
  583. if err != nil {
  584. return nil, err
  585. }
  586. githubOptions.Page = resp.NextPage
  587. githubRepos = append(githubRepos, rs...)
  588. if resp.NextPage == 0 {
  589. return githubRepos, err
  590. }
  591. } else if opts.GithubOrg != "" {
  592. rs, resp, err := githubClient.Repositories.ListByOrg(ctx, opts.GithubOrg, githubOrgOptions)
  593. if err != nil {
  594. return nil, err
  595. }
  596. githubOrgOptions.Page = resp.NextPage
  597. githubRepos = append(githubRepos, rs...)
  598. if resp.NextPage == 0 {
  599. return githubRepos, err
  600. }
  601. }
  602. for _, githubRepo := range githubRepos {
  603. log.Infof("staging repo %s", *githubRepo.Name)
  604. }
  605. }
  606. }
  607. // auditGithubRepo clones repos from github
  608. func auditGithubRepo(githubRepo *github.Repository) ([]Leak, error) {
  609. var (
  610. leaks []Leak
  611. repo *git.Repository
  612. err error
  613. )
  614. log.Infof("cloning: %s", *githubRepo.Name)
  615. if opts.Disk {
  616. ownerDir, err := ioutil.TempDir(dir, opts.GithubUser)
  617. if err != nil {
  618. return leaks, fmt.Errorf("unable to generater owner temp dir: %v", err)
  619. }
  620. if opts.IncludePrivate {
  621. if sshAuth == nil {
  622. return leaks, fmt.Errorf("no ssh auth available")
  623. }
  624. repo, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *githubRepo.Name), false, &git.CloneOptions{
  625. URL: *githubRepo.SSHURL,
  626. Auth: sshAuth,
  627. })
  628. } else {
  629. repo, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *githubRepo.Name), false, &git.CloneOptions{
  630. URL: *githubRepo.CloneURL,
  631. })
  632. }
  633. } else {
  634. if opts.IncludePrivate {
  635. if sshAuth == nil {
  636. return leaks, fmt.Errorf("no ssh auth available")
  637. }
  638. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  639. URL: *githubRepo.SSHURL,
  640. Auth: sshAuth,
  641. })
  642. } else {
  643. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  644. URL: *githubRepo.CloneURL,
  645. })
  646. }
  647. }
  648. if err != nil {
  649. return leaks, err
  650. }
  651. return auditRepo(Repo{
  652. repository: repo,
  653. name: *githubRepo.Name,
  654. })
  655. }
  656. // githubToken returns a oauth2 client for the github api to consume
  657. func githubToken() *http.Client {
  658. githubToken := os.Getenv("GITHUB_TOKEN")
  659. if githubToken == "" {
  660. return nil
  661. }
  662. ts := oauth2.StaticTokenSource(
  663. &oauth2.Token{AccessToken: githubToken},
  664. )
  665. return oauth2.NewClient(context.Background(), ts)
  666. }
  667. // discoverRepos walks all the children of `path`. If a child directory
  668. // contain a .git file then that repo will be added
  669. func discoverRepos(ownerPath string) ([]Repo, error) {
  670. var (
  671. err error
  672. repos []Repo
  673. )
  674. files, err := ioutil.ReadDir(ownerPath)
  675. if err != nil {
  676. return repos, err
  677. }
  678. for _, f := range files {
  679. if f.IsDir() {
  680. repoPath := path.Join(ownerPath, f.Name())
  681. r, err := git.PlainOpen(repoPath)
  682. if err != nil {
  683. continue
  684. }
  685. repos = append(repos, Repo{
  686. repository: r,
  687. name: f.Name(),
  688. path: repoPath,
  689. })
  690. }
  691. }
  692. return repos, err
  693. }
  694. // setLogLevel sets log level for gitleaks. Default is Warning
  695. func setLogs() {
  696. switch opts.Log {
  697. case "info":
  698. log.SetLevel(log.InfoLevel)
  699. case "debug":
  700. log.SetLevel(log.DebugLevel)
  701. case "warn":
  702. log.SetLevel(log.WarnLevel)
  703. default:
  704. log.SetLevel(log.InfoLevel)
  705. }
  706. log.SetFormatter(&log.TextFormatter{
  707. FullTimestamp: true,
  708. })
  709. }
  710. // optsGuard prevents invalid options
  711. func optsGuard() error {
  712. var err error
  713. if opts.GithubOrg != "" && opts.GithubUser != "" {
  714. return fmt.Errorf("github user and organization set")
  715. } else if opts.GithubOrg != "" && opts.OwnerPath != "" {
  716. return fmt.Errorf("github organization set and local owner path")
  717. } else if opts.GithubUser != "" && opts.OwnerPath != "" {
  718. return fmt.Errorf("github user set and local owner path")
  719. } else if opts.IncludePrivate && os.Getenv("GITHUB_TOKEN") == "" && (opts.GithubOrg != "" || opts.GithubUser != "") {
  720. return fmt.Errorf("user/organization private repos require env var GITHUB_TOKEN to be set")
  721. }
  722. // do the URL Parse and error checking here, so we can skip it later
  723. // empty string is OK, it will default to the public github URL.
  724. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  725. if !strings.HasSuffix(opts.GithubURL, "/") {
  726. opts.GithubURL += "/"
  727. }
  728. ghURL, err := url.Parse(opts.GithubURL)
  729. if err != nil {
  730. return err
  731. }
  732. tcpPort := "443"
  733. if ghURL.Scheme == "http" {
  734. tcpPort = "80"
  735. }
  736. timeout := time.Duration(1 * time.Second)
  737. _, err = net.DialTimeout("tcp", ghURL.Host+":"+tcpPort, timeout)
  738. if err != nil {
  739. return fmt.Errorf("%s unreachable, error: %s", ghURL.Host, err)
  740. }
  741. }
  742. if opts.SingleSearch != "" {
  743. singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
  744. if err != nil {
  745. return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
  746. }
  747. }
  748. return nil
  749. }
  750. // loadToml loads of the toml config containing regexes and whitelists
  751. // 1. look for config path
  752. // 2. two, look for gitleaks config env var
  753. func loadToml() error {
  754. var (
  755. config Config
  756. configPath string
  757. )
  758. if opts.ConfigPath != "" {
  759. configPath = opts.ConfigPath
  760. _, err := os.Stat(configPath)
  761. if err != nil {
  762. return fmt.Errorf("no gitleaks config at %s", configPath)
  763. }
  764. } else {
  765. configPath = os.Getenv("GITLEAKS_CONFIG")
  766. }
  767. if configPath != "" {
  768. if _, err := toml.DecodeFile(configPath, &config); err != nil {
  769. return fmt.Errorf("problem loading config: %v", err)
  770. }
  771. } else {
  772. _, err := toml.Decode(defaultConfig, &config)
  773. if err != nil {
  774. return fmt.Errorf("problem loading default config: %v", err)
  775. }
  776. }
  777. // load up regexes
  778. if singleSearchRegex != nil {
  779. // single search takes precedence over default regex
  780. regexes["singleSearch"] = singleSearchRegex
  781. } else {
  782. for _, regex := range config.Regexes {
  783. regexes[regex.Description] = regexp.MustCompile(regex.Regex)
  784. }
  785. }
  786. whiteListBranches = config.Whitelist.Branches
  787. whiteListCommits = make(map[string]bool)
  788. for _, commit := range config.Whitelist.Commits {
  789. whiteListCommits[commit] = true
  790. }
  791. for _, regex := range config.Whitelist.Files {
  792. whiteListFiles = append(whiteListFiles, regexp.MustCompile(regex))
  793. }
  794. for _, regex := range config.Whitelist.Regexes {
  795. whiteListRegexes = append(whiteListRegexes, regexp.MustCompile(regex))
  796. }
  797. return nil
  798. }
  799. // getSSHAuth generates ssh auth
  800. func getSSHAuth() (*ssh.PublicKeys, error) {
  801. var (
  802. sshKeyPath string
  803. )
  804. if opts.SSHKey != "" {
  805. sshKeyPath = opts.SSHKey
  806. } else {
  807. c, _ := user.Current()
  808. sshKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", c.HomeDir)
  809. }
  810. sshAuth, err := ssh.NewPublicKeysFromFile("git", sshKeyPath, "")
  811. if err != nil {
  812. return nil, fmt.Errorf("unable to generate ssh key: %v", err)
  813. }
  814. return sshAuth, err
  815. }
  816. func (leak *Leak) log() {
  817. b, _ := json.MarshalIndent(leak, "", " ")
  818. fmt.Println(string(b))
  819. }