main.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879
  1. package main
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/csv"
  6. "encoding/json"
  7. "fmt"
  8. "io/ioutil"
  9. "net"
  10. "net/http"
  11. "net/url"
  12. "os"
  13. "os/user"
  14. "path"
  15. "path/filepath"
  16. "regexp"
  17. "runtime"
  18. "strings"
  19. "sync"
  20. "time"
  21. "gopkg.in/src-d/go-git.v4/plumbing"
  22. "golang.org/x/oauth2"
  23. "gopkg.in/src-d/go-git.v4/plumbing/object"
  24. "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
  25. "gopkg.in/src-d/go-git.v4/storage/memory"
  26. "github.com/BurntSushi/toml"
  27. "github.com/google/go-github/github"
  28. flags "github.com/jessevdk/go-flags"
  29. log "github.com/sirupsen/logrus"
  30. git "gopkg.in/src-d/go-git.v4"
  31. )
  32. // Leak represents a leaked secret or regex match. This will be output to stdout and/or the report
  33. type Leak struct {
  34. Line string `json:"line"`
  35. Commit string `json:"commit"`
  36. Offender string `json:"offender"`
  37. Type string `json:"reason"`
  38. Message string `json:"commitMsg"`
  39. Author string `json:"author"`
  40. File string `json:"file"`
  41. Branch string `json:"branch"`
  42. Repo string `json:"repo"`
  43. }
  44. // Repo contains the actual git repository and meta data about the repo
  45. type Repo struct {
  46. path string
  47. url string
  48. name string
  49. leaks []Leak
  50. repository *git.Repository
  51. err error
  52. }
  53. // Owner contains a collection of repos. This could represent an org or user.
  54. type Owner struct {
  55. path string
  56. url string
  57. repos []Repo
  58. }
  59. // Options for gitleaks
  60. type Options struct {
  61. // remote target options
  62. Repo string `short:"r" long:"repo" description:"Repo url to audit"`
  63. GithubUser string `long:"github-user" description:"Github user to audit"`
  64. GithubOrg string `long:"github-org" description:"Github organization to audit"`
  65. GithubURL string `long:"github-url" default:"https://api.github.com/" description:"GitHub API Base URL, use for GitHub Enterprise. Example: https://github.example.com/api/v3/"`
  66. IncludePrivate bool `short:"p" long:"private" description:"Include private repos in audit"`
  67. /*
  68. TODO:
  69. GitLabUser string `long:"gitlab-user" description:"User url to audit"`
  70. GitLabOrg string `long:"gitlab-org" description:"Organization url to audit"`
  71. */
  72. Branch string `short:"b" long:"branch" description:"branch name to audit (defaults to HEAD)"`
  73. Commit string `short:"c" long:"commit" description:"sha of commit to stop at"`
  74. // local target option
  75. RepoPath string `long:"repo-path" description:"Path to repo"`
  76. OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
  77. // Process options
  78. MaxGoRoutines int `long:"max-go" description:"Maximum number of concurrent go-routines gitleaks spawns"`
  79. Disk bool `long:"disk" description:"Clones repo(s) to disk"`
  80. AuditAllRefs bool `long:"all-refs" description:"run audit on all refs"`
  81. SingleSearch string `long:"single-search" description:"single regular expression to search for"`
  82. ConfigPath string `long:"config" description:"path to gitleaks config"`
  83. SSHKey string `long:"ssh-key" description:"path to ssh key"`
  84. // TODO: IncludeMessages string `long:"messages" description:"include commit messages in audit"`
  85. // Output options
  86. Log string `short:"l" long:"log" description:"log level"`
  87. Verbose bool `short:"v" long:"verbose" description:"Show verbose output from gitleaks audit"`
  88. Report string `long:"report" description:"path to write report file"`
  89. CSV bool `long:"csv" description:"report output to csv"`
  90. Redact bool `long:"redact" description:"redact secrets from log messages and report"`
  91. Version bool `long:"version" description:"version number"`
  92. }
  93. // Config struct for regexes matching and whitelisting
  94. type Config struct {
  95. Regexes []struct {
  96. Description string
  97. Regex string
  98. }
  99. Whitelist struct {
  100. Files []string
  101. Regexes []string
  102. Commits []string
  103. Branches []string
  104. }
  105. }
  106. const defaultGithubURL = "https://api.github.com/"
  107. const version = "1.6.1"
  108. const defaultConfig = `
  109. title = "gitleaks config"
  110. # add regexes to the regex table
  111. [[regexes]]
  112. description = "AWS"
  113. regex = '''AKIA[0-9A-Z]{16}'''
  114. [[regexes]]
  115. description = "RKCS8"
  116. regex = '''-----BEGIN PRIVATE KEY-----'''
  117. [[regexes]]
  118. description = "RSA"
  119. regex = '''-----BEGIN RSA PRIVATE KEY-----'''
  120. [[regexes]]
  121. description = "Github"
  122. regex = '''(?i)github.*['\"][0-9a-zA-Z]{35,40}['\"]'''
  123. [[regexes]]
  124. description = "SSH"
  125. regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
  126. [[regexes]]
  127. description = "Facebook"
  128. regex = '''(?i)facebook.*['\"][0-9a-f]{32}['\"]'''
  129. [[regexes]]
  130. description = "Twitter"
  131. regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]'''
  132. [whitelist]
  133. #regexes = [
  134. # "AKAIMYFAKEAWKKEY",
  135. #]
  136. #files = [
  137. # "(.*?)(jpg|gif|doc|pdf|bin)$"
  138. #]
  139. #commits = [
  140. # "BADHA5H1",
  141. # "BADHA5H2",
  142. #]
  143. #branches = [
  144. # "dev/STUPDIFKNFEATURE"
  145. #]
  146. `
  147. var (
  148. opts Options
  149. regexes map[string]*regexp.Regexp
  150. singleSearchRegex *regexp.Regexp
  151. whiteListRegexes []*regexp.Regexp
  152. whiteListFiles []*regexp.Regexp
  153. whiteListCommits map[string]bool
  154. whiteListBranches []string
  155. fileDiffRegex *regexp.Regexp
  156. sshAuth *ssh.PublicKeys
  157. dir string
  158. maxGo int
  159. )
  160. func init() {
  161. log.SetOutput(os.Stdout)
  162. maxGo = runtime.GOMAXPROCS(0) / 2
  163. regexes = make(map[string]*regexp.Regexp)
  164. whiteListCommits = make(map[string]bool)
  165. }
  166. func main() {
  167. var (
  168. leaks []Leak
  169. )
  170. _, err := flags.Parse(&opts)
  171. if opts.Version {
  172. fmt.Println(version)
  173. os.Exit(0)
  174. }
  175. if err != nil {
  176. os.Exit(1)
  177. }
  178. setLogs()
  179. err = optsGuard()
  180. if err != nil {
  181. log.Fatal(err)
  182. }
  183. err = loadToml()
  184. if err != nil {
  185. log.Fatal(err)
  186. }
  187. if opts.IncludePrivate {
  188. // if including private repos use ssh as authentication
  189. sshAuth, err = getSSHAuth()
  190. if err != nil {
  191. log.Fatal(err)
  192. }
  193. }
  194. if opts.Disk {
  195. // temporary directory where all the gitleaks plain clones will reside
  196. dir, err = ioutil.TempDir("", "gitleaks")
  197. defer os.RemoveAll(dir)
  198. if err != nil {
  199. log.Fatal(err)
  200. }
  201. }
  202. leaks, err = startAudits()
  203. if err != nil {
  204. log.Fatal(err)
  205. }
  206. if opts.Report != "" {
  207. writeReport(leaks)
  208. }
  209. if len(leaks) != 0 {
  210. log.Errorf("leaks detected")
  211. os.Exit(1)
  212. }
  213. }
  214. func startAudits() ([]Leak, error) {
  215. var leaks []Leak
  216. // start audits
  217. if opts.Repo != "" || opts.RepoPath != "" {
  218. repo, err := getRepo()
  219. if err != nil {
  220. return leaks, err
  221. }
  222. return auditRepo(repo)
  223. } else if opts.OwnerPath != "" {
  224. repos, err := discoverRepos(opts.OwnerPath)
  225. if err != nil {
  226. return leaks, err
  227. }
  228. for _, repo := range repos {
  229. leaksFromRepo, err := auditRepo(repo)
  230. if err != nil {
  231. return leaks, err
  232. }
  233. leaks = append(leaksFromRepo, leaks...)
  234. }
  235. } else if opts.GithubOrg != "" || opts.GithubUser != "" {
  236. githubRepos, err := getGithubRepos()
  237. if err != nil {
  238. return leaks, err
  239. }
  240. for _, githubRepo := range githubRepos {
  241. leaksFromRepo, err := auditGithubRepo(githubRepo)
  242. if len(leaksFromRepo) == 0 {
  243. log.Infof("no leaks found for repo %s", *githubRepo.Name)
  244. } else {
  245. log.Warnf("leaks found for repo %s", *githubRepo.Name)
  246. }
  247. if err != nil {
  248. return leaks, err
  249. }
  250. leaks = append(leaks, leaksFromRepo...)
  251. }
  252. }
  253. return leaks, nil
  254. }
  255. // writeReport writes a report to opts.Report in JSON.
  256. func writeReport(leaks []Leak) error {
  257. var err error
  258. log.Infof("writing report to %s", opts.Report)
  259. if opts.CSV {
  260. f, err := os.Create(opts.Report)
  261. if err != nil {
  262. return err
  263. }
  264. defer f.Close()
  265. w := csv.NewWriter(f)
  266. w.Write([]string{"repo", "line", "commit", "offender", "reason", "commitMsg", "author", "file", "branch"})
  267. for _, leak := range leaks {
  268. w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.Type, leak.Message, leak.Author, leak.File, leak.Branch})
  269. }
  270. w.Flush()
  271. } else {
  272. reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
  273. err = ioutil.WriteFile(opts.Report, reportJSON, 0644)
  274. }
  275. return err
  276. }
  277. // getRepo is responsible for cloning a repo in-memory or to disk, or opening a local repo and creating
  278. // a repo object
  279. func getRepo() (Repo, error) {
  280. var (
  281. err error
  282. repo *git.Repository
  283. )
  284. if opts.Disk {
  285. log.Infof("cloning %s", opts.Repo)
  286. cloneTarget := fmt.Sprintf("%s/%x", dir, md5.Sum([]byte(fmt.Sprintf("%s%s", opts.GithubUser, opts.Repo))))
  287. if opts.IncludePrivate {
  288. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  289. URL: opts.Repo,
  290. Progress: os.Stdout,
  291. Auth: sshAuth,
  292. })
  293. } else {
  294. repo, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  295. URL: opts.Repo,
  296. Progress: os.Stdout,
  297. })
  298. }
  299. } else if opts.RepoPath != "" {
  300. // use existing repo
  301. log.Infof("opening %s", opts.Repo)
  302. repo, err = git.PlainOpen(opts.RepoPath)
  303. } else {
  304. log.Infof("cloning %s", opts.Repo)
  305. if opts.IncludePrivate {
  306. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  307. URL: opts.Repo,
  308. Progress: os.Stdout,
  309. Auth: sshAuth,
  310. })
  311. } else {
  312. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  313. URL: opts.Repo,
  314. Progress: os.Stdout,
  315. })
  316. }
  317. }
  318. return Repo{
  319. repository: repo,
  320. path: opts.RepoPath,
  321. url: opts.Repo,
  322. name: filepath.Base(opts.Repo),
  323. err: err,
  324. }, nil
  325. }
  326. // auditRef audits a git reference
  327. // TODO: need to add a layer of parallelism here and a cache for parent+child commits so we don't
  328. // double dip
  329. func auditRef(repo Repo, ref *plumbing.Reference, commitWg *sync.WaitGroup, commitChan chan []Leak) error {
  330. var (
  331. err error
  332. prevCommit *object.Commit
  333. semaphore chan bool
  334. repoName string
  335. )
  336. repoName = repo.name
  337. if opts.MaxGoRoutines != 0 {
  338. maxGo = opts.MaxGoRoutines
  339. }
  340. semaphore = make(chan bool, maxGo)
  341. cIter, err := repo.repository.Log(&git.LogOptions{From: ref.Hash()})
  342. if err != nil {
  343. return err
  344. }
  345. err = cIter.ForEach(func(c *object.Commit) error {
  346. if c.Hash.String() == opts.Commit {
  347. cIter.Close()
  348. }
  349. if whiteListCommits[c.Hash.String()] {
  350. log.Infof("skipping commit: %s\n", c.Hash.String())
  351. return nil
  352. }
  353. semaphore <- true
  354. commitWg.Add(1)
  355. go func(c *object.Commit, prevCommit *object.Commit) {
  356. var (
  357. leaks []Leak
  358. filePath string
  359. skipFile bool
  360. )
  361. defer func() {
  362. <-semaphore
  363. commitChan <- leaks
  364. if r := recover(); r != nil {
  365. log.Warnf("recoverying from panic on commit %s, likely large diff causing panic", c.Hash.String())
  366. }
  367. }()
  368. if prevCommit == nil {
  369. t, _ := c.Tree()
  370. files := t.Files()
  371. err := files.ForEach(func(file *object.File) error {
  372. content, err := file.Contents()
  373. if err != nil {
  374. return err
  375. }
  376. leaks = append(leaks, checkDiff(content, c, file.Name, string(ref.Name()), repoName)...)
  377. return nil
  378. })
  379. if err != nil {
  380. log.Warnf("problem generating diff for commit: %s\n", c.Hash.String())
  381. return
  382. }
  383. } else {
  384. patch, err := c.Patch(prevCommit)
  385. if err != nil {
  386. log.Warnf("problem generating patch for commit: %s\n", c.Hash.String())
  387. return
  388. }
  389. for _, f := range patch.FilePatches() {
  390. skipFile = false
  391. from, to := f.Files()
  392. filePath = "???"
  393. if from != nil {
  394. filePath = from.Path()
  395. } else if to != nil {
  396. filePath = to.Path()
  397. }
  398. for _, re := range whiteListFiles {
  399. if re.FindString(filePath) != "" {
  400. skipFile = true
  401. break
  402. }
  403. }
  404. if skipFile {
  405. continue
  406. }
  407. chunks := f.Chunks()
  408. for _, chunk := range chunks {
  409. if chunk.Type() == 1 || chunk.Type() == 2 {
  410. // only check if adding or removing
  411. leaks = append(leaks, checkDiff(chunk.Content(), prevCommit, filePath, string(ref.Name()), repoName)...)
  412. }
  413. }
  414. }
  415. }
  416. }(c, prevCommit)
  417. prevCommit = c
  418. return nil
  419. })
  420. return nil
  421. }
  422. // auditRepo performs an audit on a repository checking for regex matching and ignoring
  423. // files and regexes that are whitelisted
  424. func auditRepo(repo Repo) ([]Leak, error) {
  425. var (
  426. err error
  427. leaks []Leak
  428. commitWg sync.WaitGroup
  429. )
  430. ref, err := repo.repository.Head()
  431. if err != nil {
  432. return leaks, err
  433. }
  434. // leak messaging
  435. commitChan := make(chan []Leak, 1)
  436. if opts.AuditAllRefs {
  437. skipBranch := false
  438. refs, err := repo.repository.Storer.IterReferences()
  439. if err != nil {
  440. return leaks, err
  441. }
  442. err = refs.ForEach(func(ref *plumbing.Reference) error {
  443. for _, b := range whiteListBranches {
  444. if strings.HasSuffix(string(ref.Name()), b) {
  445. skipBranch = true
  446. }
  447. }
  448. if skipBranch {
  449. skipBranch = false
  450. return nil
  451. }
  452. auditRef(repo, ref, &commitWg, commitChan)
  453. return nil
  454. })
  455. } else {
  456. if opts.Branch != "" {
  457. foundBranch := false
  458. refs, _ := repo.repository.Storer.IterReferences()
  459. branch := strings.Split(opts.Branch, "/")[len(strings.Split(opts.Branch, "/"))-1]
  460. err = refs.ForEach(func(refBranch *plumbing.Reference) error {
  461. if strings.Split(refBranch.Name().String(), "/")[len(strings.Split(refBranch.Name().String(), "/"))-1] == branch {
  462. foundBranch = true
  463. ref = refBranch
  464. }
  465. return nil
  466. })
  467. if foundBranch == false {
  468. log.Fatalf("No branch with name", opts.Branch)
  469. return nil, nil
  470. }
  471. }
  472. auditRef(repo, ref, &commitWg, commitChan)
  473. }
  474. go func() {
  475. for commitLeaks := range commitChan {
  476. if commitLeaks != nil {
  477. for _, leak := range commitLeaks {
  478. leaks = append(leaks, leak)
  479. }
  480. }
  481. commitWg.Done()
  482. }
  483. }()
  484. commitWg.Wait()
  485. return leaks, err
  486. }
  487. // checkDiff accepts a string diff and commit object then performs a
  488. // regex check
  489. func checkDiff(diff string, commit *object.Commit, filePath string, branch string, repo string) []Leak {
  490. lines := strings.Split(diff, "\n")
  491. var (
  492. leaks []Leak
  493. skipLine bool
  494. )
  495. for _, line := range lines {
  496. skipLine = false
  497. for leakType, re := range regexes {
  498. match := re.FindString(line)
  499. if match == "" {
  500. continue
  501. }
  502. // if offender matches whitelist regex, ignore it
  503. for _, wRe := range whiteListRegexes {
  504. whitelistMatch := wRe.FindString(line)
  505. if whitelistMatch != "" {
  506. skipLine = true
  507. break
  508. }
  509. }
  510. if skipLine {
  511. break
  512. }
  513. leak := Leak{
  514. Line: line,
  515. Commit: commit.Hash.String(),
  516. Offender: match,
  517. Type: leakType,
  518. Message: commit.Message,
  519. Author: commit.Author.String(),
  520. File: filePath,
  521. Branch: branch,
  522. Repo: repo,
  523. }
  524. if opts.Redact {
  525. leak.Offender = "REDACTED"
  526. leak.Line = "REDACTED"
  527. }
  528. if opts.Verbose {
  529. leak.log()
  530. }
  531. leaks = append(leaks, leak)
  532. }
  533. }
  534. return leaks
  535. }
  536. // auditOwner audits all of the owner's(user or org) repos
  537. func getGithubRepos() ([]*github.Repository, error) {
  538. var (
  539. err error
  540. githubRepos []*github.Repository
  541. rs []*github.Repository
  542. resp *github.Response
  543. githubClient *github.Client
  544. githubOrgOptions *github.RepositoryListByOrgOptions
  545. githubOptions *github.RepositoryListOptions
  546. )
  547. ctx := context.Background()
  548. if opts.GithubOrg != "" {
  549. githubClient = github.NewClient(githubToken())
  550. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  551. ghURL, _ := url.Parse(opts.GithubURL)
  552. githubClient.BaseURL = ghURL
  553. }
  554. githubOrgOptions = &github.RepositoryListByOrgOptions{
  555. ListOptions: github.ListOptions{PerPage: 10},
  556. }
  557. } else if opts.GithubUser != "" {
  558. githubClient = github.NewClient(githubToken())
  559. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  560. ghURL, _ := url.Parse(opts.GithubURL)
  561. githubClient.BaseURL = ghURL
  562. }
  563. githubOptions = &github.RepositoryListOptions{
  564. Affiliation: "owner",
  565. ListOptions: github.ListOptions{
  566. PerPage: 10,
  567. },
  568. }
  569. }
  570. for {
  571. if opts.GithubUser != "" {
  572. if opts.IncludePrivate {
  573. rs, resp, err = githubClient.Repositories.List(ctx, "", githubOptions)
  574. } else {
  575. rs, resp, err = githubClient.Repositories.List(ctx, opts.GithubUser, githubOptions)
  576. }
  577. if err != nil {
  578. return nil, err
  579. }
  580. githubOptions.Page = resp.NextPage
  581. githubRepos = append(githubRepos, rs...)
  582. if resp.NextPage == 0 {
  583. return githubRepos, err
  584. }
  585. } else if opts.GithubOrg != "" {
  586. rs, resp, err := githubClient.Repositories.ListByOrg(ctx, opts.GithubOrg, githubOrgOptions)
  587. if err != nil {
  588. return nil, err
  589. }
  590. githubOrgOptions.Page = resp.NextPage
  591. githubRepos = append(githubRepos, rs...)
  592. if resp.NextPage == 0 {
  593. return githubRepos, err
  594. }
  595. }
  596. for _, githubRepo := range githubRepos {
  597. log.Infof("staging repo %s", *githubRepo.Name)
  598. }
  599. }
  600. }
  601. // auditGithubRepo clones repos from github
  602. func auditGithubRepo(githubRepo *github.Repository) ([]Leak, error) {
  603. var (
  604. leaks []Leak
  605. repo *git.Repository
  606. err error
  607. )
  608. log.Infof("cloning: %s", *githubRepo.Name)
  609. if opts.Disk {
  610. ownerDir, err := ioutil.TempDir(dir, opts.GithubUser)
  611. defer os.RemoveAll(fmt.Sprintf("%s/%s", ownerDir, *githubRepo.Name))
  612. if err != nil {
  613. return leaks, fmt.Errorf("unable to generater owner temp dir: %v", err)
  614. }
  615. if opts.IncludePrivate {
  616. if sshAuth == nil {
  617. return leaks, fmt.Errorf("no ssh auth available")
  618. }
  619. repo, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *githubRepo.Name), false, &git.CloneOptions{
  620. URL: *githubRepo.SSHURL,
  621. Auth: sshAuth,
  622. })
  623. } else {
  624. repo, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *githubRepo.Name), false, &git.CloneOptions{
  625. URL: *githubRepo.CloneURL,
  626. })
  627. }
  628. } else {
  629. if opts.IncludePrivate {
  630. if sshAuth == nil {
  631. return leaks, fmt.Errorf("no ssh auth available")
  632. }
  633. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  634. URL: *githubRepo.SSHURL,
  635. Auth: sshAuth,
  636. })
  637. } else {
  638. repo, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  639. URL: *githubRepo.CloneURL,
  640. })
  641. }
  642. }
  643. if err != nil {
  644. return leaks, err
  645. }
  646. return auditRepo(Repo{
  647. repository: repo,
  648. name: *githubRepo.Name,
  649. })
  650. }
  651. // githubToken returns a oauth2 client for the github api to consume
  652. func githubToken() *http.Client {
  653. githubToken := os.Getenv("GITHUB_TOKEN")
  654. if githubToken == "" {
  655. return nil
  656. }
  657. ts := oauth2.StaticTokenSource(
  658. &oauth2.Token{AccessToken: githubToken},
  659. )
  660. return oauth2.NewClient(context.Background(), ts)
  661. }
  662. // discoverRepos walks all the children of `path`. If a child directory
  663. // contain a .git file then that repo will be added
  664. func discoverRepos(ownerPath string) ([]Repo, error) {
  665. var (
  666. err error
  667. repos []Repo
  668. )
  669. files, err := ioutil.ReadDir(ownerPath)
  670. if err != nil {
  671. return repos, err
  672. }
  673. for _, f := range files {
  674. if f.IsDir() {
  675. repoPath := path.Join(ownerPath, f.Name())
  676. r, err := git.PlainOpen(repoPath)
  677. if err != nil {
  678. continue
  679. }
  680. repos = append(repos, Repo{
  681. repository: r,
  682. name: f.Name(),
  683. path: repoPath,
  684. })
  685. }
  686. }
  687. return repos, err
  688. }
  689. // setLogLevel sets log level for gitleaks. Default is Warning
  690. func setLogs() {
  691. switch opts.Log {
  692. case "info":
  693. log.SetLevel(log.InfoLevel)
  694. case "debug":
  695. log.SetLevel(log.DebugLevel)
  696. case "warn":
  697. log.SetLevel(log.WarnLevel)
  698. default:
  699. log.SetLevel(log.InfoLevel)
  700. }
  701. log.SetFormatter(&log.TextFormatter{
  702. FullTimestamp: true,
  703. })
  704. }
  705. // optsGuard prevents invalid options
  706. func optsGuard() error {
  707. var err error
  708. if opts.GithubOrg != "" && opts.GithubUser != "" {
  709. return fmt.Errorf("github user and organization set")
  710. } else if opts.GithubOrg != "" && opts.OwnerPath != "" {
  711. return fmt.Errorf("github organization set and local owner path")
  712. } else if opts.GithubUser != "" && opts.OwnerPath != "" {
  713. return fmt.Errorf("github user set and local owner path")
  714. } else if opts.IncludePrivate && os.Getenv("GITHUB_TOKEN") == "" && (opts.GithubOrg != "" || opts.GithubUser != "") {
  715. return fmt.Errorf("user/organization private repos require env var GITHUB_TOKEN to be set")
  716. }
  717. // do the URL Parse and error checking here, so we can skip it later
  718. // empty string is OK, it will default to the public github URL.
  719. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  720. if !strings.HasSuffix(opts.GithubURL, "/") {
  721. opts.GithubURL += "/"
  722. }
  723. ghURL, err := url.Parse(opts.GithubURL)
  724. if err != nil {
  725. return err
  726. }
  727. tcpPort := "443"
  728. if ghURL.Scheme == "http" {
  729. tcpPort = "80"
  730. }
  731. timeout := time.Duration(1 * time.Second)
  732. _, err = net.DialTimeout("tcp", ghURL.Host+":"+tcpPort, timeout)
  733. if err != nil {
  734. return fmt.Errorf("%s unreachable, error: %s", ghURL.Host, err)
  735. }
  736. }
  737. if opts.SingleSearch != "" {
  738. singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
  739. if err != nil {
  740. return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
  741. }
  742. }
  743. return nil
  744. }
  745. // loadToml loads of the toml config containing regexes and whitelists
  746. // 1. look for config path
  747. // 2. two, look for gitleaks config env var
  748. func loadToml() error {
  749. var (
  750. config Config
  751. configPath string
  752. )
  753. if opts.ConfigPath != "" {
  754. configPath = opts.ConfigPath
  755. _, err := os.Stat(configPath)
  756. if err != nil {
  757. return fmt.Errorf("no gitleaks config at %s", configPath)
  758. }
  759. } else {
  760. configPath = os.Getenv("GITLEAKS_CONFIG")
  761. }
  762. if configPath != "" {
  763. if _, err := toml.DecodeFile(configPath, &config); err != nil {
  764. return fmt.Errorf("problem loading config: %v", err)
  765. }
  766. } else {
  767. _, err := toml.Decode(defaultConfig, &config)
  768. if err != nil {
  769. return fmt.Errorf("problem loading default config: %v", err)
  770. }
  771. }
  772. // load up regexes
  773. if singleSearchRegex != nil {
  774. // single search takes precedence over default regex
  775. regexes["singleSearch"] = singleSearchRegex
  776. } else {
  777. for _, regex := range config.Regexes {
  778. regexes[regex.Description] = regexp.MustCompile(regex.Regex)
  779. }
  780. }
  781. whiteListBranches = config.Whitelist.Branches
  782. whiteListCommits = make(map[string]bool)
  783. for _, commit := range config.Whitelist.Commits {
  784. whiteListCommits[commit] = true
  785. }
  786. for _, regex := range config.Whitelist.Files {
  787. whiteListFiles = append(whiteListFiles, regexp.MustCompile(regex))
  788. }
  789. for _, regex := range config.Whitelist.Regexes {
  790. whiteListRegexes = append(whiteListRegexes, regexp.MustCompile(regex))
  791. }
  792. return nil
  793. }
  794. // getSSHAuth generates ssh auth
  795. func getSSHAuth() (*ssh.PublicKeys, error) {
  796. var (
  797. sshKeyPath string
  798. )
  799. if opts.SSHKey != "" {
  800. sshKeyPath = opts.SSHKey
  801. } else {
  802. c, _ := user.Current()
  803. sshKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", c.HomeDir)
  804. }
  805. sshAuth, err := ssh.NewPublicKeysFromFile("git", sshKeyPath, "")
  806. if err != nil {
  807. return nil, fmt.Errorf("unable to generate ssh key: %v", err)
  808. }
  809. return sshAuth, err
  810. }
  811. func (leak *Leak) log() {
  812. b, _ := json.MarshalIndent(leak, "", " ")
  813. fmt.Println(string(b))
  814. }