main.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868
  1. package main
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/json"
  6. "fmt"
  7. "io/ioutil"
  8. "net"
  9. "net/http"
  10. "net/url"
  11. "os"
  12. "os/user"
  13. "path"
  14. "path/filepath"
  15. "regexp"
  16. "strings"
  17. "sync"
  18. "time"
  19. "gopkg.in/src-d/go-git.v4/plumbing"
  20. "golang.org/x/oauth2"
  21. "gopkg.in/src-d/go-git.v4/plumbing/object"
  22. "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
  23. "gopkg.in/src-d/go-git.v4/storage/memory"
  24. "github.com/BurntSushi/toml"
  25. "github.com/google/go-github/github"
  26. flags "github.com/jessevdk/go-flags"
  27. log "github.com/sirupsen/logrus"
  28. git "gopkg.in/src-d/go-git.v4"
  29. )
  30. // Leak represents a leaked secret or regex match. This will be output to stdout and/or the report
  31. type Leak struct {
  32. Line string `json:"line"`
  33. Commit string `json:"commit"`
  34. Offender string `json:"offender"`
  35. Type string `json:"reason"`
  36. Message string `json:"commitMsg"`
  37. Author string `json:"author"`
  38. File string `json:"file"`
  39. Branch string `json:"branch"`
  40. }
  41. // Repo contains the actual git repository and meta data about the repo
  42. type Repo struct {
  43. path string
  44. url string
  45. name string
  46. leaks []Leak
  47. repository *git.Repository
  48. err error
  49. }
  50. // Owner contains a collection of repos. This could represent an org or user.
  51. type Owner struct {
  52. path string
  53. url string
  54. repos []Repo
  55. }
  56. // Options for gitleaks
  57. type Options struct {
  58. // remote target options
  59. Repo string `short:"r" long:"repo" description:"Repo url to audit"`
  60. GithubUser string `long:"github-user" description:"User url to audit"`
  61. GithubOrg string `long:"github-org" description:"Organization url to audit"`
  62. GithubURL string `long:"github-url" default:"https://api.github.com/" description:"GitHub API Base URL, use for GitHub Enterprise. Example: https://github.example.com/api/v3/"`
  63. IncludePrivate bool `short:"p" long:"private" description:"Include private repos in audit"`
  64. /*
  65. TODO:
  66. GitLabUser string `long:"gitlab-user" description:"User url to audit"`
  67. GitLabOrg string `long:"gitlab-org" description:"Organization url to audit"`
  68. */
  69. Branch string `short:"b" long:"branch" description:"branch name to audit (defaults to HEAD)"`
  70. Commit string `short:"c" long:"commit" description:"sha of commit to stop at"`
  71. // local target option
  72. RepoPath string `long:"repo-path" description:"Path to repo"`
  73. OwnerPath string `long:"owner-path" description:"Path to owner directory (repos discovered)"`
  74. // Process options
  75. MaxGoRoutines int `long:"max-go" description:"Maximum number of concurrent go-routines gitleaks spawns"`
  76. Disk bool `long:"disk" description:"Clones repo(s) to disk"`
  77. AuditAllRefs bool `long:"all-refs" description:"run audit on all refs"`
  78. SingleSearch string `long:"single-search" description:"single regular expression to search for"`
  79. ConfigPath string `long:"config" description:"path to gitleaks config"`
  80. SSHKey string `long:"ssh-key" description:"path to ssh key"`
  81. // TODO: IncludeMessages string `long:"messages" description:"include commit messages in audit"`
  82. // Output options
  83. Log string `short:"l" long:"log" description:"log level"`
  84. Verbose bool `short:"v" long:"verbose" description:"Show verbose output from gitleaks audit"`
  85. Report string `long:"report" description:"path to write report file"`
  86. Redact bool `long:"redact" description:"redact secrets from log messages and report"`
  87. Version bool `long:"version" description:"version number"`
  88. }
  89. // Config struct for regexes matching and whitelisting
  90. type Config struct {
  91. Regexes []struct {
  92. Description string
  93. Regex string
  94. }
  95. Whitelist struct {
  96. Files []string
  97. Regexes []string
  98. Commits []string
  99. Branches []string
  100. }
  101. }
  102. const defaultGithubURL = "https://api.github.com/"
  103. const version = "1.1.2"
  104. const defaultConfig = `
  105. title = "gitleaks config"
  106. # add regexes to the regex table
  107. [[regexes]]
  108. description = "AWS"
  109. regex = '''AKIA[0-9A-Z]{16}'''
  110. [[regexes]]
  111. description = "RKCS8"
  112. regex = '''-----BEGIN PRIVATE KEY-----'''
  113. [[regexes]]
  114. description = "RSA"
  115. regex = '''-----BEGIN RSA PRIVATE KEY-----'''
  116. [[regexes]]
  117. description = "Github"
  118. regex = '''(?i)github.*['\"][0-9a-zA-Z]{35,40}['\"]'''
  119. [[regexes]]
  120. description = "SSH"
  121. regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
  122. [[regexes]]
  123. description = "Facebook"
  124. regex = '''(?i)facebook.*['\"][0-9a-f]{32}['\"]'''
  125. [[regexes]]
  126. description = "Twitter"
  127. regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]'''
  128. [whitelist]
  129. #regexes = [
  130. # "AKAIMYFAKEAWKKEY",
  131. #]
  132. #files = [
  133. # "(.*?)(jpg|gif|doc|pdf|bin)$"
  134. #]
  135. #commits = [
  136. # "BADHA5H1",
  137. # "BADHA5H2",
  138. #]
  139. #branches = [
  140. # "dev/STUPDIFKNFEATURE"
  141. #]
  142. `
  143. var (
  144. opts Options
  145. regexes map[string]*regexp.Regexp
  146. singleSearchRegex *regexp.Regexp
  147. whiteListRegexes []*regexp.Regexp
  148. whiteListFiles []*regexp.Regexp
  149. whiteListCommits map[string]bool
  150. whiteListBranches []string
  151. fileDiffRegex *regexp.Regexp
  152. sshAuth *ssh.PublicKeys
  153. dir string
  154. )
  155. func init() {
  156. log.SetOutput(os.Stdout)
  157. regexes = make(map[string]*regexp.Regexp)
  158. whiteListCommits = make(map[string]bool)
  159. }
  160. func main() {
  161. var (
  162. leaks []Leak
  163. repos []Repo
  164. )
  165. _, err := flags.Parse(&opts)
  166. if opts.Version {
  167. fmt.Println(version)
  168. os.Exit(0)
  169. }
  170. if err != nil {
  171. os.Exit(1)
  172. }
  173. setLogs()
  174. err = optsGuard()
  175. if err != nil {
  176. log.Fatal(err)
  177. }
  178. err = loadToml()
  179. if err != nil {
  180. log.Fatal(err)
  181. }
  182. if opts.IncludePrivate {
  183. // if including private repos use ssh as authentication
  184. sshAuth, err = getSSHAuth()
  185. if err != nil {
  186. log.Fatal(err)
  187. }
  188. }
  189. if opts.Disk {
  190. // temporary directory where all the gitleaks plain clones will reside
  191. dir, err = ioutil.TempDir("", "gitleaks")
  192. defer os.RemoveAll(dir)
  193. if err != nil {
  194. log.Fatal(err)
  195. }
  196. }
  197. // start audits
  198. if opts.Repo != "" || opts.RepoPath != "" {
  199. r, err := getRepo()
  200. if err != nil {
  201. log.Fatal(err)
  202. }
  203. repos = append(repos, r)
  204. } else if ownerTarget() {
  205. repos, err = getOwnerRepos()
  206. }
  207. for _, r := range repos {
  208. if r.err != nil {
  209. log.Warnf("skipping audit for repo %s due to cloning error: %s", r.name, r.err)
  210. continue
  211. }
  212. l, err := auditRepo(r.repository)
  213. if len(l) == 0 {
  214. log.Infof("no leaks found for repo %s", r.name)
  215. } else {
  216. log.Warnf("leaks found for repo %s", r.name)
  217. }
  218. if err != nil {
  219. log.Fatalf("error during audit: %s", err)
  220. }
  221. leaks = append(leaks, l...)
  222. }
  223. if opts.Report != "" {
  224. writeReport(leaks)
  225. }
  226. if len(leaks) != 0 {
  227. log.Errorf("leaks detected")
  228. os.Exit(1)
  229. }
  230. }
  231. // writeReport writes a report to opts.Report in JSON.
  232. // TODO support yaml
  233. func writeReport(leaks []Leak) error {
  234. log.Info("writing report to %s", opts.Report)
  235. reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
  236. err := ioutil.WriteFile(opts.Report, reportJSON, 0644)
  237. return err
  238. }
  239. // getRepo is responsible for cloning a repo in-memory or to disk, or opening a local repo and creating
  240. // a repo object
  241. func getRepo() (Repo, error) {
  242. var (
  243. err error
  244. r *git.Repository
  245. )
  246. if opts.Disk {
  247. log.Infof("cloning %s", opts.Repo)
  248. cloneTarget := fmt.Sprintf("%s/%x", dir, md5.Sum([]byte(fmt.Sprintf("%s%s", opts.GithubUser, opts.Repo))))
  249. if opts.IncludePrivate {
  250. r, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  251. URL: opts.Repo,
  252. Progress: os.Stdout,
  253. Auth: sshAuth,
  254. })
  255. } else {
  256. r, err = git.PlainClone(cloneTarget, false, &git.CloneOptions{
  257. URL: opts.Repo,
  258. Progress: os.Stdout,
  259. })
  260. }
  261. } else if opts.RepoPath != "" {
  262. // use existing repo
  263. log.Infof("opening %s", opts.Repo)
  264. r, err = git.PlainOpen(opts.RepoPath)
  265. } else {
  266. log.Infof("cloning %s", opts.Repo)
  267. if opts.IncludePrivate {
  268. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  269. URL: opts.Repo,
  270. Progress: os.Stdout,
  271. Auth: sshAuth,
  272. })
  273. } else {
  274. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  275. URL: opts.Repo,
  276. Progress: os.Stdout,
  277. })
  278. }
  279. }
  280. return Repo{
  281. repository: r,
  282. path: opts.RepoPath,
  283. url: opts.Repo,
  284. name: filepath.Base(opts.Repo),
  285. err: err,
  286. }, nil
  287. }
  288. // auditRef audits a git reference
  289. // TODO: need to add a layer of parallelism here and a cache for parent+child commits so we don't
  290. // double dip
  291. func auditRef(r *git.Repository, ref *plumbing.Reference, commitWg *sync.WaitGroup, commitChan chan []Leak) error {
  292. var (
  293. err error
  294. prevCommit *object.Commit
  295. limitGoRoutines bool
  296. semaphore chan bool
  297. )
  298. // goroutine limiting
  299. if opts.MaxGoRoutines != 0 {
  300. semaphore = make(chan bool, opts.MaxGoRoutines)
  301. limitGoRoutines = true
  302. }
  303. cIter, err := r.Log(&git.LogOptions{From: ref.Hash()})
  304. if err != nil {
  305. return err
  306. }
  307. err = cIter.ForEach(func(c *object.Commit) error {
  308. if c.Hash.String() == opts.Commit {
  309. cIter.Close()
  310. }
  311. if whiteListCommits[c.Hash.String()] {
  312. log.Infof("skipping commit: %s\n", c.Hash.String())
  313. return nil
  314. }
  315. if limitGoRoutines {
  316. semaphore <- true
  317. }
  318. if prevCommit == nil {
  319. prevCommit = c
  320. }
  321. commitWg.Add(1)
  322. go func(c *object.Commit, prevCommit *object.Commit) {
  323. var (
  324. leaks []Leak
  325. filePath string
  326. skipFile bool
  327. )
  328. patch, err := c.Patch(prevCommit)
  329. if err != nil {
  330. log.Warnf("problem generating patch for commit: %s\n", c.Hash.String())
  331. if limitGoRoutines {
  332. <-semaphore
  333. }
  334. commitChan <- leaks
  335. return
  336. }
  337. for _, f := range patch.FilePatches() {
  338. skipFile = false
  339. from, to := f.Files()
  340. filePath = "???"
  341. if from != nil {
  342. filePath = from.Path()
  343. } else if to != nil {
  344. filePath = to.Path()
  345. }
  346. for _, re := range whiteListFiles {
  347. if re.FindString(filePath) != "" {
  348. skipFile = true
  349. break
  350. }
  351. }
  352. if skipFile {
  353. continue
  354. }
  355. chunks := f.Chunks()
  356. for _, chunk := range chunks {
  357. if chunk.Type() == 1 || chunk.Type() == 2 {
  358. // only check if adding or removing
  359. leaks = append(leaks, checkDiff(chunk.Content(), prevCommit, filePath, string(ref.Name()))...)
  360. }
  361. }
  362. }
  363. if limitGoRoutines {
  364. <-semaphore
  365. }
  366. commitChan <- leaks
  367. }(c, prevCommit)
  368. prevCommit = c
  369. return nil
  370. })
  371. return nil
  372. }
  373. // auditRepo performs an audit on a repository checking for regex matching and ignoring
  374. // files and regexes that are whitelisted
  375. func auditRepo(r *git.Repository) ([]Leak, error) {
  376. var (
  377. err error
  378. leaks []Leak
  379. commitWg sync.WaitGroup
  380. )
  381. ref, err := r.Head()
  382. if err != nil {
  383. return leaks, err
  384. }
  385. // leak messaging
  386. commitChan := make(chan []Leak, 1)
  387. if opts.AuditAllRefs {
  388. skipBranch := false
  389. refs, err := r.Storer.IterReferences()
  390. if err != nil {
  391. return leaks, err
  392. }
  393. err = refs.ForEach(func(ref *plumbing.Reference) error {
  394. for _, b := range whiteListBranches {
  395. if strings.HasSuffix(string(ref.Name()), b) {
  396. skipBranch = true
  397. }
  398. }
  399. if skipBranch {
  400. skipBranch = false
  401. return nil
  402. }
  403. auditRef(r, ref, &commitWg, commitChan)
  404. return nil
  405. })
  406. } else {
  407. auditRef(r, ref, &commitWg, commitChan)
  408. }
  409. go func() {
  410. for commitLeaks := range commitChan {
  411. if commitLeaks != nil {
  412. for _, leak := range commitLeaks {
  413. leaks = append(leaks, leak)
  414. }
  415. }
  416. commitWg.Done()
  417. }
  418. }()
  419. commitWg.Wait()
  420. return leaks, err
  421. }
  422. // checkDiff accepts a string diff and commit object then performs a
  423. // regex check
  424. func checkDiff(diff string, commit *object.Commit, filePath string, branch string) []Leak {
  425. lines := strings.Split(diff, "\n")
  426. var (
  427. leaks []Leak
  428. skipLine bool
  429. )
  430. for _, line := range lines {
  431. skipLine = false
  432. for leakType, re := range regexes {
  433. match := re.FindString(line)
  434. if match == "" {
  435. continue
  436. }
  437. // if offender matches whitelist regex, ignore it
  438. for _, wRe := range whiteListRegexes {
  439. whitelistMatch := wRe.FindString(line)
  440. if whitelistMatch != "" {
  441. skipLine = true
  442. break
  443. }
  444. }
  445. if skipLine {
  446. break
  447. }
  448. leak := Leak{
  449. Line: line,
  450. Commit: commit.Hash.String(),
  451. Offender: match,
  452. Type: leakType,
  453. Message: commit.Message,
  454. Author: commit.Author.String(),
  455. File: filePath,
  456. Branch: branch,
  457. }
  458. if opts.Verbose {
  459. leak.log()
  460. }
  461. if opts.Redact {
  462. leak.Offender = "REDACTED"
  463. }
  464. leaks = append(leaks, leak)
  465. }
  466. }
  467. return leaks
  468. }
  469. // auditOwner audits all of the owner's(user or org) repos
  470. func getOwnerRepos() ([]Repo, error) {
  471. var (
  472. err error
  473. repos []Repo
  474. )
  475. ctx := context.Background()
  476. if opts.OwnerPath != "" {
  477. repos, err = discoverRepos(opts.OwnerPath)
  478. } else if opts.GithubOrg != "" {
  479. githubClient := github.NewClient(githubToken())
  480. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  481. ghURL, _ := url.Parse(opts.GithubURL)
  482. githubClient.BaseURL = ghURL
  483. }
  484. githubOptions := github.RepositoryListByOrgOptions{
  485. ListOptions: github.ListOptions{PerPage: 10},
  486. }
  487. repos, err = getOrgGithubRepos(ctx, &githubOptions, githubClient)
  488. } else if opts.GithubUser != "" {
  489. githubClient := github.NewClient(githubToken())
  490. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  491. ghURL, _ := url.Parse(opts.GithubURL)
  492. githubClient.BaseURL = ghURL
  493. }
  494. githubOptions := github.RepositoryListOptions{
  495. Affiliation: "owner",
  496. ListOptions: github.ListOptions{
  497. PerPage: 10,
  498. },
  499. }
  500. repos, err = getUserGithubRepos(ctx, &githubOptions, githubClient)
  501. }
  502. return repos, err
  503. }
  504. // getUserGithubRepos grabs all github user's public and/or private repos
  505. func getUserGithubRepos(ctx context.Context, listOpts *github.RepositoryListOptions, client *github.Client) ([]Repo, error) {
  506. var (
  507. err error
  508. repos []Repo
  509. r *git.Repository
  510. rs []*github.Repository
  511. resp *github.Response
  512. )
  513. for {
  514. if opts.IncludePrivate {
  515. rs, resp, err = client.Repositories.List(ctx, "", listOpts)
  516. } else {
  517. rs, resp, err = client.Repositories.List(ctx, opts.GithubUser, listOpts)
  518. }
  519. for _, rDesc := range rs {
  520. log.Infof("cloning: %s", *rDesc.Name)
  521. if opts.Disk {
  522. ownerDir, err := ioutil.TempDir(dir, opts.GithubUser)
  523. if err != nil {
  524. return repos, fmt.Errorf("unable to generater owner temp dir: %v", err)
  525. }
  526. if opts.IncludePrivate {
  527. r, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *rDesc.Name), false, &git.CloneOptions{
  528. URL: *rDesc.SSHURL,
  529. Auth: sshAuth,
  530. })
  531. } else {
  532. r, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *rDesc.Name), false, &git.CloneOptions{
  533. URL: *rDesc.CloneURL,
  534. })
  535. }
  536. } else {
  537. if opts.IncludePrivate {
  538. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  539. URL: *rDesc.SSHURL,
  540. Auth: sshAuth,
  541. })
  542. } else {
  543. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  544. URL: *rDesc.CloneURL,
  545. })
  546. }
  547. }
  548. repos = append(repos, Repo{
  549. name: *rDesc.Name,
  550. url: *rDesc.SSHURL,
  551. repository: r,
  552. err: err,
  553. })
  554. }
  555. if resp.NextPage == 0 {
  556. break
  557. }
  558. listOpts.Page = resp.NextPage
  559. }
  560. return repos, err
  561. }
  562. // getOrgGithubRepos grabs all of org's public and/or private repos
  563. func getOrgGithubRepos(ctx context.Context, listOpts *github.RepositoryListByOrgOptions, client *github.Client) ([]Repo, error) {
  564. var (
  565. err error
  566. repos []Repo
  567. r *git.Repository
  568. ownerDir string
  569. )
  570. for {
  571. rs, resp, err := client.Repositories.ListByOrg(ctx, opts.GithubOrg, listOpts)
  572. for _, rDesc := range rs {
  573. log.Infof("cloning: %s", *rDesc.Name)
  574. if opts.Disk {
  575. ownerDir, err = ioutil.TempDir(dir, opts.GithubUser)
  576. if err != nil {
  577. return repos, fmt.Errorf("unable to generater owner temp dir: %v", err)
  578. }
  579. if opts.IncludePrivate {
  580. if sshAuth == nil {
  581. return nil, fmt.Errorf("no ssh auth available")
  582. }
  583. r, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *rDesc.Name), false, &git.CloneOptions{
  584. URL: *rDesc.SSHURL,
  585. Auth: sshAuth,
  586. })
  587. } else {
  588. r, err = git.PlainClone(fmt.Sprintf("%s/%s", ownerDir, *rDesc.Name), false, &git.CloneOptions{
  589. URL: *rDesc.CloneURL,
  590. })
  591. }
  592. } else {
  593. if opts.IncludePrivate {
  594. if sshAuth == nil {
  595. return nil, fmt.Errorf("no ssh auth available")
  596. }
  597. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  598. URL: *rDesc.SSHURL,
  599. Auth: sshAuth,
  600. })
  601. } else {
  602. r, err = git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
  603. URL: *rDesc.CloneURL,
  604. })
  605. }
  606. }
  607. repos = append(repos, Repo{
  608. url: *rDesc.SSHURL,
  609. name: *rDesc.Name,
  610. repository: r,
  611. err: err,
  612. })
  613. }
  614. if resp.NextPage == 0 {
  615. break
  616. }
  617. listOpts.Page = resp.NextPage
  618. }
  619. return repos, err
  620. }
  621. // githubToken returns a oauth2 client for the github api to consume
  622. func githubToken() *http.Client {
  623. githubToken := os.Getenv("GITHUB_TOKEN")
  624. if githubToken == "" {
  625. return nil
  626. }
  627. ts := oauth2.StaticTokenSource(
  628. &oauth2.Token{AccessToken: githubToken},
  629. )
  630. return oauth2.NewClient(context.Background(), ts)
  631. }
  632. // discoverRepos walks all the children of `path`. If a child directory
  633. // contain a .git file then that repo will be added
  634. func discoverRepos(ownerPath string) ([]Repo, error) {
  635. var (
  636. err error
  637. repos []Repo
  638. )
  639. files, err := ioutil.ReadDir(ownerPath)
  640. if err != nil {
  641. return repos, err
  642. }
  643. for _, f := range files {
  644. if f.IsDir() {
  645. repoPath := path.Join(ownerPath, f.Name())
  646. r, err := git.PlainOpen(repoPath)
  647. if err != nil {
  648. continue
  649. }
  650. repos = append(repos, Repo{
  651. repository: r,
  652. name: f.Name(),
  653. path: repoPath,
  654. })
  655. }
  656. }
  657. return repos, err
  658. }
  659. // setLogLevel sets log level for gitleaks. Default is Warning
  660. func setLogs() {
  661. switch opts.Log {
  662. case "info":
  663. log.SetLevel(log.InfoLevel)
  664. case "debug":
  665. log.SetLevel(log.DebugLevel)
  666. case "warn":
  667. log.SetLevel(log.WarnLevel)
  668. default:
  669. log.SetLevel(log.InfoLevel)
  670. }
  671. log.SetFormatter(&log.TextFormatter{
  672. FullTimestamp: true,
  673. })
  674. }
  675. // optsGuard prevents invalid options
  676. func optsGuard() error {
  677. var err error
  678. if opts.GithubOrg != "" && opts.GithubUser != "" {
  679. return fmt.Errorf("github user and organization set")
  680. } else if opts.GithubOrg != "" && opts.OwnerPath != "" {
  681. return fmt.Errorf("github organization set and local owner path")
  682. } else if opts.GithubUser != "" && opts.OwnerPath != "" {
  683. return fmt.Errorf("github user set and local owner path")
  684. } else if opts.IncludePrivate && os.Getenv("GITHUB_TOKEN") == "" && (opts.GithubOrg != "" || opts.GithubUser != "") {
  685. return fmt.Errorf("user/organization private repos require env var GITHUB_TOKEN to be set")
  686. }
  687. // do the URL Parse and error checking here, so we can skip it later
  688. // empty string is OK, it will default to the public github URL.
  689. if opts.GithubURL != "" && opts.GithubURL != defaultGithubURL {
  690. if !strings.HasSuffix(opts.GithubURL, "/") {
  691. opts.GithubURL += "/"
  692. }
  693. ghURL, err := url.Parse(opts.GithubURL)
  694. if err != nil {
  695. return err
  696. }
  697. tcpPort := "443"
  698. if ghURL.Scheme == "http" {
  699. tcpPort = "80"
  700. }
  701. timeout := time.Duration(1 * time.Second)
  702. _, err = net.DialTimeout("tcp", ghURL.Host+":"+tcpPort, timeout)
  703. if err != nil {
  704. return fmt.Errorf("%s unreachable, error: %s", ghURL.Host, err)
  705. }
  706. }
  707. if opts.SingleSearch != "" {
  708. singleSearchRegex, err = regexp.Compile(opts.SingleSearch)
  709. if err != nil {
  710. return fmt.Errorf("unable to compile regex: %s, %v", opts.SingleSearch, err)
  711. }
  712. }
  713. return nil
  714. }
  715. // loadToml loads of the toml config containing regexes and whitelists
  716. // 1. look for config path
  717. // 2. two, look for gitleaks config env var
  718. func loadToml() error {
  719. var (
  720. config Config
  721. configPath string
  722. )
  723. if opts.ConfigPath != "" {
  724. configPath = opts.ConfigPath
  725. _, err := os.Stat(configPath)
  726. if err != nil {
  727. return fmt.Errorf("no gitleaks config at %s", configPath)
  728. }
  729. } else {
  730. configPath = os.Getenv("GITLEAKS_CONFIG")
  731. }
  732. if configPath != "" {
  733. if _, err := toml.DecodeFile(configPath, &config); err != nil {
  734. return fmt.Errorf("problem loading config: %v", err)
  735. }
  736. } else {
  737. _, err := toml.Decode(defaultConfig, &config)
  738. if err != nil {
  739. return fmt.Errorf("problem loading default config: %v", err)
  740. }
  741. }
  742. // load up regexes
  743. if singleSearchRegex != nil {
  744. // single search takes precedence over default regex
  745. regexes["singleSearch"] = singleSearchRegex
  746. } else {
  747. for _, regex := range config.Regexes {
  748. regexes[regex.Description] = regexp.MustCompile(regex.Regex)
  749. }
  750. }
  751. whiteListBranches = config.Whitelist.Branches
  752. whiteListCommits = make(map[string]bool)
  753. for _, commit := range config.Whitelist.Commits {
  754. whiteListCommits[commit] = true
  755. }
  756. for _, regex := range config.Whitelist.Files {
  757. whiteListFiles = append(whiteListFiles, regexp.MustCompile(regex))
  758. }
  759. for _, regex := range config.Whitelist.Regexes {
  760. whiteListRegexes = append(whiteListRegexes, regexp.MustCompile(regex))
  761. }
  762. return nil
  763. }
  764. // ownerTarget checks if we are dealing with a remote owner
  765. func ownerTarget() bool {
  766. if opts.GithubOrg != "" ||
  767. opts.GithubUser != "" ||
  768. opts.OwnerPath != "" {
  769. return true
  770. }
  771. return false
  772. }
  773. // getSSHAuth generates ssh auth
  774. func getSSHAuth() (*ssh.PublicKeys, error) {
  775. var (
  776. sshKeyPath string
  777. )
  778. if opts.SSHKey != "" {
  779. sshKeyPath = opts.SSHKey
  780. } else {
  781. c, _ := user.Current()
  782. sshKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", c.HomeDir)
  783. }
  784. sshAuth, err := ssh.NewPublicKeysFromFile("git", sshKeyPath, "")
  785. if err != nil {
  786. return nil, fmt.Errorf("unable to generate ssh key: %v", err)
  787. }
  788. return sshAuth, err
  789. }
  790. func (leak *Leak) log() {
  791. b, _ := json.MarshalIndent(leak, "", " ")
  792. fmt.Println(string(b))
  793. }