package main import ( "bytes" "encoding/json" "fmt" "io/ioutil" "log" "os" "os/exec" "os/signal" "path/filepath" "strings" "sync" "syscall" ) // LeakElem contains the line and commit of a leak type LeakElem struct { Line string `json:"line"` Commit string `json:"commit"` Offender string `json:"string"` Reason string `json:"reason"` Msg string `json:"commitMsg"` Time string `json:"time"` Author string `json:"author"` File string `json:"file"` RepoURL string `json:"repoURL"` } type Commit struct { Hash string Author string Time string Msg string } func rmTmp(owner *Owner) { if _, err := os.Stat(owner.path); err == nil { err := os.RemoveAll(owner.path) log.Printf("\nCleaning up tmp repos in %s\n", owner.path) if err != nil { log.Printf("failed to properly remove tmp gitleaks dir: %v", err) } } os.Exit(1) } // start func start(repos []RepoDesc, owner *Owner, opts *Options) { var report []LeakElem if opts.Tmp { defer rmTmp(owner) } // interrupt handling c := make(chan os.Signal, 2) signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { <-c if opts.Tmp { rmTmp(owner) } os.Exit(1) }() // run checks on repos for _, repo := range repos { dotGitPath := filepath.Join(repo.path, ".git") if _, err := os.Stat(dotGitPath); err == nil { if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil { log.Fatal(err) } // use pre-cloned repo fmt.Printf("Checking \x1b[37;1m%s\x1b[0m...\n", repo.url) err := exec.Command("git", "fetch").Run() if err != nil { log.Printf("failed to fetch repo %v", err) return } report = getLeaks(repo, owner, opts) } else { // no repo present, clone it if err := os.Chdir(fmt.Sprintf(owner.path)); err != nil { log.Fatal(err) } fmt.Printf("Cloning \x1b[37;1m%s\x1b[0m...\n", repo.url) err := exec.Command("git", "clone", repo.url).Run() if err != nil { fmt.Printf("failed to clone repo %v", err) return } report = getLeaks(repo, owner, opts) } if len(report) == 0 { fmt.Printf("No Leaks detected for \x1b[35;2m%s\x1b[0m...\n\n", repo.url) } if opts.EnableJSON { outputGitLeaksReport(report, repo, opts) } } } // outputGitLeaksReport func outputGitLeaksReport(report []LeakElem, repo RepoDesc, opts *Options) { reportJSON, _ := json.MarshalIndent(report, "", "\t") if _, err := os.Stat(repo.owner.reportPath); os.IsNotExist(err) { os.Mkdir(repo.owner.reportPath, os.ModePerm) } reportFileName := fmt.Sprintf("%s_leaks.json", repo.name) reportFile := filepath.Join(repo.owner.reportPath, reportFileName) err := ioutil.WriteFile(reportFile, reportJSON, 0644) if err != nil { log.Fatalf("Can't write to file: %s", err) } fmt.Printf("Report written to %s\n", reportFile) } // getLeaks will attempt to find gitleaks func getLeaks(repo RepoDesc, owner *Owner, opts *Options) []LeakElem { var ( out []byte err error commitWG sync.WaitGroup gitLeakReceiverWG sync.WaitGroup gitLeaks = make(chan LeakElem) report []LeakElem ) semaphoreChan := make(chan struct{}, opts.Concurrency) go func(commitWG *sync.WaitGroup, gitLeakReceiverWG *sync.WaitGroup) { for gitLeak := range gitLeaks { b, err := json.MarshalIndent(gitLeak, "", " ") if err != nil { fmt.Println("failed to output leak:", err) } fmt.Println(string(b)) report = append(report, gitLeak) gitLeakReceiverWG.Done() } }(&commitWG, &gitLeakReceiverWG) if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil { log.Fatal(err) } gitFormat := "--format=%H%n%an%n%s%n%ci" out, err = exec.Command("git", "rev-list", "--all", "--remotes", "--topo-order", gitFormat).Output() if err != nil { log.Fatalf("error retrieving commits%v\n", err) } revListLines := bytes.Split(out, []byte("\n")) commits := parseFormattedRevList(revListLines) for _, commit := range commits { if commit.Hash == "" { continue } commitWG.Add(1) go func(currCommit Commit, repoName string, commitWG *sync.WaitGroup, gitLeakReceiverWG *sync.WaitGroup, opts *Options) { defer commitWG.Done() if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil { log.Fatal(err) } commitCmp := fmt.Sprintf("%s^!", currCommit.Hash) semaphoreChan <- struct{}{} out, err := exec.Command("git", "diff", commitCmp).Output() <-semaphoreChan if err != nil { if strings.Contains(err.Error(), "too many files open") { log.Printf("error retrieving diff for commit %s. Try turning concurrency down. %v\n", currCommit, err) } if opts.Tmp { rmTmp(owner) } } leaks := doChecks(string(out), currCommit, opts, repo) if len(leaks) == 0 { return } for _, leak := range leaks { gitLeakReceiverWG.Add(1) gitLeaks <- leak } }(commit, repo.name, &commitWG, &gitLeakReceiverWG, opts) if commit.Hash == opts.SinceCommit { break } } commitWG.Wait() gitLeakReceiverWG.Wait() return report } func parseFormattedRevList(revList [][]byte) []Commit { var commits []Commit for i := 0; i < len(revList)-1; i = i + 5 { commit := Commit{ Hash: string(revList[i+1]), Author: string(revList[i+2]), Msg: string(revList[i+3]), Time: string(revList[i+4]), } commits = append(commits, commit) } return commits }