gitleaks/repo.go

package main

import (
	"path/filepath"
	"os"
	"fmt"
	"os/exec"
	"sync"
	"bytes"
	"encoding/json"
	"log"
	"strings"
	"io/ioutil"
)

type Repo struct {
	name  string
	url   string
	path  string
	status string // TODO
	leaks  []Leak
	owner *Owner
}

type Leak struct {
	Line     string `json:"line"`
	Commit   string `json:"commit"`
	Offender string `json:"string"`
	Reason   string `json:"reason"`
	Msg      string `json:"commitMsg"`
	Time     string `json:"time"`
	Author   string `json:"author"`
	File     string `json:"file"`
	RepoURL  string `json:"repoURL"`
}

type Commit struct {
	Hash   string
	Author string
	Time   string
	Msg    string
}

func newRepo(owner *Owner, name string, url string) *Repo {
	repo := &Repo{
		name: name,
		url: url,
		path: owner.path + "/" + name,
	}
	return repo
}

// Audit operates on a single repo and searches the full or partial history of the repo.
// A semaphore is declared for every repo to bind concurrency. If unbounded, the system will throw a
// `too many open files` error. Eventually, gitleaks should use src-d/go-git to avoid shelling out
// commands so that users could opt for doing all clones/diffs in memory.
// Audit also declares two WaitGroups, one for distributing regex/entropy checks, and one for receiving
// the leaks if there are any. This could be done a little more elegantly in the future.
func (repo *Repo) audit(owner *Owner, opts *Options) error {
	var (
		out               []byte
		err               error
		commitWG          sync.WaitGroup
		gitLeakReceiverWG sync.WaitGroup
		gitLeaksChan      = make(chan Leak)
		leaks             []Leak
		semaphoreChan     = make(chan struct{}, opts.Concurrency)
	)

	dotGitPath := filepath.Join(repo.path, ".git")

	// Navigate to proper location to being audit. Clone repo
	// if not present, otherwise fetch for new changes.
	if _, err := os.Stat(dotGitPath); os.IsNotExist(err) {
		// no repo present, clone it
		fmt.Printf("Cloning \x1b[37;1m%s\x1b[0m into %s...\n", repo.url, repo.path)
		err = exec.Command("git", "clone", repo.url, repo.path).Run()
		if err != nil{
			fmt.Println("can run clonse")
		}
	} else {
		fmt.Printf("Checking \x1b[37;1m%s\x1b[0m from %s...\n", repo.url, repo.path)
		err = exec.Command("git", "fetch").Run()
		if err != nil{
			fmt.Println("can run fetch")
		}
	}

	if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil {
		fmt.Println("cant chdir")
	}

	gitFormat := "--format=%H%n%an%n%s%n%ci"
	out, err = exec.Command("git", "rev-list", "--all",
		"--remotes", "--topo-order", gitFormat).Output()

	if err != nil {
		fmt.Println("problem with rev list")
	}

	revListLines := bytes.Split(out, []byte("\n"))
	commits := parseRevList(revListLines)

	for _, commit := range commits {
		if commit.Hash == "" {
			continue
		}

		commitWG.Add(1)
		go auditDiff(commit, repo, &commitWG, &gitLeakReceiverWG, opts,
			semaphoreChan, gitLeaksChan)

		if commit.Hash == opts.SinceCommit {
			break
		}
	}
	go reportAggregator(&gitLeakReceiverWG, gitLeaksChan, &leaks)
	commitWG.Wait()
	gitLeakReceiverWG.Wait()

	// repo audit has finished
	repo.leaks = leaks

	if opts.EnableJSON && len(leaks) != 0 {
		repo.writeReport(owner)
	}

	return nil
}

func (repo *Repo) log() {

}

// Used by audit, writeReport will generate a report and write it out to
// $GITLEAKS_HOME/report/<owner>/<repo>. No report will be generated if
// no leaks have been found
func (repo *Repo) writeReport(owner *Owner) {
	reportJSON, _ := json.MarshalIndent(repo.leaks, "", "\t")
	if _, err := os.Stat(owner.reportPath); os.IsNotExist(err) {
		os.Mkdir(repo.owner.reportPath, os.ModePerm)
	}

	reportFileName := fmt.Sprintf("%s_leaks.json", repo.name)
	reportFile := filepath.Join(owner.reportPath, reportFileName)
	err := ioutil.WriteFile(reportFile, reportJSON, 0644)
	if err != nil {
		fmt.Println("cant write report")
	}
	fmt.Printf("Report written to %s\n", reportFile)
}

// parseRevList is responsible for parsing the output of
// $ `git rev-list --all -remotes --topo-order --format=%H%n%an%n%s%n%ci`
// sample output from the above command looks like:
//		...
// 		SHA
// 		Author Name
// 		Commit Msg
// 		Commit Date
//		...
// Used by audit
func parseRevList(revList [][]byte) []Commit {
	var commits []Commit
	for i := 0; i < len(revList)-1; i = i + 5 {
		commit := Commit{
			Hash:   string(revList[i+1]),
			Author: string(revList[i+2]),
			Msg:    string(revList[i+3]),
			Time:   string(revList[i+4]),
		}
		commits = append(commits, commit)
	}
	return commits
}

// reportAggregator is a go func responsible for ...
func reportAggregator(gitLeakReceiverWG *sync.WaitGroup, gitLeaks chan Leak, leaks *[]Leak) {
	for gitLeak := range gitLeaks {
		b, err := json.MarshalIndent(gitLeak, "", "   ")
		if err != nil {
			fmt.Println("failed to output leak:", err)
		}
		fmt.Println(string(b))
		*leaks = append(*leaks, gitLeak)
		gitLeakReceiverWG.Done()
	}
}

// Used by audit, auditDiff is a go func responsible for diffing and auditing a commit.
// Three channels are input here: 1. a semaphore to bind gitleaks, 2. a leak stream, 3. error handling (TODO)
// This func performs a diff and runs regexes checks on each line of the diff.
func auditDiff(currCommit Commit, repo *Repo, commitWG *sync.WaitGroup,
	gitLeakReceiverWG *sync.WaitGroup, opts *Options, semaphoreChan chan struct{},
	gitLeaks chan Leak) {
	// signal to WG this diff is done being audited
	defer commitWG.Done()

	if err := os.Chdir(fmt.Sprintf(repo.path)); err != nil {
		log.Fatal(err)
	}

	commitCmp := fmt.Sprintf("%s^!", currCommit.Hash)
	semaphoreChan <- struct{}{}
	out, err := exec.Command("git", "diff", commitCmp).Output()
	<-semaphoreChan

	if err != nil {
		// TODO
		if strings.Contains(err.Error(), "too many files open") {
			log.Printf("error retrieving diff for commit %s. Try turning concurrency down. %v\n", currCommit, err)
		}
	}

	leaks := doChecks(string(out), currCommit, opts, repo)
	if len(leaks) == 0 {
		return
	}
	for _, leak := range leaks {
		gitLeakReceiverWG.Add(1)
		gitLeaks <- leak
	}
}