Merge branch 'master' of https://github.com/w0rmr1d3r/gitleaks into update_pre-commit_example

 Conflicts:

	README.md

Dockerfile

@@ -4,7 +4,7 @@ ARG ldflags
 COPY . .
 RUN GO111MODULE=on CGO_ENABLED=0 go build -o bin/gitleaks -ldflags "-X="${ldflags} *.go 
 
-FROM alpine:3.14.1
+FROM alpine:3.14.2
 RUN adduser -D gitleaks && \
     apk add --no-cache bash git openssh-client
 COPY --from=build /go/src/github.com/zricethezav/gitleaks/bin/* /usr/bin/

Makefile

@@ -35,6 +35,7 @@ release-builds:
 	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips" $(LDFLAGS)
 	env GOOS="linux" GOARCH="mips" go build -o "build/gitleaks-linux-mips" $(LDFLAGS)
 	env GOOS="darwin" GOARCH="amd64" go build -o "build/gitleaks-darwin-amd64" $(LDFLAGS)
+	env GOOS="darwin" GOARCH="arm64" go build -o "build/gitleaks-darwin-arm64" $(LDFLAGS)
 
 deploy:
 	@echo "$(DOCKER_PASSWORD)" | docker login -u "$(DOCKER_USERNAME)" --password-stdin

README.md

@@ -62,6 +62,15 @@ Go 1.16+ required.
 GO111MODULE=on go get github.com/zricethezav/gitleaks/v7
 ```
 
+##### From bin:
+
+```bash
+GITLEAKS_VERSION=$(curl -s https://api.github.com/repos/zricethezav/gitleaks/releases/latest |  grep -oP '"tag_name": "\K(.*)(?=")') && wget https://github.com/zricethezav/gitleaks/releases/download/$GITLEAKS_VERSION/gitleaks-linux-amd64
+mv gitleaks-linux-amd64 gitleaks
+chmod +x gitleaks
+sudo mv gitleaks /usr/local/bin/
+```
+
 ##### As a pre-commit hook
 
 See [pre-commit](https://github.com/pre-commit/pre-commit) for instructions.
@@ -163,8 +172,8 @@ gitleaks --path=path/to/local/repo/main.go -v --no-git
 ```
 
 #### Scan unstaged changes:
-If you have unstaged changes are currently at the root of the repo, you can run `gitleaks` with no `--path` or `--repo-url` specified which will run a scan on your uncommitted changes. Or if you want to specify a
-path, you can run:
+If you have unstaged changes that are currently at the root of the repo, you can run `gitleaks` with no `--path` or `--repo-url` specified which will run a scan on your uncommitted changes.  
+Or if you want to specify a path, you can run:
 ```bash
 gitleaks --path=path/to/local/repo -v --unstaged
 ```

config/config_test.go

@@ -30,95 +30,95 @@ func TestParse(t *testing.T) {
 		{
 			description: "test successful load",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "aws_key.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "aws_key.toml")),
 			},
 		},
 		{
 			description: "test bad toml",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_aws_key.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_aws_key.toml")),
 			},
 			wantErr: fmt.Errorf("Near line 7 (last key parsed 'rules.description'): expected value but found \"AWS\" instead"),
 		},
 		{
 			description: "test bad regex",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_regex_aws_key.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_regex_aws_key.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: invalid nested repetition operator: `???`"),
 		},
 		{
 			description: "test bad global allowlist file regex",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_aws_key_global_allowlist_file.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_aws_key_global_allowlist_file.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
 		},
 		{
 			description: "test bad global file regex",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_aws_key_file_regex.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_aws_key_file_regex.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: error parsing regexp: missing argument to repetition operator: `??`"),
 		},
 		{
 			description: "test successful load big ol thing",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "large.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "large.toml")),
 			},
 		},
 		{
 			description: "test load entropy",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "entropy.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "entropy.toml")),
 			},
 		},
 		{
 			description: "test entropy bad range",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_1.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_1.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: entropy Min value cannot be higher than Max value"),
 		},
 		{
 			description: "test entropy value max",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_2.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_2.toml")),
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value min",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_3.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_3.toml")),
 			},
 			wantErr: fmt.Errorf("strconv.ParseFloat: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_4.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_4.toml")),
 			},
 			wantErr: fmt.Errorf("strconv.ParseInt: parsing \"x\": invalid syntax"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_5.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_5.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: group cannot be lower than 0"),
 		},
 		{
 			description: "test entropy value group",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_6.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_6.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: group cannot be higher than number of groups in regexp"),
 		},
 		{
 			description: "test entropy range limits",
 			opts: options.Options{
-				ConfigPath: filepath.Join(configPath, "bad_entropy_7.toml"),
+				ConfigPath: filepath.ToSlash(filepath.Join(configPath, "bad_entropy_7.toml")),
 			},
 			wantErr: fmt.Errorf("problem loading config: invalid entropy ranges, must be within 0.0-8.0"),
 		},
@@ -200,7 +200,7 @@ func findRuleByDescription(rules []config.Rule, description string) (*config.Rul
 }
 
 func writeTestConfig(toml string) (string, error) {
-	tmpfile, err := ioutil.TempFile("", "testConfig")
+	tmpfile, err := ioutil.TempFile(".", "testConfig")
 	if err != nil {
 		return "", fmt.Errorf("Couldn't create test config got: %w", err)
 	}

config/gitleaks.toml

@@ -75,7 +75,7 @@ title = "gitleaks config"
 
 [[rules]]
     description = "Google API key"
-    regex = '''AIza[0-9A-Za-z\\-_]{35}'''
+    regex = '''AIza[0-9A-Za-z\-_]{35}'''
     tags = ["key", "Google"]
 
 [[rules]]
@@ -130,7 +130,7 @@ title = "gitleaks config"
 
 [[rules]]
     description = "Square OAuth secret"
-    regex = '''sq0csp-[0-9A-Za-z\\-_]{43}'''
+    regex = '''sq0csp-[0-9A-Za-z\-_]{43}'''
     tags = ["key", "square"]
 
 [[rules]]

scan/report.go

@@ -4,6 +4,7 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -57,13 +58,15 @@ func WriteReport(report Report, opts options.Options, cfg config.Config) error {
 				return err
 			}
 		case "csv":
+			newLineRegex := regexp.MustCompile("[\r]*\n")
 			w := csv.NewWriter(file)
 			err = w.Write([]string{"repo", "line", "commit", "offender", "leakURL", "rule", "tags", "commitMsg", "author", "email", "file", "date"})
 			if err != nil {
 				return err
 			}
 			for _, leak := range report.Leaks {
-				err := w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.LeakURL, leak.Rule, leak.Tags, leak.Message, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
+				commitFirstLine := newLineRegex.ReplaceAllString(leak.Message, " ")
+				err := w.Write([]string{leak.Repo, leak.Line, leak.Commit, leak.Offender, leak.LeakURL, leak.Rule, leak.Tags, commitFirstLine, leak.Author, leak.Email, leak.File, leak.Date.Format(time.RFC3339)})
 				if err != nil {
 					return err
 				}

scan/scan_test.go

@@ -5,13 +5,71 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"testing"
 
+	"github.com/zricethezav/gitleaks/v7/config"
+	"github.com/zricethezav/gitleaks/v7/options"
 	"github.com/zricethezav/gitleaks/v7/scan"
 )
 
 const repoBasePath = "../testdata/repos/"
 const expectPath = "../testdata/expect/"
 
+func TestScan(t *testing.T) {
+	err := moveDotGit("dotGit", ".git")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer moveDotGit(".git", "dotGit")
+	tests := []struct {
+		description string
+		opts        options.Options
+		wantPath    string
+	}{
+		{
+			description: "test google api key leak AND square oauth leak",
+			opts: options.Options{
+				Path:         filepath.Join(repoBasePath, "with_square_and_google"),
+				Report:       filepath.Join(expectPath, "results_square_and_google.json.got"),
+				ReportFormat: "json",
+				NoGit:				true,
+			},
+			wantPath: filepath.Join(expectPath, "results_square_and_google.json"),
+		},
+	}
+
+	for _, test := range tests {
+		cfg, err := config.NewConfig(test.opts)
+		if err != nil {
+			t.Error(err)
+		}
+
+		scanner, err := scan.NewScanner(test.opts, cfg)
+		if err != nil {
+			t.Error(test.description, err)
+		}
+
+		scannerReport, err := scanner.Scan()
+		if err != nil {
+			t.Fatal(test.description, err)
+		}
+
+		err = scan.WriteReport(scannerReport, test.opts, cfg)
+		if err != nil {
+			t.Error(test.description, err)
+		}
+
+		if test.wantPath != "" {
+			err := fileCheck(test.wantPath, test.opts.Report)
+			if err != nil {
+				t.Error(test.description, err)
+			}
+		}
+	}
+}
+
+
 func moveDotGit(from, to string) error {
 	repoDirs, err := ioutil.ReadDir("../testdata/repos")
 	if err != nil {

scan/utils.go

@@ -212,6 +212,8 @@ func extractLine(patchContent string, leak Leak, lineLookup map[string]bool) int
 			if _, ok := lineLookup[fmt.Sprintf("%s%s%d%s", leak.Offender, leak.Line, lineNumber, leak.File)]; !ok {
 				lineLookup[fmt.Sprintf("%s%s%d%s", leak.Offender, leak.Line, lineNumber, leak.File)] = true
 				return lineNumber
+			} else if ok {
+				return lineNumber
 			}
 		}
 		currLine++

testdata/expect/results_square_and_google.json

@@ -0,0 +1,36 @@
+[
+  {
+   "line": "SQUARE_OAUTH=sq0csp-QZxqK3fHycIxFsjZ7_HoqRN08qY2by3wLXcfJCfsKEg",
+   "lineNumber": 1,
+   "offender": "sq0csp-QZxqK3fHycIxFsjZ7_HoqRN08qY2by3wLXcfJCfsKEg",
+   "offenderEntropy": -1,
+   "commit": "",
+   "repo": "",
+   "repoURL": "",
+   "leakURL": "",
+   "rule": "Square OAuth secret",
+   "commitMessage": "",
+   "author": "",
+   "email": "",
+   "file": ".",
+   "date": "0001-01-01T00:00:00Z",
+   "tags": "key, square"
+  },
+  {
+   "line": "GOOGLE_MAPS_KEY=AIzaSyDtz3Di0ZrzeU5ET3UFU0p-xP4bx2FkSrg",
+   "lineNumber": 3,
+   "offender": "AIzaSyDtz3Di0ZrzeU5ET3UFU0p-xP4bx2FkSrg",
+   "offenderEntropy": -1,
+   "commit": "",
+   "repo": "",
+   "repoURL": "",
+   "leakURL": "",
+   "rule": "Google API key",
+   "commitMessage": "",
+   "author": "",
+   "email": "",
+   "file": ".",
+   "date": "0001-01-01T00:00:00Z",
+   "tags": "key, Google"
+  }
+ ]

testdata/repos/with_square_and_google/env

@@ -0,0 +1,3 @@
+SQUARE_OAUTH=sq0csp-QZxqK3fHycIxFsjZ7_HoqRN08qY2by3wLXcfJCfsKEg
+
+GOOGLE_MAPS_KEY=AIzaSyDtz3Di0ZrzeU5ET3UFU0p-xP4bx2FkSrg