fix deduplication issue caused by clobbered findings (#742)

* fix deduplication issue caused by clobbered findings

* fix index

* remove indexing, slow is better than wrong

cmd/detect.go

@@ -31,7 +31,7 @@ func runDetect(cmd *cobra.Command, args []string) {
 	initConfig()
 	var (
 		vc       config.ViperConfig
-		findings []*report.Finding
+		findings []report.Finding
 		err      error
 	)
 

config/gitleaks.toml

@@ -273,14 +273,19 @@ description = "Dropbox API secret/key"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
 
 [[rules]]
-id = "dropbox-sl-api-token"
+id = "dropbox--api-key"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox-short-lived-api-token"
 description = "Dropbox short lived API token"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
 
 [[rules]]
-id = "dropbox-ll-api-token"
+id = "dropbox-long-lived-api-token"
 description = "Dropbox long lived API token"
-regex = '''(?i)(dropbox)(.{0,20})['\"](?i)[a-z0-9]{11}(AAAAAAAAAA)[a-z0-9-_=]{43}['\"]'''
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
 
 [[rules]]
 id = "duffel-api-token"

detect/detect.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog/log"
+
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/report"
 )

detect/files.go

@@ -15,9 +15,9 @@ import (
 
 // FromFiles opens the directory or file specified in source and checks each file against the rules
 // from the configuration. If any secrets are found, they are added to the list of findings.
-func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*report.Finding, error) {
+func FromFiles(source string, cfg config.Config, outputOptions Options) ([]report.Finding, error) {
 	var (
-		findings []*report.Finding
+		findings []report.Finding
 		mu       sync.Mutex
 	)
 	g, _ := errgroup.WithContext(context.Background())
@@ -51,7 +51,6 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*repo
 			}
 			fis := DetectFindings(cfg, b, p, "")
 			for _, fi := range fis {
-				fi.File = p
 				if outputOptions.Redact {
 					fi.Redact()
 				}
@@ -59,7 +58,7 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*repo
 					printFinding(fi)
 				}
 				mu.Lock()
-				findings = append(findings, &fi)
+				findings = append(findings, fi)
 				mu.Unlock()
 			}
 			return nil

detect/files_test.go

@@ -17,12 +17,12 @@ func TestFromFiles(t *testing.T) {
 		cfgName          string
 		opts             Options
 		source           string
-		expectedFindings []*report.Finding
+		expectedFindings []report.Finding
 	}{
 		{
 			source:  filepath.Join(repoBasePath, "nogit"),
 			cfgName: "simple",
-			expectedFindings: []*report.Finding{
+			expectedFindings: []report.Finding{
 				{
 					Description: "AWS Access Key",
 					StartLine:   19,
@@ -40,7 +40,7 @@ func TestFromFiles(t *testing.T) {
 		{
 			source:  filepath.Join(repoBasePath, "nogit", "main.go"),
 			cfgName: "simple",
-			expectedFindings: []*report.Finding{
+			expectedFindings: []report.Finding{
 				{
 					Description: "AWS Access Key",
 					StartLine:   19,

detect/git.go

@@ -15,8 +15,8 @@ import (
 // FromGit accepts a gitdiff.File channel (structure output from `git log -p`) and a configuration
 // struct. Files from the gitdiff.File channel are then checked against each rule in the configuration to
 // check for secrets. If any secrets are found, they are added to the list of findings.
-func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Options) []*report.Finding {
-	var findings []*report.Finding
+func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Options) []report.Finding {
+	var findings []report.Finding
 	mu := sync.Mutex{}
 	wg := sync.WaitGroup{}
 	commitMap := make(map[string]bool)
@@ -82,7 +82,7 @@ func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Option
 						printFinding(fi)
 					}
 					mu.Lock()
-					findings = append(findings, &fi)
+					findings = append(findings, fi)
 					mu.Unlock()
 
 				}

detect/git_test.go

@@ -26,13 +26,13 @@ func TestFromGit(t *testing.T) {
 		source           string
 		logOpts          string
 		expected         string
-		expectedFindings []*report.Finding
+		expectedFindings []report.Finding
 	}{
 		{
 			source:   filepath.Join(repoBasePath, "small"),
 			expected: filepath.Join(expectPath, "git", "small.txt"),
 			cfgName:  "simple",
-			expectedFindings: []*report.Finding{
+			expectedFindings: []report.Finding{
 				{
 					Description: "AWS Access Key",
 					StartLine:   20,
@@ -40,15 +40,15 @@ func TestFromGit(t *testing.T) {
 					StartColumn: 19,
 					EndColumn:   38,
 					Secret:      "AKIALALEMEL33243OLIA",
+					Match:       "AKIALALEMEL33243OLIA",
 					File:        "main.go",
-					// Line:        "\tawsToken := \"AKIALALEMEL33243OLIA\"",
-					Date:    "2021-11-02T23:37:53Z",
-					Commit:  "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587",
-					Author:  "Zachary Rice",
-					Email:   "zricer@protonmail.com",
-					Message: "Accidentally add a secret",
-					RuleID:  "aws-access-key",
-					Tags:    []string{"key", "AWS"},
+					Date:        "2021-11-02T23:37:53Z",
+					Commit:      "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587",
+					Author:      "Zachary Rice",
+					Email:       "zricer@protonmail.com",
+					Message:     "Accidentally add a secret",
+					RuleID:      "aws-access-key",
+					Tags:        []string{"key", "AWS"},
 				},
 				{
 					Description: "AWS Access Key",
@@ -57,15 +57,15 @@ func TestFromGit(t *testing.T) {
 					StartColumn: 17,
 					EndColumn:   36,
 					Secret:      "AKIALALEMEL33243OLIA",
+					Match:       "AKIALALEMEL33243OLIA",
 					File:        "foo/foo.go",
-					// Line:        "\taws_token := \"AKIALALEMEL33243OLIA\"",
-					Date:    "2021-11-02T23:48:06Z",
-					Commit:  "491504d5a31946ce75e22554cc34203d8e5ff3ca",
-					Author:  "Zach Rice",
-					Email:   "zricer@protonmail.com",
-					Message: "adding foo package with secret",
-					RuleID:  "aws-access-key",
-					Tags:    []string{"key", "AWS"},
+					Date:        "2021-11-02T23:48:06Z",
+					Commit:      "491504d5a31946ce75e22554cc34203d8e5ff3ca",
+					Author:      "Zach Rice",
+					Email:       "zricer@protonmail.com",
+					Message:     "adding foo package with secret",
+					RuleID:      "aws-access-key",
+					Tags:        []string{"key", "AWS"},
 				},
 			},
 		},
@@ -74,7 +74,7 @@ func TestFromGit(t *testing.T) {
 			expected: filepath.Join(expectPath, "git", "small-branch-foo.txt"),
 			logOpts:  "--all foo...",
 			cfgName:  "simple",
-			expectedFindings: []*report.Finding{
+			expectedFindings: []report.Finding{
 				{
 					Description: "AWS Access Key",
 					StartLine:   9,
@@ -82,15 +82,15 @@ func TestFromGit(t *testing.T) {
 					StartColumn: 17,
 					EndColumn:   36,
 					Secret:      "AKIALALEMEL33243OLIA",
-					// Line:        "\taws_token := \"AKIALALEMEL33243OLIA\"",
-					Date:    "2021-11-02T23:48:06Z",
-					File:    "foo/foo.go",
-					Commit:  "491504d5a31946ce75e22554cc34203d8e5ff3ca",
-					Author:  "Zach Rice",
-					Email:   "zricer@protonmail.com",
-					Message: "adding foo package with secret",
-					RuleID:  "aws-access-key",
-					Tags:    []string{"key", "AWS"},
+					Match:       "AKIALALEMEL33243OLIA",
+					Date:        "2021-11-02T23:48:06Z",
+					File:        "foo/foo.go",
+					Commit:      "491504d5a31946ce75e22554cc34203d8e5ff3ca",
+					Author:      "Zach Rice",
+					Email:       "zricer@protonmail.com",
+					Message:     "adding foo package with secret",
+					RuleID:      "aws-access-key",
+					Tags:        []string{"key", "AWS"},
 				},
 			},
 		},

report/csv.go

@@ -7,7 +7,7 @@ import (
 )
 
 // writeCsv writes the list of findings to a writeCloser.
-func writeCsv(f []*Finding, w io.WriteCloser) error {
+func writeCsv(f []Finding, w io.WriteCloser) error {
 	if len(f) == 0 {
 		return nil
 	}

report/csv_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestWriteCSV(t *testing.T) {
 	tests := []struct {
-		findings       []*Finding
+		findings       []Finding
 		testReportName string
 		expected       string
 		wantEmpty      bool
@@ -17,7 +17,7 @@ func TestWriteCSV(t *testing.T) {
 		{
 			testReportName: "simple",
 			expected:       filepath.Join(expectPath, "report", "csv_simple.csv"),
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID:      "test-rule",
 					Match:       "line containing secret",
@@ -39,7 +39,7 @@ func TestWriteCSV(t *testing.T) {
 			wantEmpty:      true,
 			testReportName: "empty",
 			expected:       filepath.Join(expectPath, "report", "this_should_not_exist.csv"),
-			findings:       []*Finding{}},
+			findings:       []Finding{}},
 	}
 
 	for _, test := range tests {

report/finding.go

@@ -1,6 +1,9 @@
 package report
 
-import "strings"
+import (
+	"strconv"
+	"strings"
+)
 
 // Finding contains information about strings that
 // have been captured by a tree-sitter query.
@@ -40,3 +43,10 @@ func (f *Finding) Redact() {
 	f.Match = strings.Replace(f.Match, f.Secret, "REDACTED", -1)
 	f.Secret = "REDACT"
 }
+
+func (f *Finding) Hash() string {
+	return f.Secret + f.Commit +
+		strconv.Itoa(f.EndLine) +
+		strconv.Itoa(f.StartLine)
+
+}

report/json.go

@@ -5,7 +5,7 @@ import (
 	"io"
 )
 
-func writeJson(findings []*Finding, w io.WriteCloser) error {
+func writeJson(findings []Finding, w io.WriteCloser) error {
 	if len(findings) == 0 {
 		return nil
 	}

report/json_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestWriteJSON(t *testing.T) {
 	tests := []struct {
-		findings       []*Finding
+		findings       []Finding
 		testReportName string
 		expected       string
 		wantEmpty      bool
@@ -17,7 +17,7 @@ func TestWriteJSON(t *testing.T) {
 		{
 			testReportName: "simple",
 			expected:       filepath.Join(expectPath, "report", "json_simple.json"),
-			findings: []*Finding{
+			findings: []Finding{
 				{
 
 					Description: "",
@@ -42,7 +42,7 @@ func TestWriteJSON(t *testing.T) {
 			wantEmpty:      true,
 			testReportName: "empty",
 			expected:       filepath.Join(expectPath, "report", "this_should_not_exist.json"),
-			findings:       []*Finding{}},
+			findings:       []Finding{}},
 	}
 
 	for _, test := range tests {

report/report.go

@@ -13,7 +13,7 @@ const (
 	CWE_DESCRIPTION = "Use of Hard-coded Credentials"
 )
 
-func Write(findings []*Finding, cfg config.Config, ext string, reportPath string) error {
+func Write(findings []Finding, cfg config.Config, ext string, reportPath string) error {
 	if len(findings) == 0 {
 		return nil
 	}

report/report_test.go

@@ -16,13 +16,13 @@ const (
 
 func TestReport(t *testing.T) {
 	tests := []struct {
-		findings  []*Finding
+		findings  []Finding
 		ext       string
 		wantEmpty bool
 	}{
 		{
 			ext: "json",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},
@@ -30,7 +30,7 @@ func TestReport(t *testing.T) {
 		},
 		{
 			ext: ".json",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},
@@ -38,7 +38,7 @@ func TestReport(t *testing.T) {
 		},
 		{
 			ext: ".jsonj",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},
@@ -47,7 +47,7 @@ func TestReport(t *testing.T) {
 		},
 		{
 			ext: ".csv",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},
@@ -55,7 +55,7 @@ func TestReport(t *testing.T) {
 		},
 		{
 			ext: "csv",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},
@@ -63,7 +63,7 @@ func TestReport(t *testing.T) {
 		},
 		{
 			ext: "CSV",
-			findings: []*Finding{
+			findings: []Finding{
 				{
 					RuleID: "test-rule",
 				},

report/sarif.go

@@ -8,7 +8,7 @@ import (
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 
-func writeSarif(cfg config.Config, findings []*Finding, w io.WriteCloser) error {
+func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error {
 	sarif := Sarif{
 		Schema:  "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json",
 		Version: "2.1.0",
@@ -20,7 +20,7 @@ func writeSarif(cfg config.Config, findings []*Finding, w io.WriteCloser) error
 	return encoder.Encode(sarif)
 }
 
-func getRuns(cfg config.Config, findings []*Finding) []Runs {
+func getRuns(cfg config.Config, findings []Finding) []Runs {
 	return []Runs{
 		{
 			Tool:    getTool(cfg),
@@ -54,7 +54,7 @@ func getRules(cfg config.Config) []Rules {
 	return rules
 }
 
-func messageText(f *Finding) string {
+func messageText(f Finding) string {
 	if f.Commit == "" {
 		return fmt.Sprintf("%s has detected secret for file %s.", f.RuleID, f.File)
 	}
@@ -63,7 +63,7 @@ func messageText(f *Finding) string {
 
 }
 
-func getResults(findings []*Finding) []Results {
+func getResults(findings []Finding) []Results {
 	var results []Results
 	for _, f := range findings {
 		r := Results{
@@ -87,7 +87,7 @@ func getResults(findings []*Finding) []Results {
 	return results
 }
 
-func getLocation(f *Finding) []Locations {
+func getLocation(f Finding) []Locations {
 	return []Locations{
 		{
 			PhysicalLocation: PhysicalLocation{

report/sarif_test.go

@@ -14,7 +14,7 @@ const configPath = "../testdata/config/"
 
 func TestWriteSarif(t *testing.T) {
 	tests := []struct {
-		findings       []*Finding
+		findings       []Finding
 		testReportName string
 		expected       string
 		wantEmpty      bool
@@ -24,7 +24,7 @@ func TestWriteSarif(t *testing.T) {
 			cfgName:        "simple",
 			testReportName: "simple",
 			expected:       filepath.Join(expectPath, "report", "sarif_simple.sarif"),
-			findings: []*Finding{
+			findings: []Finding{
 				{
 
 					Description: "",