duplicate leak logic includes the line in hash (#345)

audit/util.go

@@ -98,7 +98,6 @@ func shannonEntropy(data string) (entropy float64) {
 	return entropy
 }
 
-
 // Checks if the given rule has a regex
 func ruleContainRegex(rule config.Rule) bool {
 	if rule.Regex == nil {
@@ -189,7 +188,7 @@ func InspectFile(content string, fullpath string, c *object.Commit, repo *Repo)
 		if isFileNameWhiteListed(filename, rule.Whitelist) || isFilePathWhiteListed(path, rule.Whitelist) {
 			continue
 		}
-    
+
 		// If it has fileNameRegex and it doesnt match we continue to next rule
 		if ruleContainFileNameRegex(rule) && !fileMatched(filename, rule.FileNameRegex) {
 			continue

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/sergi/go-diff v1.0.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/xanzy/go-gitlab v0.21.0
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/src-d/go-billy.v4 v4.3.2
 	gopkg.in/src-d/go-git.v4 v4.13.1

go.sum

@@ -76,6 +76,8 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

manager/manager.go

@@ -156,7 +156,7 @@ func (manager *Manager) SendLeaks(l Leak) {
 		l.Offender = l.Offender[0:maxLineLen-1] + "..."
 	}
 	h := sha1.New()
-	h.Write([]byte(l.Commit + l.Offender + l.File))
+	h.Write([]byte(l.Commit + l.Offender + l.File + l.Line))
 	l.lookupHash = hex.EncodeToString(h.Sum(nil))
 	manager.leakWG.Add(1)
 	manager.leakChan <- l

test_data/test_local_owner_aws_leak.json

@@ -116,6 +116,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",

test_data/test_local_repo_three_leaks.json

@@ -77,6 +77,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_3",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",

test_data/test_local_repo_two_leaks.json

@@ -103,6 +103,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",

test_data/test_local_repo_two_leaks_commit_from.json

@@ -51,6 +51,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",

test_data/test_local_repo_two_leaks_commit_range.json

@@ -50,5 +50,18 @@
   "file": "secrets.md",
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, API, generic"
+ },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "AKIALALEMEL33243OLIA",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "AWS Manager ID",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, AWS"
  }
 ]