Fixing merge commit scans and large patch rule checking (#414)

* Fixing merge commit scans and large patch rule checking

* Adding tests for merge request scans

scan/rule.go

@@ -200,7 +200,15 @@ func extractAndInjectLineNumber(leak *manager.Leak, bundle *Bundle, repo *Repo)
 		if bundle.Patch == "" {
 			return
 		}
+
+		// This is needed as some patches generate strings that are larger than
+		// scanners max size (MaxScanTokenSize = 64 * 1024)
+		// https://github.com/zricethezav/gitleaks/issues/413
+		buf := make([]byte, len(bundle.Patch))
 		scanner := bufio.NewScanner(strings.NewReader(bundle.Patch))
+		scanner.Buffer(buf, len(bundle.Patch))
+		scanner.Split(bufio.ScanLines)
+
 		currFile := ""
 		currLine := 0
 		currStartDiffLine := 0

scan/scan.go

@@ -113,41 +113,46 @@ func (repo *Repo) Scan() error {
 		// increase Commit counter
 		cc++
 
-		err = c.Parents().ForEach(func(parent *object.Commit) error {
-			defer func() {
-				if err := recover(); err != nil {
-					// sometimes the Patch generation will fail due to a known bug in
-					// sergi's go-diff: https://github.com/sergi/go-diff/issues/89.
-					// Once a fix has been merged I will remove this recover.
-					return
-				}
-			}()
-			if repo.timeoutReached() {
-				return nil
-			}
-			if parent == nil {
-				// shouldn't reach this point but just in case
-				return nil
-			}
+		// inspect first parent only as all other parents will be eventually reached
+		// (they exist as the tip of other branches, etc)
+		// See https://github.com/zricethezav/gitleaks/issues/413 for details
+		parent, err := c.Parent(0)
+		if err != nil {
+			return err
+		}
 
-			start := time.Now()
-			patch, err := parent.Patch(c)
-			if err != nil {
-				return fmt.Errorf("could not generate Patch")
+		defer func() {
+			if err := recover(); err != nil {
+				// sometimes the Patch generation will fail due to a known bug in
+				// sergi's go-diff: https://github.com/sergi/go-diff/issues/89.
+				// Once a fix has been merged I will remove this recover.
+				return
 			}
-			repo.Manager.RecordTime(manager.PatchTime(howLong(start)))
-			wg.Add(1)
-			semaphore <- true
-			go func(c *object.Commit, patch *object.Patch) {
-				defer func() {
-					<-semaphore
-					wg.Done()
-				}()
-				scanPatch(patch, c, repo)
-			}(c, patch)
-
+		}()
+		if repo.timeoutReached() {
 			return nil
-		})
+		}
+		if parent == nil {
+			// shouldn't reach this point but just in case
+			return nil
+		}
+
+		start := time.Now()
+		patch, err := parent.Patch(c)
+		if err != nil {
+			return fmt.Errorf("could not generate Patch")
+		}
+		repo.Manager.RecordTime(manager.PatchTime(howLong(start)))
+		wg.Add(1)
+		semaphore <- true
+		go func(c *object.Commit, patch *object.Patch) {
+			defer func() {
+				<-semaphore
+				wg.Done()
+			}()
+			scanPatch(patch, c, repo)
+		}(c, patch)
+
 		if c.Hash.String() == repo.Manager.Opts.CommitTo {
 			return storer.ErrStop
 		}

scan/scan_test.go

@@ -376,6 +376,15 @@ func TestScan(t *testing.T) {
 			},
 			wantPath: "../test_data/test_local_repo_two_leaks_deletion.json",
 		},
+		{
+			description: "test local repo eight (merges)",
+			opts: options.Options{
+				RepoPath:     "../test_data/test_repos/test_repo_8",
+				Report:       "../test_data/test_local_repo_eight.json.got",
+				ReportFormat: "json",
+			},
+			wantPath: "../test_data/test_local_repo_eight.json",
+		},
 	}
 
 	for _, test := range tests {

test_data/test_local_owner_aws_leak.json

@@ -133,5 +133,50 @@
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS",
   "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:44:48-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 2,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "ce7e8177bbf8a172c06b6a1e370a374d5c19f660",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "dummy.txt w/ text",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:39:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "9267bc86ec1497471cbc6f3308f3527f7ef34b9d",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Update dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:41:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
  }
 ]

test_data/test_local_owner_aws_leak_allowlist_repo.json

@@ -133,5 +133,50 @@
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS",
   "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:44:48-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 2,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "ce7e8177bbf8a172c06b6a1e370a374d5c19f660",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "dummy.txt w/ text",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:39:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "9267bc86ec1497471cbc6f3308f3527f7ef34b9d",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Update dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:41:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
  }
 ]

test_data/test_local_owner_aws_leak_depth_2.json

@@ -73,5 +73,20 @@
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS",
   "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:44:48-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
  }
 ]

test_data/test_local_repo_eight.json

@@ -0,0 +1,47 @@
+[
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "748f11eaf2c38c3cf0ac6a22e44208777e79fa6f",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Merge pull request #1 from zricethezav/additional-secret-branch\n\nUpdate dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:44:48-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 2,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "ce7e8177bbf8a172c06b6a1e370a374d5c19f660",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "dummy.txt w/ text",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:39:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "lineNumber": 6,
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "9267bc86ec1497471cbc6f3308f3527f7ef34b9d",
+  "repo": "test_repo_8",
+  "rule": "AWS Manager ID",
+  "commitMessage": "Update dummy.txt",
+  "author": "Zachary Rice",
+  "email": "zricer@protonmail.com",
+  "file": "dummy.txt",
+  "date": "2020-07-25T14:41:11-04:00",
+  "tags": "key, AWS",
+  "operation": "addition"
+ }
+]

test_data/test_repos/test_repo_8/README.md

@@ -0,0 +1,2 @@
+# test_repo_8
+test repo for 413

test_data/test_repos/test_repo_8/dotGit/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

test_data/test_repos/test_repo_8/dotGit/config

@@ -0,0 +1,13 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true
+[remote "origin"]
+	url = https://github.com/zricethezav/test_repo_8.git
+	fetch = +refs/heads/*:refs/remotes/origin/*
+[branch "master"]
+	remote = origin
+	merge = refs/heads/master

test_data/test_repos/test_repo_8/dotGit/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

test_data/test_repos/test_repo_8/dotGit/info/exclude

@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~

test_data/test_repos/test_repo_8/dotGit/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 748f11eaf2c38c3cf0ac6a22e44208777e79fa6f Zach Rice <zrice@gitlab.com> 1595702825 -0400	clone: from https://github.com/zricethezav/test_repo_8.git

test_data/test_repos/test_repo_8/dotGit/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 748f11eaf2c38c3cf0ac6a22e44208777e79fa6f Zach Rice <zrice@gitlab.com> 1595702825 -0400	clone: from https://github.com/zricethezav/test_repo_8.git

test_data/test_repos/test_repo_8/dotGit/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 748f11eaf2c38c3cf0ac6a22e44208777e79fa6f Zach Rice <zrice@gitlab.com> 1595702825 -0400	clone: from https://github.com/zricethezav/test_repo_8.git

test_data/test_repos/test_repo_8/dotGit/objects/0b/aadf22eda42ba55c4fec3e14973edc6cec783a

@@ -0,0 +1,4 @@
+x
uRÉnã8œ³¾‚@�ÄZ¨
è„TdYqäEŠ×K@RÔb[–LQV¬¯O7èÃÌ»ÔC¡^
…z¬®ªRG×þ’‚s 2èºÐ´lJufÛ”SmÇrRƒfZšêŽ�ê*4u¥!‚_$ åÌ6 �šaÚ:*µÓâªî:)· KÌŒgü­· …†ûpÊtj¸nFM]s8‡�e®ªiZjd6#.QH'‹Z€aw—Œƒïƒx€xiD-ëKEÊó3««¿�fº¦­ê–j�'ªªò`©$ (å´£àû¥¼9ß_òRýŸ³¼ÉÛ2Oÿöƒp–Á$a0GëØÿÉ+@
}‹F{­ðê-›Nµ…ã™]áâfˆ°G(�†!òÄ~ð4áÒ‘úfLÔû—~9š>¶›Ó¹Û|xóþ€	Ê°Ãw
iï«à–ònN‰#Í>¾ÎǘàÅy»vŠt¾Šö´‰fazñ�.CŠ¢‰w5o{ý}t�,'Fñ ¡Äóߨ–Å_ìÓXÏB¯â÷­y<Ö¢e¥)¨lb3Q@\‡ñ¨1d†ü#´ÛÛr¶ÓG˜«�÷±WΤìó­iä§0y_ÌÔà.Û4êG:ZìÛ`2™( ›M§$zõÔè÷E8>ýk»,o²º-†
+LJ'zÇîv¸.ñán9m±ø¥En) ×‹ä}2ŽŽFP¿n/Á»7
%J®î~—0'¯¨»	ö;˜Ÿ®«‰9ЈR»ðf±ì“y'ò
+øAOh­üêÌŸ¿þwcJÄEÎAÓ�Ï@ðkÇ[	¾é u~þ”,ø@nã?ö§†HV<iŠ²nR"9貺ý¬Ê¼�Ÿ”
+NÎÏòKþ&¾

test_data/test_repos/test_repo_8/dotGit/objects/3c/cb6c2b338c176bbb7de1154cd7b61d0f6155ce

@@ -0,0 +1,4 @@
+x
5‘QJÄ@†}î)r€RQ¼€.º
+*‹�éLÚ§™Ì´êÓ^CÐËíÏæ¥Iþ|ÿŸ>Hçg›“�À�Ä‘p„‡	½¬ÚÂ=ÅŒ£ÄãáSá©Ûvw<Eΰçà_d&ØrôG…ûÒvM³•²˜‚'‡žZÈÁk’1áÜB”rq/ämŸ²§ßòÝ0°#�
v¸`€ë˜)):˜PÁ³º¢jcó»fJ,E�ÃÔ
+îªòÀVïÓP™ºÆ¾«Äö\âÂæ# æ$¯Ó»²cÍ°N«¤
+dk´ôNbNè²YÁè+ M�ýÕ2ÆƒUª«ÿ”þús	ÊÆ
§›ÍEŠ\%0·•Skh•Œ…Þp戙%VçUpÆj
ƒšgoÍ0`²ÇÁè²X¢V¶™ßÐ8YCI¶¨BÖŽ‰çÚe .”šüñðµ·uR‚¯—̦o‡³+ÚœRXH»ãáÛRZeý ¤Mãy«›Êîù±ù
�Ã6

test_data/test_repos/test_repo_8/dotGit/objects/74/8f11eaf2c38c3cf0ac6a22e44208777e79fa6f

@@ -0,0 +1,5 @@
+x
u▓I▐⌡@┘sФW╢■ё51п╛рL4мЮaЯ2╤1фЖ╜i╨
ш,n/ЭЗL&й-y≈▓>╫*╘Тi╙╙юRЮ7а)┼иТлd√i╚z Ал0m≤й▄≤B c╘
ц░Zлi-─°b°1U╔жтК:я%░* mB ┐PbZЪУjPзи,╛и╡╔°)*╤≤Fl┐Й√LTU√┴"А^
L
+л`]
+·Ч9Ьkкят.о?HSЩ┼nК╕╛√·До╚р'ЩЭJP╪RЬ}
+·К├сЖЭxмKQТИжР6Ой<Щ√3Я┌XzKчmБУД▀K@╥н!BнB+g2ъW▀╥╣35▀ёЖq┘<╦!■ЫA─°╡█
шe■мp┘╝с╫}╧█$`L°у)Qё≈╟╦дХ^р┘u8. А·Ts@Б8F├╚ьц⌡йг≈УUсЫ╤┴NLКnЕС│Шм=иX▐?|Вяi╔÷┬⌠DiбМn}Ю╫≈zrXo╨8!^ЁXmгm╘≈K e╣⌡╦╖B⌡╫╨Т╣ц║вs╕T≈+┼xnВСЯuВЭ≤Тс╙∙Е5W╫Р╗и╨≈─=╚╤║zЮGyё≈МЧ█ы▒ФN{\<NхЯТ▐5g▀²╤яv)┌L@}╦g
1GЭ╫╧≥0А²║hХн}
+∙YS╟▒√лрCэшtyы╩О┐IН┤.оG║K╡▐Зh╩тr^т≥©3:КE/5щJ2⌡,э'&м)о)hШСpzИi'юw0чTЮ╚S╒═╬▌q√∙╒lj|~Й(АT<╔в╓░╓╦м╟═ К╚ЙЯCэе/ВT

test_data/test_repos/test_repo_8/dotGit/packed-refs

@@ -0,0 +1,4 @@
+# pack-refs with: peeled fully-peeled sorted 
+ce6e0398cd8a40085cad12a8f4c96e580c2200c1 refs/remotes/origin/additional-secret-branch
+748f11eaf2c38c3cf0ac6a22e44208777e79fa6f refs/remotes/origin/master
+64b4391ddf2b399fb5218ee44cf90111d3f7ca9a refs/remotes/origin/zricethezav-patch-1

test_data/test_repos/test_repo_8/dotGit/refs/heads/master

@@ -0,0 +1 @@
+748f11eaf2c38c3cf0ac6a22e44208777e79fa6f

test_data/test_repos/test_repo_8/dotGit/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+ref: refs/remotes/origin/master

test_data/test_repos/test_repo_8/dummy.txt

@@ -0,0 +1,8 @@
+Adding a secret
+aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
+
+
+Adding a secret on another branch!!!
+aws_access_key_id='AKIAIO5FODNN7EXAMPLE',
+
+Some more content

test_data/test_repos/test_repo_8/ufos_might_be_real.txt

@@ -0,0 +1,19 @@
+No Longer in Shadows, Pentagon’s U.F.O. Unit Will Make Some Findings Public
+
+For over a decade, the program, now tucked inside the Office of Naval Intelligence, has discussed mysterious events in classified briefings.
+
+
+
+Eric W. Davis, an astrophysicist who worked as a subcontractor and then a consultant for the Pentagon U.F.O. program since 2007, said that, in some cases, examination of the materials had so far failed to determine their source and led him to conclude, “We couldn’t make it ourselves.”
+
+
+wowzers
+
+credit to NYT
+
+
+
+uhmmmmmmm
+
+Some more stuff
+