Improve default pattern configuration (#356)

* Add GCP, fix Mailgun format, and loosen format for other keys (don't require quotes)

* Add more match cases to generic credential pattern

* Fix quotation for valid toml

* Update tests

config/default.go

@@ -13,7 +13,7 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "AWS Secret Key"
-	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
+	regex = '''(?i)aws(.{0,20})?(?-i)[0-9a-zA-Z\/+]{40}'''
 	tags = ["key", "AWS"]
 
 [[rules]]
@@ -23,37 +23,37 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "Facebook Secret Key"
-	regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]'''
+	regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)[0-9a-f]{32}'''
 	tags = ["key", "Facebook"]
 
 [[rules]]
 	description = "Facebook Client ID"
-	regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]'''
+	regex = '''(?i)(facebook|fb)(.{0,20})?[0-9]{13,17}'''
 	tags = ["key", "Facebook"]
 
 [[rules]]
 	description = "Twitter Secret Key"
-	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]'''
+	regex = '''(?i)twitter(.{0,20})?[0-9a-z]{35,44}'''
 	tags = ["key", "Twitter"]
 
 [[rules]]
 	description = "Twitter Client ID"
-	regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]'''
+	regex = '''(?i)twitter(.{0,20})?[0-9a-z]{18,25}'''
 	tags = ["client", "Twitter"]
 
 [[rules]]
 	description = "Github"
-	regex = '''(?i)github(.{0,20})?(?-i)['\"][0-9a-zA-Z]{35,40}['\"]'''
+	regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}'''
 	tags = ["key", "Github"]
 
 [[rules]]
 	description = "LinkedIn Client ID"
-	regex = '''(?i)linkedin(.{0,20})?(?-i)['\"][0-9a-z]{12}['\"]'''
+	regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}'''
 	tags = ["client", "LinkedIn"]
 
 [[rules]]
 	description = "LinkedIn Secret Key"
-	regex = '''(?i)linkedin(.{0,20})?['\"][0-9a-z]{16}['\"]'''
+	regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}'''
 	tags = ["secret", "LinkedIn"]
 
 [[rules]]
@@ -68,7 +68,7 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "Generic Credential"
-	regex = '''(?i)(api_key|apikey|secret)(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]'''
+	regex = '''(?i)(api_key|apikey|secret|password|pass|pw|key)(.{0,20})?[0-9a-zA-Z]{16,45}'''
 	tags = ["key", "API", "generic"]
 
 [[rules]]
@@ -76,20 +76,24 @@ title = "gitleaks config"
 	regex = '''AIza[0-9A-Za-z\\-_]{35}'''
 	tags = ["key", "Google"]
 
+[[rules]]
+	description = "Google (GCP) Service Account"
+	regex = '''"type": "service_account"'''
+	tags = ["key", "Google"]
 
 [[rules]]
 	description = "Heroku API key"
-	regex = '''(?i)heroku(.{0,20})?['"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]'''
+	regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'''
 	tags = ["key", "Heroku"]
 
 [[rules]]
 	description = "MailChimp API key"
-	regex = '''(?i)(mailchimp|mc)(.{0,20})?['"][0-9a-f]{32}-us[0-9]{1,2}['"]'''
+	regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}'''
 	tags = ["key", "Mailchimp"]
 
 [[rules]]
 	description = "Mailgun API key"
-	regex = '''(?i)(mailgun|mg)(.{0,20})?['"][0-9a-z]{32}['"]'''
+	regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}'''
 	tags = ["key", "Mailgun"]
 
 [[rules]]
@@ -114,7 +118,7 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "Stripe API key"
-	regex = '''(?i)stripe(.{0,20})?['\"][sk|rk]_live_[0-9a-zA-Z]{24}'''
+	regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}'''
 	tags = ["key", "Stripe"]
 
 [[rules]]
@@ -129,7 +133,7 @@ title = "gitleaks config"
 
 [[rules]]
 	description = "Twilio API key"
-	regex = '''(?i)twilio(.{0,20})?['\"][0-9a-f]{32}['\"]'''
+	regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}'''
 	tags = ["key", "twilio"]
 
 [whitelist]

test_data/test_local_owner_aws_leak.json

@@ -12,6 +12,32 @@
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
+  "offender": "secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
   "offender": "AKIALALEMEL33243OLIB",
@@ -25,9 +51,22 @@
   "date": "2019-10-25T13:12:32-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -51,9 +90,22 @@
   "date": "2019-10-25T13:12:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -79,7 +131,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "996865bb912f3bc45898a370a13aadb315014b55",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -105,7 +157,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -129,6 +181,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",
@@ -142,6 +207,19 @@
   "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
   "offender": "AKIALALEMEL33243OLIA",
@@ -155,9 +233,22 @@
   "date": "2019-10-25T13:54:26-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIAE",
+  "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
+  "repo": "test_repo_3",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:26-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -181,9 +272,22 @@
   "date": "2019-10-25T13:54:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIAE",
+  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
+  "repo": "test_repo_3",
+  "rule": "Generic Credential",
+  "commitMessage": "more secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -209,7 +313,7 @@
  },
  {
   "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "64cfcee9aad1c84581631636bfc54f2050718d1a",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -235,7 +339,7 @@
  },
  {
   "line": "AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "deea550dd6c7acaf0e59432600593533984a2125",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -248,7 +352,7 @@
  },
  {
   "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
-  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231",
   "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
   "repo": "test_repo_5",
   "rule": "Generic Credential",
@@ -272,6 +376,19 @@
   "date": "2020-02-01T10:08:04-05:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "\naws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPL2",
+  "commit": "ca71fcdeda15f25f0cc661d90e8785c255925c27",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "introduce more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:08:04-05:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
   "offender": "AKIAIO5FODNN7EXAMPLE",
@@ -285,6 +402,32 @@
   "date": "2020-02-01T10:07:34-05:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "1f2a4abc47dabf991e6af6f9770867ce0ac1f360",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "introduce secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:07:34-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "db_password: verySecretProductionPassword",
+  "offender": "password: verySecretProductionPassword",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic Credential",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "application.yaml",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
   "offender": "AKIAIO5FODNN7EXAMPLE",
@@ -297,5 +440,18 @@
   "file": "config/application.properties",
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "offender": "key_id=AKIAIO5FODNN7EXAMPLE",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic Credential",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_owner_aws_leak_depth_2.json

@@ -12,6 +12,32 @@
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
+  "offender": "secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
   "offender": "AKIALALEMEL33243OLIB",
@@ -25,9 +51,22 @@
   "date": "2019-10-25T13:12:32-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -51,9 +90,22 @@
   "date": "2019-10-25T13:12:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -77,9 +129,22 @@
   "date": "2019-10-25T13:54:26-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIAE",
+  "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
+  "repo": "test_repo_3",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:26-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "cd5eb8bef855f73c46b97b4c088badffdc40ebe9",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -103,9 +168,22 @@
   "date": "2019-10-25T13:54:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "const AWSKEY = \"AKIALALEMEL33243OLIAE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIAE",
+  "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
+  "repo": "test_repo_3",
+  "rule": "Generic Credential",
+  "commitMessage": "more secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:54:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "84ac4e80d4dbf2c968b64e9d4005f5079795bb81",
   "repo": "test_repo_3",
   "rule": "Generic Credential",
@@ -118,7 +196,7 @@
  },
  {
   "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
-  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231",
   "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
   "repo": "test_repo_5",
   "rule": "Generic Credential",
@@ -142,6 +220,32 @@
   "date": "2020-02-01T10:08:04-05:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "\naws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPL2",
+  "commit": "ca71fcdeda15f25f0cc661d90e8785c255925c27",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "introduce more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:08:04-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "db_password: verySecretProductionPassword",
+  "offender": "password: verySecretProductionPassword",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic Credential",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "application.yaml",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
   "offender": "AKIAIO5FODNN7EXAMPLE",
@@ -154,5 +258,18 @@
   "file": "config/application.properties",
   "date": "2020-02-24T14:13:15-05:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id=AKIAIO5FODNN7EXAMPLE",
+  "offender": "key_id=AKIAIO5FODNN7EXAMPLE",
+  "commit": "98b6c7cb3fb29a5993c4c95c56a2dc53050b9247",
+  "repo": "test_repo_6",
+  "rule": "Generic Credential",
+  "commitMessage": "Adding some secrets in config folder\n\n",
+  "author": "Noel Algora",
+  "email": "noealgigu@gmail.com",
+  "file": "config/application.properties",
+  "date": "2020-02-24T14:13:15-05:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_five_commit.json

@@ -1,7 +1,7 @@
 [
  {
   "line": "\nmore_secrets = '99432bfewaf823ec3294e231'",
-  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231",
   "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
   "repo": "test_repo_5",
   "rule": "Generic Credential",

test_data/test_local_repo_five_files_at_commit.json

@@ -25,9 +25,35 @@
   "date": "2020-02-01T10:30:22-05:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPL2",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "more_secrets = '99432bfewaf823ec3294e231'",
-  "offender": "secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231",
   "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
   "repo": "test_repo_5",
   "rule": "Generic Credential",

test_data/test_local_repo_five_files_at_latest_commit.json

@@ -0,0 +1,67 @@
+[
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "AKIAIO5FODNN7EXAMPLE",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "AKIAIO5FODNN7EXAMPL2",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "AWS Manager ID",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, AWS"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPLE'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "aws_access_key_id='AKIAIO5FODNN7EXAMPL2'",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPL2",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "more_secrets = '99432bfewaf823ec3294e231'",
+  "offender": "secrets = '99432bfewaf823ec3294e231",
+  "commit": "a4c9fb737d5552fd96fce5cc7eedb23353ba9ed0",
+  "repo": "test_repo_5",
+  "rule": "Generic Credential",
+  "commitMessage": "even more secrets\n",
+  "author": "Zach Rice",
+  "email": "zrice@gitlab.com",
+  "file": "secrets.py",
+  "date": "2020-02-01T10:30:22-05:00",
+  "tags": "key, API, generic"
+ }
+]

test_data/test_local_repo_one_aws_leak.json

@@ -11,5 +11,31 @@
   "file": "server.test.py",
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
+  "offender": "secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_one_aws_leak_commit.json

@@ -11,5 +11,31 @@
   "file": "server.test.py",
   "date": "2019-10-24T09:29:27-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "    aws_access_key_id='AKIAIO5FODNN7EXAMPLE',",
+  "offender": "key_id='AKIAIO5FODNN7EXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
+ },
+ {
+  "line": "    aws_secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE'",
+  "offender": "secret_access_key='ABCDEF+c2L7yXeGvUyrPgYsDnWRRC1AYEXAMPLE",
+  "commit": "6557c92612d3b35979bd426d429255b3bf9fab74",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "commit 1 with secrets\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "server.test.py",
+  "date": "2019-10-24T09:29:27-04:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_one_aws_leak_uncommitted.json

@@ -11,5 +11,18 @@
   "file": "server.test.py",
   "date": "1970-01-01T00:00:00Z",
   "tags": "key, AWS"
+ },
+ {
+  "line": " aws_access_key_id='AKIAIO5FODNN7DXAMPLE'",
+  "offender": "key_id='AKIAIO5FODNN7DXAMPLE",
+  "commit": "0000000000000000000000000000000000000000",
+  "repo": "test_repo_1",
+  "rule": "Generic Credential",
+  "commitMessage": "***STAGED CHANGES***",
+  "author": "",
+  "email": "",
+  "file": "server.test.py",
+  "date": "1970-01-01T00:00:00Z",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_two_leaks.json

@@ -12,9 +12,22 @@
   "date": "2019-10-25T13:12:32-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -38,9 +51,22 @@
   "date": "2019-10-25T13:12:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -66,7 +92,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "996865bb912f3bc45898a370a13aadb315014b55",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -92,7 +118,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -116,6 +142,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",
@@ -128,5 +167,18 @@
   "file": "secrets.md",
   "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_two_leaks_commit_from.json

@@ -14,7 +14,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "996865bb912f3bc45898a370a13aadb315014b55",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -40,7 +40,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -64,6 +64,19 @@
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
   "offender": "AKIALALEMEL33243OLIA",
@@ -76,5 +89,18 @@
   "file": "secrets.md",
   "date": "2019-10-25T12:58:39-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "\nHere's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "b10b3e2cb320a8c211fda94c4567299d37de7776",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding aws key\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T12:58:39-04:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_two_leaks_commit_range.json

@@ -14,7 +14,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "996865bb912f3bc45898a370a13aadb315014b55",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -40,7 +40,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -63,5 +63,18 @@
   "file": "secrets.md",
   "date": "2019-10-25T13:01:27-04:00",
   "tags": "key, AWS"
+ },
+ {
+  "line": "Here's an AWS secret: AKIALALEMEL33243OLIAE",
+  "offender": "secret: AKIALALEMEL33243OLIAE",
+  "commit": "17471a5fda722a9e423f1a0d3f0d267ea009d41c",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "wait this is actually adding an aws secret\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:01:27-04:00",
+  "tags": "key, API, generic"
  }
 ]

test_data/test_local_repo_two_leaks_commit_to.json

@@ -12,9 +12,22 @@
   "date": "2019-10-25T13:12:32-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "rm secrets again\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:32-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "f61cd8587b7ac1d75a89a0c9af870a2f24c60263",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -38,9 +51,22 @@
   "date": "2019-10-25T13:12:08-04:00",
   "tags": "key, AWS"
  },
+ {
+  "line": "    const AWSKEY = \"AKIALALEMEL33243OLIBE\"",
+  "offender": "KEY = \"AKIALALEMEL33243OLIBE",
+  "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
+  "repo": "test_repo_2",
+  "rule": "Generic Credential",
+  "commitMessage": "adding another one\n",
+  "author": "zach rice",
+  "email": "zricer@protonmail.com",
+  "file": "secrets.md",
+  "date": "2019-10-25T13:12:08-04:00",
+  "tags": "key, API, generic"
+ },
  {
   "line": "    const AWSSECRET = \"99432bfewaf823ec3294e231\"",
-  "offender": "SECRET = \"99432bfewaf823ec3294e231\"",
+  "offender": "SECRET = \"99432bfewaf823ec3294e231",
   "commit": "b2eb34a61c988afd9b4aaa9dd58c8dd7d5f14dba",
   "repo": "test_repo_2",
   "rule": "Generic Credential",
@@ -66,7 +92,7 @@
  },
  {
   "line": "Here's an AWS secret: \"AKIALALEMEL33243OLIAE\"",
-  "offender": "secret: \"AKIALALEMEL33243OLIAE\"",
+  "offender": "secret: \"AKIALALEMEL33243OLIAE",
   "commit": "996865bb912f3bc45898a370a13aadb315014b55",
   "repo": "test_repo_2",
   "rule": "Generic Credential",