fix(rules): add entropy (#1580)

cmd/generate/config/rules/discord.go

@@ -9,8 +9,8 @@ import (
 func DiscordAPIToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord.",
 		RuleID:      "discord-api-token",
+		Description: "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord.",
 		Regex:       utils.GenerateSemiGenericRegex([]string{"discord"}, utils.Hex("64"), true),
 		Keywords:    []string{"discord"},
 	}
@@ -25,9 +25,10 @@ func DiscordAPIToken() *config.Rule {
 func DiscordClientID() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications.",
 		RuleID:      "discord-client-id",
+		Description: "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications.",
 		Regex:       utils.GenerateSemiGenericRegex([]string{"discord"}, utils.Numeric("18"), true),
+		Entropy:     2,
 		Keywords:    []string{"discord"},
 	}
 
@@ -35,15 +36,20 @@ func DiscordClientID() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("discord", secrets.NewSecret(utils.Numeric("18"))),
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		// Low entropy
+		`discord=000000000000000000`,
+	}
+	return utils.Validate(r, tps, fps)
 }
 
 func DiscordClientSecret() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks.",
 		RuleID:      "discord-client-secret",
+		Description: "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks.",
 		Regex:       utils.GenerateSemiGenericRegex([]string{"discord"}, utils.AlphaNumericExtended("32"), true),
+		Entropy:     2,
 		Keywords:    []string{"discord"},
 	}
 
@@ -51,5 +57,11 @@ func DiscordClientSecret() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("discord", secrets.NewSecret(utils.Numeric("32"))),
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		// Low entropy
+		`discord=00000000000000000000000000000000`,
+		// TODO:
+		//`discord=01234567890123456789012345678901`,
+	}
+	return utils.Validate(r, tps, fps)
 }

cmd/generate/config/rules/gitlab.go

@@ -11,9 +11,10 @@ import (
 func GitlabPat() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure.",
 		RuleID:      "gitlab-pat",
-		Regex:       regexp.MustCompile(`glpat-[0-9a-zA-Z\-\_]{20}`),
+		Description: "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure.",
+		Regex:       regexp.MustCompile(`glpat-[\w-]{20}`),
+		Entropy:     3,
 		Keywords:    []string{"glpat-"},
 	}
 
@@ -21,15 +22,19 @@ func GitlabPat() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("gitlab", "glpat-"+secrets.NewSecret(utils.AlphaNumeric("20"))),
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		"glpat-XXXXXXXXXXX-XXXXXXXX",
+	}
+	return utils.Validate(r, tps, fps)
 }
 
 func GitlabPipelineTriggerToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security.",
 		RuleID:      "gitlab-ptt",
+		Description: "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security.",
 		Regex:       regexp.MustCompile(`glptt-[0-9a-f]{40}`),
+		Entropy:     3,
 		Keywords:    []string{"glptt-"},
 	}
 
@@ -37,15 +42,19 @@ func GitlabPipelineTriggerToken() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("gitlab", "glptt-"+secrets.NewSecret(utils.Hex("40"))),
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		"glptt-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+	}
+	return utils.Validate(r, tps, fps)
 }
 
 func GitlabRunnerRegistrationToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access.",
 		RuleID:      "gitlab-rrt",
-		Regex:       regexp.MustCompile(`GR1348941[0-9a-zA-Z\-\_]{20}`),
+		Description: "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access.",
+		Regex:       regexp.MustCompile(`GR1348941[\w-]{20}`),
+		Entropy:     3,
 		Keywords:    []string{"GR1348941"},
 	}
 
@@ -53,5 +62,9 @@ func GitlabRunnerRegistrationToken() *config.Rule {
 	tps := []string{
 		utils.GenerateSampleSecret("gitlab", "GR1348941"+secrets.NewSecret(utils.AlphaNumeric("20"))),
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		"GR134894112312312312312312312",
+		"GR1348941XXXXXXXXXXXXXXXXXXXX",
+	}
+	return utils.Validate(r, tps, fps)
 }

cmd/generate/config/rules/grafana.go

@@ -9,11 +9,10 @@ import (
 func GrafanaApiKey() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics.",
 		RuleID:      "grafana-api-key",
-
-		Regex:    utils.GenerateUniqueTokenRegex(`eyJrIjoi[A-Za-z0-9]{70,400}={0,2}`, true),
-		Keywords: []string{"eyJrIjoi"},
+		Description: "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics.",
+		Regex:       utils.GenerateUniqueTokenRegex(`eyJrIjoi[A-Za-z0-9]{70,400}={0,3}`, true),
+		Keywords:    []string{"eyJrIjoi"},
 	}
 
 	// validate
@@ -28,11 +27,11 @@ func GrafanaApiKey() *config.Rule {
 func GrafanaCloudApiToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure.",
 		RuleID:      "grafana-cloud-api-token",
-
-		Regex:    utils.GenerateUniqueTokenRegex(`glc_[A-Za-z0-9+/]{32,400}={0,2}`, true),
-		Keywords: []string{"glc_"},
+		Description: "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure.",
+		Regex:       utils.GenerateUniqueTokenRegex(`glc_[A-Za-z0-9+/]{32,400}={0,3}`, true),
+		Entropy:     3,
+		Keywords:    []string{"glc_"},
 	}
 
 	// validate
@@ -40,18 +39,35 @@ func GrafanaCloudApiToken() *config.Rule {
 		utils.GenerateSampleSecret("grafana-cloud-api-token",
 			"glc_"+
 				secrets.NewSecret(utils.AlphaNumeric("32"))),
+		`loki_key: glc_eyJvIjoiNzQ0NTg3IiwibiI7InN0YWlrLTQ3NTgzMC1obC13cml0ZS1oYW5kc29uJG9raSIsImsiOiI4M2w3cmdYUlBoMTUyMW1lMU023nl5UDUiLCJtIjp7IOIiOiJ1cyJ9fQ==`,
+		// TODO:
+		//`  loki:
+		//endpoint: https://322137:glc_eyJvIjoiNzQ0NTg3IiwibiI7InN0YWlrLTQ3NTgzMC1obC13cml0ZS1oYW5kc29uJG9raSIsImsiOiI4M2w3cmdYUlBoMTUyMW1lMU023nl5UDUiLCJtIjp7IOIiOiJ1cyJ9fQ==@logs-prod4.grafana.net/loki/api/v1/push`,
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		// Low entropy.
+		`glc_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
+		`   API_KEY="glc_111111111111111111111111111111111111111111="`,
+		// Invalid.
+		`static void GLC_CreateLightmapTextureArray(void);
+static void GLC_CreateLightmapTexturesIndividual(void);
+
+void GLC_UploadLightmap(int textureUnit, int lightmapnum);`,
+		`// Alias models
+void GLC_StateBeginUnderwaterAliasModelCaustics(texture_ref base_texture, texture_ref caustics_texture)
+{`,
+	}
+	return utils.Validate(r, tps, fps)
 }
 
 func GrafanaServiceAccountToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity.",
 		RuleID:      "grafana-service-account-token",
-
-		Regex:    utils.GenerateUniqueTokenRegex(`glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8}`, true),
-		Keywords: []string{"glsa_"},
+		Description: "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity.",
+		Regex:       utils.GenerateUniqueTokenRegex(`glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8}`, true),
+		Entropy:     3,
+		Keywords:    []string{"glsa_"},
 	}
 
 	// validate
@@ -61,6 +77,10 @@ func GrafanaServiceAccountToken() *config.Rule {
 				secrets.NewSecret(utils.AlphaNumeric("32"))+
 				"_"+
 				secrets.NewSecret((utils.Hex("8")))),
+		`'Authorization': 'Bearer glsa_pITqMOBIfNH2KL4PkXJqmTyQl0D9QGxF_486f63e1'`,
 	}
-	return utils.Validate(r, tps, nil)
+	fps := []string{
+		"glsa_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_AAAAAAAA",
+	}
+	return utils.Validate(r, tps, fps)
 }

cmd/generate/config/rules/infracost.go

@@ -9,24 +9,34 @@ import (
 func InfracostAPIToken() *config.Rule {
 	// define rule
 	r := config.Rule{
-		// Human readable description of the rule
+		RuleID:      "infracost-api-token",
 		Description: "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data.",
-
-		// Unique ID for the rule
-		RuleID: "infracost-api-token",
-
-		// Regex capture group for the actual secret
-
-		// Regex used for detecting secrets. See regex section below for more details
-		Regex: utils.GenerateUniqueTokenRegex(`ico-[a-zA-Z0-9]{32}`, true),
-
-		// Keywords used for string matching on fragments (think of this as a prefilter)
-		Keywords: []string{"ico-"},
+		Regex:       utils.GenerateUniqueTokenRegex(`ico-[a-zA-Z0-9]{32}`, false),
+		Entropy:     3,
+		Keywords:    []string{"ico-"},
 	}
 
 	// validate
 	tps := []string{
 		utils.GenerateSampleSecret("ico", "ico-"+secrets.NewSecret("[A-Za-z0-9]{32}")),
+		`  variable {
+    name = "INFRACOST_API_KEY"
+    secret_value = "ico-mlCr1Mn3SRcRiZMObUZOTHLcgtH2Lpgt"
+    is_secret = true
+  }`,
+		// TODO: New format with longer keys?
+		//	`    headers = {
+		//'X-Api-Key': 'ico-EeDdSfctrmjD14f45f45te5gJ7l6lw4o6M36sXT62a6',
+		//'Content-Type': 'application/json',
+		//}`,
+	}
+	fps := []string{
+		// Low entropy
+		`ico-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX`,
+		// Invalid
+		`http://assets.r7.com/assets/media_box_tv_tres_colunas/video_box.ico-7a388b69018576d24b59331fd60aab0c.png`,
+		`https://explosivelab.notion.site/Pianificazione-Nerdz-Ng-pubblico-1bc826ecc0994dd8915be97fc3489cde?pvs=74`,
+		`http://ece252-2.uwaterloo.ca:2540/image?q=gAAAAABdHkoqb9ZaJ3q4dlzEvTgG9WYwKcD9Aw7OUXeFicO-5M5IdNDjHBpKw7KBK3nCVqtuga4yzUaFEpJn8BqA1LzZprIJBw==`,
 	}
-	return utils.Validate(r, tps, nil)
+	return utils.Validate(r, tps, fps)
 }

config/gitleaks.toml

@@ -332,12 +332,14 @@ keywords = ["discord"]
 id = "discord-client-id"
 description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications."
 regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9]{18})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 2
 keywords = ["discord"]
 
 [[rules]]
 id = "discord-client-secret"
 description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks."
 regex = '''(?i)(?:discord)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-z0-9=_\-]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 2
 keywords = ["discord"]
 
 [[rules]]
@@ -2035,19 +2037,22 @@ keywords = ["ghr_"]
 [[rules]]
 id = "gitlab-pat"
 description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure."
-regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+regex = '''glpat-[\w-]{20}'''
+entropy = 3
 keywords = ["glpat-"]
 
 [[rules]]
 id = "gitlab-ptt"
 description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security."
 regex = '''glptt-[0-9a-f]{40}'''
+entropy = 3
 keywords = ["glptt-"]
 
 [[rules]]
 id = "gitlab-rrt"
 description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access."
-regex = '''GR1348941[0-9a-zA-Z\-\_]{20}'''
+regex = '''GR1348941[\w-]{20}'''
+entropy = 3
 keywords = ["gr1348941"]
 
 [[rules]]
@@ -2068,19 +2073,21 @@ keywords = [
 [[rules]]
 id = "grafana-api-key"
 description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics."
-regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
 keywords = ["eyjrijoi"]
 
 [[rules]]
 id = "grafana-cloud-api-token"
 description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure."
-regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,2})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 3
 keywords = ["glc_"]
 
 [[rules]]
 id = "grafana-service-account-token"
 description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity."
 regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 3
 keywords = ["glsa_"]
 
 [[rules]]
@@ -2138,7 +2145,8 @@ keywords = ["api_org_"]
 [[rules]]
 id = "infracost-api-token"
 description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data."
-regex = '''(?i)\b(ico-[a-zA-Z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+regex = '''\b(ico-[a-zA-Z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 3
 keywords = ["ico-"]
 
 [[rules]]