gitleaks/config/config_test.go

package config

import (
	"fmt"
	"github.com/google/go-cmp/cmp/cmpopts"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/zricethezav/gitleaks/v8/regexp"
)

const configPath = "../testdata/config/"

func TestTranslate(t *testing.T) {
	tests := []struct {
		// Configuration file basename to load, from `../testdata/config/`.
		cfgName string
		// Expected result.
		cfg Config
		// Rules to compare.
		rules []string
		// Error to expect.
		wantError error
	}{
		{
			cfgName: "allowlist_old_compat",
			cfg: Config{
				Rules: map[string]Rule{"example": {
					RuleID:   "example",
					Regex:    regexp.MustCompile(`example\d+`),
					Tags:     []string{},
					Keywords: []string{},
					Allowlists: []*Allowlist{
						{
							MatchCondition: AllowlistMatchOr,
							Regexes:        []*regexp.Regexp{regexp.MustCompile("123")},
						},
					},
				}},
			},
		},
		{
			cfgName:   "allowlist_invalid_empty",
			cfg:       Config{},
			wantError: fmt.Errorf("example: [[rules.allowlists]] must contain at least one check for: commits, paths, regexes, or stopwords"),
		},
		{
			cfgName:   "allowlist_invalid_old_and_new",
			cfg:       Config{},
			wantError: fmt.Errorf("example: [rules.allowlist] is deprecated, it cannot be used alongside [[rules.allowlist]]"),
		},
		{
			cfgName:   "allowlist_invalid_regextarget",
			cfg:       Config{},
			wantError: fmt.Errorf("example: unknown allowlist |regexTarget| 'mtach' (expected 'match', 'line')"),
		},
		{
			cfgName: "allow_aws_re",
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
					Allowlists: []*Allowlist{
						{
							MatchCondition: AllowlistMatchOr,
							Regexes:        []*regexp.Regexp{regexp.MustCompile("AKIALALEMEL33243OLIA")},
						},
					},
				}},
			},
		},
		{
			cfgName: "allow_commit",
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
					Allowlists: []*Allowlist{
						{
							MatchCondition: AllowlistMatchOr,
							Commits:        []string{"allowthiscommit"},
						},
					},
				}},
			},
		},
		{
			cfgName: "allow_path",
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
					Allowlists: []*Allowlist{
						{
							MatchCondition: AllowlistMatchOr,
							Paths:          []*regexp.Regexp{regexp.MustCompile(".go")},
						},
					},
				}},
			},
		},
		{
			cfgName: "entropy_group",
			cfg: Config{
				Rules: map[string]Rule{"discord-api-key": {
					RuleID:      "discord-api-key",
					Description: "Discord API key",
					Regex:       regexp.MustCompile(`(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]`),
					Entropy:     3.5,
					SecretGroup: 3,
					Keywords:    []string{},
					Tags:        []string{},
				}},
			},
		},
		{
			cfgName:   "missing_id",
			cfg:       Config{},
			wantError: fmt.Errorf("rule |id| is missing or empty, regex: (?i)(discord[a-z0-9_ .\\-,]{0,25})(=|>|:=|\\|\\|:|<=|=>|:).{0,5}['\\\"]([a-h0-9]{64})['\\\"]"),
		},
		{
			cfgName:   "no_regex_or_path",
			cfg:       Config{},
			wantError: fmt.Errorf("discord-api-key: both |regex| and |path| are empty, this rule will have no effect"),
		},
		{
			cfgName:   "bad_entropy_group",
			cfg:       Config{},
			wantError: fmt.Errorf("discord-api-key: invalid regex secret group 5, max regex secret group 3"),
		},
		{
			cfgName: "base",
			cfg: Config{
				Rules: map[string]Rule{
					"aws-access-key": {
						RuleID:      "aws-access-key",
						Description: "AWS Access Key",
						Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
					},
					"aws-secret-key": {
						RuleID:      "aws-secret-key",
						Description: "AWS Secret Key",
						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
					},
					"aws-secret-key-again": {
						RuleID:      "aws-secret-key-again",
						Description: "AWS Secret Key",
						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
					},
				},
			},
		},
		{
			cfgName: "extend_rule_allowlist_or",
			cfg: Config{
				Rules: map[string]Rule{
					"aws-secret-key-again-again": {
						RuleID:      "aws-secret-key-again-again",
						Description: "AWS Secret Key",
						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
						Allowlists: []*Allowlist{
							{
								MatchCondition: AllowlistMatchOr,
								StopWords:      []string{"fake"},
							},
							{
								MatchCondition: AllowlistMatchOr,
								Commits:        []string{"abcdefg1"},
								Paths:          []*regexp.Regexp{regexp.MustCompile(`ignore\.xaml`)},
								Regexes:        []*regexp.Regexp{regexp.MustCompile(`foo.+bar`)},
								RegexTarget:    "line",
								StopWords:      []string{"example"},
							},
						},
					},
				},
			},
		},
		{
			cfgName: "extend_rule_allowlist_and",
			cfg: Config{
				Rules: map[string]Rule{
					"aws-secret-key-again-again": {
						RuleID:      "aws-secret-key-again-again",
						Description: "AWS Secret Key",
						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
						Allowlists: []*Allowlist{
							{
								MatchCondition: AllowlistMatchOr,
								StopWords:      []string{"fake"},
							},
							{
								MatchCondition: AllowlistMatchAnd,
								Commits:        []string{"abcdefg1"},
								Paths:          []*regexp.Regexp{regexp.MustCompile(`ignore\.xaml`)},
								Regexes:        []*regexp.Regexp{regexp.MustCompile(`foo.+bar`)},
								RegexTarget:    "line",
								StopWords:      []string{"example"},
							},
						},
					},
				},
			},
		},
		{
			cfgName: "extend_empty_regexpath",
			cfg: Config{
				Rules: map[string]Rule{
					"aws-secret-key-again-again": {
						RuleID:      "aws-secret-key-again-again",
						Description: "AWS Secret Key",
						Regex:       regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Keywords:    []string{},
						Tags:        []string{"key", "AWS"},
						Allowlists: []*Allowlist{
							{
								Description:    "False positive. Keys used for colors match the rule, and should be excluded.",
								MatchCondition: AllowlistMatchOr,
								Paths:          []*regexp.Regexp{regexp.MustCompile(`something.py`)},
							},
						},
					},
				},
			},
		},
		{
			cfgName: "override_description",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "Puppy Doggy",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "override_entropy",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Entropy:     999.0,
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "override_secret_group",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:a)(?:a)"),
					SecretGroup: 2,
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "override_regex",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:a)"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "override_path",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Path:        regexp.MustCompile("(?:puppy)"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "override_tags",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{},
					Tags:        []string{"key", "AWS", "puppy"},
				},
				},
			},
		},
		{
			cfgName: "override_keywords",
			rules:   []string{"aws-access-key"},
			cfg: Config{
				Rules: map[string]Rule{"aws-access-key": {
					RuleID:      "aws-access-key",
					Description: "AWS Access Key",
					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
					Keywords:    []string{"puppy"},
					Tags:        []string{"key", "AWS"},
				},
				},
			},
		},
		{
			cfgName: "extend_disabled",
			cfg: Config{
				Rules: map[string]Rule{
					"aws-secret-key": {
						RuleID:   "aws-secret-key",
						Regex:    regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
						Tags:     []string{"key", "AWS"},
						Keywords: []string{},
					},
					"pypi-upload-token": {
						RuleID:   "pypi-upload-token",
						Regex:    regexp.MustCompile(`pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}`),
						Tags:     []string{},
						Keywords: []string{},
					},
				},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.cfgName, func(t *testing.T) {
			t.Cleanup(func() {
				extendDepth = 0
				viper.Reset()
			})

			viper.AddConfigPath(configPath)
			viper.SetConfigName(tt.cfgName)
			viper.SetConfigType("toml")
			err := viper.ReadInConfig()
			require.NoError(t, err)

			var vc ViperConfig
			err = viper.Unmarshal(&vc)
			require.NoError(t, err)
			cfg, err := vc.Translate()
			if err != nil && !assert.EqualError(t, tt.wantError, err.Error()) {
				return
			}

			if len(tt.rules) > 0 {
				rules := make(map[string]Rule)
				for _, name := range tt.rules {
					rules[name] = cfg.Rules[name]
				}
				cfg.Rules = rules
			}

			var regexComparer = func(x, y *regexp.Regexp) bool {
				if x == nil || y == nil {
					return x == y
				}
				return x.String() == y.String()
			}
			opts := cmp.Options{
				cmp.Comparer(regexComparer),
				cmpopts.IgnoreUnexported(Rule{}, Allowlist{}),
			}
			if diff := cmp.Diff(tt.cfg.Rules, cfg.Rules, opts); diff != "" {
				t.Errorf("%s diff: (-want +got)\n%s", tt.cfgName, diff)
			}
		})
	}
}

func TestExtendedRuleKeywordsAreDowncase(t *testing.T) {
	tests := []struct {
		name             string
		cfgName          string
		expectedKeywords string
	}{
		{
			name:             "Extend base rule that includes AWS keyword with new attribute",
			cfgName:          "extend_base_rule_including_keysword_with_attribute",
			expectedKeywords: "aws",
		},
		{
			name:             "Extend base with a new rule with CMS keyword",
			cfgName:          "extend_with_new_rule",
			expectedKeywords: "cms",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Cleanup(func() {
				viper.Reset()
			})

			viper.AddConfigPath(configPath)
			viper.SetConfigName(tt.cfgName)
			viper.SetConfigType("toml")
			err := viper.ReadInConfig()
			require.NoError(t, err)

			var vc ViperConfig
			err = viper.Unmarshal(&vc)
			require.NoError(t, err)
			cfg, err := vc.Translate()
			require.NoError(t, err)

			_, exists := cfg.Keywords[tt.expectedKeywords]
			require.Truef(t, exists, "The expected keyword %s did not exist as a key of cfg.Keywords", tt.expectedKeywords)
		})
	}
}