Diagnostics (#1856)

* init diagnostics

* update usage readme

README.md

@@ -144,7 +144,6 @@ Usage:
   gitleaks [command]
 
 Available Commands:
-  completion  generate the autocompletion script for the specified shell
   dir         scan directories or files for secrets
   git         scan git repositories for secrets
   help        Help about any command
@@ -160,6 +159,8 @@ Flags:
                                       3. env var GITLEAKS_CONFIG_TOML with the file content
                                       4. (target path)/.gitleaks.toml
                                       If none of the four options are used, then gitleaks will use the default config
+      --diagnostics string            enable diagnostics (comma-separated list: cpu,mem,trace). cpu=CPU profiling, mem=memory profiling, trace=execution tracing
+      --diagnostics-dir string        directory to store diagnostics output files (defaults to current directory)
       --enable-rule strings           only enable specific rules by id
       --exit-code int                 exit code when leaks have been encountered (default 1)
   -i, --gitleaks-ignore-path string   path to .gitleaksignore file or folder containing one (default ".")
@@ -171,7 +172,7 @@ Flags:
       --no-banner                     suppress banner
       --no-color                      turn off color for verbose output
       --redact uint[=100]             redact secrets from logs and stdout. To redact only parts of the secret just apply a percent value from 0..100. For example --redact=20 (default 100%)
-  -f, --report-format string          output format (json, csv, junit, sarif) (default "json")
+  -f, --report-format string          output format (json, csv, junit, sarif, template)
   -r, --report-path string            report file
       --report-template string        template file used to generate the report (implies --report-format=template)
   -v, --verbose                       show verbose output from scan

cmd/detect.go

@@ -55,6 +55,7 @@ func runDetect(cmd *cobra.Command, args []string) {
 
 	// setup config (aka, the thing that defines rules)
 	initConfig(source)
+	initDiagnostics()
 	cfg := Config(cmd)
 
 	// create detector

cmd/diagnostics.go

@@ -0,0 +1,209 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"strings"
+
+	"github.com/zricethezav/gitleaks/v8/logging"
+)
+
+// DiagnosticsManager manages various types of diagnostics
+type DiagnosticsManager struct {
+	Enabled      bool
+	DiagTypes    []string
+	OutputDir    string
+	cpuProfile   *os.File
+	memProfile   string
+	traceProfile *os.File
+}
+
+// NewDiagnosticsManager creates a new DiagnosticsManager instance
+func NewDiagnosticsManager(diagnosticsFlag string, diagnosticsDir string) (*DiagnosticsManager, error) {
+	if diagnosticsFlag == "" {
+		return &DiagnosticsManager{Enabled: false}, nil
+	}
+
+	dm := &DiagnosticsManager{
+		Enabled:   true,
+		DiagTypes: strings.Split(diagnosticsFlag, ","),
+		OutputDir: diagnosticsDir,
+	}
+
+	// If no output directory is specified, use the current directory
+	if dm.OutputDir == "" {
+		var err error
+		dm.OutputDir, err = os.Getwd()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get current directory: %w", err)
+		}
+		logging.Debug().Msgf("No diagnostics directory specified, using current directory: %s", dm.OutputDir)
+	}
+
+	// Create the output directory if it doesn't exist
+	if err := os.MkdirAll(dm.OutputDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create diagnostics directory: %w", err)
+	}
+
+	// Make sure the output directory is absolute
+	if !filepath.IsAbs(dm.OutputDir) {
+		absPath, err := filepath.Abs(dm.OutputDir)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get absolute path for diagnostics directory: %w", err)
+		}
+		dm.OutputDir = absPath
+	}
+
+	logging.Debug().Msgf("Diagnostics enabled: %s", strings.Join(dm.DiagTypes, ","))
+	logging.Debug().Msgf("Diagnostics output directory: %s", dm.OutputDir)
+
+	return dm, nil
+}
+
+// StartDiagnostics starts all enabled diagnostics
+func (dm *DiagnosticsManager) StartDiagnostics() error {
+	if !dm.Enabled {
+		return nil
+	}
+
+	var err error
+
+	for _, diagType := range dm.DiagTypes {
+		diagType = strings.TrimSpace(diagType)
+		switch diagType {
+		case "cpu":
+			if err = dm.StartCPUProfile(); err != nil {
+				return err
+			}
+		case "mem":
+			if err = dm.SetupMemoryProfile(); err != nil {
+				return err
+			}
+		case "trace":
+			if err = dm.StartTraceProfile(); err != nil {
+				return err
+			}
+		default:
+			logging.Warn().Msgf("Unknown diagnostics type: %s", diagType)
+		}
+	}
+
+	return nil
+}
+
+// StopDiagnostics stops all started diagnostics
+func (dm *DiagnosticsManager) StopDiagnostics() {
+	if !dm.Enabled {
+		return
+	}
+
+	logging.Debug().Msg("Stopping diagnostics and writing profiling data...")
+
+	for _, diagType := range dm.DiagTypes {
+		diagType = strings.TrimSpace(diagType)
+		switch diagType {
+		case "cpu":
+			dm.StopCPUProfile()
+		case "mem":
+			dm.WriteMemoryProfile()
+		case "trace":
+			dm.StopTraceProfile()
+		}
+	}
+}
+
+// StartCPUProfile starts CPU profiling
+func (dm *DiagnosticsManager) StartCPUProfile() error {
+	cpuProfilePath := filepath.Join(dm.OutputDir, "cpu.pprof")
+	f, err := os.Create(cpuProfilePath)
+	if err != nil {
+		return fmt.Errorf("could not create CPU profile at %s: %w", cpuProfilePath, err)
+	}
+
+	if err := pprof.StartCPUProfile(f); err != nil {
+		f.Close()
+		return fmt.Errorf("could not start CPU profile: %w", err)
+	}
+
+	dm.cpuProfile = f
+	return nil
+}
+
+// StopCPUProfile stops CPU profiling
+func (dm *DiagnosticsManager) StopCPUProfile() {
+	if dm.cpuProfile != nil {
+		pprof.StopCPUProfile()
+		if err := dm.cpuProfile.Close(); err != nil {
+			logging.Error().Err(err).Msg("Error closing CPU profile file")
+		}
+		logging.Info().Msgf("CPU profile written to: %s", dm.cpuProfile.Name())
+		dm.cpuProfile = nil
+	}
+}
+
+// SetupMemoryProfile sets up memory profiling to be written when StopDiagnostics is called
+func (dm *DiagnosticsManager) SetupMemoryProfile() error {
+	memProfilePath := filepath.Join(dm.OutputDir, "mem.pprof")
+	dm.memProfile = memProfilePath
+	return nil
+}
+
+// WriteMemoryProfile writes the memory profile to disk
+func (dm *DiagnosticsManager) WriteMemoryProfile() {
+	if dm.memProfile == "" {
+		return
+	}
+
+	f, err := os.Create(dm.memProfile)
+	if err != nil {
+		logging.Error().Err(err).Msgf("Could not create memory profile at %s", dm.memProfile)
+		return
+	}
+
+	// Get memory profile
+	runtime.GC() // Run GC before taking the memory profile
+	if err := pprof.WriteHeapProfile(f); err != nil {
+		logging.Error().Err(err).Msg("Could not write memory profile")
+	} else {
+		logging.Info().Msgf("Memory profile written to: %s", dm.memProfile)
+	}
+
+	if err := f.Close(); err != nil {
+		logging.Error().Err(err).Msg("Error closing memory profile file")
+	}
+
+	dm.memProfile = ""
+}
+
+// StartTraceProfile starts execution tracing
+func (dm *DiagnosticsManager) StartTraceProfile() error {
+	traceProfilePath := filepath.Join(dm.OutputDir, "trace.out")
+	f, err := os.Create(traceProfilePath)
+	if err != nil {
+		return fmt.Errorf("could not create trace profile at %s: %w", traceProfilePath, err)
+	}
+
+	if err := trace.Start(f); err != nil {
+		f.Close()
+		return fmt.Errorf("could not start trace profile: %w", err)
+	}
+
+	dm.traceProfile = f
+	return nil
+}
+
+// StopTraceProfile stops execution tracing
+func (dm *DiagnosticsManager) StopTraceProfile() {
+	if dm.traceProfile != nil {
+		trace.Stop()
+		if err := dm.traceProfile.Close(); err != nil {
+			logging.Error().Err(err).Msg("Error closing trace profile file")
+		}
+		logging.Info().Msgf("Trace profile written to: %s", dm.traceProfile.Name())
+		dm.traceProfile = nil
+	}
+}

cmd/directory.go

@@ -31,7 +31,10 @@ func runDirectory(cmd *cobra.Command, args []string) {
 			source = "."
 		}
 	}
+
 	initConfig(source)
+	initDiagnostics()
+
 	var (
 		findings []report.Finding
 		err      error

cmd/git.go

@@ -42,6 +42,8 @@ func runGit(cmd *cobra.Command, args []string) {
 
 	// setup config (aka, the thing that defines rules)
 	initConfig(source)
+	initDiagnostics()
+
 	cfg := Config(cmd)
 
 	// create detector

cmd/protect.go

@@ -33,6 +33,8 @@ func runProtect(cmd *cobra.Command, args []string) {
 
 	// setup config (aka, the thing that defines rules)
 	initConfig(source)
+	initDiagnostics()
+
 	cfg := Config(cmd)
 
 	// create detector

cmd/root.go

@@ -38,11 +38,17 @@ order of precedence:
 4. (target path)/.gitleaks.toml
 If none of the four options are used, then gitleaks will use the default config`
 
-var rootCmd = &cobra.Command{
-	Use:     "gitleaks",
-	Short:   "Gitleaks scans code, past or present, for secrets",
-	Version: Version,
-}
+var (
+	rootCmd = &cobra.Command{
+		Use:     "gitleaks",
+		Short:   "Gitleaks scans code, past or present, for secrets",
+		Version: Version,
+	}
+
+	// diagnostics manager is global to ensure it can be started before a scan begins
+	// and stopped after a scan completes
+	diagnosticsManager *DiagnosticsManager
+)
 
 const (
 	BYTE     = 1.0
@@ -71,6 +77,10 @@ func init() {
 	rootCmd.PersistentFlags().StringP("gitleaks-ignore-path", "i", ".", "path to .gitleaksignore file or folder containing one")
 	rootCmd.PersistentFlags().Int("max-decode-depth", 0, "allow recursive decoding up to this depth (default \"0\", no decoding is done)")
 
+	// Add diagnostics flags
+	rootCmd.PersistentFlags().String("diagnostics", "", "enable diagnostics (comma-separated list: cpu,mem,trace). cpu=CPU profiling, mem=memory profiling, trace=execution tracing")
+	rootCmd.PersistentFlags().String("diagnostics-dir", "", "directory to store diagnostics output files (defaults to current directory)")
+
 	err := viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
 	if err != nil {
 		logging.Fatal().Msgf("err binding config %s", err.Error())
@@ -169,6 +179,33 @@ func initConfig(source string) {
 	}
 }
 
+func initDiagnostics() {
+	// Initialize diagnostics manager
+	diagnosticsFlag, err := rootCmd.PersistentFlags().GetString("diagnostics")
+	if err != nil {
+		logging.Fatal().Err(err).Msg("Error getting diagnostics flag")
+	}
+
+	diagnosticsDir, err := rootCmd.PersistentFlags().GetString("diagnostics-dir")
+	if err != nil {
+		logging.Fatal().Err(err).Msg("Error getting diagnostics-dir flag")
+	}
+
+	var diagErr error
+	diagnosticsManager, diagErr = NewDiagnosticsManager(diagnosticsFlag, diagnosticsDir)
+	if diagErr != nil {
+		logging.Fatal().Err(diagErr).Msg("Error initializing diagnostics")
+	}
+
+	if diagnosticsManager.Enabled {
+		logging.Info().Msg("Starting diagnostics...")
+		if diagErr := diagnosticsManager.StartDiagnostics(); diagErr != nil {
+			logging.Fatal().Err(diagErr).Msg("Failed to start diagnostics")
+		}
+	}
+
+}
+
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		if strings.Contains(err.Error(), "unknown flag") {
@@ -379,6 +416,11 @@ func bytesConvert(bytes uint64) string {
 }
 
 func findingSummaryAndExit(detector *detect.Detector, findings []report.Finding, exitCode int, start time.Time, err error) {
+	if diagnosticsManager.Enabled {
+		logging.Debug().Msg("Finalizing diagnostics...")
+		diagnosticsManager.StopDiagnostics()
+	}
+
 	totalBytes := detector.TotalBytes.Load()
 	bytesMsg := fmt.Sprintf("scanned ~%d bytes (%s)", totalBytes, bytesConvert(totalBytes))
 	if err == nil {

cmd/stdin.go

@@ -25,6 +25,8 @@ func runStdIn(cmd *cobra.Command, _ []string) {
 
 	// setup config (aka, the thing that defines rules)
 	initConfig(".")
+	initDiagnostics()
+
 	cfg := Config(cmd)
 
 	// create detector