limit number of goroutines for historic scanning as well (#761)

detect/detect.go

@@ -17,6 +17,8 @@ type Options struct {
 	Redact  bool
 }
 
+const MAXGOROUTINES = 4
+
 func DetectFindings(cfg config.Config, b []byte, filePath string, commit string) []report.Finding {
 	var findings []report.Finding
 	linePairs := regexp.MustCompile("\n").FindAllIndex(b, -1)

detect/files.go

@@ -20,7 +20,7 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor
 		findings []report.Finding
 		mu       sync.Mutex
 	)
-	concurrentGoroutines := make(chan struct{}, 4)
+	concurrentGoroutines := make(chan struct{}, MAXGOROUTINES)
 	g, _ := errgroup.WithContext(context.Background())
 	paths := make(chan string)
 	g.Go(func() error {
@@ -41,16 +41,17 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor
 	})
 	for pa := range paths {
 		p := pa
+		concurrentGoroutines <- struct{}{}
 		g.Go(func() error {
-			concurrentGoroutines <- struct{}{}
+			defer func() {
+				<-concurrentGoroutines
+			}()
 			b, err := os.ReadFile(p)
 			if err != nil {
-				<-concurrentGoroutines
 				return err
 			}
 
 			if !godocutil.IsText(b) {
-				<-concurrentGoroutines
 				return nil
 			}
 			fis := DetectFindings(cfg, b, p, "")
@@ -69,7 +70,6 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor
 				findings = append(findings, fi)
 				mu.Unlock()
 			}
-			<-concurrentGoroutines
 			return nil
 		})
 	}

detect/git.go

@@ -19,16 +19,20 @@ func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Option
 	var findings []report.Finding
 	mu := sync.Mutex{}
 	wg := sync.WaitGroup{}
+	concurrentGoroutines := make(chan struct{}, MAXGOROUTINES)
 	commitMap := make(map[string]bool)
 	for f := range files {
 		// keep track of commits for logging
 		if f.PatchHeader != nil {
 			commitMap[f.PatchHeader.SHA] = true
 		}
-
 		wg.Add(1)
+		concurrentGoroutines <- struct{}{}
 		go func(f *gitdiff.File) {
-			defer wg.Done()
+			defer func() {
+				wg.Done()
+				<-concurrentGoroutines
+			}()
 			if f.IsBinary {
 				return
 			}