Просмотр исходного кода

fix memory issue when writing json report (#168)

Previously, gitleaks.writeReport used json.MarshallIndent on the entire
set of leaks when writing json reports, which would cause failures due
to excessive memory consumption when a large number of leaks were
identified.

This PR addresses the problem by doing incremental marshalling and
writing of leaks to the report file, and reduces the memory footprint by
several GB in tests with >30k leaks.
Milo Minderbinder 7 лет назад
Родитель
Сommit
0ba9dda0f0
1 измененных файлов с 35 добавлено и 2 удалено
  1. 35 2
      main.go

+ 35 - 2
main.go

@@ -365,8 +365,41 @@ func writeReport(leaks []Leak) error {
 		}
 		}
 		w.Flush()
 		w.Flush()
 	} else {
 	} else {
-		reportJSON, _ := json.MarshalIndent(leaks, "", "\t")
-		err = ioutil.WriteFile(opts.Report, reportJSON, 0644)
+		var (
+			f       *os.File
+			encoder *json.Encoder
+		)
+		f, err := os.Create(opts.Report)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		encoder = json.NewEncoder(f)
+		encoder.SetIndent("", "\t")
+		if _, err := f.WriteString("[\n"); err != nil {
+			return err
+		}
+		for i := 0; i < len(leaks); i++ {
+			if err := encoder.Encode(leaks[i]); err != nil {
+				return err
+			}
+			// for all but the last leak, seek back and overwrite the newline appended by Encode() with comma & newline
+			if i+1 < len(leaks) {
+				if _, err := f.Seek(-1, 1); err != nil {
+					return err
+				}
+				if _, err := f.WriteString(",\n"); err != nil {
+					return err
+				}
+			}
+		}
+		if _, err := f.WriteString("]"); err != nil {
+			return err
+		}
+		if err := f.Sync(); err != nil {
+			log.Error(err)
+			return err
+		}
 	}
 	}
 	return err
 	return err
 }
 }