main.go 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. package main
  2. import (
  3. "bufio"
  4. "context"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "io"
  9. "os"
  10. "os/exec"
  11. "path/filepath"
  12. "strings"
  13. "time"
  14. log "github.com/sirupsen/logrus"
  15. )
  16. const (
  17. defaultLogFile = "flakey-test-runs.log"
  18. defaultJSONLFile = "flakey-test-runs.jsonl"
  19. )
  20. type testEvent struct {
  21. Time time.Time `json:"Time"`
  22. Action string `json:"Action"`
  23. Package string `json:"Package"`
  24. Test string `json:"Test"`
  25. Output string `json:"Output"`
  26. Elapsed float64 `json:"Elapsed"`
  27. }
  28. type testFailure struct {
  29. Package string `json:"package"`
  30. Test string `json:"test"`
  31. Output string `json:"output"`
  32. }
  33. type runSummary struct {
  34. Passes int `json:"passes"`
  35. Failures int `json:"failures"`
  36. Skipped int `json:"skipped"`
  37. }
  38. type jsonlRecord struct {
  39. Timestamp string `json:"timestamp"`
  40. FailureDetails []testFailure `json:"failureDetails"`
  41. Run int `json:"run"`
  42. ExitCode int `json:"exitCode"`
  43. DurationMs int64 `json:"durationMs"`
  44. Passes int `json:"passes"`
  45. Failures int `json:"failures"`
  46. Skipped int `json:"skipped"`
  47. }
  48. type testRunState struct {
  49. failureOutput map[string]*strings.Builder
  50. failures []testFailure
  51. summary runSummary
  52. }
  53. func initLog() {
  54. logFormat := os.Getenv("OLIVETIN_LOG_FORMAT")
  55. if logFormat == "json" {
  56. log.SetFormatter(&log.JSONFormatter{})
  57. } else {
  58. log.SetFormatter(&log.TextFormatter{
  59. FullTimestamp: true,
  60. ForceQuote: true,
  61. })
  62. }
  63. log.SetLevel(log.InfoLevel)
  64. }
  65. func hasGoMod(dir string) bool {
  66. stat, err := os.Stat(filepath.Join(dir, "go.mod"))
  67. return err == nil && !stat.IsDir()
  68. }
  69. func rootFromExecutable() (string, bool) {
  70. exe, err := os.Executable()
  71. if err != nil {
  72. return "", false
  73. }
  74. candidate := filepath.Join(filepath.Dir(exe), "..", "..")
  75. if hasGoMod(candidate) {
  76. return candidate, true
  77. }
  78. return "", false
  79. }
  80. func rootFromWorkingDir() string {
  81. wd, err := os.Getwd()
  82. if err != nil {
  83. return "."
  84. }
  85. if hasGoMod(wd) {
  86. return wd
  87. }
  88. return filepath.Join(wd, "..")
  89. }
  90. func serviceRoot() string {
  91. if root, ok := rootFromExecutable(); ok {
  92. return root
  93. }
  94. return rootFromWorkingDir()
  95. }
  96. func envOrDefault(name, fallback string) string {
  97. if value := strings.TrimSpace(os.Getenv(name)); value != "" {
  98. return value
  99. }
  100. return fallback
  101. }
  102. func formatFailure(failure testFailure) string {
  103. lines := []string{
  104. fmt.Sprintf("FAILURE: %s", failure.Test),
  105. fmt.Sprintf(" package: %s", failure.Package),
  106. }
  107. output := strings.TrimSpace(failure.Output)
  108. if output == "" {
  109. output = "unknown"
  110. }
  111. lines = append(lines, fmt.Sprintf(" output: %s", output))
  112. return strings.Join(lines, "\n")
  113. }
  114. func formatRunCounts(summary runSummary) string {
  115. return fmt.Sprintf("%d pass %d fail %d skip", summary.Passes, summary.Failures, summary.Skipped)
  116. }
  117. func runResultLabel(exitCode int) string {
  118. if exitCode == 0 {
  119. return "PASS"
  120. }
  121. return "FAIL"
  122. }
  123. func buildRunLogBlock(run int, timestamp string, exitCode int, summary runSummary, failures []testFailure, durationMs int64) string {
  124. block := []string{
  125. fmt.Sprintf(
  126. "=== RUN %d | %s | %s | %d pass %d fail %d skip | %.1fs ===",
  127. run,
  128. timestamp,
  129. runResultLabel(exitCode),
  130. summary.Passes,
  131. summary.Failures,
  132. summary.Skipped,
  133. float64(durationMs)/1000,
  134. ),
  135. }
  136. for _, failure := range failures {
  137. block = append(block, formatFailure(failure))
  138. }
  139. block = append(block, "")
  140. return strings.Join(block, "\n") + "\n"
  141. }
  142. func appendJSONLRecord(jsonlFile string, run int, timestamp string, exitCode int, durationMs int64, summary runSummary, failures []testFailure) error {
  143. record := jsonlRecord{
  144. Run: run,
  145. Timestamp: timestamp,
  146. ExitCode: exitCode,
  147. DurationMs: durationMs,
  148. Passes: summary.Passes,
  149. Failures: summary.Failures,
  150. Skipped: summary.Skipped,
  151. FailureDetails: failures,
  152. }
  153. encoded, err := json.Marshal(record)
  154. if err != nil {
  155. return err
  156. }
  157. return appendFile(jsonlFile, string(encoded)+"\n")
  158. }
  159. func appendRunLog(logFile, jsonlFile string, run, exitCode int, summary runSummary, failures []testFailure, durationMs int64) error {
  160. timestamp := time.Now().UTC().Format(time.RFC3339)
  161. if err := appendFile(logFile, buildRunLogBlock(run, timestamp, exitCode, summary, failures, durationMs)); err != nil {
  162. return err
  163. }
  164. return appendJSONLRecord(jsonlFile, run, timestamp, exitCode, durationMs, summary, failures)
  165. }
  166. func appendFile(path, content string) error {
  167. file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
  168. if err != nil {
  169. return err
  170. }
  171. _, writeErr := file.WriteString(content)
  172. closeErr := file.Close()
  173. if writeErr != nil {
  174. return writeErr
  175. }
  176. return closeErr
  177. }
  178. func newTestRunState() *testRunState {
  179. return &testRunState{
  180. failures: make([]testFailure, 0),
  181. failureOutput: make(map[string]*strings.Builder),
  182. }
  183. }
  184. func failureKey(pkg, test string) string {
  185. return pkg + "\x00" + test
  186. }
  187. func (state *testRunState) handlePass(event testEvent) {
  188. if event.Test == "" {
  189. return
  190. }
  191. state.summary.Passes++
  192. }
  193. func (state *testRunState) handleSkip(event testEvent) {
  194. if event.Test == "" {
  195. return
  196. }
  197. state.summary.Skipped++
  198. }
  199. func (state *testRunState) handleFail(event testEvent) {
  200. if event.Test == "" {
  201. return
  202. }
  203. state.summary.Failures++
  204. key := failureKey(event.Package, event.Test)
  205. output := ""
  206. if builder, ok := state.failureOutput[key]; ok {
  207. output = strings.TrimSpace(builder.String())
  208. } else {
  209. state.failureOutput[key] = &strings.Builder{}
  210. }
  211. state.failures = append(state.failures, testFailure{
  212. Package: event.Package,
  213. Test: event.Test,
  214. Output: output,
  215. })
  216. }
  217. func (state *testRunState) handleOutput(event testEvent) {
  218. if event.Test == "" {
  219. return
  220. }
  221. key := failureKey(event.Package, event.Test)
  222. builder, ok := state.failureOutput[key]
  223. if !ok {
  224. builder = &strings.Builder{}
  225. state.failureOutput[key] = builder
  226. }
  227. builder.WriteString(event.Output)
  228. }
  229. type testEventHandler func(*testRunState, testEvent)
  230. var testEventHandlers = map[string]testEventHandler{
  231. "pass": (*testRunState).handlePass,
  232. "fail": (*testRunState).handleFail,
  233. "skip": (*testRunState).handleSkip,
  234. "output": (*testRunState).handleOutput,
  235. }
  236. func (state *testRunState) processEvent(event testEvent) {
  237. handler, ok := testEventHandlers[event.Action]
  238. if !ok {
  239. return
  240. }
  241. handler(state, event)
  242. }
  243. func (state *testRunState) finalizeFailureOutputs() {
  244. for index := range state.failures {
  245. key := failureKey(state.failures[index].Package, state.failures[index].Test)
  246. if builder, ok := state.failureOutput[key]; ok {
  247. state.failures[index].Output = strings.TrimSpace(builder.String())
  248. }
  249. }
  250. }
  251. func scanTestEvents(stdout io.Reader, state *testRunState) error {
  252. scanner := bufio.NewScanner(stdout)
  253. scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
  254. for scanner.Scan() {
  255. var event testEvent
  256. if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
  257. continue
  258. }
  259. state.processEvent(event)
  260. }
  261. return scanner.Err()
  262. }
  263. func finishTestCommand(cmd *exec.Cmd, state *testRunState) (int, runSummary, []testFailure, error) {
  264. if err := cmd.Wait(); err != nil {
  265. if errExit, ok := errors.AsType[*exec.ExitError](err); ok {
  266. state.finalizeFailureOutputs()
  267. return errExit.ExitCode(), state.summary, state.failures, nil
  268. }
  269. return 1, state.summary, state.failures, err
  270. }
  271. return 0, state.summary, state.failures, nil
  272. }
  273. func runTestsOnce(rootDir string) (int, runSummary, []testFailure, error) {
  274. ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
  275. defer cancel()
  276. cmd := exec.CommandContext(ctx, "go", "test", "./...", "-count=1", "-json")
  277. cmd.Dir = rootDir
  278. stdout, err := cmd.StdoutPipe()
  279. if err != nil {
  280. return 1, runSummary{}, nil, err
  281. }
  282. if err := cmd.Start(); err != nil {
  283. return 1, runSummary{}, nil, err
  284. }
  285. state := newTestRunState()
  286. if err := scanTestEvents(stdout, state); err != nil {
  287. return 1, state.summary, state.failures, err
  288. }
  289. return finishTestCommand(cmd, state)
  290. }
  291. func buildLogHeader(logFile, jsonlFile string) string {
  292. return strings.Join([]string{
  293. fmt.Sprintf("# Flaky test run log started %s", time.Now().UTC().Format(time.RFC3339)),
  294. fmt.Sprintf("# Log file: %s", logFile),
  295. fmt.Sprintf("# JSONL file: %s", jsonlFile),
  296. "",
  297. }, "\n") + "\n"
  298. }
  299. func initRunFiles(logFile, jsonlFile string) error {
  300. if err := os.WriteFile(logFile, []byte(buildLogHeader(logFile, jsonlFile)), 0o644); err != nil {
  301. return err
  302. }
  303. return os.WriteFile(jsonlFile, nil, 0o644)
  304. }
  305. func logRunResult(run, exitCode int, summary runSummary, durationMs int64) {
  306. log.Infof(
  307. "Run %d: %s | %s (%.1fs) — logged",
  308. run,
  309. runResultLabel(exitCode),
  310. formatRunCounts(summary),
  311. float64(durationMs)/1000,
  312. )
  313. }
  314. func executeRun(run int, rootDir, logFile, jsonlFile string) (exitCode int, stop bool) {
  315. start := time.Now()
  316. exitCode, summary, failures, err := runTestsOnce(rootDir)
  317. durationMs := time.Since(start).Milliseconds()
  318. if err != nil {
  319. log.WithError(err).Errorf("Run %d failed to execute: %s", run, formatRunCounts(summary))
  320. _ = appendRunLog(logFile, jsonlFile, run, 1, summary, failures, durationMs)
  321. os.Exit(1)
  322. }
  323. if logErr := appendRunLog(logFile, jsonlFile, run, exitCode, summary, failures, durationMs); logErr != nil {
  324. log.WithError(logErr).Fatal("failed to append run log")
  325. }
  326. logRunResult(run, exitCode, summary, durationMs)
  327. return exitCode, exitCode != 0
  328. }
  329. func main() {
  330. initLog()
  331. rootDir := serviceRoot()
  332. logFile := envOrDefault("FLAKEY_LOG_FILE", filepath.Join(rootDir, defaultLogFile))
  333. jsonlFile := envOrDefault("FLAKEY_JSONL_FILE", filepath.Join(rootDir, defaultJSONLFile))
  334. if err := initRunFiles(logFile, jsonlFile); err != nil {
  335. log.WithError(err).Fatal("failed to initialize output files")
  336. }
  337. log.Infof("Logging flaky test runs to %s", logFile)
  338. log.Infof("Structured run data: %s", jsonlFile)
  339. run := 0
  340. for {
  341. run++
  342. exitCode, stop := executeRun(run, rootDir, logFile, jsonlFile)
  343. if stop {
  344. log.Errorf("Failure on run %d, stopping. See %s", run, logFile)
  345. os.Exit(exitCode)
  346. }
  347. }
  348. }