OliveTin/integration-tests/scripts/find-flakey-tests-inf.mjs

#!/usr/bin/env node
import { spawn } from 'node:child_process'
import {
  appendFileSync,
  writeFileSync,
  readFileSync,
  unlinkSync,
} from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { tmpdir } from 'node:os'
import { randomUUID } from 'node:crypto'

const rootDir = join(dirname(fileURLToPath(import.meta.url)), '..')
const logFile = process.env.FLAKEY_LOG_FILE || join(rootDir, 'flakey-test-runs.log')
const jsonlFile = process.env.FLAKEY_JSONL_FILE || join(rootDir, 'flakey-test-runs.jsonl')

function formatFailure (failure) {
  const err = failure.err || {}
  const lines = [
    `FAILURE: ${failure.fullTitle || failure.title}`,
    `  file: ${failure.file || 'unknown'}`,
    `  message: ${(err.message || 'unknown').trim()}`,
  ]

  if (err.stack) {
    lines.push('  stack:')
    for (const line of err.stack.split('\n').slice(0, 8)) {
      lines.push(`    ${line}`)
    }
  }

  return lines.join('\n')
}

function appendRunLog (run, exitCode, report, durationMs, spawnError) {
  const timestamp = new Date().toISOString()
  const stats = report?.stats || {}
  const passes = stats.passes ?? '?'
  const failures = stats.failures ?? '?'
  const pending = stats.pending ?? 0
  const passed = exitCode === 0 && !spawnError
  const result = passed ? 'PASS' : 'FAIL'
  const durationSec = (durationMs / 1000).toFixed(1)

  const block = [
    `=== RUN ${run} | ${timestamp} | ${result} | ${passes} pass ${failures} fail ${pending} pending | ${durationSec}s ===`,
  ]

  if (spawnError) {
    block.push(`SPAWN_ERROR: ${spawnError}`)
  }

  if (report?.failures?.length) {
    for (const failure of report.failures) {
      block.push(formatFailure(failure))
    }
  } else if (!passed && !report) {
    block.push('No JSON report captured (mocha may have crashed before writing results)')
  }

  block.push('')
  appendFileSync(logFile, `${block.join('\n')}\n`)

  const jsonl = {
    run,
    timestamp,
    exitCode,
    durationMs,
    passes,
    failures,
    pending,
    failureDetails: (report?.failures || []).map((failure) => ({
      fullTitle: failure.fullTitle || failure.title,
      file: failure.file,
      message: failure.err?.message,
      stack: failure.err?.stack,
    })),
  }
  appendFileSync(jsonlFile, `${JSON.stringify(jsonl)}\n`)
}

function runMochaOnce () {
  const reportPath = join(tmpdir(), `mocha-flakey-${randomUUID()}.json`)

  return new Promise((resolve) => {
    const proc = spawn('npx', [
      'mocha',
      'tests',
      '--recursive',
      '-t',
      '10000',
      '--reporter',
      'json',
      '--reporter-option',
      `output=${reportPath}`,
    ], {
      cwd: rootDir,
      stdio: ['ignore', 'inherit', 'inherit'],
    })

    proc.on('close', (exitCode) => {
      let report = null
      try {
        report = JSON.parse(readFileSync(reportPath, 'utf8'))
      } catch {
        report = null
      }

      try {
        unlinkSync(reportPath)
      } catch {
        // ignore missing temp report
      }

      resolve({ exitCode: exitCode ?? 1, report })
    })

    proc.on('error', (spawnError) => {
      resolve({ exitCode: 1, report: null, spawnError: spawnError.message })
    })
  })
}

async function main () {
  const header = [
    `# Flaky test run log started ${new Date().toISOString()}`,
    `# Log file: ${logFile}`,
    `# JSONL file: ${jsonlFile}`,
    '',
  ].join('\n')

  writeFileSync(logFile, `${header}\n`)
  writeFileSync(jsonlFile, '')

  console.log(`Logging flaky test runs to ${logFile}`)
  console.log(`Structured run data: ${jsonlFile}`)

  let run = 0

  while (true) {
    run += 1
    console.log(`\n--- Starting run ${run} ---`)

    const start = Date.now()
    const { exitCode, report, spawnError } = await runMochaOnce()
    const durationMs = Date.now() - start

    appendRunLog(run, exitCode, report, durationMs, spawnError)

    const summary = exitCode === 0 ? 'PASS' : 'FAIL'
    console.log(`Run ${run}: ${summary} (${(durationMs / 1000).toFixed(1)}s) — logged`)

    if (exitCode !== 0) {
      console.log(`Failure on run ${run}, stopping. See ${logFile}`)
      process.exit(exitCode)
    }
  }
}

main().catch((err) => {
  console.error(err)
  process.exit(1)
})