|
|
@@ -0,0 +1,164 @@
|
|
|
+#!/usr/bin/env node
|
|
|
+import { spawn } from 'node:child_process'
|
|
|
+import {
|
|
|
+ appendFileSync,
|
|
|
+ writeFileSync,
|
|
|
+ readFileSync,
|
|
|
+ unlinkSync,
|
|
|
+} from 'node:fs'
|
|
|
+import { dirname, join } from 'node:path'
|
|
|
+import { fileURLToPath } from 'node:url'
|
|
|
+import { tmpdir } from 'node:os'
|
|
|
+import { randomUUID } from 'node:crypto'
|
|
|
+
|
|
|
+const rootDir = join(dirname(fileURLToPath(import.meta.url)), '..')
|
|
|
+const logFile = process.env.FLAKEY_LOG_FILE || join(rootDir, 'flakey-test-runs.log')
|
|
|
+const jsonlFile = process.env.FLAKEY_JSONL_FILE || join(rootDir, 'flakey-test-runs.jsonl')
|
|
|
+
|
|
|
+function formatFailure (failure) {
|
|
|
+ const err = failure.err || {}
|
|
|
+ const lines = [
|
|
|
+ `FAILURE: ${failure.fullTitle || failure.title}`,
|
|
|
+ ` file: ${failure.file || 'unknown'}`,
|
|
|
+ ` message: ${(err.message || 'unknown').trim()}`,
|
|
|
+ ]
|
|
|
+
|
|
|
+ if (err.stack) {
|
|
|
+ lines.push(' stack:')
|
|
|
+ for (const line of err.stack.split('\n').slice(0, 8)) {
|
|
|
+ lines.push(` ${line}`)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return lines.join('\n')
|
|
|
+}
|
|
|
+
|
|
|
+function appendRunLog (run, exitCode, report, durationMs, spawnError) {
|
|
|
+ const timestamp = new Date().toISOString()
|
|
|
+ const stats = report?.stats || {}
|
|
|
+ const passes = stats.passes ?? '?'
|
|
|
+ const failures = stats.failures ?? '?'
|
|
|
+ const pending = stats.pending ?? 0
|
|
|
+ const passed = exitCode === 0 && !spawnError
|
|
|
+ const result = passed ? 'PASS' : 'FAIL'
|
|
|
+ const durationSec = (durationMs / 1000).toFixed(1)
|
|
|
+
|
|
|
+ const block = [
|
|
|
+ `=== RUN ${run} | ${timestamp} | ${result} | ${passes} pass ${failures} fail ${pending} pending | ${durationSec}s ===`,
|
|
|
+ ]
|
|
|
+
|
|
|
+ if (spawnError) {
|
|
|
+ block.push(`SPAWN_ERROR: ${spawnError}`)
|
|
|
+ }
|
|
|
+
|
|
|
+ if (report?.failures?.length) {
|
|
|
+ for (const failure of report.failures) {
|
|
|
+ block.push(formatFailure(failure))
|
|
|
+ }
|
|
|
+ } else if (!passed && !report) {
|
|
|
+ block.push('No JSON report captured (mocha may have crashed before writing results)')
|
|
|
+ }
|
|
|
+
|
|
|
+ block.push('')
|
|
|
+ appendFileSync(logFile, `${block.join('\n')}\n`)
|
|
|
+
|
|
|
+ const jsonl = {
|
|
|
+ run,
|
|
|
+ timestamp,
|
|
|
+ exitCode,
|
|
|
+ durationMs,
|
|
|
+ passes,
|
|
|
+ failures,
|
|
|
+ pending,
|
|
|
+ failureDetails: (report?.failures || []).map((failure) => ({
|
|
|
+ fullTitle: failure.fullTitle || failure.title,
|
|
|
+ file: failure.file,
|
|
|
+ message: failure.err?.message,
|
|
|
+ stack: failure.err?.stack,
|
|
|
+ })),
|
|
|
+ }
|
|
|
+ appendFileSync(jsonlFile, `${JSON.stringify(jsonl)}\n`)
|
|
|
+}
|
|
|
+
|
|
|
+function runMochaOnce () {
|
|
|
+ const reportPath = join(tmpdir(), `mocha-flakey-${randomUUID()}.json`)
|
|
|
+
|
|
|
+ return new Promise((resolve) => {
|
|
|
+ const proc = spawn('npx', [
|
|
|
+ 'mocha',
|
|
|
+ 'tests',
|
|
|
+ '--recursive',
|
|
|
+ '-t',
|
|
|
+ '10000',
|
|
|
+ '--reporter',
|
|
|
+ 'json',
|
|
|
+ '--reporter-option',
|
|
|
+ `output=${reportPath}`,
|
|
|
+ ], {
|
|
|
+ cwd: rootDir,
|
|
|
+ stdio: ['ignore', 'inherit', 'inherit'],
|
|
|
+ })
|
|
|
+
|
|
|
+ proc.on('close', (exitCode) => {
|
|
|
+ let report = null
|
|
|
+ try {
|
|
|
+ report = JSON.parse(readFileSync(reportPath, 'utf8'))
|
|
|
+ } catch {
|
|
|
+ report = null
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ unlinkSync(reportPath)
|
|
|
+ } catch {
|
|
|
+ // ignore missing temp report
|
|
|
+ }
|
|
|
+
|
|
|
+ resolve({ exitCode: exitCode ?? 1, report })
|
|
|
+ })
|
|
|
+
|
|
|
+ proc.on('error', (spawnError) => {
|
|
|
+ resolve({ exitCode: 1, report: null, spawnError: spawnError.message })
|
|
|
+ })
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+async function main () {
|
|
|
+ const header = [
|
|
|
+ `# Flaky test run log started ${new Date().toISOString()}`,
|
|
|
+ `# Log file: ${logFile}`,
|
|
|
+ `# JSONL file: ${jsonlFile}`,
|
|
|
+ '',
|
|
|
+ ].join('\n')
|
|
|
+
|
|
|
+ writeFileSync(logFile, `${header}\n`)
|
|
|
+ writeFileSync(jsonlFile, '')
|
|
|
+
|
|
|
+ console.log(`Logging flaky test runs to ${logFile}`)
|
|
|
+ console.log(`Structured run data: ${jsonlFile}`)
|
|
|
+
|
|
|
+ let run = 0
|
|
|
+
|
|
|
+ while (true) {
|
|
|
+ run += 1
|
|
|
+ console.log(`\n--- Starting run ${run} ---`)
|
|
|
+
|
|
|
+ const start = Date.now()
|
|
|
+ const { exitCode, report, spawnError } = await runMochaOnce()
|
|
|
+ const durationMs = Date.now() - start
|
|
|
+
|
|
|
+ appendRunLog(run, exitCode, report, durationMs, spawnError)
|
|
|
+
|
|
|
+ const summary = exitCode === 0 ? 'PASS' : 'FAIL'
|
|
|
+ console.log(`Run ${run}: ${summary} (${(durationMs / 1000).toFixed(1)}s) — logged`)
|
|
|
+
|
|
|
+ if (exitCode !== 0) {
|
|
|
+ console.log(`Failure on run ${run}, stopping. See ${logFile}`)
|
|
|
+ process.exit(exitCode)
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+main().catch((err) => {
|
|
|
+ console.error(err)
|
|
|
+ process.exit(1)
|
|
|
+})
|