openvibers/openvibe.shared test-runner.js
test-runner.js
| 1 | ; |
| 2 | // ═══════════════════════════════════════════════════════════════ |
| 3 | // openvibe-shared/test-runner — the `npm test` runner every service uses (test/run.js), with no fake green |
| 4 | // (roadmap WS-Q task 7: "skipped counts as yellow; nothing may be marked green by being synthesised from |
| 5 | // skipped parts"). |
| 6 | // |
| 7 | // // test/run.js |
| 8 | // require('openvibe-shared/test-runner').main({ dir: __dirname, timeoutMs: 120000, hide: /^\[DB\] / }); |
| 9 | // |
| 10 | // npm test # every *.test.js in dir, each in its own process, a few at a time |
| 11 | // npm test -- quota cache # only files whose name contains one of the words |
| 12 | // npm test -- --strict # a skip fails the run too (or OV_TEST_STRICT=1) |
| 13 | // npm test -- page --allow-empty # exit 0 when no file matches (or OV_TEST_ALLOW_EMPTY=1): |
| 14 | // # the pin-only CI subset on a repository without those files |
| 15 | // |
| 16 | // Each test file runs in its own process and owns its database: openvibe-sdk/testing `createTestDb` gives it its |
| 17 | // own PGlite, or under `test:pg` its own PostgreSQL schema, roles and port. Files therefore parallelize safely and |
| 18 | // a repo needs no `parallel: 1` pin. `parallel` is a number (a pin), `'auto'` (min(8, cpus-1), at least 1) or unset |
| 19 | // (min(4, cpus-1), at least 1). |
| 20 | // |
| 21 | // A test that cannot run something (no ffmpeg, no Chrome, no git history) prints one line per thing it |
| 22 | // skipped, `<label>: skipped (<why>)`, and still exits 0. Such a file is listed with ○ and its reasons, |
| 23 | // and is NOT counted as passed: the summary reads `148/150 test files passed, 2 skipped (…)`. Only a run |
| 24 | // with nothing skipped prints `N/N test files passed`. Exit status: 1 when a file failed (or, strict, when |
| 25 | // one skipped), else 0. |
| 26 | // ═══════════════════════════════════════════════════════════════ |
| 27 | const { spawn } = require('child_process'); |
| 28 | const fs = require('fs'); |
| 29 | const os = require('os'); |
| 30 | const path = require('path'); |
| 31 | |
| 32 | /** `<label>: skipped (<why>)` on a line of its own (the convention OpenVibe.Host's acceptance runner reads too). */ |
| 33 | const SKIP_RE = /^[ \t]*[\w .,'()/+#-]{1,120}: skipped \((.+)\)[ \t]*$/gm; |
| 34 | |
| 35 | /** The reasons a test's output gives for what it skipped, in order, without repeats. */ |
| 36 | function skipsIn(output) { |
| 37 | const out = []; |
| 38 | for (const m of String(output || '').matchAll(SKIP_RE)) { |
| 39 | const line = m[0].trim(); |
| 40 | if (!out.includes(line)) out.push(line); |
| 41 | } |
| 42 | return out; |
| 43 | } |
| 44 | |
| 45 | /** How many files run at once: a numeric pin as given (at least 1), 'auto' by CPUs (1..8), else the default (1..4). */ |
| 46 | function resolveParallel(parallel) { |
| 47 | const cpus = os.cpus().length; |
| 48 | if (parallel === 'auto') return Math.max(1, Math.min(8, cpus - 1)); |
| 49 | const n = Math.floor(Number(parallel)); |
| 50 | if (Number.isFinite(n) && n >= 1) return n; |
| 51 | return Math.max(1, Math.min(4, cpus - 1)); |
| 52 | } |
| 53 | |
| 54 | function runOne(file, { dir, cwd, env, timeoutMs, nodeArgs }) { |
| 55 | return new Promise((resolve) => { |
| 56 | const started = Date.now(); |
| 57 | const child = spawn(process.execPath, [...nodeArgs, path.join(dir, file)], { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }); |
| 58 | let output = ''; |
| 59 | child.stdout.on('data', (c) => { output += c; }); |
| 60 | child.stderr.on('data', (c) => { output += c; }); |
| 61 | const timer = setTimeout(() => { output += `\n[run] timed out after ${timeoutMs}ms`; child.kill('SIGKILL'); }, timeoutMs); |
| 62 | child.on('close', (code, signal) => { |
| 63 | clearTimeout(timer); |
| 64 | const ok = code === 0; |
| 65 | const skips = ok ? skipsIn(output) : []; |
| 66 | resolve({ file, ok, state: !ok ? 'fail' : skips.length ? 'skip' : 'pass', skips, code: code ?? signal, ms: Date.now() - started, output }); |
| 67 | }); |
| 68 | }); |
| 69 | } |
| 70 | |
| 71 | /** |
| 72 | * Run the tests; resolves { results, passed, skipped, failed, strict, exitCode, summary } and prints as it goes. |
| 73 | * opts: dir (required: the test directory), argv (default process.argv.slice(2)), cwd (default dir/..), |
| 74 | * timeoutMs (60000), parallel (a number, 'auto' = min(8, cpus-1), default min(4, cpus-1); resolved value returned), env (added to process.env; NODE_ENV=test by default), |
| 75 | * hide (a RegExp: output lines of a failed file left out, e.g. /^\[DB\] /), pad (file-name column, 44), |
| 76 | * match (which files, default /\.test\.js$/), serial (files to run alone after parallel files), |
| 77 | * retry (a RegExp: a failed file that matches is run once more; a pass on the second try counts, and is named in the summary), |
| 78 | * nodeArgs ([]), log (console.log). |
| 79 | */ |
| 80 | async function run(opts = {}) { |
| 81 | const dir = opts.dir; |
| 82 | if (!dir) throw new TypeError('test-runner: dir required'); |
| 83 | const argv = opts.argv || process.argv.slice(2); |
| 84 | const log = opts.log || console.log; |
| 85 | const strict = argv.includes('--strict') || process.env.OV_TEST_STRICT === '1'; |
| 86 | const allowEmpty = argv.includes('--allow-empty') || process.env.OV_TEST_ALLOW_EMPTY === '1'; |
| 87 | const filters = argv.filter((a) => !a.startsWith('-')); |
| 88 | const match = opts.match || /\.test\.js$/; |
| 89 | const files = fs.readdirSync(dir).filter((f) => match.test(f)) |
| 90 | .filter((f) => !filters.length || filters.some((w) => f.includes(w))).sort(); |
| 91 | if (!files.length) { |
| 92 | if (allowEmpty) { |
| 93 | log('no test files matched (allowed)'); |
| 94 | return { results: [], passed: 0, skipped: 0, failed: 0, strict, exitCode: 0, summary: 'no test files matched (allowed)' }; |
| 95 | } |
| 96 | log('no test files matched'); |
| 97 | return { results: [], passed: 0, skipped: 0, failed: 0, strict, exitCode: 1, summary: 'no test files matched' }; |
| 98 | } |
| 99 | const cfg = { |
| 100 | dir, cwd: opts.cwd || path.join(dir, '..'), timeoutMs: opts.timeoutMs || 60000, nodeArgs: opts.nodeArgs || [], |
| 101 | env: { ...process.env, NODE_ENV: 'test', ...(opts.env || {}) }, |
| 102 | }; |
| 103 | const parallel = resolveParallel(opts.parallel); |
| 104 | const pad = opts.pad || 44; |
| 105 | const queue = [], serial = []; |
| 106 | for (const file of files) (opts.serial && opts.serial.test(file) ? serial : queue).push(file); |
| 107 | const results = []; |
| 108 | // A file the caller marks as worth a second try (browser tests on a slow machine) gets one. |
| 109 | const runFile = async (file) => { |
| 110 | let r = await runOne(file, cfg); |
| 111 | if (r.state === 'fail' && opts.retry && opts.retry.test(file)) { |
| 112 | log(`↻ ${file.padEnd(pad)} failed once (exit ${r.code}); a second try`); |
| 113 | const again = await runOne(file, cfg); |
| 114 | r = { ...again, ms: r.ms + again.ms, retried: true }; |
| 115 | } |
| 116 | return r; |
| 117 | }; |
| 118 | await Promise.all(Array.from({ length: parallel }, async () => { |
| 119 | while (queue.length) { |
| 120 | const r = await runFile(queue.shift()); |
| 121 | results.push(r); |
| 122 | const mark = r.state === 'pass' ? '✓' : r.state === 'skip' ? '○' : '✗'; |
| 123 | log(`${mark} ${r.file.padEnd(pad)} ${String(r.ms).padStart(6)}ms${r.state === 'skip' ? ` ${r.skips.join('; ')}` : ''}`); |
| 124 | } |
| 125 | })); |
| 126 | for (const file of serial) { |
| 127 | const r = await runFile(file); |
| 128 | results.push(r); |
| 129 | const mark = r.state === 'pass' ? '✓' : r.state === 'skip' ? '○' : '✗'; |
| 130 | log(`${mark} ${r.file.padEnd(pad)} ${String(r.ms).padStart(6)}ms${r.state === 'skip' ? ` ${r.skips.join('; ')}` : ''}`); |
| 131 | } |
| 132 | results.sort((a, b) => a.file.localeCompare(b.file)); |
| 133 | const failed = results.filter((r) => r.state === 'fail'); |
| 134 | const skipped = results.filter((r) => r.state === 'skip'); |
| 135 | for (const r of failed) { |
| 136 | log(`\n── ${r.file} (exit ${r.code}) ──`); |
| 137 | log(r.output.split('\n').filter((l) => !(opts.hide && opts.hide.test(l))).slice(-40).join('\n')); |
| 138 | } |
| 139 | const passed = results.length - failed.length - skipped.length; |
| 140 | const second = results.filter((r) => r.retried && r.state === 'pass').map((r) => r.file); |
| 141 | const summary = (skipped.length |
| 142 | ? `${passed}/${results.length} test files passed, ${skipped.length} skipped (${skipped.map((r) => `${r.file}: ${r.skips.join('; ')}`).join(' | ')})${strict ? ' — strict: skips fail the run' : ''}` |
| 143 | : `${passed}/${results.length} test files passed`) |
| 144 | + (second.length ? `; passed on a second try: ${second.join(', ')}` : ''); |
| 145 | log(`\n${summary}`); |
| 146 | const exitCode = failed.length || (strict && skipped.length) ? 1 : 0; |
| 147 | return { results, passed, skipped: skipped.length, failed: failed.length, strict, exitCode, summary, parallel, secondTry: second }; |
| 148 | } |
| 149 | |
| 150 | /** run(), then exit with its status: what a test/run.js calls. */ |
| 151 | function main(opts) { |
| 152 | run(opts).then((r) => process.exit(r.exitCode), (err) => { console.error(err); process.exit(1); }); |
| 153 | } |
| 154 | |
| 155 | module.exports = { run, main, skipsIn, resolveParallel, SKIP_RE }; |
| 156 |