openvibers/openvibe.host Commits

browser-watch: a release is judged once its process has run 3 minutes (/release.json booted_at), and the confirming second run of a failure waits a minute first. On 2026-09-26 a check minutes after a Live restart failed on 4 console errors from reconnecting sockets and paged; a run a few minutes later passed. --force still checks at once.

OP OpenVibers committed 26 Sept 2026 · 87bca98b15 · 2 files, +49 −20


Changes

ddiff --git a/lib/browser-watch.js b/lib/browser-watch.js
iindex a78e478..818aacf 100644
---- a/lib/browser-watch.js
++++ b/lib/browser-watch.js
@@@ -7,8 +7,9 @@
* ovhost browser-watch [--sites live,network] [--force] (openvibe-browsercheck.timer: every 5 minutes)
*
* Per site: GET <origin>/release.json. A release not checked yet, or a last check older than 24 hours, runs the
-- * check as the unprivileged `ovcheck` account (Chrome never runs as root). A failing check runs once more before
-- * it counts (a single network blip is not a regression). The outcome is kept in
++ * check as the unprivileged `ovcheck` account (Chrome never runs as root), once its process has run 3 minutes
++ * (/release.json booted_at). A failing check runs once more, a minute later, before it counts (a single network blip or
++ * a restart's reconnects are not a regression). The outcome is kept in
* <stateDir>/browser-watch/<site>.json, and every site's latest outcome is written to the textfile collector
* (openvibe_browsercheck.prom):
*
@@@ -25,6 +26,8 @@ const path = require('path');
const { runBrowserCheck } = require('./browser-check-hook');
const DAY_MS = 24 * 3600 * 1000;
++const SETTLE_MS = 3 * 60 * 1000; // a release is checked once its process has run this long
++const RETRY_DELAY_MS = 60 * 1000; // before the confirming second run
const TEXTFILE_DIR = '/var/lib/prometheus/node-exporter';
const SITE_RE = /^[a-z][a-z0-9-]{1,39}$/;
const RELEASE_RE = /^[0-9A-Za-z._+-]{1,64}$/;
@@@ -38,14 +41,16 @@ function originOf(inv, id, contracts) {
return m && m.publicOrigin ? String(m.publicOrigin).replace(/\/+$/, '') : null;
}
--async function currentRelease(exec, origin) {
++async function currentRelease(exec, origin, { withAge = false } = {}) {
const r = await exec.request(`${origin}/release.json`, { timeoutMs: 8000 });
-- if (r.status !== 200) return null;
++ if (r.status !== 200) return withAge ? { release: null, bootedAt: null } : null;
try {
const m = JSON.parse(r.body);
const rel = String(m.release || m.commit || '').slice(0, 12);
-- return RELEASE_RE.test(rel) ? rel : null;
-- } catch { return null; }
++ const release = RELEASE_RE.test(rel) ? rel : null;
++ const booted = Date.parse(m.booted_at || m.released_at || '');
++ return withAge ? { release, bootedAt: Number.isFinite(booted) ? booted : null } : release;
++ } catch { return withAge ? { release: null, bootedAt: null } : null; }
}
/** The failing checks of a --json report list: [{ check, fail }]. */
@@@ -94,7 +99,7 @@ async function writeMetrics(exec, inv) {
* One pass over the sites. Answers { sites: [{ site, release, action: 'checked'|'skipped'|'error', ok?, failing?, reason? }] }.
* opts: sites (ids), force, asUser (default ovcheck), home, now, check (the runBrowserCheck to use; tests).
*/
--async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck', home = '/var/lib/ovcheck', now = Date.now(), check = runBrowserCheck, contracts = null } = {}) {
++async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck', home = '/var/lib/ovcheck', now = Date.now(), check = runBrowserCheck, contracts = null, settleMs = SETTLE_MS, retryDelayMs = RETRY_DELAY_MS, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
const { exec, inv } = ctx;
const stateDir = path.join(inv.stateDir, 'browser-watch');
await exec.mkdir(stateDir, { mode: 0o750 });
@@@ -103,7 +108,10 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
if (!SITE_RE.test(site)) { out.push({ site, action: 'error', reason: 'not a service id' }); continue; }
const origin = originOf(inv, site, contracts);
if (!origin) { out.push({ site, action: 'error', reason: 'no origin (inventory origin or manifest publicOrigin)' }); continue; }
-- const release = await currentRelease(exec, origin);
++ const { release, bootedAt } = await currentRelease(exec, origin, { withAge: true });
++ // A process that just started (a deploy, a restart) gets a few minutes before it is judged: sockets
++ // reconnect and caches warm, and the console errors of those minutes are not the release's.
++ if (!force && bootedAt && now - bootedAt < settleMs) { out.push({ site, release, action: 'skipped', reason: 'the release is settling' }); continue; }
const file = path.join(stateDir, `${site}.json`);
let prev = null;
try { prev = JSON.parse((await exec.readFile(file)) || 'null'); } catch { prev = null; }
@@@ -112,6 +120,7 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
let r = null;
let reports = null;
for (let attempt = 1; attempt <= 2; attempt++) {
++ if (attempt === 2 && retryDelayMs > 0) await sleep(retryDelayMs); // a transient blip gets a minute to pass
r = await check(site, { args: ['--json'], asUser, home });
try { reports = JSON.parse(r.stdout || 'null'); } catch { reports = null; }
if (r.code === 0 || r.code >= 2 || !reports) break; // a pass, or a run that broke: no second attempt
@@@ -126,4 +135,4 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
return { sites: out };
}
--module.exports = { watch, failingChecks, writeMetrics, originOf, currentRelease, DAY_MS };
++module.exports = { watch, failingChecks, writeMetrics, originOf, currentRelease, DAY_MS, SETTLE_MS };
ddiff --git a/test/browser-watch.test.js b/test/browser-watch.test.js
iindex e896409..986f521 100644
---- a/test/browser-watch.test.js
++++ b/test/browser-watch.test.js
@@@ -19,7 +19,7 @@ function site({ release = 'abc1234def56' } = {}) {
host.inv.services.live.origin = 'https://openvibe.live';
host.put('/var/lib/prometheus/node-exporter/.keep', '');
host.release = release;
-- host.http.set('https://openvibe.live/release.json', () => ({ status: 200, body: { service: 'live', release: host.release } }));
++ host.http.set('https://openvibe.live/release.json', () => ({ status: 200, body: { service: 'live', release: host.release, booted_at: host.bootedAt || '2026-09-01T00:00:00Z' } }));
host.runs = [];
host.answers = [];
host.check = async (s, opts) => { host.runs.push({ s, opts }); const a = host.answers.shift() || { code: 0, reports: pass }; return { code: a.code, stdout: a.reports ? JSON.stringify(a.reports) : '', error: a.error || null }; };
@@@ -30,48 +30,68 @@ runTests([
test('a new release is checked as ovcheck with --json; the same release is skipped; a day later it is checked again', async () => {
const host = site();
const now = Date.parse('2026-09-26T14:00:00Z');
-- let r = await watch(host.ctx(), { check: host.check, now });
++ let r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now });
assert.deepStrictEqual(r.sites.map((x) => [x.site, x.release, x.action, x.ok]), [['live', 'abc1234def56', 'checked', true]]);
assert.deepStrictEqual(host.runs.map((x) => [x.s, x.opts.asUser, x.opts.home, x.opts.args]), [['live', 'ovcheck', '/var/lib/ovcheck', ['--json']]]);
assert.match(host.files.get(PROM).content, /^openvibe_browser_check_ok\{service="live"\} 1$/m);
assert.match(host.files.get(PROM).content, /^openvibe_browser_check_release_info\{service="live",release="abc1234def56"\} 1$/m);
assert.match(host.files.get(PROM).content, /^openvibe_browser_check_failing_checks\{service="live"\} 0$/m);
-- r = await watch(host.ctx(), { check: host.check, now: now + 60000 });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: now + 60000 });
assert.strictEqual(r.sites[0].action, 'skipped');
assert.strictEqual(host.runs.length, 1);
-- r = await watch(host.ctx(), { check: host.check, now: now + DAY_MS + 1 });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: now + DAY_MS + 1 });
assert.strictEqual(r.sites[0].action, 'checked', 'daily even without a release');
host.release = 'fff0000aaa11';
-- r = await watch(host.ctx(), { check: host.check, now: now + DAY_MS + 60000 });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: now + DAY_MS + 60000 });
assert.deepStrictEqual([r.sites[0].action, r.sites[0].release], ['checked', 'fff0000aaa11']);
-- r = await watch(host.ctx(), { check: host.check, now: now + DAY_MS + 120000, force: true });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: now + DAY_MS + 120000, force: true });
assert.strictEqual(r.sites[0].action, 'checked', '--force');
}),
test('a failure is confirmed by a second run before it counts; one flake then a pass is a pass', async () => {
let host = site();
host.answers.push({ code: 1, reports: fail }, { code: 1, reports: fail });
-- let r = await watch(host.ctx(), { check: host.check });
++ let r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
assert.strictEqual(host.runs.length, 2);
assert.deepStrictEqual([r.sites[0].ok, r.sites[0].failing], [false, [{ check: 'console', fail: 2 }, { check: 'overflow', fail: 1 }]]);
assert.match(host.files.get(PROM).content, /^openvibe_browser_check_ok\{service="live"\} 0$/m);
assert.match(host.files.get(PROM).content, /^openvibe_browser_check_failing_checks\{service="live"\} 2$/m);
host = site();
host.answers.push({ code: 1, reports: fail }, { code: 0, reports: pass });
-- r = await watch(host.ctx(), { check: host.check });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
assert.deepStrictEqual([host.runs.length, r.sites[0].ok], [2, true]);
}),
++ test('a release that just booted settles for 3 minutes before it is judged; --force does not wait', async () => {
++ const host = site();
++ const now = Date.parse('2026-09-26T16:30:00Z');
++ host.bootedAt = '2026-09-26T16:29:00Z';
++ let r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now });
++ assert.deepStrictEqual([r.sites[0].action, r.sites[0].reason, host.runs.length], ['skipped', 'the release is settling', 0]);
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: now + 3 * 60 * 1000 });
++ assert.strictEqual(r.sites[0].action, 'checked');
++ host.bootedAt = '2026-09-26T16:40:00Z';
++ host.release = 'bbb1111ccc22';
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: Date.parse('2026-09-26T16:40:30Z'), force: true });
++ assert.strictEqual(r.sites[0].action, 'checked');
++ }),
++ test('the confirming second run waits first', async () => {
++ const host = site();
++ host.answers.push({ code: 1, reports: fail }, { code: 0, reports: pass });
++ const waits = [];
++ await watch(host.ctx(), { check: host.check, retryDelayMs: 60000, sleep: async (ms) => { waits.push(ms); } });
++ assert.deepStrictEqual(waits, [60000]);
++ }),
test('a run that broke (exit 2) records nothing and is retried by the next timer run', async () => {
const host = site();
host.answers.push({ code: 2, error: 'Chrome not found' });
-- let r = await watch(host.ctx(), { check: host.check });
++ let r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
assert.deepStrictEqual([r.sites[0].action, r.sites[0].reason, host.runs.length], ['error', 'Chrome not found', 1]);
assert.ok(!host.files.has('/var/lib/openvibe-host/browser-watch/live.json'));
-- r = await watch(host.ctx(), { check: host.check });
++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
assert.strictEqual(r.sites[0].action, 'checked');
}),
test('bad ids and sites without an origin are errors, not checks; the CLI exits 1 on a failed check and 2 on an error', async () => {
const host = site();
-- const r = await watch(host.ctx(), { check: host.check, sites: ['Bad!', 'nosuchservice'], contracts: { services: { get: () => null } } });
++ const r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, sites: ['Bad!', 'nosuchservice'], contracts: { services: { get: () => null } } });
assert.deepStrictEqual(r.sites.map((x) => x.action), ['error', 'error']);
assert.strictEqual(host.runs.length, 0);
const c = await scenario().cli('browser-watch', '--sites', 'Bad!');