openvibers/openvibe.host Commits
Fewer false operator pages. OpenVibeDeployDrift reads max_over_time(openvibe_deploy_drift_seconds[2h]): a Network restart drops the series for about a minute, which reset the rule's for: 1h and made the alert resolve and fire again every few hours (2026-09-28). browser-watch checks a failed release again after 30 minutes instead of keeping it red until the next release or day, so a transient failure clears on its own; a pass still holds for a day.
Changes
@@@ -47,8 +47,10 @@ groups:annotations:summary: "No successful nightly backup for more than 30 hours (or no backup metrics at all)"# Production drift (OpenVibe.Network registry/deploy-drift.js, hourly against GitHub; WS-S task 7).++ # max_over_time: Network restarts drop the series for a minute, which reset `for` and made the alert++ # resolve and fire again every few hours (2026-09-28). Documentation-only drift is not counted.- alert: OpenVibeDeployDrift-- expr: openvibe_deploy_drift_seconds > 24 * 3600++ expr: max_over_time(openvibe_deploy_drift_seconds[2h]) > 24 * 3600for: 1hlabels: { severity: ticket }annotations:@@@ -25,6 +25,7 @@const path = require('path');const { runBrowserCheck } = require('./browser-check-hook');++const RECHECK_FAILED_MS = 30 * 60 * 1000;const DAY_MS = 24 * 3600 * 1000;const SETTLE_MS = 3 * 60 * 1000; // a release is checked once its process has run this longconst RETRY_DELAY_MS = 60 * 1000; // before the confirming second run@@@ -137,7 +138,8 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',const file = path.join(stateDir, `${site}.json`);let prev = null;try { prev = JSON.parse((await exec.readFile(file)) || 'null'); } catch { prev = null; }-- const fresh = prev && prev.release === release && now - Date.parse(prev.checkedAt) < DAY_MS;++ // A pass holds for a day; a failure is checked again after 30 minutes, so a transient one clears itself.++ const fresh = prev && prev.release === release && now - Date.parse(prev.checkedAt) < (prev.ok ? DAY_MS : RECHECK_FAILED_MS);if (fresh && !force) { out.push({ site, release, action: 'skipped', ok: prev.ok, reason: 'this release was checked' }); continue; }let r = null;let reports = null;@@@ -157,4 +159,4 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',return { sites: out };}--module.exports = { watch, failingChecks, failureDetails, writeMetrics, originOf, currentRelease, DAY_MS, SETTLE_MS };++module.exports = { watch, failingChecks, failureDetails, writeMetrics, originOf, currentRelease, DAY_MS, SETTLE_MS, RECHECK_FAILED_MS };@@@ -62,6 +62,13 @@ runTests([{ route: '/', check: 'errors', width: 390, message: 'WebSocket closed' },{ route: '/', check: 'overflow', width: 390, offenders: ['.wide'] },], 'what failed is kept for the operator');++ // The same failed release: left alone for 30 minutes, then checked again (a transient failure clears itself).++ const t0 = Date.parse(st.checkedAt);++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: t0 + 10 * 60000 });++ assert.deepStrictEqual([r.sites[0].action, host.runs.length], ['skipped', 2]);++ host.answers.push({ code: 0, reports: pass });++ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: t0 + 31 * 60000 });++ assert.deepStrictEqual([r.sites[0].action, r.sites[0].ok, host.runs.length], ['checked', true, 3]);host = site();host.answers.push({ code: 1, reports: fail }, { code: 0, reports: pass });r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
@@ -47,8 +47,10 @@ groups:
@@ -47,8 +47,10 @@ groups:
annotations:
annotations:
summary: "No successful nightly backup for more than 30 hours (or no backup metrics at all)"
summary: "No successful nightly backup for more than 30 hours (or no backup metrics at all)"
# Production drift (OpenVibe.Network registry/deploy-drift.js, hourly against GitHub; WS-S task 7).
# Production drift (OpenVibe.Network registry/deploy-drift.js, hourly against GitHub; WS-S task 7).
+ # max_over_time: Network restarts drop the series for a minute, which reset `for` and made the alert
+ # resolve and fire again every few hours (2026-09-28). Documentation-only drift is not counted.
- alert: OpenVibeDeployDrift
- alert: OpenVibeDeployDrift
- expr: openvibe_deploy_drift_seconds > 24 * 3600
+ expr: max_over_time(openvibe_deploy_drift_seconds[2h]) > 24 * 3600
for: 1h
for: 1h
labels: { severity: ticket }
labels: { severity: ticket }
annotations:
annotations:
@@ -25,6 +25,7 @@
@@ -25,6 +25,7 @@
const path = require('path');
const path = require('path');
const { runBrowserCheck } = require('./browser-check-hook');
const { runBrowserCheck } = require('./browser-check-hook');
+const RECHECK_FAILED_MS = 30 * 60 * 1000;
const DAY_MS = 24 * 3600 * 1000;
const DAY_MS = 24 * 3600 * 1000;
const SETTLE_MS = 3 * 60 * 1000; // a release is checked once its process has run this long
const SETTLE_MS = 3 * 60 * 1000; // a release is checked once its process has run this long
const RETRY_DELAY_MS = 60 * 1000; // before the confirming second run
const RETRY_DELAY_MS = 60 * 1000; // before the confirming second run
@@ -137,7 +138,8 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
@@ -137,7 +138,8 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
const file = path.join(stateDir, `${site}.json`);
const file = path.join(stateDir, `${site}.json`);
let prev = null;
let prev = null;
try { prev = JSON.parse((await exec.readFile(file)) || 'null'); } catch { prev = null; }
try { prev = JSON.parse((await exec.readFile(file)) || 'null'); } catch { prev = null; }
- const fresh = prev && prev.release === release && now - Date.parse(prev.checkedAt) < DAY_MS;
+ // A pass holds for a day; a failure is checked again after 30 minutes, so a transient one clears itself.
+ const fresh = prev && prev.release === release && now - Date.parse(prev.checkedAt) < (prev.ok ? DAY_MS : RECHECK_FAILED_MS);
if (fresh && !force) { out.push({ site, release, action: 'skipped', ok: prev.ok, reason: 'this release was checked' }); continue; }
if (fresh && !force) { out.push({ site, release, action: 'skipped', ok: prev.ok, reason: 'this release was checked' }); continue; }
let r = null;
let r = null;
let reports = null;
let reports = null;
@@ -157,4 +159,4 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
@@ -157,4 +159,4 @@ async function watch(ctx, { sites = ['live'], force = false, asUser = 'ovcheck',
return { sites: out };
return { sites: out };
}
}
-module.exports = { watch, failingChecks, failureDetails, writeMetrics, originOf, currentRelease, DAY_MS, SETTLE_MS };
+module.exports = { watch, failingChecks, failureDetails, writeMetrics, originOf, currentRelease, DAY_MS, SETTLE_MS, RECHECK_FAILED_MS };
@@ -62,6 +62,13 @@ runTests([
@@ -62,6 +62,13 @@ runTests([
{ route: '/', check: 'errors', width: 390, message: 'WebSocket closed' },
{ route: '/', check: 'errors', width: 390, message: 'WebSocket closed' },
{ route: '/', check: 'overflow', width: 390, offenders: ['.wide'] },
{ route: '/', check: 'overflow', width: 390, offenders: ['.wide'] },
], 'what failed is kept for the operator');
], 'what failed is kept for the operator');
+ // The same failed release: left alone for 30 minutes, then checked again (a transient failure clears itself).
+ const t0 = Date.parse(st.checkedAt);
+ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: t0 + 10 * 60000 });
+ assert.deepStrictEqual([r.sites[0].action, host.runs.length], ['skipped', 2]);
+ host.answers.push({ code: 0, reports: pass });
+ r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check, now: t0 + 31 * 60000 });
+ assert.deepStrictEqual([r.sites[0].action, r.sites[0].ok, host.runs.length], ['checked', true, 3]);
host = site();
host = site();
host.answers.push({ code: 1, reports: fail }, { code: 0, reports: pass });
host.answers.push({ code: 1, reports: fail }, { code: 0, reports: pass });
r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });
r = await watch(host.ctx(), { retryDelayMs: 0, check: host.check });