openvibers/openvibe.network Commits
Operator alerts page once per episode and stop piling up (the bell had Alert/Resolved pairs every few hours). An opened alert creates one notification per operator; a critical one is pushed, a warning stays in the bell (create() takes silent). When it resolves, the same notifications are revised in place to Resolved, marked read, with how long it fired, instead of a new entry. One that fires again within 12 hours of resolving revises them back (Fired again n time(s)) and creates or pushes nothing. Only critical alerts get the 24-hour reminder, and info alerts are listed, never notified. The notification service gains revise(id, userId, fields); operator_alerts gains notices and reopened. Deploy drift: main ahead by documentation and tests only (README, STATUS.json, docs/*.md, tests, .github/: the files the deploy controller skips) is current, not drift.
Changes
@@@ -508,8 +508,9 @@ router.post('/operator/alerts', principals.guard('network.operator.alert', { legif (!v.valid) return res.status(400).json({ error: 'The body does not match network.operator-alerts-request@1', details: (v.errors || []).slice(0, 5) });const notifService = req.app.locals.notificationService;const notify = notifService ? (userId, n) => notifService.create({ user_id: userId, ...n }) : null;++ const revise = notifService ? (userId, id, fields) => notifService.revise(id, userId, fields) : null;try {-- res.json(require('../operator/alerts').receive(getDb(req), req.body, { notify }));++ res.json(require('../operator/alerts').receive(getDb(req), req.body, { notify, revise }));} catch (err) {console.error('[Internal] Operator alerts error:', err.message);res.status(500).json({ error: 'Operator alerts failed' });@@@ -194,8 +194,8 @@ class NotificationService {// Wake the relay (it reads after the outermost transaction commits: better-sqlite3 is synchronous).if (announced) { const live = eventRelay.outboxFor(this.db); if (live) live.kick(); }-- // Fire browser push notification (async, non-blocking)-- try {++ // Fire browser push notification (async, non-blocking); `silent` keeps it to the bell.++ if (!data.silent) try {const pushService = require('../push/push-service');pushService.sendPush(data.user_id, {title,@@@ -340,6 +340,18 @@ class NotificationService {// ─── Update ────────────────────────────────────────────────++ /** Edit one of a user's notifications in place (an operator alert that resolves or fires again). */++ revise(id, userId, { title, message, icon, is_read: isRead } = {}) {++ const sets = []; const args = [];++ if (title != null) { sets.push('title = ?'); args.push(String(title).slice(0, 200)); }++ if (message != null) { sets.push('message = ?'); args.push(String(message)); }++ if (icon != null) { sets.push('icon = ?'); args.push(String(icon)); }++ if (isRead != null) { sets.push('is_read = ?'); args.push(isRead ? 1 : 0); }++ if (!sets.length) return false;++ const r = this.db.prepare(`UPDATE notifications SET ${sets.join(', ')} WHERE id = ? AND user_id = ?`).run(...args, id, userId);++ return r.changes > 0;++ }++markRead(id, userId) {return this._markRead.run(id, userId).changes > 0;}@@@ -5,11 +5,16 @@* The production host's Prometheus evaluates alert rules (backups, the developer path, the Tools job proof,* the browser check), but nothing delivered them. Host relays the complete set of alerts firing now* (`ovhost alerts relay`, every two minutes) to POST /internal/operator/alerts, and this module turns the-- * difference into notifications for the operator:++ * difference into notifications for the operator. One notification per alert episode, kept up to date in place,++ * so a flapping or resolving alert never piles up new entries (the bell had open/resolved pairs every few hours):*-- * opened an alert not firing before critical → priority critical, else high-- * reminded still firing 24 h after the last page same priority-- * resolved firing before, absent from this set priority normal++ * opened an alert not firing before critical: a pushed notification; warning: in the bell only++ * reopened fires again within 12 h of resolving the same notification is revised (Alert again, times counted);++ * nothing new is created or pushed++ * reminded a critical still firing 24 h after a new pushed notification (warnings are not reminded)++ * the last page++ * resolved firing before, absent from this set its notifications are revised to Resolved and marked read++ * info never notified; listed only** The operator is the owner account (OWNER_USERNAME), plus any usernames in OPERATOR_ALERT_USERNAMES* (comma-separated). The notification is type OPERATOR_ALERT in category `admin`, so it reaches the bell,@@@ -18,6 +23,7 @@*/const REMIND_MS = 24 * 3600 * 1000;++const FLAP_MS = 12 * 3600 * 1000;const KEEP_RESOLVED_MS = 30 * 24 * 3600 * 1000;const STATUS_URL = 'https://openvibe.network/status';@@@ -38,6 +44,10 @@ function ensureTables(db) {resolved_at INTEGER,PRIMARY KEY (source, fingerprint))`);++ // The notifications of the current episode ([[userId, notificationId], …]) and how often it fired again.++ for (const col of ['notices TEXT', 'reopened INTEGER NOT NULL DEFAULT 0']) {++ try { db.exec(`ALTER TABLE operator_alerts ADD COLUMN ${col}`); } catch { /* already there */ }++ }}function operatorUserIds(db, env = process.env) {@@@ -51,59 +61,87 @@ function message(a) {return [a.description || a.summary, a.service ? `Service: ${a.service}.` : null, `Firing since ${a.started_at}.`].filter(Boolean).join(' ');}++const ago = (ms) => (ms < 3600e3 ? `${Math.max(1, Math.round(ms / 60e3))} min` : `${Math.round(ms / 3600e3)} h`);++/*** Apply one report. `body` has passed the request contract. Answers the result document.-- * notify(userId, notification) creates one notification (the notification service's create()).++ * notify(userId, notification) creates one notification (the notification service's create(); `silent` skips++ * web push); revise(userId, notificationId, fields) edits one in place (title, message, icon, is_read).*/--function receive(db, body, { notify, now = Date.now(), env = process.env } = {}) {++function receive(db, body, { notify, revise = null, now = Date.now(), env = process.env } = {}) {ensureTables(db);const source = body.source;const current = new Map(body.alerts.map((a) => [a.fingerprint, a]));const rows = db.prepare('SELECT * FROM operator_alerts WHERE source = ?').all(source);const byFp = new Map(rows.map((r) => [r.fingerprint, r]));-- const pages = [];++ const work = [];const out = { ok: true, firing: current.size, opened: 0, reminded: 0, resolved: 0, notified: 0 };-- const upsert = db.prepare(`INSERT INTO operator_alerts (source, fingerprint, name, severity, summary, description, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at)-- VALUES (@source, @fingerprint, @name, @severity, @summary, @description, @service, 'firing', @started_at, @now, @now, @notified_at, NULL)++ const upsert = db.prepare(`INSERT INTO operator_alerts (source, fingerprint, name, severity, summary, description, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at, reopened)++ VALUES (@source, @fingerprint, @name, @severity, @summary, @description, @service, 'firing', @started_at, @now, @now, @notified_at, NULL, 0)ON CONFLICT (source, fingerprint) DO UPDATE SET name = @name, severity = @severity, summary = @summary, description = @description, service = @service,state = 'firing', started_at = @started_at, last_seen_at = @now, notified_at = @notified_at, resolved_at = NULL,-- opened_at = CASE WHEN operator_alerts.state = 'resolved' THEN @now ELSE operator_alerts.opened_at END`);-- const resolve = db.prepare("UPDATE operator_alerts SET state = 'resolved', resolved_at = ?, notified_at = ? WHERE source = ? AND fingerprint = ?");++ reopened = operator_alerts.reopened + @reopen,++ opened_at = CASE WHEN operator_alerts.state = 'resolved' AND @reopen = 0 THEN @now ELSE operator_alerts.opened_at END`);++ const resolve = db.prepare("UPDATE operator_alerts SET state = 'resolved', resolved_at = ? WHERE source = ? AND fingerprint = ?");db.transaction(() => {for (const a of current.values()) {const prev = byFp.get(a.fingerprint);let kind = null;-- if (!prev || prev.state === 'resolved') kind = 'opened';-- else if (!prev.notified_at || now - prev.notified_at >= REMIND_MS) kind = 'reminded';++ if (!prev || prev.state === 'resolved') {++ kind = prev && prev.resolved_at && now - prev.resolved_at < FLAP_MS && prev.notices ? 'reopened' : 'opened';++ out.opened += 1;++ } else if (a.severity === 'critical' && (!prev.notified_at || now - prev.notified_at >= REMIND_MS)) {++ kind = 'reminded';++ out.reminded += 1;++ }++ const paged = kind === 'opened' || kind === 'reminded';upsert.run({ source, fingerprint: a.fingerprint, name: a.name, severity: a.severity, summary: a.summary, description: a.description || null,-- service: a.service || null, started_at: a.started_at, now, notified_at: kind ? now : (prev && prev.notified_at) || null });-- if (kind) { out[kind] += 1; pages.push({ kind, a }); }++ service: a.service || null, started_at: a.started_at, now, reopen: kind === 'reopened' ? 1 : 0,++ notified_at: paged ? now : (prev && prev.notified_at) || null });++ if (kind) work.push({ kind, a, prev });}for (const r of rows) {if (r.state !== 'firing' || current.has(r.fingerprint)) continue;-- resolve.run(now, now, source, r.fingerprint);++ resolve.run(now, source, r.fingerprint);out.resolved += 1;-- pages.push({ kind: 'resolved', a: r });++ work.push({ kind: 'resolved', a: r, prev: r });}db.prepare("DELETE FROM operator_alerts WHERE state = 'resolved' AND resolved_at < ?").run(now - KEEP_RESOLVED_MS);})();-- if (pages.length && typeof notify === 'function') {-- const users = operatorUserIds(db, env);-- for (const { kind, a } of pages) {-- const resolved = kind === 'resolved';++ const notices = (row) => { try { return JSON.parse((row && row.notices) || '[]'); } catch { return []; } };++ const saveNotices = db.prepare('UPDATE operator_alerts SET notices = ? WHERE source = ? AND fingerprint = ?');++ const users = work.length ? operatorUserIds(db, env) : [];++ for (const { kind, a, prev } of work) {++ if (a.severity === 'info') continue;++ const critical = a.severity === 'critical';++ if (kind === 'opened' || kind === 'reminded') {++ if (typeof notify !== 'function') continue;const n = {type: 'OPERATOR_ALERT', category: 'admin', service: 'host', url: STATUS_URL,-- priority: resolved ? 'normal' : (a.severity === 'critical' ? 'critical' : 'high'),-- icon: resolved ? '✅' : (a.severity === 'critical' ? '🚨' : '⚠️'),-- title: `${resolved ? 'Resolved' : kind === 'reminded' ? 'Still firing' : 'Alert'}: ${a.name}`.slice(0, 120),-- message: (resolved ? `${a.summary} (resolved).` : `${a.summary}. ${message(a)}`).slice(0, 500),++ priority: critical ? 'critical' : 'high', icon: critical ? '🚨' : '⚠️', silent: !critical,++ title: `${kind === 'reminded' ? 'Still firing' : 'Alert'}: ${a.name}`.slice(0, 120),++ message: `${a.summary}. ${message(a)}`.slice(0, 500),};++ const kept = kind === 'reminded' ? notices(prev) : [];for (const uid of users) {-- try { if (notify(uid, n)) out.notified += 1; } catch (e) { console.warn('[operator-alerts] notify failed:', e.message); }++ try {++ const created = notify(uid, n);++ if (created) out.notified += 1;++ if (created && created.id) kept.push([uid, created.id]);++ } catch (e) { console.warn('[operator-alerts] notify failed:', e.message); }}++ saveNotices.run(JSON.stringify(kept), source, a.fingerprint);++ continue;++ }++ if (typeof revise !== 'function') continue;++ const fields = kind === 'resolved'++ ? { title: `Resolved: ${a.name}`.slice(0, 120), icon: '✅', is_read: 1, message: `${a.summary} (resolved after ${ago(now - (prev.opened_at || now))}${prev.reopened ? `; it fired ${prev.reopened + 1} times` : ''}).`.slice(0, 500) }++ : { title: `Alert: ${a.name}`.slice(0, 120), icon: critical ? '🚨' : '⚠️', is_read: critical ? 0 : 1, message: `${a.summary}. ${message(a)} Fired again ${(prev.reopened || 0) + 1} time(s) within 12 h of resolving.`.slice(0, 500) };++ for (const [uid, id] of notices(prev)) {++ try { revise(uid, id, fields); } catch (e) { console.warn('[operator-alerts] revise failed:', e.message); }}}return out;@@@ -115,4 +153,4 @@ function list(db, { limit = 100 } = {}) {return db.prepare("SELECT source, fingerprint, name, severity, summary, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at FROM operator_alerts ORDER BY state = 'firing' DESC, COALESCE(resolved_at, last_seen_at) DESC LIMIT ?").all(limit);}--module.exports = { ensureTables, receive, list, operatorUserIds, REMIND_MS };++module.exports = { ensureTables, receive, list, operatorUserIds, REMIND_MS, FLAP_MS };@@@ -10,12 +10,16 @@** Read hourly with GitHub's compare API (GET /repos/{repo}/compare/{deployed}...main, one call per* running service; GITHUB_TOKEN or the admin GitHub token raises the rate limit). A failure keeps the-- * last answer. `since` is the committer date of the oldest undeployed commit, so the drift age is how++ * last answer. Main ahead by documentation and tests only (README, STATUS.json, docs/*.md, tests, .github/) counts++ * as current: nothing that runs differs. `since` is the committer date of the oldest undeployed commit, so the drift age is how* long main has had something production does not run. Shown on /status and in* /api/v1/registry/releases; openvibe_deploy_drift_seconds{service} feeds the Prometheus alert* OpenVibeDeployDrift (24 hours; OpenVibe.Host deploy/prometheus/openvibe-rules.yml).*/const SHA_RE = /^[0-9a-f]{7,40}$/;++// Files that document or test a service and are not part of what it runs (OpenVibe.Host lib/reconcile.js, the++// deploy controller, does not deploy them either): main ahead by only these is not drift.++const DOCS_OR_TESTS = /^(README\.md|STATUS\.json|CHANGELOG\.md|SECURITY\.md|docs\/.*\.md|tests?\/|\.github\/)|\.test\.[cm]?[jt]s$/;const REPO_RE = /^OpenVibers\/[A-Za-z0-9._-]{1,100}$/;const state = new Map(); // service id -> { repo, deployed, state, behind_by, since, main, checked_at, error }@@@ -61,7 +65,9 @@ function createDeployDrift({ services, fetchImpl = globalThis.fetch, token = ()let row;if (st === 'behind' || st === 'diverged') row = { state: 'diverged', behind_by: ahead };else if (st === 'identical' || ahead === 0) row = { state: 'current', behind_by: 0 };-- else {++ else if (Array.isArray(body.files) && body.files.length && body.files.length < 300 && body.files.every((f) => DOCS_OR_TESTS.test(String(f.filename || '')))) {++ row = { state: 'current', behind_by: 0, docs_only_ahead: ahead };++ } else {const first = commits[0] && commits[0].commit && (commits[0].commit.committer || commits[0].commit.author);row = { state: 'behind', behind_by: ahead, since: first && first.date ? new Date(first.date).toISOString() : null };}@@@ -16,6 +16,11 @@ const answers = {] } },'OpenVibers/OpenVibe.Media/compare/ccccccc3...main': { status: 200, body: { status: 'diverged', ahead_by: 1, behind_by: 2, commits: [] } },'OpenVibers/OpenVibe.Tools/compare/ddddddd4...main': { status: 404, body: { message: 'Not Found' } },++ // Main ahead by documentation and tests only: nothing that runs differs, so it is current.++ 'OpenVibers/OpenVibe.Wiki/compare/eeeeeee5...main': { status: 200, body: { ahead_by: 2, behind_by: 0, files: [{ filename: 'README.md' }, { filename: 'docs/api.md' }, { filename: 'test/x.test.js' }, { filename: 'STATUS.json' }], commits: [++ { sha: 'e1', commit: { committer: { date: '2026-09-20T10:00:00Z' } } }, { sha: 'e2', commit: { committer: { date: '2026-09-21T10:00:00Z' } } }] } },++ 'OpenVibers/OpenVibe.Blog/compare/fffffff6...main': { status: 200, body: { ahead_by: 1, behind_by: 0, files: [{ filename: 'README.md' }, { filename: 'server/index.js' }], commits: [++ { sha: 'f1', commit: { committer: { date: '2026-09-26T10:00:00Z' } } }] } },};let failing = false;const asked = [];@@@ -34,10 +39,12 @@ const fetchImpl = async (url, opts) => {{ id: 'media', release: 'ccccccc3', repository: 'OpenVibers/OpenVibe.Media' },{ id: 'tools', release: 'ddddddd4', repository: 'OpenVibers/OpenVibe.Tools' },{ id: 'sites', release: 'not-a-sha', repository: 'OpenVibers/OpenVibe.Sites' },++ { id: 'wiki', release: 'eeeeeee5', repository: 'OpenVibers/OpenVibe.Wiki' },++ { id: 'blog', release: 'fffffff6', repository: 'OpenVibers/OpenVibe.Blog' },];const d = drift.createDeployDrift({ services: () => running, fetchImpl, token: 'ghp_test', now: () => NOW, log: null });await d.refresh();-- assert.strictEqual(asked.length, 4, 'a release that is not a commit is not looked up');++ assert.strictEqual(asked.length, 6, 'a release that is not a commit is not looked up');assert.strictEqual(asked[0].auth, 'Bearer ghp_test');assert.strictEqual(drift.current('live').state, 'current');const chat = drift.current('chat');@@@ -46,7 +53,10 @@ const fetchImpl = async (url, opts) => {assert.deepStrictEqual([drift.current('tools').state, drift.current('tools').error], ['unknown', 'commit not on GitHub']);assert.strictEqual(drift.current('sites'), null);const g = Object.fromEntries(drift.driftSeconds(NOW).map((x) => [x.labels.service, x.value]));-- assert.deepStrictEqual(g, { live: 0, chat: 26 * 3600 }, 'seconds since the oldest undeployed commit; unknown and diverged are not measured');++ assert.deepStrictEqual([drift.current('wiki').state, drift.current('wiki').docs_only_ahead], ['current', 2], 'docs and tests only: not drift');++ assert.strictEqual(drift.current('blog').state, 'behind', 'one runtime file makes it drift');++ delete g.blog;++ assert.deepStrictEqual(g, { live: 0, chat: 26 * 3600, wiki: 0 }, 'seconds since the oldest undeployed commit; unknown and diverged are not measured');failing = true;await d.refresh();@@@ -33,7 +33,8 @@ app.locals.db = db;app.locals.config = { internalKey: 'legacy-key', jwt: { issuer: 'https://openvibe.network', accessTokenExpiry: '1h' } };app.locals.privateKey = keys.privateKey;app.locals.publicKey = keys.publicKey;--app.locals.notificationService = { create: (n) => { sent.push(n); return { id: `n${sent.length}` }; } };++const revised = [];++app.locals.notificationService = { create: (n) => { sent.push(n); return { id: `n${sent.length}` }; }, revise: (id, uid, f) => { revised.push({ id, uid, ...f }); return true; } };app.use('/oauth', require('../server/auth/oauth-routes'));app.use('/internal', require('../server/internal/routes'));const server = http.createServer(app);@@@ -76,6 +77,9 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').assert.deepStrictEqual([backup.type, backup.category, backup.priority, backup.url], ['OPERATOR_ALERT', 'admin', 'critical', 'https://openvibe.network/status']);assert.match(backup.message, /No successful backup in 26 hours\. Service: host\. Firing since 2026-09-26T05:00:00Z\./);assert.strictEqual(sent.find((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').priority, 'high', 'a warning pages at high');++ assert.deepStrictEqual([backup.silent, sent.find((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').silent], [false, true], 'only a critical alert is pushed');++ const warnIds = sent.filter((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').map((n, i) => n).length;++ assert.strictEqual(warnIds, 2);// The same set again: nothing new.sent.length = 0;@@@ -83,31 +87,61 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').assert.deepStrictEqual(r.body, { ok: true, firing: 2, opened: 0, reminded: 0, resolved: 0, notified: 0 });assert.strictEqual(sent.length, 0);-- // One drops out: resolved notice.++ // One drops out: its notifications are revised to Resolved and marked read; nothing new is created.r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, auth);-- assert.deepStrictEqual(r.body, { ok: true, firing: 1, opened: 0, reminded: 0, resolved: 1, notified: 2 });-- assert.deepStrictEqual([...new Set(sent.map((n) => `${n.title}|${n.priority}`))], ['Resolved: OpenVibeBrowserCheckFailed|normal']);++ assert.deepStrictEqual(r.body, { ok: true, firing: 1, opened: 0, reminded: 0, resolved: 1, notified: 0 });++ assert.strictEqual(sent.length, 0, 'no Resolved entry piles up');++ assert.strictEqual(revised.length, 2, 'one revision per operator');++ assert.ok(revised.every((x) => x.title === 'Resolved: OpenVibeBrowserCheckFailed' && x.icon === '✅' && x.is_read === 1), JSON.stringify(revised));++ assert.match(revised[0].message, /\(resolved after \d+ min\)\./);++ const warnNotices = revised.map((x) => x.id).sort();-- // A day later, still firing: one reminder; then quiet again.++ // A day later, a critical still firing: one reminder; then quiet again.sent.length = 0;const later = Date.now() + alerts.REMIND_MS + 1000;-- let out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: (u, n) => { sent.push({ user_id: u, ...n }); return true; }, now: later });++ let out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: (u, n) => { sent.push({ user_id: u, ...n }); return { id: `r${sent.length}` }; }, now: later });assert.deepStrictEqual(out, { ok: true, firing: 1, opened: 0, reminded: 1, resolved: 0, notified: 2 });assert.strictEqual(sent[0].title, 'Still firing: OpenVibeBackupMissed');out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => true, now: later + 60000 });assert.strictEqual(out.reminded, 0);-- // An alert that resolved and fires again opens again.-- sent.length = 0;++ // A warning is never reminded.++ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('Warned', { severity: 'warning' })] }, { notify: () => ({ id: 'w' }), now: later + 120000 });++ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('Warned', { severity: 'warning' })] }, { notify: () => ({ id: 'w' }), now: later + 2 * alerts.REMIND_MS });++ assert.strictEqual(out.reminded, 1, 'only the critical one');++ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null, now: later + 2 * alerts.REMIND_MS + 1 });++++ // Fires again within 12 h of resolving: the same notifications are revised; nothing is created or pushed.++ sent.length = 0; revised.length = 0;r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('OpenVibeBrowserCheckFailed', { severity: 'warning' })] }, auth);-- assert.strictEqual(r.body.opened, 1);-- assert.ok(sent.some((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed'));++ assert.deepStrictEqual([r.body.opened, r.body.notified], [1, 0]);++ assert.strictEqual(sent.length, 0);++ assert.deepStrictEqual(revised.map((x) => x.id).sort(), warnNotices, 'the episode keeps its notifications');++ assert.ok(revised.every((x) => x.title === 'Alert: OpenVibeBrowserCheckFailed' && /Fired again 1 time\(s\) within 12 h/.test(x.message)));++ revised.length = 0;++ await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, auth);++ assert.match(revised[0].message, /it fired 2 times/);++++ // Info alerts are listed, never notified.++ sent.length = 0;++ r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('JustInfo', { severity: 'info' })] }, auth);++ assert.deepStrictEqual([r.body.opened, r.body.notified, sent.length], [1, 0, 0]);++ assert.ok(alerts.list(db).some((x) => x.name === 'JustInfo' && x.state === 'firing'));++++ // After 12 h resolved, firing again is a new episode with new notifications.++ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null });++ const much = Date.now() + alerts.FLAP_MS + 60000;++ sent.length = 0;++ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('OpenVibeBrowserCheckFailed', { severity: 'warning' })] }, { notify: (u, n) => { sent.push(n); return { id: `x${sent.length}` }; }, now: much });++ assert.deepStrictEqual([out.opened, out.notified], [1, 2]);++ assert.ok(sent.every((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed'));++ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null, now: much + 1000 });// An empty set resolves everything from that source; the list keeps them.r = await post({ source: 'prometheus', alerts: [] }, auth);-- assert.deepStrictEqual([r.body.firing, r.body.resolved], [0, 2]);++ assert.deepStrictEqual([r.body.firing, r.body.resolved], [0, 1]);const rows = alerts.list(db);-- assert.deepStrictEqual(rows.map((x) => x.state), ['resolved', 'resolved']);++ assert.ok(rows.every((x) => x.state === 'resolved'), JSON.stringify(rows.map((x) => [x.name, x.state])));assert.ok(rows.every((x) => !('description' in x) || x.description === undefined));// No operator account: nothing is sent, nothing breaks.
@@ -508,8 +508,9 @@ router.post('/operator/alerts', principals.guard('network.operator.alert', { leg
@@ -508,8 +508,9 @@ router.post('/operator/alerts', principals.guard('network.operator.alert', { leg
if (!v.valid) return res.status(400).json({ error: 'The body does not match network.operator-alerts-request@1', details: (v.errors || []).slice(0, 5) });
if (!v.valid) return res.status(400).json({ error: 'The body does not match network.operator-alerts-request@1', details: (v.errors || []).slice(0, 5) });
const notifService = req.app.locals.notificationService;
const notifService = req.app.locals.notificationService;
const notify = notifService ? (userId, n) => notifService.create({ user_id: userId, ...n }) : null;
const notify = notifService ? (userId, n) => notifService.create({ user_id: userId, ...n }) : null;
+ const revise = notifService ? (userId, id, fields) => notifService.revise(id, userId, fields) : null;
try {
try {
- res.json(require('../operator/alerts').receive(getDb(req), req.body, { notify }));
+ res.json(require('../operator/alerts').receive(getDb(req), req.body, { notify, revise }));
} catch (err) {
} catch (err) {
console.error('[Internal] Operator alerts error:', err.message);
console.error('[Internal] Operator alerts error:', err.message);
res.status(500).json({ error: 'Operator alerts failed' });
res.status(500).json({ error: 'Operator alerts failed' });
@@ -194,8 +194,8 @@ class NotificationService {
@@ -194,8 +194,8 @@ class NotificationService {
// Wake the relay (it reads after the outermost transaction commits: better-sqlite3 is synchronous).
// Wake the relay (it reads after the outermost transaction commits: better-sqlite3 is synchronous).
if (announced) { const live = eventRelay.outboxFor(this.db); if (live) live.kick(); }
if (announced) { const live = eventRelay.outboxFor(this.db); if (live) live.kick(); }
- // Fire browser push notification (async, non-blocking)
- try {
+ // Fire browser push notification (async, non-blocking); `silent` keeps it to the bell.
+ if (!data.silent) try {
const pushService = require('../push/push-service');
const pushService = require('../push/push-service');
pushService.sendPush(data.user_id, {
pushService.sendPush(data.user_id, {
title,
title,
@@ -340,6 +340,18 @@ class NotificationService {
@@ -340,6 +340,18 @@ class NotificationService {
// ─── Update ────────────────────────────────────────────────
// ─── Update ────────────────────────────────────────────────
+ /** Edit one of a user's notifications in place (an operator alert that resolves or fires again). */
+ revise(id, userId, { title, message, icon, is_read: isRead } = {}) {
+ const sets = []; const args = [];
+ if (title != null) { sets.push('title = ?'); args.push(String(title).slice(0, 200)); }
+ if (message != null) { sets.push('message = ?'); args.push(String(message)); }
+ if (icon != null) { sets.push('icon = ?'); args.push(String(icon)); }
+ if (isRead != null) { sets.push('is_read = ?'); args.push(isRead ? 1 : 0); }
+ if (!sets.length) return false;
+ const r = this.db.prepare(`UPDATE notifications SET ${sets.join(', ')} WHERE id = ? AND user_id = ?`).run(...args, id, userId);
+ return r.changes > 0;
+ }
+
markRead(id, userId) {
markRead(id, userId) {
return this._markRead.run(id, userId).changes > 0;
return this._markRead.run(id, userId).changes > 0;
}
}
@@ -5,11 +5,16 @@
@@ -5,11 +5,16 @@
* The production host's Prometheus evaluates alert rules (backups, the developer path, the Tools job proof,
* The production host's Prometheus evaluates alert rules (backups, the developer path, the Tools job proof,
* the browser check), but nothing delivered them. Host relays the complete set of alerts firing now
* the browser check), but nothing delivered them. Host relays the complete set of alerts firing now
* (`ovhost alerts relay`, every two minutes) to POST /internal/operator/alerts, and this module turns the
* (`ovhost alerts relay`, every two minutes) to POST /internal/operator/alerts, and this module turns the
- * difference into notifications for the operator:
+ * difference into notifications for the operator. One notification per alert episode, kept up to date in place,
+ * so a flapping or resolving alert never piles up new entries (the bell had open/resolved pairs every few hours):
*
*
- * opened an alert not firing before critical → priority critical, else high
- * reminded still firing 24 h after the last page same priority
- * resolved firing before, absent from this set priority normal
+ * opened an alert not firing before critical: a pushed notification; warning: in the bell only
+ * reopened fires again within 12 h of resolving the same notification is revised (Alert again, times counted);
+ * nothing new is created or pushed
+ * reminded a critical still firing 24 h after a new pushed notification (warnings are not reminded)
+ * the last page
+ * resolved firing before, absent from this set its notifications are revised to Resolved and marked read
+ * info never notified; listed only
*
*
* The operator is the owner account (OWNER_USERNAME), plus any usernames in OPERATOR_ALERT_USERNAMES
* The operator is the owner account (OWNER_USERNAME), plus any usernames in OPERATOR_ALERT_USERNAMES
* (comma-separated). The notification is type OPERATOR_ALERT in category `admin`, so it reaches the bell,
* (comma-separated). The notification is type OPERATOR_ALERT in category `admin`, so it reaches the bell,
@@ -18,6 +23,7 @@
@@ -18,6 +23,7 @@
*/
*/
const REMIND_MS = 24 * 3600 * 1000;
const REMIND_MS = 24 * 3600 * 1000;
+const FLAP_MS = 12 * 3600 * 1000;
const KEEP_RESOLVED_MS = 30 * 24 * 3600 * 1000;
const KEEP_RESOLVED_MS = 30 * 24 * 3600 * 1000;
const STATUS_URL = 'https://openvibe.network/status';
const STATUS_URL = 'https://openvibe.network/status';
@@ -38,6 +44,10 @@ function ensureTables(db) {
@@ -38,6 +44,10 @@ function ensureTables(db) {
resolved_at INTEGER,
resolved_at INTEGER,
PRIMARY KEY (source, fingerprint)
PRIMARY KEY (source, fingerprint)
)`);
)`);
+ // The notifications of the current episode ([[userId, notificationId], …]) and how often it fired again.
+ for (const col of ['notices TEXT', 'reopened INTEGER NOT NULL DEFAULT 0']) {
+ try { db.exec(`ALTER TABLE operator_alerts ADD COLUMN ${col}`); } catch { /* already there */ }
+ }
}
}
function operatorUserIds(db, env = process.env) {
function operatorUserIds(db, env = process.env) {
@@ -51,59 +61,87 @@ function message(a) {
@@ -51,59 +61,87 @@ function message(a) {
return [a.description || a.summary, a.service ? `Service: ${a.service}.` : null, `Firing since ${a.started_at}.`].filter(Boolean).join(' ');
return [a.description || a.summary, a.service ? `Service: ${a.service}.` : null, `Firing since ${a.started_at}.`].filter(Boolean).join(' ');
}
}
+const ago = (ms) => (ms < 3600e3 ? `${Math.max(1, Math.round(ms / 60e3))} min` : `${Math.round(ms / 3600e3)} h`);
+
/**
/**
* Apply one report. `body` has passed the request contract. Answers the result document.
* Apply one report. `body` has passed the request contract. Answers the result document.
- * notify(userId, notification) creates one notification (the notification service's create()).
+ * notify(userId, notification) creates one notification (the notification service's create(); `silent` skips
+ * web push); revise(userId, notificationId, fields) edits one in place (title, message, icon, is_read).
*/
*/
-function receive(db, body, { notify, now = Date.now(), env = process.env } = {}) {
+function receive(db, body, { notify, revise = null, now = Date.now(), env = process.env } = {}) {
ensureTables(db);
ensureTables(db);
const source = body.source;
const source = body.source;
const current = new Map(body.alerts.map((a) => [a.fingerprint, a]));
const current = new Map(body.alerts.map((a) => [a.fingerprint, a]));
const rows = db.prepare('SELECT * FROM operator_alerts WHERE source = ?').all(source);
const rows = db.prepare('SELECT * FROM operator_alerts WHERE source = ?').all(source);
const byFp = new Map(rows.map((r) => [r.fingerprint, r]));
const byFp = new Map(rows.map((r) => [r.fingerprint, r]));
- const pages = [];
+ const work = [];
const out = { ok: true, firing: current.size, opened: 0, reminded: 0, resolved: 0, notified: 0 };
const out = { ok: true, firing: current.size, opened: 0, reminded: 0, resolved: 0, notified: 0 };
- const upsert = db.prepare(`INSERT INTO operator_alerts (source, fingerprint, name, severity, summary, description, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at)
- VALUES (@source, @fingerprint, @name, @severity, @summary, @description, @service, 'firing', @started_at, @now, @now, @notified_at, NULL)
+ const upsert = db.prepare(`INSERT INTO operator_alerts (source, fingerprint, name, severity, summary, description, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at, reopened)
+ VALUES (@source, @fingerprint, @name, @severity, @summary, @description, @service, 'firing', @started_at, @now, @now, @notified_at, NULL, 0)
ON CONFLICT (source, fingerprint) DO UPDATE SET name = @name, severity = @severity, summary = @summary, description = @description, service = @service,
ON CONFLICT (source, fingerprint) DO UPDATE SET name = @name, severity = @severity, summary = @summary, description = @description, service = @service,
state = 'firing', started_at = @started_at, last_seen_at = @now, notified_at = @notified_at, resolved_at = NULL,
state = 'firing', started_at = @started_at, last_seen_at = @now, notified_at = @notified_at, resolved_at = NULL,
- opened_at = CASE WHEN operator_alerts.state = 'resolved' THEN @now ELSE operator_alerts.opened_at END`);
- const resolve = db.prepare("UPDATE operator_alerts SET state = 'resolved', resolved_at = ?, notified_at = ? WHERE source = ? AND fingerprint = ?");
+ reopened = operator_alerts.reopened + @reopen,
+ opened_at = CASE WHEN operator_alerts.state = 'resolved' AND @reopen = 0 THEN @now ELSE operator_alerts.opened_at END`);
+ const resolve = db.prepare("UPDATE operator_alerts SET state = 'resolved', resolved_at = ? WHERE source = ? AND fingerprint = ?");
db.transaction(() => {
db.transaction(() => {
for (const a of current.values()) {
for (const a of current.values()) {
const prev = byFp.get(a.fingerprint);
const prev = byFp.get(a.fingerprint);
let kind = null;
let kind = null;
- if (!prev || prev.state === 'resolved') kind = 'opened';
- else if (!prev.notified_at || now - prev.notified_at >= REMIND_MS) kind = 'reminded';
+ if (!prev || prev.state === 'resolved') {
+ kind = prev && prev.resolved_at && now - prev.resolved_at < FLAP_MS && prev.notices ? 'reopened' : 'opened';
+ out.opened += 1;
+ } else if (a.severity === 'critical' && (!prev.notified_at || now - prev.notified_at >= REMIND_MS)) {
+ kind = 'reminded';
+ out.reminded += 1;
+ }
+ const paged = kind === 'opened' || kind === 'reminded';
upsert.run({ source, fingerprint: a.fingerprint, name: a.name, severity: a.severity, summary: a.summary, description: a.description || null,
upsert.run({ source, fingerprint: a.fingerprint, name: a.name, severity: a.severity, summary: a.summary, description: a.description || null,
- service: a.service || null, started_at: a.started_at, now, notified_at: kind ? now : (prev && prev.notified_at) || null });
- if (kind) { out[kind] += 1; pages.push({ kind, a }); }
+ service: a.service || null, started_at: a.started_at, now, reopen: kind === 'reopened' ? 1 : 0,
+ notified_at: paged ? now : (prev && prev.notified_at) || null });
+ if (kind) work.push({ kind, a, prev });
}
}
for (const r of rows) {
for (const r of rows) {
if (r.state !== 'firing' || current.has(r.fingerprint)) continue;
if (r.state !== 'firing' || current.has(r.fingerprint)) continue;
- resolve.run(now, now, source, r.fingerprint);
+ resolve.run(now, source, r.fingerprint);
out.resolved += 1;
out.resolved += 1;
- pages.push({ kind: 'resolved', a: r });
+ work.push({ kind: 'resolved', a: r, prev: r });
}
}
db.prepare("DELETE FROM operator_alerts WHERE state = 'resolved' AND resolved_at < ?").run(now - KEEP_RESOLVED_MS);
db.prepare("DELETE FROM operator_alerts WHERE state = 'resolved' AND resolved_at < ?").run(now - KEEP_RESOLVED_MS);
})();
})();
- if (pages.length && typeof notify === 'function') {
- const users = operatorUserIds(db, env);
- for (const { kind, a } of pages) {
- const resolved = kind === 'resolved';
+ const notices = (row) => { try { return JSON.parse((row && row.notices) || '[]'); } catch { return []; } };
+ const saveNotices = db.prepare('UPDATE operator_alerts SET notices = ? WHERE source = ? AND fingerprint = ?');
+ const users = work.length ? operatorUserIds(db, env) : [];
+ for (const { kind, a, prev } of work) {
+ if (a.severity === 'info') continue;
+ const critical = a.severity === 'critical';
+ if (kind === 'opened' || kind === 'reminded') {
+ if (typeof notify !== 'function') continue;
const n = {
const n = {
type: 'OPERATOR_ALERT', category: 'admin', service: 'host', url: STATUS_URL,
type: 'OPERATOR_ALERT', category: 'admin', service: 'host', url: STATUS_URL,
- priority: resolved ? 'normal' : (a.severity === 'critical' ? 'critical' : 'high'),
- icon: resolved ? '✅' : (a.severity === 'critical' ? '🚨' : '⚠️'),
- title: `${resolved ? 'Resolved' : kind === 'reminded' ? 'Still firing' : 'Alert'}: ${a.name}`.slice(0, 120),
- message: (resolved ? `${a.summary} (resolved).` : `${a.summary}. ${message(a)}`).slice(0, 500),
+ priority: critical ? 'critical' : 'high', icon: critical ? '🚨' : '⚠️', silent: !critical,
+ title: `${kind === 'reminded' ? 'Still firing' : 'Alert'}: ${a.name}`.slice(0, 120),
+ message: `${a.summary}. ${message(a)}`.slice(0, 500),
};
};
+ const kept = kind === 'reminded' ? notices(prev) : [];
for (const uid of users) {
for (const uid of users) {
- try { if (notify(uid, n)) out.notified += 1; } catch (e) { console.warn('[operator-alerts] notify failed:', e.message); }
+ try {
+ const created = notify(uid, n);
+ if (created) out.notified += 1;
+ if (created && created.id) kept.push([uid, created.id]);
+ } catch (e) { console.warn('[operator-alerts] notify failed:', e.message); }
}
}
+ saveNotices.run(JSON.stringify(kept), source, a.fingerprint);
+ continue;
+ }
+ if (typeof revise !== 'function') continue;
+ const fields = kind === 'resolved'
+ ? { title: `Resolved: ${a.name}`.slice(0, 120), icon: '✅', is_read: 1, message: `${a.summary} (resolved after ${ago(now - (prev.opened_at || now))}${prev.reopened ? `; it fired ${prev.reopened + 1} times` : ''}).`.slice(0, 500) }
+ : { title: `Alert: ${a.name}`.slice(0, 120), icon: critical ? '🚨' : '⚠️', is_read: critical ? 0 : 1, message: `${a.summary}. ${message(a)} Fired again ${(prev.reopened || 0) + 1} time(s) within 12 h of resolving.`.slice(0, 500) };
+ for (const [uid, id] of notices(prev)) {
+ try { revise(uid, id, fields); } catch (e) { console.warn('[operator-alerts] revise failed:', e.message); }
}
}
}
}
return out;
return out;
@@ -115,4 +153,4 @@ function list(db, { limit = 100 } = {}) {
@@ -115,4 +153,4 @@ function list(db, { limit = 100 } = {}) {
return db.prepare("SELECT source, fingerprint, name, severity, summary, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at FROM operator_alerts ORDER BY state = 'firing' DESC, COALESCE(resolved_at, last_seen_at) DESC LIMIT ?").all(limit);
return db.prepare("SELECT source, fingerprint, name, severity, summary, service, state, started_at, opened_at, last_seen_at, notified_at, resolved_at FROM operator_alerts ORDER BY state = 'firing' DESC, COALESCE(resolved_at, last_seen_at) DESC LIMIT ?").all(limit);
}
}
-module.exports = { ensureTables, receive, list, operatorUserIds, REMIND_MS };
+module.exports = { ensureTables, receive, list, operatorUserIds, REMIND_MS, FLAP_MS };
@@ -10,12 +10,16 @@
@@ -10,12 +10,16 @@
*
*
* Read hourly with GitHub's compare API (GET /repos/{repo}/compare/{deployed}...main, one call per
* Read hourly with GitHub's compare API (GET /repos/{repo}/compare/{deployed}...main, one call per
* running service; GITHUB_TOKEN or the admin GitHub token raises the rate limit). A failure keeps the
* running service; GITHUB_TOKEN or the admin GitHub token raises the rate limit). A failure keeps the
- * last answer. `since` is the committer date of the oldest undeployed commit, so the drift age is how
+ * last answer. Main ahead by documentation and tests only (README, STATUS.json, docs/*.md, tests, .github/) counts
+ * as current: nothing that runs differs. `since` is the committer date of the oldest undeployed commit, so the drift age is how
* long main has had something production does not run. Shown on /status and in
* long main has had something production does not run. Shown on /status and in
* /api/v1/registry/releases; openvibe_deploy_drift_seconds{service} feeds the Prometheus alert
* /api/v1/registry/releases; openvibe_deploy_drift_seconds{service} feeds the Prometheus alert
* OpenVibeDeployDrift (24 hours; OpenVibe.Host deploy/prometheus/openvibe-rules.yml).
* OpenVibeDeployDrift (24 hours; OpenVibe.Host deploy/prometheus/openvibe-rules.yml).
*/
*/
const SHA_RE = /^[0-9a-f]{7,40}$/;
const SHA_RE = /^[0-9a-f]{7,40}$/;
+// Files that document or test a service and are not part of what it runs (OpenVibe.Host lib/reconcile.js, the
+// deploy controller, does not deploy them either): main ahead by only these is not drift.
+const DOCS_OR_TESTS = /^(README\.md|STATUS\.json|CHANGELOG\.md|SECURITY\.md|docs\/.*\.md|tests?\/|\.github\/)|\.test\.[cm]?[jt]s$/;
const REPO_RE = /^OpenVibers\/[A-Za-z0-9._-]{1,100}$/;
const REPO_RE = /^OpenVibers\/[A-Za-z0-9._-]{1,100}$/;
const state = new Map(); // service id -> { repo, deployed, state, behind_by, since, main, checked_at, error }
const state = new Map(); // service id -> { repo, deployed, state, behind_by, since, main, checked_at, error }
@@ -61,7 +65,9 @@ function createDeployDrift({ services, fetchImpl = globalThis.fetch, token = ()
@@ -61,7 +65,9 @@ function createDeployDrift({ services, fetchImpl = globalThis.fetch, token = ()
let row;
let row;
if (st === 'behind' || st === 'diverged') row = { state: 'diverged', behind_by: ahead };
if (st === 'behind' || st === 'diverged') row = { state: 'diverged', behind_by: ahead };
else if (st === 'identical' || ahead === 0) row = { state: 'current', behind_by: 0 };
else if (st === 'identical' || ahead === 0) row = { state: 'current', behind_by: 0 };
- else {
+ else if (Array.isArray(body.files) && body.files.length && body.files.length < 300 && body.files.every((f) => DOCS_OR_TESTS.test(String(f.filename || '')))) {
+ row = { state: 'current', behind_by: 0, docs_only_ahead: ahead };
+ } else {
const first = commits[0] && commits[0].commit && (commits[0].commit.committer || commits[0].commit.author);
const first = commits[0] && commits[0].commit && (commits[0].commit.committer || commits[0].commit.author);
row = { state: 'behind', behind_by: ahead, since: first && first.date ? new Date(first.date).toISOString() : null };
row = { state: 'behind', behind_by: ahead, since: first && first.date ? new Date(first.date).toISOString() : null };
}
}
@@ -16,6 +16,11 @@ const answers = {
@@ -16,6 +16,11 @@ const answers = {
] } },
] } },
'OpenVibers/OpenVibe.Media/compare/ccccccc3...main': { status: 200, body: { status: 'diverged', ahead_by: 1, behind_by: 2, commits: [] } },
'OpenVibers/OpenVibe.Media/compare/ccccccc3...main': { status: 200, body: { status: 'diverged', ahead_by: 1, behind_by: 2, commits: [] } },
'OpenVibers/OpenVibe.Tools/compare/ddddddd4...main': { status: 404, body: { message: 'Not Found' } },
'OpenVibers/OpenVibe.Tools/compare/ddddddd4...main': { status: 404, body: { message: 'Not Found' } },
+ // Main ahead by documentation and tests only: nothing that runs differs, so it is current.
+ 'OpenVibers/OpenVibe.Wiki/compare/eeeeeee5...main': { status: 200, body: { ahead_by: 2, behind_by: 0, files: [{ filename: 'README.md' }, { filename: 'docs/api.md' }, { filename: 'test/x.test.js' }, { filename: 'STATUS.json' }], commits: [
+ { sha: 'e1', commit: { committer: { date: '2026-09-20T10:00:00Z' } } }, { sha: 'e2', commit: { committer: { date: '2026-09-21T10:00:00Z' } } }] } },
+ 'OpenVibers/OpenVibe.Blog/compare/fffffff6...main': { status: 200, body: { ahead_by: 1, behind_by: 0, files: [{ filename: 'README.md' }, { filename: 'server/index.js' }], commits: [
+ { sha: 'f1', commit: { committer: { date: '2026-09-26T10:00:00Z' } } }] } },
};
};
let failing = false;
let failing = false;
const asked = [];
const asked = [];
@@ -34,10 +39,12 @@ const fetchImpl = async (url, opts) => {
@@ -34,10 +39,12 @@ const fetchImpl = async (url, opts) => {
{ id: 'media', release: 'ccccccc3', repository: 'OpenVibers/OpenVibe.Media' },
{ id: 'media', release: 'ccccccc3', repository: 'OpenVibers/OpenVibe.Media' },
{ id: 'tools', release: 'ddddddd4', repository: 'OpenVibers/OpenVibe.Tools' },
{ id: 'tools', release: 'ddddddd4', repository: 'OpenVibers/OpenVibe.Tools' },
{ id: 'sites', release: 'not-a-sha', repository: 'OpenVibers/OpenVibe.Sites' },
{ id: 'sites', release: 'not-a-sha', repository: 'OpenVibers/OpenVibe.Sites' },
+ { id: 'wiki', release: 'eeeeeee5', repository: 'OpenVibers/OpenVibe.Wiki' },
+ { id: 'blog', release: 'fffffff6', repository: 'OpenVibers/OpenVibe.Blog' },
];
];
const d = drift.createDeployDrift({ services: () => running, fetchImpl, token: 'ghp_test', now: () => NOW, log: null });
const d = drift.createDeployDrift({ services: () => running, fetchImpl, token: 'ghp_test', now: () => NOW, log: null });
await d.refresh();
await d.refresh();
- assert.strictEqual(asked.length, 4, 'a release that is not a commit is not looked up');
+ assert.strictEqual(asked.length, 6, 'a release that is not a commit is not looked up');
assert.strictEqual(asked[0].auth, 'Bearer ghp_test');
assert.strictEqual(asked[0].auth, 'Bearer ghp_test');
assert.strictEqual(drift.current('live').state, 'current');
assert.strictEqual(drift.current('live').state, 'current');
const chat = drift.current('chat');
const chat = drift.current('chat');
@@ -46,7 +53,10 @@ const fetchImpl = async (url, opts) => {
@@ -46,7 +53,10 @@ const fetchImpl = async (url, opts) => {
assert.deepStrictEqual([drift.current('tools').state, drift.current('tools').error], ['unknown', 'commit not on GitHub']);
assert.deepStrictEqual([drift.current('tools').state, drift.current('tools').error], ['unknown', 'commit not on GitHub']);
assert.strictEqual(drift.current('sites'), null);
assert.strictEqual(drift.current('sites'), null);
const g = Object.fromEntries(drift.driftSeconds(NOW).map((x) => [x.labels.service, x.value]));
const g = Object.fromEntries(drift.driftSeconds(NOW).map((x) => [x.labels.service, x.value]));
- assert.deepStrictEqual(g, { live: 0, chat: 26 * 3600 }, 'seconds since the oldest undeployed commit; unknown and diverged are not measured');
+ assert.deepStrictEqual([drift.current('wiki').state, drift.current('wiki').docs_only_ahead], ['current', 2], 'docs and tests only: not drift');
+ assert.strictEqual(drift.current('blog').state, 'behind', 'one runtime file makes it drift');
+ delete g.blog;
+ assert.deepStrictEqual(g, { live: 0, chat: 26 * 3600, wiki: 0 }, 'seconds since the oldest undeployed commit; unknown and diverged are not measured');
failing = true;
failing = true;
await d.refresh();
await d.refresh();
@@ -33,7 +33,8 @@ app.locals.db = db;
@@ -33,7 +33,8 @@ app.locals.db = db;
app.locals.config = { internalKey: 'legacy-key', jwt: { issuer: 'https://openvibe.network', accessTokenExpiry: '1h' } };
app.locals.config = { internalKey: 'legacy-key', jwt: { issuer: 'https://openvibe.network', accessTokenExpiry: '1h' } };
app.locals.privateKey = keys.privateKey;
app.locals.privateKey = keys.privateKey;
app.locals.publicKey = keys.publicKey;
app.locals.publicKey = keys.publicKey;
-app.locals.notificationService = { create: (n) => { sent.push(n); return { id: `n${sent.length}` }; } };
+const revised = [];
+app.locals.notificationService = { create: (n) => { sent.push(n); return { id: `n${sent.length}` }; }, revise: (id, uid, f) => { revised.push({ id, uid, ...f }); return true; } };
app.use('/oauth', require('../server/auth/oauth-routes'));
app.use('/oauth', require('../server/auth/oauth-routes'));
app.use('/internal', require('../server/internal/routes'));
app.use('/internal', require('../server/internal/routes'));
const server = http.createServer(app);
const server = http.createServer(app);
@@ -76,6 +77,9 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').
@@ -76,6 +77,9 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').
assert.deepStrictEqual([backup.type, backup.category, backup.priority, backup.url], ['OPERATOR_ALERT', 'admin', 'critical', 'https://openvibe.network/status']);
assert.deepStrictEqual([backup.type, backup.category, backup.priority, backup.url], ['OPERATOR_ALERT', 'admin', 'critical', 'https://openvibe.network/status']);
assert.match(backup.message, /No successful backup in 26 hours\. Service: host\. Firing since 2026-09-26T05:00:00Z\./);
assert.match(backup.message, /No successful backup in 26 hours\. Service: host\. Firing since 2026-09-26T05:00:00Z\./);
assert.strictEqual(sent.find((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').priority, 'high', 'a warning pages at high');
assert.strictEqual(sent.find((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').priority, 'high', 'a warning pages at high');
+ assert.deepStrictEqual([backup.silent, sent.find((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').silent], [false, true], 'only a critical alert is pushed');
+ const warnIds = sent.filter((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed').map((n, i) => n).length;
+ assert.strictEqual(warnIds, 2);
// The same set again: nothing new.
// The same set again: nothing new.
sent.length = 0;
sent.length = 0;
@@ -83,31 +87,61 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').
@@ -83,31 +87,61 @@ const alert = (name, extra = {}) => ({ fingerprint: crypto.createHash('sha256').
assert.deepStrictEqual(r.body, { ok: true, firing: 2, opened: 0, reminded: 0, resolved: 0, notified: 0 });
assert.deepStrictEqual(r.body, { ok: true, firing: 2, opened: 0, reminded: 0, resolved: 0, notified: 0 });
assert.strictEqual(sent.length, 0);
assert.strictEqual(sent.length, 0);
- // One drops out: resolved notice.
+ // One drops out: its notifications are revised to Resolved and marked read; nothing new is created.
r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, auth);
r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, auth);
- assert.deepStrictEqual(r.body, { ok: true, firing: 1, opened: 0, reminded: 0, resolved: 1, notified: 2 });
- assert.deepStrictEqual([...new Set(sent.map((n) => `${n.title}|${n.priority}`))], ['Resolved: OpenVibeBrowserCheckFailed|normal']);
+ assert.deepStrictEqual(r.body, { ok: true, firing: 1, opened: 0, reminded: 0, resolved: 1, notified: 0 });
+ assert.strictEqual(sent.length, 0, 'no Resolved entry piles up');
+ assert.strictEqual(revised.length, 2, 'one revision per operator');
+ assert.ok(revised.every((x) => x.title === 'Resolved: OpenVibeBrowserCheckFailed' && x.icon === '✅' && x.is_read === 1), JSON.stringify(revised));
+ assert.match(revised[0].message, /\(resolved after \d+ min\)\./);
+ const warnNotices = revised.map((x) => x.id).sort();
- // A day later, still firing: one reminder; then quiet again.
+ // A day later, a critical still firing: one reminder; then quiet again.
sent.length = 0;
sent.length = 0;
const later = Date.now() + alerts.REMIND_MS + 1000;
const later = Date.now() + alerts.REMIND_MS + 1000;
- let out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: (u, n) => { sent.push({ user_id: u, ...n }); return true; }, now: later });
+ let out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: (u, n) => { sent.push({ user_id: u, ...n }); return { id: `r${sent.length}` }; }, now: later });
assert.deepStrictEqual(out, { ok: true, firing: 1, opened: 0, reminded: 1, resolved: 0, notified: 2 });
assert.deepStrictEqual(out, { ok: true, firing: 1, opened: 0, reminded: 1, resolved: 0, notified: 2 });
assert.strictEqual(sent[0].title, 'Still firing: OpenVibeBackupMissed');
assert.strictEqual(sent[0].title, 'Still firing: OpenVibeBackupMissed');
out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => true, now: later + 60000 });
out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => true, now: later + 60000 });
assert.strictEqual(out.reminded, 0);
assert.strictEqual(out.reminded, 0);
- // An alert that resolved and fires again opens again.
- sent.length = 0;
+ // A warning is never reminded.
+ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('Warned', { severity: 'warning' })] }, { notify: () => ({ id: 'w' }), now: later + 120000 });
+ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('Warned', { severity: 'warning' })] }, { notify: () => ({ id: 'w' }), now: later + 2 * alerts.REMIND_MS });
+ assert.strictEqual(out.reminded, 1, 'only the critical one');
+ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null, now: later + 2 * alerts.REMIND_MS + 1 });
+
+ // Fires again within 12 h of resolving: the same notifications are revised; nothing is created or pushed.
+ sent.length = 0; revised.length = 0;
r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('OpenVibeBrowserCheckFailed', { severity: 'warning' })] }, auth);
r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('OpenVibeBrowserCheckFailed', { severity: 'warning' })] }, auth);
- assert.strictEqual(r.body.opened, 1);
- assert.ok(sent.some((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed'));
+ assert.deepStrictEqual([r.body.opened, r.body.notified], [1, 0]);
+ assert.strictEqual(sent.length, 0);
+ assert.deepStrictEqual(revised.map((x) => x.id).sort(), warnNotices, 'the episode keeps its notifications');
+ assert.ok(revised.every((x) => x.title === 'Alert: OpenVibeBrowserCheckFailed' && /Fired again 1 time\(s\) within 12 h/.test(x.message)));
+ revised.length = 0;
+ await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, auth);
+ assert.match(revised[0].message, /it fired 2 times/);
+
+ // Info alerts are listed, never notified.
+ sent.length = 0;
+ r = await post({ source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('JustInfo', { severity: 'info' })] }, auth);
+ assert.deepStrictEqual([r.body.opened, r.body.notified, sent.length], [1, 0, 0]);
+ assert.ok(alerts.list(db).some((x) => x.name === 'JustInfo' && x.state === 'firing'));
+
+ // After 12 h resolved, firing again is a new episode with new notifications.
+ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null });
+ const much = Date.now() + alerts.FLAP_MS + 60000;
+ sent.length = 0;
+ out = alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed'), alert('OpenVibeBrowserCheckFailed', { severity: 'warning' })] }, { notify: (u, n) => { sent.push(n); return { id: `x${sent.length}` }; }, now: much });
+ assert.deepStrictEqual([out.opened, out.notified], [1, 2]);
+ assert.ok(sent.every((n) => n.title === 'Alert: OpenVibeBrowserCheckFailed'));
+ alerts.receive(db, { source: 'prometheus', alerts: [alert('OpenVibeBackupMissed')] }, { notify: () => null, now: much + 1000 });
// An empty set resolves everything from that source; the list keeps them.
// An empty set resolves everything from that source; the list keeps them.
r = await post({ source: 'prometheus', alerts: [] }, auth);
r = await post({ source: 'prometheus', alerts: [] }, auth);
- assert.deepStrictEqual([r.body.firing, r.body.resolved], [0, 2]);
+ assert.deepStrictEqual([r.body.firing, r.body.resolved], [0, 1]);
const rows = alerts.list(db);
const rows = alerts.list(db);
- assert.deepStrictEqual(rows.map((x) => x.state), ['resolved', 'resolved']);
+ assert.ok(rows.every((x) => x.state === 'resolved'), JSON.stringify(rows.map((x) => [x.name, x.state])));
assert.ok(rows.every((x) => !('description' in x) || x.description === undefined));
assert.ok(rows.every((x) => !('description' in x) || x.description === undefined));
// No operator account: nothing is sent, nothing breaks.
// No operator account: nothing is sent, nothing breaks.