openvibers/openvibe.host Commits

Backup alerts (WS-S task 3): ovhost backup --all (and offsite push) writes the run's outcome to node_exporter's textfile collector (last run, ok, last success, per-status counts, off-site ok); ovhost backup-metrics rewrites it from a recorded run; Prometheus rules OpenVibeBackupFailed (the last run failed) and OpenVibeBackupMissed (no success in 30 hours, or no metrics)

OP OpenVibers committed 25 Sept 2026 · eab60f606d · 5 files, +107 −2


Changes

ddiff --git a/deploy/prometheus/openvibe-rules.yml b/deploy/prometheus/openvibe-rules.yml
iindex daa6f32..385ca40 100644
---- a/deploy/prometheus/openvibe-rules.yml
++++ b/deploy/prometheus/openvibe-rules.yml
@@@ -30,3 +30,17 @@ groups:
labels: { severity: ticket }
annotations:
summary: "Less than 8% of memory available for 10 minutes"
++ # Nightly backups (ovhost backup --all, openvibe-backup.timer; metrics from lib/commands/backup.js
++ # through node_exporter's textfile collector; roadmap WS-S task 3).
++ - alert: OpenVibeBackupFailed
++ expr: openvibe_backup_last_run_ok == 0
++ for: 5m
++ labels: { severity: page }
++ annotations:
++ summary: "The last ovhost backup --all run failed ({{ with query \"openvibe_backup_services{status='failed'}\" }}{{ . | first | value }}{{ end }} service(s) failed, or the off-site copy)"
++ - alert: OpenVibeBackupMissed
++ expr: time() - openvibe_backup_last_success_timestamp_seconds > 30 * 3600 or absent(openvibe_backup_last_run_timestamp_seconds)
++ for: 30m
++ labels: { severity: page }
++ annotations:
++ summary: "No successful nightly backup for more than 30 hours (or no backup metrics at all)"
ddiff --git a/lib/cli.js b/lib/cli.js
iindex 37bf205..4116944 100644
---- a/lib/cli.js
++++ b/lib/cli.js
@@@ -37,6 +37,8 @@ Usage: ovhost <command> [args] [--inventory <file>] [--json]
others; keeps the last 7 daily + 4 weekly good backups per
service; JSON summary in <stateDir>/backup-runs/<run>.json;
--offsite then encrypts and uploads the run (root)
++ backup-metrics [--run <run>] rewrite the Prometheus textfile (node_exporter collector) from a
++ recorded backup --all run (default: the latest)
offsite push [--run <run>] encrypt + upload a backup --all run (default: the latest)
offsite list [<service>] [--from-host <name>]
runs stored off-host, newest first
@@@ -272,6 +274,15 @@ async function main(argv, { exec = lib.createSystemExecutor(), out = (s) => proc
if (json) emit(r); else for (const f of r.files) out(f.skipped ? `skipped ${f.source}: ${f.skipped}` : `${f.source} → ${f.dest} (${f.bytes} bytes)`);
return 0;
}
++ case 'backup-metrics': {
++ // Rewrite the Prometheus textfile from a recorded run (the latest by default): after an
++ // install, or when the collector directory appeared after the last run.
++ const summary = await lib.readRun(exec, inv, opts.run || null);
++ if (!summary) throw new UsageError(`no backup --all run recorded in ${inv.stateDir}/backup-runs`);
++ const wrote = await lib.writeMetrics(ctx, summary);
++ if (json) emit({ run: summary.run, written: wrote }); else out(wrote ? `backup metrics written from run ${summary.run}` : 'no textfile collector directory: nothing written');
++ return 0;
++ }
case 'offsite': {
const sub = pos[1];
if (!['push', 'list', 'check'].includes(sub)) throw new UsageError('usage: ovhost offsite push [--run <run>] | offsite list [<service>] | offsite check');
@@@ -295,6 +306,7 @@ async function main(argv, { exec = lib.createSystemExecutor(), out = (s) => proc
summary.offsite = await lib.offsite.push(ctx, summary, { cfg, client });
if (!summary.offsite.ok) summary.ok = false;
await lib.writeRun(exec, summary);
++ try { await lib.writeMetrics(ctx, summary); } catch { /* metrics never fail a push */ }
const o = summary.offsite;
if (json) emit(o);
else out(`OFFSITE ${o.ok ? 'ok' : 'FAILED'}: ${o.uploaded.length} file(s) to s3://${o.bucket}/${o.location}${o.failures.length ? `; failures: ${o.failures.map((f) => `${f.service ? `${f.service}/` : ''}${f.file}: ${f.error}`).join('; ')}` : ''}`);
ddiff --git a/lib/commands/backup.js b/lib/commands/backup.js
iindex 6dfecbb..729a1d1 100644
---- a/lib/commands/backup.js
++++ b/lib/commands/backup.js
@@@ -204,11 +204,60 @@ async function backupAll(ctx, { daily = 7, weekly = 4, prune: doPrune = true, of
await exec.mkdir(path.join(inv.stateDir, 'backup-runs'), { mode: 0o750 });
await exec.writeFile(file, `${JSON.stringify(summary, null, 2)}\n`, { mode: 0o640 });
await held.release();
++ // The metrics never fail a backup (WS-S task 3).
++ try { await writeMetrics(ctx, summary); } catch (err) { log(`backup metrics not written: ${err.message}`); }
}
summary.file = file;
return summary;
}
++const TEXTFILE_DIR = '/var/lib/prometheus/node-exporter';
++
++/**
++ * The last backup --all run for Prometheus, through node_exporter's textfile collector (WS-S task 3;
++ * alerts OpenVibeBackupFailed and OpenVibeBackupMissed in deploy/prometheus/openvibe-rules.yml):
++ *
++ * openvibe_backup_last_run_timestamp_seconds when the last run finished
++ * openvibe_backup_last_run_ok 1 when every service (and the off-site copy) succeeded
++ * openvibe_backup_last_success_timestamp_seconds the last run that succeeded (read back from backup-runs)
++ * openvibe_backup_services{status} services ok, failed and skipped in the last run
++ * openvibe_backup_offsite_ok 1/0 for the last run's off-site copy, when it made one
++ *
++ * Written only where the collector's directory exists (inventory `textfileDir`, default the Debian
++ * package's). A small file written in place: a scrape that caught it half-written would only miss once.
++ */
++async function writeMetrics(ctx, summary) {
++ const { exec, inv } = ctx;
++ const dir = inv.textfileDir || TEXTFILE_DIR;
++ if (!(await exec.readdir(dir))) return false;
++ const at = (iso) => Math.floor(Date.parse(iso) / 1000);
++ let lastOk = summary.ok ? summary.finishedAt : null;
++ if (!lastOk) {
++ const runs = ((await exec.readdir(path.join(inv.stateDir, 'backup-runs'))) || []).map((e) => e.name).filter((n) => n.endsWith('.json')).sort().reverse();
++ for (const name of runs) {
++ let r = null;
++ try { r = JSON.parse((await exec.readFile(path.join(inv.stateDir, 'backup-runs', name))) || 'null'); } catch { r = null; }
++ if (r && r.ok && r.finishedAt) { lastOk = r.finishedAt; break; }
++ }
++ }
++ const count = (st) => summary.services.filter((x) => x.status === st).length;
++ const lines = [
++ '# HELP openvibe_backup_last_run_timestamp_seconds When the last ovhost backup --all run finished.',
++ '# TYPE openvibe_backup_last_run_timestamp_seconds gauge',
++ `openvibe_backup_last_run_timestamp_seconds ${at(summary.finishedAt)}`,
++ '# HELP openvibe_backup_last_run_ok 1 when every service and the off-site copy succeeded in the last run.',
++ '# TYPE openvibe_backup_last_run_ok gauge',
++ `openvibe_backup_last_run_ok ${summary.ok ? 1 : 0}`,
++ '# HELP openvibe_backup_services Services by outcome in the last run.',
++ '# TYPE openvibe_backup_services gauge',
++ ...['ok', 'failed', 'skipped'].map((st) => `openvibe_backup_services{status="${st}"} ${count(st)}`),
++ ];
++ if (lastOk) lines.push('# HELP openvibe_backup_last_success_timestamp_seconds When the last fully successful run finished.', '# TYPE openvibe_backup_last_success_timestamp_seconds gauge', `openvibe_backup_last_success_timestamp_seconds ${at(lastOk)}`);
++ if (summary.offsite) lines.push('# HELP openvibe_backup_offsite_ok 1 when the last run\'s off-site copy succeeded.', '# TYPE openvibe_backup_offsite_ok gauge', `openvibe_backup_offsite_ok ${summary.offsite.ok ? 1 : 0}`);
++ await exec.writeFile(path.join(dir, 'openvibe_backup.prom'), `${lines.join('\n')}\n`, { mode: 0o644 });
++ return true;
++}
++
/** The summary of a backup --all run (the latest when run is null). */
async function readRun(exec, inv, run = null) {
const dir = path.join(inv.stateDir, 'backup-runs');
@@@ -231,4 +280,4 @@ async function writeRun(exec, summary) {
await exec.writeFile(file, `${JSON.stringify(rest, null, 2)}\n`, { mode: 0o640 });
}
--module.exports = { backup, backupAll, prune, records, readRun, writeRun, stamp, lockDown, BackupError };
++module.exports = { backup, backupAll, prune, records, readRun, writeRun, writeMetrics, stamp, lockDown, BackupError };
ddiff --git a/lib/index.js b/lib/index.js
iindex 601eb36..95dbf76 100644
---- a/lib/index.js
++++ b/lib/index.js
@@@ -7,7 +7,7 @@ const inventory = require('./inventory');
const ops = require('./release-ops');
const { validate, envNames } = require('./commands/validate');
const { status } = require('./commands/status');
--const { backup, backupAll, readRun, writeRun, BackupError } = require('./commands/backup');
++const { backup, backupAll, readRun, writeRun, writeMetrics, BackupError } = require('./commands/backup');
const offsite = require('./offsite');
const { snapshot } = require('./commands/snapshot');
const { drill, DrillError } = require('./commands/drill');
@@@ -29,6 +29,7 @@ module.exports = {
backupAll,
readRun,
writeRun,
++ writeMetrics,
BackupError,
offsite,
snapshot,
ddiff --git a/test/backup-offsite.test.js b/test/backup-offsite.test.js
iindex 19c8ce8..6ac1154 100644
---- a/test/backup-offsite.test.js
++++ b/test/backup-offsite.test.js
@@@ -238,6 +238,35 @@ runTests([
assert.deepStrictEqual(dirsOf(host, '/var/backups/openvibe/ai'), ['20260922-120000', '20260924-120000'], 'the failed one went once a newer good one existed');
}),
++ test('backup --all leaves its outcome for Prometheus (textfile collector): last run, ok, last success, per-status counts', async () => {
++ const host = backupHost();
++ const PROM = '/var/lib/prometheus/node-exporter/openvibe_backup.prom';
++ await host.cli('backup', '--all');
++ assert.strictEqual(host.files.has(PROM), false, 'no collector directory: nothing written');
++ host.put('/var/lib/prometheus/node-exporter/.keep', '');
++ host.advance(DAY);
++ await host.cli('backup', '--all');
++ const metric = (name) => { const m = new RegExp(`^${name} (\\S+)$`, 'm').exec(host.read(PROM)); return m ? Number(m[1]) : null; };
++ const goodAt = metric('openvibe_backup_last_run_timestamp_seconds');
++ assert.ok(goodAt > 0);
++ assert.strictEqual(metric('openvibe_backup_last_run_ok'), 1);
++ assert.strictEqual(metric('openvibe_backup_last_success_timestamp_seconds'), goodAt);
++ assert.strictEqual(metric('openvibe_backup_services\\{status="failed"\\}'), 0);
++ host.advance(DAY);
++ host.onSqliteBackup = (db) => (db.endsWith('extra.db') ? new Error('disk I/O error') : null);
++ await host.cli('backup', '--all');
++ host.onSqliteBackup = null;
++ assert.strictEqual(metric('openvibe_backup_last_run_ok'), 0);
++ assert.strictEqual(metric('openvibe_backup_services\\{status="failed"\\}'), 1);
++ assert.ok(metric('openvibe_backup_last_run_timestamp_seconds') > goodAt);
++ assert.strictEqual(metric('openvibe_backup_last_success_timestamp_seconds'), goodAt, 'the last success is still the good run');
++ host.files.delete(PROM);
++ const r = await host.cli('backup-metrics');
++ assert.strictEqual(r.code, 0, r.out);
++ assert.strictEqual(metric('openvibe_backup_last_run_ok'), 0, 'rewritten from the latest recorded run');
++ assert.strictEqual(metric('openvibe_backup_last_success_timestamp_seconds'), goodAt);
++ }),
++
// ── failure isolation and the summary ─────────────────────────────────────
test('one failing service never stops the others; exit code 2; JSON summary per run', async () => {
const host = backupHost();