openvibers/openvibe.host Commits

Data role alerts: a group openvibe-data with DataPostgresDown, DataPgBouncerDown and DataValkeyDown (page after 2 minutes), DataWalArchiveFailing (page: point-in-time recovery at risk), DataPostgresConnectionsHigh (over 80% of max_connections), DataBackupStale (no good pgBackRest backup for 26 hours) and DataValkeyMemoryHigh (over 90% of maxmemory). A successful backup writes openvibe_pgbackrest_last_success_timestamp_seconds to the textfile collector through a helper run by ExecStartPost, which systemd runs only when the backup itself succeeded.

OP OpenVibers committed 28 Sept 2026 · 61e8a95926 · 5 files, +63 −0


Changes

ddiff --git a/deploy/prometheus/openvibe-rules.yml b/deploy/prometheus/openvibe-rules.yml
iindex 615e686..f444ead 100644
---- a/deploy/prometheus/openvibe-rules.yml
++++ b/deploy/prometheus/openvibe-rules.yml
@@@ -101,3 +101,48 @@ groups:
labels: { severity: page }
annotations:
summary: "Operator alerts were not delivered for more than 15 minutes (ovhost alerts relay: Prometheus, the token or Network)"
++ # The data role (roles/data, ADR-035): PostgreSQL, PgBouncer, Valkey and pgBackRest.
++ - name: openvibe-data
++ rules:
++ - alert: DataPostgresDown
++ expr: pg_up == 0
++ for: 2m
++ labels: { severity: page }
++ annotations:
++ summary: "PostgreSQL does not answer its exporter: every migrated service is down"
++ - alert: DataPgBouncerDown
++ expr: pgbouncer_up == 0
++ for: 2m
++ labels: { severity: page }
++ annotations:
++ summary: "PgBouncer does not answer: services reach PostgreSQL through it"
++ - alert: DataValkeyDown
++ expr: redis_up == 0
++ for: 2m
++ labels: { severity: page }
++ annotations:
++ summary: "Valkey does not answer: limits, caches, queues and fan-out are degraded"
++ - alert: DataPostgresConnectionsHigh
++ expr: sum(pg_stat_database_numbackends) / scalar(pg_settings_max_connections) > 0.8
++ for: 10m
++ labels: { severity: ticket }
++ annotations:
++ summary: "PostgreSQL connections above 80% of max_connections: check PgBouncer pool sizes"
++ - alert: DataWalArchiveFailing
++ expr: increase(pg_stat_archiver_failed_count[15m]) > 0
++ for: 15m
++ labels: { severity: page }
++ annotations:
++ summary: "WAL archiving to the pgBackRest repository is failing: point-in-time recovery is at risk"
++ - alert: DataBackupStale
++ expr: time() - max(openvibe_pgbackrest_last_success_timestamp_seconds) > 26 * 3600
++ for: 30m
++ labels: { severity: ticket }
++ annotations:
++ summary: "No successful pgBackRest backup for more than 26 hours"
++ - alert: DataValkeyMemoryHigh
++ expr: redis_memory_used_bytes / clamp_min(redis_memory_max_bytes, 1) > 0.9
++ for: 15m
++ labels: { severity: ticket }
++ annotations:
++ summary: "Valkey above 90% of maxmemory: only TTL keys can be evicted"
ddiff --git a/roles/data/files/backup-metric.sh b/roles/data/files/backup-metric.sh
nnew file mode 100644
iindex 0000000..07a29fe
---- /dev/null
++++ b/roles/data/files/backup-metric.sh
@@@ -0,0 +1,8 @@
++#!/bin/sh
++# Records the time of the last successful pgBackRest backup of type $1 (full or diff) for the node-exporter
++# textfile collector (alert DataBackupStale). Run by [email protected] only after a successful backup.
++set -eu
++dir=/var/lib/prometheus/node-exporter
++f="$dir/pgbackrest_$1.prom"
++printf 'openvibe_pgbackrest_last_success_timestamp_seconds{type="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp"
++chmod 644 "$f.tmp" && mv "$f.tmp" "$f"
ddiff --git a/roles/data/files/[email protected] b/roles/data/files/[email protected]
iindex 1f53ca4..b17e695 100644
---- a/roles/data/files/[email protected]
++++ b/roles/data/files/[email protected]
@@@ -7,5 +7,7 @@ Type=oneshot
User=postgres
Group=postgres
ExecStart=/usr/bin/pgbackrest --stanza=openvibe --type=%i backup
++# On success only: the time of the last good backup, for the DataBackupStale alert (textfile collector).
++ExecStartPost=+/usr/local/lib/openvibe-data/backup-metric.sh %i
Nice=10
IOSchedulingClass=idle
ddiff --git a/roles/data/provision.sh b/roles/data/provision.sh
iindex ee6ca39..03459d2 100755
---- a/roles/data/provision.sh
++++ b/roles/data/provision.sh
@@@ -88,6 +88,7 @@ CONF
)
mkdir -p -m 755 /etc/pgbackrest # umask 077 would make it 700, and postgres must read the file inside
install_file "$TMP/pgbackrest.conf" /etc/pgbackrest/pgbackrest.conf 640 root:postgres || true
++ install_file "$F/backup-metric.sh" /usr/local/lib/openvibe-data/backup-metric.sh 755 root:root || true
for u in [email protected] pgbackrest-full.timer pgbackrest-diff.timer; do
install_file "$F/$u" "/etc/systemd/system/$u" 644 root:root && RELOAD_UNITS=1
done
ddiff --git a/test/data-role.test.js b/test/data-role.test.js
iindex a1bd0cf..bd91930 100644
---- a/test/data-role.test.js
++++ b/test/data-role.test.js
@@@ -38,6 +38,13 @@ runTests([
assert.match(vk, /^aclfile /m);
assert.ok(vk.split('\n').every((l) => l.startsWith('#') || !/\s#/.test(l)), 'Valkey takes no end-of-line comments');
}),
++ test('alerts cover the data role, and the backup metric is written only after a successful backup', () => {
++ const rules = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'prometheus', 'openvibe-rules.yml'), 'utf8');
++ for (const a of ['DataPostgresDown', 'DataPgBouncerDown', 'DataValkeyDown', 'DataPostgresConnectionsHigh', 'DataWalArchiveFailing', 'DataBackupStale', 'DataValkeyMemoryHigh']) assert.ok(rules.includes(`alert: ${a}`), a);
++ const unit = read('files/[email protected]');
++ assert.match(unit, /^ExecStartPost=\+\/usr\/local\/lib\/openvibe-data\/backup-metric\.sh %i$/m, 'ExecStartPost runs only after ExecStart succeeded');
++ assert.match(read('files/backup-metric.sh'), /openvibe_pgbackrest_last_success_timestamp_seconds/);
++ }),
test('exporters bind loopback, archiving goes through pgBackRest, and secrets are never echoed', () => {
const p = read('provision.sh');
for (const port of ['9187', '9121', '9127']) assert.match(p, new RegExp(`127\\.0\\.0\\.1:${port}|127\\.0\\.0\\.1 *:?${port}`));