openvibers/openvibe.host Commits
Data role alerts: a group openvibe-data with DataPostgresDown, DataPgBouncerDown and DataValkeyDown (page after 2 minutes), DataWalArchiveFailing (page: point-in-time recovery at risk), DataPostgresConnectionsHigh (over 80% of max_connections), DataBackupStale (no good pgBackRest backup for 26 hours) and DataValkeyMemoryHigh (over 90% of maxmemory). A successful backup writes openvibe_pgbackrest_last_success_timestamp_seconds to the textfile collector through a helper run by ExecStartPost, which systemd runs only when the backup itself succeeded.
Changes
@@@ -101,3 +101,48 @@ groups:labels: { severity: page }annotations:summary: "Operator alerts were not delivered for more than 15 minutes (ovhost alerts relay: Prometheus, the token or Network)"++ # The data role (roles/data, ADR-035): PostgreSQL, PgBouncer, Valkey and pgBackRest.++ - name: openvibe-data++ rules:++ - alert: DataPostgresDown++ expr: pg_up == 0++ for: 2m++ labels: { severity: page }++ annotations:++ summary: "PostgreSQL does not answer its exporter: every migrated service is down"++ - alert: DataPgBouncerDown++ expr: pgbouncer_up == 0++ for: 2m++ labels: { severity: page }++ annotations:++ summary: "PgBouncer does not answer: services reach PostgreSQL through it"++ - alert: DataValkeyDown++ expr: redis_up == 0++ for: 2m++ labels: { severity: page }++ annotations:++ summary: "Valkey does not answer: limits, caches, queues and fan-out are degraded"++ - alert: DataPostgresConnectionsHigh++ expr: sum(pg_stat_database_numbackends) / scalar(pg_settings_max_connections) > 0.8++ for: 10m++ labels: { severity: ticket }++ annotations:++ summary: "PostgreSQL connections above 80% of max_connections: check PgBouncer pool sizes"++ - alert: DataWalArchiveFailing++ expr: increase(pg_stat_archiver_failed_count[15m]) > 0++ for: 15m++ labels: { severity: page }++ annotations:++ summary: "WAL archiving to the pgBackRest repository is failing: point-in-time recovery is at risk"++ - alert: DataBackupStale++ expr: time() - max(openvibe_pgbackrest_last_success_timestamp_seconds) > 26 * 3600++ for: 30m++ labels: { severity: ticket }++ annotations:++ summary: "No successful pgBackRest backup for more than 26 hours"++ - alert: DataValkeyMemoryHigh++ expr: redis_memory_used_bytes / clamp_min(redis_memory_max_bytes, 1) > 0.9++ for: 15m++ labels: { severity: ticket }++ annotations:++ summary: "Valkey above 90% of maxmemory: only TTL keys can be evicted"nnew file mode 100644@@@ -0,0 +1,8 @@++#!/bin/sh++# Records the time of the last successful pgBackRest backup of type $1 (full or diff) for the node-exporter++# textfile collector (alert DataBackupStale). Run by [email protected] only after a successful backup.++set -eu++dir=/var/lib/prometheus/node-exporter++f="$dir/pgbackrest_$1.prom"++printf 'openvibe_pgbackrest_last_success_timestamp_seconds{type="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp"++chmod 644 "$f.tmp" && mv "$f.tmp" "$f"@@@ -7,5 +7,7 @@ Type=oneshotUser=postgresGroup=postgresExecStart=/usr/bin/pgbackrest --stanza=openvibe --type=%i backup++# On success only: the time of the last good backup, for the DataBackupStale alert (textfile collector).++ExecStartPost=+/usr/local/lib/openvibe-data/backup-metric.sh %iNice=10IOSchedulingClass=idle@@@ -88,6 +88,7 @@ CONF)mkdir -p -m 755 /etc/pgbackrest # umask 077 would make it 700, and postgres must read the file insideinstall_file "$TMP/pgbackrest.conf" /etc/pgbackrest/pgbackrest.conf 640 root:postgres || true++ install_file "$F/backup-metric.sh" /usr/local/lib/openvibe-data/backup-metric.sh 755 root:root || truefor u in [email protected] pgbackrest-full.timer pgbackrest-diff.timer; doinstall_file "$F/$u" "/etc/systemd/system/$u" 644 root:root && RELOAD_UNITS=1done@@@ -38,6 +38,13 @@ runTests([assert.match(vk, /^aclfile /m);assert.ok(vk.split('\n').every((l) => l.startsWith('#') || !/\s#/.test(l)), 'Valkey takes no end-of-line comments');}),++ test('alerts cover the data role, and the backup metric is written only after a successful backup', () => {++ const rules = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'prometheus', 'openvibe-rules.yml'), 'utf8');++ for (const a of ['DataPostgresDown', 'DataPgBouncerDown', 'DataValkeyDown', 'DataPostgresConnectionsHigh', 'DataWalArchiveFailing', 'DataBackupStale', 'DataValkeyMemoryHigh']) assert.ok(rules.includes(`alert: ${a}`), a);++ const unit = read('files/[email protected]');++ assert.match(unit, /^ExecStartPost=\+\/usr\/local\/lib\/openvibe-data\/backup-metric\.sh %i$/m, 'ExecStartPost runs only after ExecStart succeeded');++ assert.match(read('files/backup-metric.sh'), /openvibe_pgbackrest_last_success_timestamp_seconds/);++ }),test('exporters bind loopback, archiving goes through pgBackRest, and secrets are never echoed', () => {const p = read('provision.sh');for (const port of ['9187', '9121', '9127']) assert.match(p, new RegExp(`127\\.0\\.0\\.1:${port}|127\\.0\\.0\\.1 *:?${port}`));
@@ -101,3 +101,48 @@ groups:
@@ -101,3 +101,48 @@ groups:
labels: { severity: page }
labels: { severity: page }
annotations:
annotations:
summary: "Operator alerts were not delivered for more than 15 minutes (ovhost alerts relay: Prometheus, the token or Network)"
summary: "Operator alerts were not delivered for more than 15 minutes (ovhost alerts relay: Prometheus, the token or Network)"
+ # The data role (roles/data, ADR-035): PostgreSQL, PgBouncer, Valkey and pgBackRest.
+ - name: openvibe-data
+ rules:
+ - alert: DataPostgresDown
+ expr: pg_up == 0
+ for: 2m
+ labels: { severity: page }
+ annotations:
+ summary: "PostgreSQL does not answer its exporter: every migrated service is down"
+ - alert: DataPgBouncerDown
+ expr: pgbouncer_up == 0
+ for: 2m
+ labels: { severity: page }
+ annotations:
+ summary: "PgBouncer does not answer: services reach PostgreSQL through it"
+ - alert: DataValkeyDown
+ expr: redis_up == 0
+ for: 2m
+ labels: { severity: page }
+ annotations:
+ summary: "Valkey does not answer: limits, caches, queues and fan-out are degraded"
+ - alert: DataPostgresConnectionsHigh
+ expr: sum(pg_stat_database_numbackends) / scalar(pg_settings_max_connections) > 0.8
+ for: 10m
+ labels: { severity: ticket }
+ annotations:
+ summary: "PostgreSQL connections above 80% of max_connections: check PgBouncer pool sizes"
+ - alert: DataWalArchiveFailing
+ expr: increase(pg_stat_archiver_failed_count[15m]) > 0
+ for: 15m
+ labels: { severity: page }
+ annotations:
+ summary: "WAL archiving to the pgBackRest repository is failing: point-in-time recovery is at risk"
+ - alert: DataBackupStale
+ expr: time() - max(openvibe_pgbackrest_last_success_timestamp_seconds) > 26 * 3600
+ for: 30m
+ labels: { severity: ticket }
+ annotations:
+ summary: "No successful pgBackRest backup for more than 26 hours"
+ - alert: DataValkeyMemoryHigh
+ expr: redis_memory_used_bytes / clamp_min(redis_memory_max_bytes, 1) > 0.9
+ for: 15m
+ labels: { severity: ticket }
+ annotations:
+ summary: "Valkey above 90% of maxmemory: only TTL keys can be evicted"
new file mode 100644
new file mode 100644
@@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
+#!/bin/sh
+# Records the time of the last successful pgBackRest backup of type $1 (full or diff) for the node-exporter
+# textfile collector (alert DataBackupStale). Run by [email protected] only after a successful backup.
+set -eu
+dir=/var/lib/prometheus/node-exporter
+f="$dir/pgbackrest_$1.prom"
+printf 'openvibe_pgbackrest_last_success_timestamp_seconds{type="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp"
+chmod 644 "$f.tmp" && mv "$f.tmp" "$f"
@@ -7,5 +7,7 @@ Type=oneshot
@@ -7,5 +7,7 @@ Type=oneshot
User=postgres
User=postgres
Group=postgres
Group=postgres
ExecStart=/usr/bin/pgbackrest --stanza=openvibe --type=%i backup
ExecStart=/usr/bin/pgbackrest --stanza=openvibe --type=%i backup
+# On success only: the time of the last good backup, for the DataBackupStale alert (textfile collector).
+ExecStartPost=+/usr/local/lib/openvibe-data/backup-metric.sh %i
Nice=10
Nice=10
IOSchedulingClass=idle
IOSchedulingClass=idle
@@ -88,6 +88,7 @@ CONF
@@ -88,6 +88,7 @@ CONF
)
)
mkdir -p -m 755 /etc/pgbackrest # umask 077 would make it 700, and postgres must read the file inside
mkdir -p -m 755 /etc/pgbackrest # umask 077 would make it 700, and postgres must read the file inside
install_file "$TMP/pgbackrest.conf" /etc/pgbackrest/pgbackrest.conf 640 root:postgres || true
install_file "$TMP/pgbackrest.conf" /etc/pgbackrest/pgbackrest.conf 640 root:postgres || true
+ install_file "$F/backup-metric.sh" /usr/local/lib/openvibe-data/backup-metric.sh 755 root:root || true
for u in [email protected] pgbackrest-full.timer pgbackrest-diff.timer; do
for u in [email protected] pgbackrest-full.timer pgbackrest-diff.timer; do
install_file "$F/$u" "/etc/systemd/system/$u" 644 root:root && RELOAD_UNITS=1
install_file "$F/$u" "/etc/systemd/system/$u" 644 root:root && RELOAD_UNITS=1
done
done
@@ -38,6 +38,13 @@ runTests([
@@ -38,6 +38,13 @@ runTests([
assert.match(vk, /^aclfile /m);
assert.match(vk, /^aclfile /m);
assert.ok(vk.split('\n').every((l) => l.startsWith('#') || !/\s#/.test(l)), 'Valkey takes no end-of-line comments');
assert.ok(vk.split('\n').every((l) => l.startsWith('#') || !/\s#/.test(l)), 'Valkey takes no end-of-line comments');
}),
}),
+ test('alerts cover the data role, and the backup metric is written only after a successful backup', () => {
+ const rules = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'prometheus', 'openvibe-rules.yml'), 'utf8');
+ for (const a of ['DataPostgresDown', 'DataPgBouncerDown', 'DataValkeyDown', 'DataPostgresConnectionsHigh', 'DataWalArchiveFailing', 'DataBackupStale', 'DataValkeyMemoryHigh']) assert.ok(rules.includes(`alert: ${a}`), a);
+ const unit = read('files/[email protected]');
+ assert.match(unit, /^ExecStartPost=\+\/usr\/local\/lib\/openvibe-data\/backup-metric\.sh %i$/m, 'ExecStartPost runs only after ExecStart succeeded');
+ assert.match(read('files/backup-metric.sh'), /openvibe_pgbackrest_last_success_timestamp_seconds/);
+ }),
test('exporters bind loopback, archiving goes through pgBackRest, and secrets are never echoed', () => {
test('exporters bind loopback, archiving goes through pgBackRest, and secrets are never echoed', () => {
const p = read('provision.sh');
const p = read('provision.sh');
for (const port of ['9187', '9121', '9127']) assert.match(p, new RegExp(`127\\.0\\.0\\.1:${port}|127\\.0\\.0\\.1 *:?${port}`));
for (const port of ['9187', '9121', '9127']) assert.match(p, new RegExp(`127\\.0\\.0\\.1:${port}|127\\.0\\.0\\.1 *:?${port}`));