openvibers/openvibe.host Commits
Alert rules (Track O): deploy/prometheus/openvibe-rules.yml (target down 5 min, 5xx above 5% for 10 min, root disk under 10%, memory under 8%), loaded through rule_files in the rendered config; firing alerts at 127.0.0.1:9090/api/v1/alerts
Changes
nnew file mode 100644@@@ -0,0 +1,32 @@++# OpenVibe alert rules (roadmap Track O), loaded by the host's Prometheus (rule_files in the config++# scripts/prometheus-config.js renders). Firing alerts are read at GET 127.0.0.1:9090/api/v1/alerts;++# no notifier is configured yet.++groups:++ - name: openvibe++ rules:++ - alert: TargetDown++ expr: up == 0++ for: 5m++ labels: { severity: page }++ annotations:++ summary: "{{ $labels.process }} ({{ $labels.instance }}) has not answered /metrics for 5 minutes"++ - alert: HighErrorRate++ expr: |++ sum by (service) (rate(http_requests_total{status=~"5.."}[10m]))++ / clamp_min(sum by (service) (rate(http_requests_total[10m])), 0.05) > 0.05++ for: 10m++ labels: { severity: ticket }++ annotations:++ summary: "{{ $labels.service }}: more than 5% of requests answered 5xx for 10 minutes"++ - alert: DiskAlmostFull++ expr: node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} < 0.10++ for: 10m++ labels: { severity: page }++ annotations:++ summary: "Root filesystem has less than 10% free ({{ $value | humanizePercentage }})"++ - alert: MemoryLow++ expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.08++ for: 10m++ labels: { severity: ticket }++ annotations:++ summary: "Less than 8% of memory available for 10 minutes"@@@ -8,7 +8,8 @@* "sites": { …, "metrics": false } nothing to scrape** Every target is 127.0.0.1 (the services answer /metrics only to direct loopback callers), and-- * node-exporter (host CPU, memory, disk) is scraped on 127.0.0.1:9100.++ * node-exporter (host CPU, memory, disk) is scraped on 127.0.0.1:9100. Alert rules are read from++ * /etc/prometheus/rules/*.yml (deploy/prometheus/openvibe-rules.yml).** node scripts/prometheus-config.js [--inventory /etc/openvibe/host.json] [--interval 30s] > prometheus.yml*/@@@ -40,6 +41,8 @@ function render(inventory, { interval = '30s' } = {}) {'global:',` scrape_interval: ${interval}`,' scrape_timeout: 10s',++ 'rule_files:',++ " - '/etc/prometheus/rules/*.yml'",'scrape_configs:',' - job_name: node',' static_configs:',@@@ -11,6 +11,7 @@ const out = render({ services: {noport: {},} }, { interval: '15s' });assert.ok(out.includes('scrape_interval: 15s'));++assert.ok(out.includes("rule_files:\n - '/etc/prometheus/rules/*.yml'"), 'alert rules');assert.ok(out.includes("- targets: ['127.0.0.1:9100']"), 'node-exporter');assert.ok(out.includes(" - job_name: live\n metrics_path: /metrics\n static_configs:\n - targets: ['127.0.0.1:3000']\n labels:\n service: live\n process: live\n"));assert.ok(out.includes(" - targets: ['127.0.0.1:4012']\n labels:\n service: tools\n process: tools-img\n"), 'declared extra process');
new file mode 100644
new file mode 100644
@@ -0,0 +1,32 @@
@@ -0,0 +1,32 @@
+# OpenVibe alert rules (roadmap Track O), loaded by the host's Prometheus (rule_files in the config
+# scripts/prometheus-config.js renders). Firing alerts are read at GET 127.0.0.1:9090/api/v1/alerts;
+# no notifier is configured yet.
+groups:
+ - name: openvibe
+ rules:
+ - alert: TargetDown
+ expr: up == 0
+ for: 5m
+ labels: { severity: page }
+ annotations:
+ summary: "{{ $labels.process }} ({{ $labels.instance }}) has not answered /metrics for 5 minutes"
+ - alert: HighErrorRate
+ expr: |
+ sum by (service) (rate(http_requests_total{status=~"5.."}[10m]))
+ / clamp_min(sum by (service) (rate(http_requests_total[10m])), 0.05) > 0.05
+ for: 10m
+ labels: { severity: ticket }
+ annotations:
+ summary: "{{ $labels.service }}: more than 5% of requests answered 5xx for 10 minutes"
+ - alert: DiskAlmostFull
+ expr: node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} < 0.10
+ for: 10m
+ labels: { severity: page }
+ annotations:
+ summary: "Root filesystem has less than 10% free ({{ $value | humanizePercentage }})"
+ - alert: MemoryLow
+ expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.08
+ for: 10m
+ labels: { severity: ticket }
+ annotations:
+ summary: "Less than 8% of memory available for 10 minutes"
@@ -8,7 +8,8 @@
@@ -8,7 +8,8 @@
* "sites": { …, "metrics": false } nothing to scrape
* "sites": { …, "metrics": false } nothing to scrape
*
*
* Every target is 127.0.0.1 (the services answer /metrics only to direct loopback callers), and
* Every target is 127.0.0.1 (the services answer /metrics only to direct loopback callers), and
- * node-exporter (host CPU, memory, disk) is scraped on 127.0.0.1:9100.
+ * node-exporter (host CPU, memory, disk) is scraped on 127.0.0.1:9100. Alert rules are read from
+ * /etc/prometheus/rules/*.yml (deploy/prometheus/openvibe-rules.yml).
*
*
* node scripts/prometheus-config.js [--inventory /etc/openvibe/host.json] [--interval 30s] > prometheus.yml
* node scripts/prometheus-config.js [--inventory /etc/openvibe/host.json] [--interval 30s] > prometheus.yml
*/
*/
@@ -40,6 +41,8 @@ function render(inventory, { interval = '30s' } = {}) {
@@ -40,6 +41,8 @@ function render(inventory, { interval = '30s' } = {}) {
'global:',
'global:',
` scrape_interval: ${interval}`,
` scrape_interval: ${interval}`,
' scrape_timeout: 10s',
' scrape_timeout: 10s',
+ 'rule_files:',
+ " - '/etc/prometheus/rules/*.yml'",
'scrape_configs:',
'scrape_configs:',
' - job_name: node',
' - job_name: node',
' static_configs:',
' static_configs:',
@@ -11,6 +11,7 @@ const out = render({ services: {
@@ -11,6 +11,7 @@ const out = render({ services: {
noport: {},
noport: {},
} }, { interval: '15s' });
} }, { interval: '15s' });
assert.ok(out.includes('scrape_interval: 15s'));
assert.ok(out.includes('scrape_interval: 15s'));
+assert.ok(out.includes("rule_files:\n - '/etc/prometheus/rules/*.yml'"), 'alert rules');
assert.ok(out.includes("- targets: ['127.0.0.1:9100']"), 'node-exporter');
assert.ok(out.includes("- targets: ['127.0.0.1:9100']"), 'node-exporter');
assert.ok(out.includes(" - job_name: live\n metrics_path: /metrics\n static_configs:\n - targets: ['127.0.0.1:3000']\n labels:\n service: live\n process: live\n"));
assert.ok(out.includes(" - job_name: live\n metrics_path: /metrics\n static_configs:\n - targets: ['127.0.0.1:3000']\n labels:\n service: live\n process: live\n"));
assert.ok(out.includes(" - targets: ['127.0.0.1:4012']\n labels:\n service: tools\n process: tools-img\n"), 'declared extra process');
assert.ok(out.includes(" - targets: ['127.0.0.1:4012']\n labels:\n service: tools\n process: tools-img\n"), 'declared extra process');