alerts: when AI throttles free use hard, open a review item (#305)

OP OpenVibers committed 7 days ago · 70a5d7dbd3 · 2 files, +23 −0

Plan E5. OpenVibe.AI exposes ai_abuse_throttles{reason} (throttles in the last 15 minutes) from server/abuse/; this adds OpenVibeAiAbuseThrottling (severity ticket) above 20, pointing the operator at /console/abuse. The earlier AI balance and GPU alerts are untouched.

Changes

ddiff --git a/deploy/prometheus/openvibe-rules.yml b/deploy/prometheus/openvibe-rules.yml
iindex 68b0c1f..00e8876 100644
---- a/deploy/prometheus/openvibe-rules.yml
++++ b/deploy/prometheus/openvibe-rules.yml
@@@ -45,6 +45,15 @@ groups:
          labels: { severity: page }
          annotations:
            summary: 'OpenVibe.AI could not give a {{ $labels.marketplace }} GPU instance back ({{ $value }} tries in a row): it is still running and still being paid for'
++      # OpenVibe.AI's free-tier abuse controls (AI server/abuse/, plan E5): a burst of throttles means free capacity is
++      # being pushed much harder than usual from one network, device or repeated request. The throttle itself is
++      # automatic; this opens a review queue item, with the page at /console/abuse on the AI operator console.
++      - alert: OpenVibeAiAbuseThrottling
++        expr: sum(ai_abuse_throttles) > 20
++        for: 10m
++        labels: { severity: ticket }
++        annotations:
++          summary: 'OpenVibe.AI is throttling free use hard ({{ $value }} requests in 15 minutes): review /console/abuse'
        - alert: HighErrorRate
          expr: |
            sum by (service) (rate(http_requests_total{status=~"5.."}[10m]))
ddiff --git a/test/prometheus-rules.test.js b/test/prometheus-rules.test.js
nnew file mode 100644
iindex 0000000..0873b68
---- /dev/null
++++ b/test/prometheus-rules.test.js
@@@ -0,0 +1,14 @@
++'use strict';
++// The alert rules shipped in deploy/prometheus/openvibe-rules.yml (loaded by Prometheus' rule_files; `ovhost alerts
++// relay` sends what fires). Promtool is not installed on every agent box, so this checks the contract the AI gauge
++// and the rule share: the alert names the gauge and carries a severity.
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++
++const rules = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'prometheus', 'openvibe-rules.yml'), 'utf8');
++assert.ok(/alert: OpenVibeAiAbuseThrottling\n/.test(rules), 'the abuse alert exists');
++assert.ok(/expr: sum\(ai_abuse_throttles\) > 20/.test(rules), 'it reads the AI gauge');
++assert.ok(/alert: OpenVibeAiAbuseThrottling[\s\S]{0,200}severity: ticket/.test(rules), 'it opens a review item, never a page');
++assert.ok(/alert: OpenVibeAiProviderBalanceLow\n/.test(rules) && /alert: OpenVibeAiGpuInstanceNotDestroyed\n/.test(rules), 'the earlier AI alerts are untouched');
++console.log('prometheus rules: all checks passed');

diff --git a/deploy/prometheus/openvibe-rules.yml b/deploy/prometheus/openvibe-rules.yml

index 68b0c1f..00e8876 100644

--- a/deploy/prometheus/openvibe-rules.yml

+++ b/deploy/prometheus/openvibe-rules.yml

@@ -45,6 +45,15 @@ groups:

labels: { severity: page }

annotations:

summary: 'OpenVibe.AI could not give a {{ $labels.marketplace }} GPU instance back ({{ $value }} tries in a row): it is still running and still being paid for'

+ # OpenVibe.AI's free-tier abuse controls (AI server/abuse/, plan E5): a burst of throttles means free capacity is

+ # being pushed much harder than usual from one network, device or repeated request. The throttle itself is

+ # automatic; this opens a review queue item, with the page at /console/abuse on the AI operator console.

+ - alert: OpenVibeAiAbuseThrottling

+ expr: sum(ai_abuse_throttles) > 20

+ for: 10m

+ labels: { severity: ticket }

+ annotations:

+ summary: 'OpenVibe.AI is throttling free use hard ({{ $value }} requests in 15 minutes): review /console/abuse'

- alert: HighErrorRate

expr: |

sum by (service) (rate(http_requests_total{status=~"5.."}[10m]))

diff --git a/test/prometheus-rules.test.js b/test/prometheus-rules.test.js

new file mode 100644

index 0000000..0873b68

--- /dev/null

+++ b/test/prometheus-rules.test.js

@@ -0,0 +1,14 @@

+'use strict';

+// The alert rules shipped in deploy/prometheus/openvibe-rules.yml (loaded by Prometheus' rule_files; `ovhost alerts

+// relay` sends what fires). Promtool is not installed on every agent box, so this checks the contract the AI gauge

+// and the rule share: the alert names the gauge and carries a severity.

+const assert = require('assert');

+const fs = require('fs');

+const path = require('path');

+const rules = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'prometheus', 'openvibe-rules.yml'), 'utf8');

+assert.ok(/alert: OpenVibeAiAbuseThrottling\n/.test(rules), 'the abuse alert exists');

+assert.ok(/expr: sum$ai_abuse_throttles$ > 20/.test(rules), 'it reads the AI gauge');

+assert.ok(/alert: OpenVibeAiAbuseThrottling[\s\S]{0,200}severity: ticket/.test(rules), 'it opens a review item, never a page');

+assert.ok(/alert: OpenVibeAiProviderBalanceLow\n/.test(rules) && /alert: OpenVibeAiGpuInstanceNotDestroyed\n/.test(rules), 'the earlier AI alerts are untouched');

+console.log('prometheus rules: all checks passed');