openvibers/openvibe.ai Commits

Track O: /metrics for direct loopback callers and a truthful /api/ready (openvibe-shared v1.3.0)

OP OpenVibers committed 23 Sept 2026 · 96dc8a3d9c · 8 files, +356 −15

Adds openvibe-shared v1.3.0. /metrics (openvibe-shared/metrics): golden signals labelled by route
template, process metrics, release_info, ai_runs{state="queued"|"running"} with the concurrency and
queue caps, ai_provider_circuit{provider,state} and ai_provider_credentials_missing{provider}; a
request carrying X-Forwarded-For, X-Real-IP or CF-Connecting-IP gets 404. /api/ready
(openvibe-shared/ready): the database, an active workflow and the Network key are required (503);
provider configuration (an active real provider without its credentials or with an open circuit,
or no real provider at all) is optional and degrades it, with each provider's state and the queue in
the body. GET /release.json. nginx answers /metrics with 404. test/observability.test.js.

Changes

ddiff --git a/README.md b/README.md
iindex 978b7ed..b9fd9f7 100644
---- a/README.md
++++ b/README.md
@@@ -60,7 +60,13 @@ idempotent replay):
  | `POST /api/v1/{templates,workflows,routes}/:key/versions`, `…/versions/:v/status` | `ai.workflow.manage` |
  | `POST/PATCH /api/v1/providers…`, `…/disable\|enable\|reset`, `POST/PATCH /api/v1/models…`, `POST /api/v1/quotas`, `DELETE /api/v1/cache` | `ai.provider.manage` |
  | `GET /api/v1/status\|usage\|quotas\|requests\|audit\|cache\|providers\|models` | `ai.usage.read` |
--| `GET /api/health`, `GET /api/ready` | public |
++| `GET /api/health`, `GET /api/ready`, `GET /release.json` | public |
++| `GET /metrics` | direct loopback callers only (a request carrying X-Forwarded-For gets 404) |
++
++`/api/ready` (openvibe-shared/ready) is 503 when the database, the active workflows or the Network
++key fail; provider configuration (an active provider without its credentials, or with an open
++circuit) degrades it. `/metrics` (openvibe-shared/metrics) carries golden signals by route template,
++`ai_runs{state="queued"|"running"}` and `ai_provider_circuit{provider,state}`.
  
  Errors are `application/problem+json` (openvibe-contracts `http.problem`): `input.invalid` (with the
  schema errors), `workflow.not_found`, `idempotency.conflict`, `quota.exceeded` (429 +
@@@ -180,7 +186,8 @@ installed: `ai.openvibe.network` still serves the Sites placeholder.
    translations cannot become cache entries (they have no source text) and stay in Live.
  - No fallback is declared on any production route, so an outage of the one real provider fails every
    Live AI feature; `live.*` and `network.site_copy` outputs carry no citations or gaps (0 citation rows).
--- `/metrics` (404); the host has a backup of `ai.db` but no restore drill has run for it.
++- The host has a backup of `ai.db` but no restore drill has run for it; `/metrics` is built but not
++  deployed yet.
  
  ## Launch rule
  
ddiff --git a/deploy/nginx/ai.openvibe.network.conf b/deploy/nginx/ai.openvibe.network.conf
iindex 36ac6eb..6d77e1a 100644
---- a/deploy/nginx/ai.openvibe.network.conf
++++ b/deploy/nginx/ai.openvibe.network.conf
@@@ -7,6 +7,7 @@
  ##
  ## The Node service (port 4700, systemd unit openvibe-ai) serves:
  ##   /api/health, /api/ready
++##   /metrics    Prometheus text, answered only to direct loopback callers (never proxied)
  ##   /api/v1/*   service-to-service API (Network service tokens, audience openvibe.ai). Services on
  ##               this host call 127.0.0.1:4700 directly, so the public vhost refuses it; open the
  ##               location below only when a service on another host needs it (it still needs a token).
@@@ -44,6 +45,9 @@ server {
      location = /api/health { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      location = /api/ready  { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
  
++    # Metrics are for host-local scrapers only (http://127.0.0.1:4700/metrics).
++    location = /metrics { return 404; }
++
      # ── Service API: host-local only (see header) ───────────
      location /api/ {
          return 403;
ddiff --git a/package-lock.json b/package-lock.json
iindex 13d5991..b22bc6e 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -13,7 +13,8 @@
          "better-sqlite3": "^11.10.0",
          "dotenv": "^16.4.7",
          "express": "^4.21.2",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.29.0"
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.29.0",
++        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
        },
        "engines": {
          "node": ">=20.0.0"
@@@ -544,6 +545,12 @@
          "ieee754": "^1.1.13"
        }
      },
++    "node_modules/buffer-equal-constant-time": {
++      "version": "1.0.1",
++      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
++      "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
++      "license": "BSD-3-Clause"
++    },
      "node_modules/bytes": {
        "version": "3.1.2",
        "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@@ -756,6 +763,15 @@
          "node": ">= 0.4"
        }
      },
++    "node_modules/ecdsa-sig-formatter": {
++      "version": "1.0.11",
++      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
++      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
++      "license": "Apache-2.0",
++      "dependencies": {
++        "safe-buffer": "^5.0.1"
++      }
++    },
      "node_modules/ee-first": {
        "version": "1.1.1",
        "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@@ -1124,6 +1140,97 @@
        "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
        "license": "MIT"
      },
++    "node_modules/jsonwebtoken": {
++      "version": "9.0.3",
++      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz",
++      "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==",
++      "license": "MIT",
++      "dependencies": {
++        "jws": "^4.0.1",
++        "lodash.includes": "^4.3.0",
++        "lodash.isboolean": "^3.0.3",
++        "lodash.isinteger": "^4.0.4",
++        "lodash.isnumber": "^3.0.3",
++        "lodash.isplainobject": "^4.0.6",
++        "lodash.isstring": "^4.0.1",
++        "lodash.once": "^4.0.0",
++        "ms": "^2.1.1",
++        "semver": "^7.5.4"
++      },
++      "engines": {
++        "node": ">=12",
++        "npm": ">=6"
++      }
++    },
++    "node_modules/jsonwebtoken/node_modules/ms": {
++      "version": "2.1.3",
++      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
++      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
++      "license": "MIT"
++    },
++    "node_modules/jwa": {
++      "version": "2.0.1",
++      "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",
++      "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==",
++      "license": "MIT",
++      "dependencies": {
++        "buffer-equal-constant-time": "^1.0.1",
++        "ecdsa-sig-formatter": "1.0.11",
++        "safe-buffer": "^5.0.1"
++      }
++    },
++    "node_modules/jws": {
++      "version": "4.0.1",
++      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
++      "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==",
++      "license": "MIT",
++      "dependencies": {
++        "jwa": "^2.0.1",
++        "safe-buffer": "^5.0.1"
++      }
++    },
++    "node_modules/lodash.includes": {
++      "version": "4.3.0",
++      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
++      "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isboolean": {
++      "version": "3.0.3",
++      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
++      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isinteger": {
++      "version": "4.0.4",
++      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
++      "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isnumber": {
++      "version": "3.0.3",
++      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
++      "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isplainobject": {
++      "version": "4.0.6",
++      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
++      "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isstring": {
++      "version": "4.0.1",
++      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
++      "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.once": {
++      "version": "4.1.1",
++      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
++      "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
++      "license": "MIT"
++    },
      "node_modules/math-intrinsics": {
        "version": "1.1.0",
        "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@@ -1302,6 +1409,15 @@
          "node": ">=20"
        }
      },
++    "node_modules/openvibe-shared": {
++      "version": "1.3.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
++      "integrity": "sha512-w8yl/5+9eend8I2O/DzAJnEAL2N3GTUY7c7pzK+MLJuLAGD8FmJJQ9gwbgyrq70Snjt3ebrg3TlpORhRAp2kgA==",
++      "license": "MIT",
++      "dependencies": {
++        "jsonwebtoken": "^9.0.0"
++      }
++    },
      "node_modules/parseurl": {
        "version": "1.3.3",
        "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
ddiff --git a/package.json b/package.json
iindex 7624a40..6a2b3ce 100644
---- a/package.json
++++ b/package.json
@@@ -19,7 +19,8 @@
      "better-sqlite3": "^11.10.0",
      "dotenv": "^16.4.7",
      "express": "^4.21.2",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.29.0"
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.29.0",
++    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
    },
    "optionalDependencies": {
      "sharp": "^0.33.5"
ddiff --git a/server/app.js b/server/app.js
iindex 7b54f75..5d80da6 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -1,7 +1,11 @@
  'use strict';
--/** Express app: request context, health/readiness, the v1 run API and the admin/registry API. */
++/** Express app: request context, health/readiness, metrics, the v1 run API and the admin/registry API. */
++const path = require('path');
  const express = require('express');
  const { http } = require('openvibe-contracts');
++const { instrument } = require('openvibe-shared/metrics');
++const { createRelease } = require('openvibe-shared/release');
++const { createAiReadiness, registerAiGauges } = require('./observability');
  const { runsRouter } = require('./api/runs');
  const { adminRouter } = require('./api/admin');
  const pkg = require('../package.json');
@@@ -10,6 +14,11 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
      const app = express();
      app.disable('x-powered-by');
      app.set('trust proxy', 'loopback');
++    const release = createRelease({ service: 'ai', root: path.join(__dirname, '..') });
++    // HTTP golden signals by route template, process metrics, release_info and the AI gauges;
++    // GET /metrics answers direct loopback callers only (Track O).
++    const metrics = instrument(app, { service: 'ai', release: release.release });
++    registerAiGauges(metrics.registry, { runs, registry, pool });
      app.use(http.middleware());
      app.use((req, res, next) => {
          res.setHeader('X-Content-Type-Options', 'nosniff');
@@@ -23,14 +32,11 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
          res.json({ status: 'ok', service: 'openvibe-ai', version: pkg.version });
      });
  
--    app.get('/api/ready', (_req, res) => {
--        let dbOk = false;
--        try { dbOk = db.prepare('SELECT 1 AS x').get().x === 1; } catch { dbOk = false; }
--        const checks = { db: dbOk, key: keys.loaded(), workflows: dbOk && registry.listWorkflows().length > 0 };
--        const ready = Object.values(checks).every(Boolean);
--        const providers = dbOk ? registry.listProviders().map(p => ({ key: p.key, kind: p.kind, status: p.status, health: pool.health(p.key).state })) : [];
--        res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'not_ready', checks, providers, inflight: runs.inflight.size });
--    });
++    // Readiness (openvibe-shared/ready): 503 when the database, the workflows or the Network key
++    // fail; provider configuration is optional and degrades it (see observability.js).
++    const readiness = createAiReadiness({ db, registry, pool, keys, runs, release: release.release });
++    app.get('/api/ready', readiness.handler);
++    app.get('/release.json', release.handler);
  
      app.use(runsRouter({ runs, registry, auth, config, log }));
      app.use(adminRouter({ db, registry, pool, quotas, cache, runs, auth, log }));
@@@ -43,7 +49,7 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
              'GET  /api/v1/runs/:id    POST /api/v1/runs/:id/cancel|retry',
              'POST /api/v1/{chat,generate,summarize,classify,extract,enrich,embed}',
              'GET  /api/v1/workflows | templates | routes | providers | models | quotas | usage | audit',
--            'GET  /api/health, /api/ready',
++            'GET  /api/health, /api/ready, /release.json',
              '',
              'Callers authenticate with OpenVibe.Network service tokens (audience openvibe.ai).',
              'AI output is a draft/evidence package attributed to a workflow, model and run, never to a person.',
@@@ -64,6 +70,7 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
          return http.sendProblem(res, 500, 'ai.internal', { detail: 'internal error', ctx: req.ov });
      });
  
++    app.locals.metrics = metrics;
      return app;
  }
  
ddiff --git a/server/observability.js b/server/observability.js
nnew file mode 100644
iindex 0000000..cbaf738
---- /dev/null
++++ b/server/observability.js
@@@ -0,0 +1,85 @@
++'use strict';
++/**
++ * Track O: truthful readiness for GET /api/ready and the AI gauges on GET /metrics
++ * (openvibe-shared/ready and openvibe-shared/metrics).
++ *
++ *   db            required  a real query on AI's SQLite (the provider registry answers)
++ *   workflows     required  at least one active workflow: without one there is nothing to run
++ *   network_jwks  required  the Network signing key has loaded. Every endpoint but health/ready
++ *                           needs a verified service token, so without it AI serves nothing
++ *   providers     optional  provider configuration: every active non-stub provider has the
++ *                           credentials it needs and a circuit that is not open. A missing key or an
++ *                           open circuit sends runs to fallbacks (or to a 503 provider.unavailable),
++ *                           so it degrades the service rather than taking it out
++ *
++ * Gauges: runs waiting for a slot and running (ai_runs), and each provider's circuit state
++ * (ai_provider_circuit{provider,state} = 1 for the current state, 0 for the others).
++ */
++const { createReadiness } = require('openvibe-shared/ready');
++
++const CIRCUIT_STATES = ['closed', 'half_open', 'open'];
++
++/** Active providers with what they need to answer: credentials and circuit state. */
++function providerView(registry, pool) {
++    return registry.listProviders().filter(p => p.status === 'active').map((p) => {
++        const pub = registry.publicProvider(p);
++        return { key: p.key, kind: p.kind, credentials: pub.credentials, circuit: pool.health(p.key).state };
++    });
++}
++
++function createAiReadiness({ db, registry, pool, keys, runs, release = null }) {
++    return createReadiness({
++        service: 'ai',
++        release,
++        checks: [
++            { name: 'db', required: true, check: () => (db.prepare('SELECT COUNT(*) AS n FROM providers').get().n > 0 ? true : 'the provider registry is empty') },
++            {
++                name: 'workflows', required: true,
++                check: () => {
++                    const n = db.prepare("SELECT COUNT(DISTINCT key) AS n FROM workflows WHERE status = 'active'").get().n;
++                    return n > 0 ? { ok: true, detail: { active: n } } : 'no active workflow';
++                },
++            },
++            { name: 'network_jwks', required: true, check: () => keys.loaded() || 'Network signing key not loaded yet: no service token can be verified' },
++            {
++                name: 'providers', required: false,
++                check: () => {
++                    const list = providerView(registry, pool);
++                    const real = list.filter(p => p.kind !== 'stub');
++                    const detail = { active: list.length, usable: real.filter(p => p.credentials !== 'missing' && p.circuit !== 'open').length };
++                    if (!real.length) return { ok: false, error: 'no real provider is active: only synthetic stub output is available', detail };
++                    const problems = real.filter(p => p.credentials === 'missing' || p.circuit === 'open')
++                        .map(p => `${p.key}: ${p.credentials === 'missing' ? 'credentials missing' : 'circuit open'}`);
++                    return problems.length ? { ok: false, error: problems.join('; '), detail } : { ok: true, detail };
++                },
++            },
++        ],
++        details: (body) => ({
++            providers: body.checks.db.status === 'ok' ? providerView(registry, pool) : null,
++            runs: runs.stats(),
++        }),
++    });
++}
++
++/** AI gauges on the openvibe-shared/metrics registry. */
++function registerAiGauges(registry, { runs, registry: ai, pool }) {
++    registry.gauge({
++        name: 'ai_runs', help: 'Runs waiting for a slot (queued) and holding one (running)', labelNames: ['state'],
++        collect: () => {
++            const s = runs.stats();
++            return [{ labels: { state: 'queued' }, value: s.queued }, { labels: { state: 'running' }, value: s.running }];
++        },
++    });
++    registry.gauge({ name: 'ai_runs_max_concurrent', help: 'Runs allowed to execute at once (AI_MAX_CONCURRENT_RUNS)', collect: () => runs.stats().max_concurrent });
++    registry.gauge({ name: 'ai_runs_max_queued', help: 'Runs allowed to wait for a slot (AI_MAX_QUEUED_RUNS)', collect: () => runs.stats().max_queued });
++    registry.gauge({
++        name: 'ai_provider_circuit', help: 'Circuit-breaker state of each active provider (1 = the current state)', labelNames: ['provider', 'state'],
++        collect: () => providerView(ai, pool).flatMap(p => CIRCUIT_STATES.map(state => ({ labels: { provider: p.key, state }, value: p.circuit === state ? 1 : 0 }))),
++    });
++    registry.gauge({
++        name: 'ai_provider_credentials_missing', help: 'Active providers whose secret reference does not resolve (1 = missing)', labelNames: ['provider'],
++        collect: () => providerView(ai, pool).filter(p => p.credentials !== 'not_required').map(p => ({ labels: { provider: p.key }, value: p.credentials === 'missing' ? 1 : 0 })),
++    });
++}
++
++module.exports = { createAiReadiness, registerAiGauges };
ddiff --git a/server/runs.js b/server/runs.js
iindex 9cba4ca..7adc65d 100644
---- a/server/runs.js
++++ b/server/runs.js
@@@ -382,7 +382,10 @@ function createRuns({ db, registry, engine, cache, quotas, config, clock = { now
          await Promise.allSettled([...inflight.values()].map(h => h.promise));
      }
  
--    return { create, wait, get, list, cancel, retry, citations, addCitations, requestsFor, recoverInterrupted, prune, drain, inflight, TERMINAL, decode };
++    /** Queue depth for /metrics and /api/ready: runs waiting for a slot, and runs holding one. */
++    function stats() { return { queued: queue.length, running: active, inflight: inflight.size, max_concurrent: config.runs.maxConcurrent, max_queued: config.runs.maxQueued }; }
++
++    return { create, wait, get, list, cancel, retry, citations, addCitations, requestsFor, recoverInterrupted, prune, drain, stats, inflight, TERMINAL, decode };
  }
  
  module.exports = { createRuns, entityKey, subjectKey };
ddiff --git a/test/observability.test.js b/test/observability.test.js
nnew file mode 100644
iindex 0000000..982341e
---- /dev/null
++++ b/test/observability.test.js
@@@ -0,0 +1,118 @@
++'use strict';
++// Track O: GET /metrics answers direct loopback callers only, labels requests by route template and
++// carries the AI gauges (queued/running runs, provider circuit state); /api/ready is 503 when the
++// database fails, and provider configuration (missing credentials, an open circuit) degrades it.
++
++const assert = require('assert');
++const nodeHttp = require('http');
++const { boot, request, token, suite, ALL } = require('./helpers');
++
++const t = suite('observability');
++const admin = token('ops', ALL);
++const caller = token('live', ['ai.run.create', 'ai.run.read']);
++let h;
++
++function get(base, p, headers = {}) {
++    return new Promise((resolve, reject) => nodeHttp.get(base + p, { headers }, (res) => {
++        let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b }));
++    }).on('error', reject));
++}
++const metric = (text, re) => { const m = text.match(re); return m ? Number(m[1]) : null; };
++
++t.test('boot with one run slot, a configured shared key and a slow provider', async () => {
++    h = await boot({ env: { AI_API_KEY: 'sk-test-not-real', AI_MAX_CONCURRENT_RUNS: '1', AI_STUB_FALLBACK: 'false' } });
++    await h.keyLoaded;
++    const p = await request(h.base, 'POST', '/api/v1/providers', { tok: admin, body: { key: 'slowstub', kind: 'stub', auth_mode: 'none', capabilities: ['chat', 'generate', 'summarize', 'json'], metadata: { delay_ms: 1500 } } });
++    assert.strictEqual(p.status, 201, p.text);
++    const r = await request(h.base, 'POST', '/api/v1/routes/default.chat/versions', { tok: admin, body: { primary: { provider: 'slowstub' }, fallbacks: [] } });
++    assert.strictEqual(r.status, 201, r.text);
++});
++
++t.test('/api/ready: every check reports; providers is the only optional one', async () => {
++    const r = await request(h.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 200, r.text);
++    assert.strictEqual(r.body.status, 'ready');
++    assert.strictEqual(r.body.service, 'ai');
++    assert.deepStrictEqual(Object.keys(r.body.checks), ['db', 'workflows', 'network_jwks', 'providers']);
++    for (const [name, c] of Object.entries(r.body.checks)) {
++        assert.strictEqual(c.status, 'ok', `${name}: ${c.error}`);
++        assert.strictEqual(c.required, name !== 'providers', name);
++        assert.strictEqual(typeof c.latency_ms, 'number');
++        assert.ok(Date.parse(c.checked_at));
++    }
++    assert.ok(r.body.providers.some(p => p.key === 'shared' && p.credentials === 'configured' && p.circuit === 'closed'));
++    assert.strictEqual(r.body.runs.queued, 0);
++    assert.ok(!r.text.includes('sk-test-not-real'));
++});
++
++t.test('/metrics: 404 through a proxy; queued and running runs, route templates, circuit state direct', async () => {
++    const ids = [];
++    for (let i = 0; i < 3; i++) {
++        const c = await request(h.base, 'POST', '/api/v1/runs', { tok: caller, body: { workflow: 'ai.generate', input: { prompt: `metrics ${i}` }, options: { cache: false } } });
++        assert.strictEqual(c.status, 202, c.text);
++        ids.push(c.body.run.id);
++    }
++    await request(h.base, 'GET', `/api/v1/runs/${ids[0]}`, { tok: caller });
++    for (const hdr of [{ 'X-Forwarded-For': '203.0.113.7' }, { 'X-Real-IP': '203.0.113.7' }, { 'CF-Connecting-IP': '203.0.113.7' }]) {
++        const m = await get(h.base, '/metrics', hdr);
++        assert.strictEqual(m.status, 404, JSON.stringify(hdr));
++        assert.ok(!m.body.includes('ai_runs'));
++    }
++    const m = await get(h.base, '/metrics');
++    assert.strictEqual(m.status, 200);
++    const text = m.body;
++    assert.strictEqual(metric(text, /\nai_runs\{state="queued"\} (\d+)\n/), 2, 'two runs wait for the one slot');
++    assert.strictEqual(metric(text, /\nai_runs\{state="running"\} (\d+)\n/), 1);
++    assert.strictEqual(metric(text, /\nai_runs_max_concurrent (\d+)\n/), 1);
++    assert.ok(/http_requests_total\{method="POST",route="\/api\/v1\/runs",status_class="2xx"\} 3\n/.test(text));
++    assert.ok(/http_requests_total\{method="GET",route="\/api\/v1\/runs\/:id",status_class="2xx"\} 1\n/.test(text), 'route template');
++    assert.ok(!text.includes(ids[0]), 'no run id in any label');
++    assert.ok(/http_request_duration_seconds_count\{method="GET",route="\/api\/ready"\} 1\n/.test(text));
++    assert.ok(/\nprocess_resident_memory_bytes \d+\n/.test(text));
++    assert.ok(/release_info\{service="ai",release="[^"]+"\} 1\n/.test(text));
++    assert.ok(/ai_provider_circuit\{provider="shared",state="closed"\} 1\n/.test(text));
++    assert.ok(/ai_provider_circuit\{provider="shared",state="open"\} 0\n/.test(text));
++    assert.ok(/ai_provider_credentials_missing\{provider="shared"\} 0\n/.test(text));
++    for (const id of ids) await request(h.base, 'POST', `/api/v1/runs/${id}/cancel`, { tok: caller });
++});
++
++t.test('an open circuit on a real provider degrades /api/ready (still 200) and shows on /metrics', async () => {
++    h.db.prepare("INSERT INTO provider_health (provider_key, state, consecutive_failures, opened_at, last_error) VALUES ('shared', 'open', 3, ?, 'test')").run(Date.now());
++    const r = await request(h.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 200, r.text);
++    assert.strictEqual(r.body.status, 'degraded');
++    assert.deepStrictEqual(r.body.degraded, ['providers']);
++    assert.strictEqual(r.body.checks.providers.error, 'shared: circuit open');
++    const text = (await get(h.base, '/metrics')).body;
++    assert.ok(/ai_provider_circuit\{provider="shared",state="open"\} 1\n/.test(text));
++    assert.ok(/ai_provider_circuit\{provider="shared",state="closed"\} 0\n/.test(text));
++    h.pool.resetHealth('shared');
++});
++
++t.test('a broken database makes the service unready (503); /metrics still answers', async () => {
++    await h.stop();
++    const fresh = await boot({ env: { AI_API_KEY: 'sk-test-not-real' } });
++    fresh.db.close();
++    const r = await request(fresh.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 503, r.text);
++    assert.strictEqual(r.body.ready, false);
++    assert.ok(r.body.failed.includes('db'), r.text);
++    assert.strictEqual(r.body.providers, null);
++    const m = await get(fresh.base, '/metrics');
++    assert.strictEqual(m.status, 200);
++    assert.ok(!/ai_provider_circuit\{/.test(m.body), 'a gauge that cannot be read is left out, not invented');
++    try { await fresh.stop(); } catch { /* the database is already closed */ }
++});
++
++t.test('no shared key: ready but degraded, and says which provider lacks credentials', async () => {
++    const d = await boot();
++    const r = await request(d.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 200, r.text);
++    assert.strictEqual(r.body.status, 'degraded');
++    assert.match(r.body.checks.providers.error, /shared: credentials missing/);
++    const text = (await get(d.base, '/metrics')).body;
++    assert.ok(/ai_provider_credentials_missing\{provider="shared"\} 1\n/.test(text));
++    await d.stop();
++});
++
++t.run();