openvibers/openvibe.network Commits
nginx generator: services can ship location includes for another host; openvibe.live includes OpenVibe.Chat's chat locations (glob, before /api/)
Changes
@@@ -53,6 +53,9 @@ server {add_header X-Frame-Options SAMEORIGIN always;add_header Referrer-Policy strict-origin-when-cross-origin always;++ # Prometheus metrics are for loopback scrapers only (the app also 404s any proxied request).++ location = /metrics { return 404; }++# ── Auth endpoints (strict rate limit) ──────────────────location ~ ^/(api/auth/(register|login|anon-session)|oauth/(token|authorize)) {limit_req zone=openvibenetwork_auth burst=5 nodelay;nnew file mode 100644@@@ -0,0 +1,463 @@++From a4186199f8ae7cf519307669f36e5864aca5f2f1 Mon Sep 17 00:00:00 2001++From: OpenVibers <[email protected]>++Date: Tue, 22 Sep 2026 19:03:59 -0700++Subject: [PATCH] Metrics and truthful readiness (Track O): GET /metrics++ (loopback only) with golden signals by route template, release_info, live++ streams, WebSocket connections per server and outbox depth; /api/ready checks++ boot + db (required) and SFU, Media, Network key (optional, reported as++ degraded); openvibe-shared v1.3.0++++---++ deploy/nginx/openvibe.live.conf | 3 +++ docs/deploy.md | 6 ++++ package-lock.json | 7 +-++ package.json | 2 +-++ server/index.js | 56 +++++-----++ server/web/observability.js | 108 ++++++++++++++++++++++ test/observability.test.js | 175 ++++++++++++++++++++++++++++++++++ 7 files changed, 323 insertions(+), 34 deletions(-)++ create mode 100644 server/web/observability.js++ create mode 100644 test/observability.test.js++++diff --git a/deploy/nginx/openvibe.live.conf b/deploy/nginx/openvibe.live.conf++index 07b075d..90a2414 100644++--- a/deploy/nginx/openvibe.live.conf++@@ -72,6 +72,9 @@ server {++ # ceiling ordinary traffic cannot reach.++ limit_conn addr_limit 256;+++++ # Prometheus metrics are for local scrapers only; Node also 404s any proxied request to it.+++ location = /metrics { return 404; }+++++ location = /api/auth/register {++ limit_req zone=login_limit burst=5 nodelay;++ proxy_pass http://127.0.0.1:3000;++diff --git a/docs/deploy.md b/docs/deploy.md++index 5af9ee8..65ad04d 100644++--- a/docs/deploy.md++@@ -47,6 +47,12 @@ restart. A rollback resets the checkout but cannot restore previous `node_module++ `OV_APP_ROOT=/opt/openvibe.live/current`, so static files and docs are read through the symlink and++ a static-only switch needs no restart.++ - If the restarted release never becomes ready, `current` is switched back and restarted (exit 3).+++ "Ready" is `GET /api/ready` → 200: boot finished and the database answers. The WebRTC SFU,+++ OpenVibe.Media and the Network signing key are optional checks: when one is missing the answer is+++ still 200 with `"status": "degraded"` and the check named in `degraded`, never a silent pass.+++- `GET /metrics` (Prometheus text: requests by route template, latency, in-flight, process,+++ `release_info`, live streams, WebSocket connections per server, outbox) answers only+++ `curl http://127.0.0.1:3000/metrics` on the host; through nginx it is a 404.++ - `--rollback` selects the previous release with **its own** `node_modules`.++ - The last 5 releases are kept.++ - Content-hashed assets from the previous release are still served under their old hashes, so a page++diff --git a/package-lock.json b/package-lock.json++index 7a0b166..0c980ff 100644++--- a/package-lock.json++@@ -27,7 +27,7 @@++ "onnxruntime-node": "^1.20.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.2.2",++- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",+++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "sanitize-html": "^2.17.7",++ "sdp-transform": "^3.0.0",++ "sharp": "^0.33.0",++@@ -2839,9 +2839,8 @@++ }++ },++ "node_modules/openvibe-shared": {++- "version": "1.2.1",++- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",++- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",+++ "version": "1.3.0",+++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "license": "MIT",++ "dependencies": {++ "jsonwebtoken": "^9.0.0"++diff --git a/package.json b/package.json++index 48e0cb1..d2c3c21 100644++--- a/package.json++@@ -45,7 +45,7 @@++ "onnxruntime-node": "^1.20.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.2.2",++- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",+++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "sanitize-html": "^2.17.7",++ "sdp-transform": "^3.0.0",++ "sharp": "^0.33.0",++diff --git a/server/index.js b/server/index.js++index 7a4c359..4bba504 100644++--- a/server/index.js++@@ -131,6 +131,13 @@ const whipHandler = require('./streaming/whip-handler');++ // ── Express App ──────────────────────────────────────────────++ const app = express();++ const server = http.createServer(app);++++++// What this server runs (ADR-016): GET /release.json below, and release_info in /metrics.+++const release = require('openvibe-shared/release').createRelease({ service: 'live', root: path.join(__dirname, '..') });+++// Metrics first, so every request is counted (by route template, never by raw URL). GET /metrics+++// answers direct loopback callers only; through nginx it is a 404 (server/web/observability.js).+++const observability = require('./web/observability');+++const { registry: metricsRegistry } = observability.mountMetrics(app, { release });++ const vibeCodingPublishServer = new VibeCodingPublishServer(chatServer, db);++++ function normalizeOrigin(origin) {++@@ -723,36 +730,27 @@ app.get('/api/health', (req, res) => {++ * genuine gate.++ */++ let _bootComplete = false;++-// What this server runs (ADR-016). The shared navbar loads /shared/release-watch.js, which polls++-// this and prompts open tabs after a deploy; it never reloads a tab that is watching, broadcasting++-// or typing.++-{++- const release = require('openvibe-shared/release').createRelease({ service: 'live', root: path.join(__dirname, '..') });++- app.get('/release.json', release.handler);++-}+++// The shared navbar loads /shared/release-watch.js, which polls this and prompts open tabs after a+++// deploy; it never reloads a tab that is watching, broadcasting or typing.+++app.get('/release.json', release.handler);++++++// Required: boot finished and the database answers. Optional (a failure is "degraded", still 200):+++// the WebRTC SFU, OpenVibe.Media and the Network signing key — Live serves channels, chat and RTMP+++// without them, so they must neither pass silently nor take the whole service out of rotation.+++const readiness = observability.createLiveReadiness({+++ release,+++ bootComplete: () => _bootComplete,+++ dbQuery: () => db.get('SELECT 1 AS ok'),+++ sfuReady: () => webrtcSFU.ready === true && !!webrtcSFU.worker,+++ mediaUrl: mediaClient.MEDIA_URL,+++ networkKey: () => require('./auth/auth').getNetworkPublicKey(),+++});+++app.get('/api/ready', readiness.handler);++++-app.get('/api/ready', (req, res) => {++- res.set('Cache-Control', 'no-store');++- const checks = { boot: _bootComplete, db: false };++- try {++- // Cheapest possible proof that the connection is open and the schema exists.++- db.get('SELECT 1 AS ok');++- checks.db = true;++- } catch (err) {++- checks.dbError = String(err && err.message || err).slice(0, 200);++- }++- const ready = checks.boot && checks.db;++- res.status(ready ? 200 : 503).json({++- ready,++- checks,++- uptime: Math.round(process.uptime()),++- // Advisory only — a deploy gate should not wait on optional subsystems, which can be++- // legitimately unavailable (no mediasoup worker on a box that only relays RTMP, say).++- optional: {++- sfu: (() => { try { return !!webrtcSFU.isReady?.(); } catch { return null; } })(),++- media: (() => { try { return require('./media-client').lastOk === true; } catch { return null; } })(),++- },++- });+++observability.registerDomainGauges(metricsRegistry, {+++ liveStreams: () => db.getLiveStreams().length,+++ wsServers: { chat: chatServer, broadcast: broadcastServer, control: controlServer, call: callServer },+++ outboxStatus: () => require('./events/stream-events').status(),++ });++++ // ── Updates / Changelog ──────────────────────────────────────++diff --git a/server/web/observability.js b/server/web/observability.js++new file mode 100644++index 0000000..f1170c4++--- /dev/null++@@ -0,0 +1,108 @@+++'use strict';+++/**+++ * Metrics and readiness for Live (roadmap Track O, §15.19), on openvibe-shared/metrics and+++ * openvibe-shared/ready.+++ *+++ * GET /metrics Prometheus text, direct loopback callers only (404 through nginx or any proxy):+++ * http_requests_total{method,route,status_class}, http_request_duration_seconds,+++ * http_requests_in_flight, process/event-loop metrics, release_info{service,release},+++ * live_streams_live, live_ws_connections{server}, live_events_outbox{status}+++ * GET /api/ready 200 while Live can serve (boot finished, the database answers); 503 otherwise.+++ * Media, the WebRTC SFU and the Network signing key are optional: when one is+++ * missing the answer stays 200 with status "degraded" and names it, because Live+++ * still serves channels, chat and RTMP without them.+++ *+++ * Everything the checks and gauges read is injected, so the test drives this module without booting+++ * the whole server.+++ */+++const metrics = require('openvibe-shared/metrics');+++const { createReadiness } = require('openvibe-shared/ready');++++++// Long-lived responses whose "duration" is a viewing session, not a request.+++const STREAMING_PATHS = [/^\/api\/live-events$/, /^\/api\/streams\/rtmp-proxy\//];+++// Static mounts: served by express.static (no route), labelled by mount instead of 'unmatched'.+++const STATIC_MOUNT = /^\/(js|css|fragments|shared|assets|data|media|docs-assets|images|img|fonts|sounds)\//;++++++function normalize(req) {+++ if (req.route) return null; // the matched route's template (openvibe-shared/metrics routeLabel)+++ const m = STATIC_MOUNT.exec(String(req.originalUrl || req.url || '').split('?')[0]);+++ return m ? `/${m[1]}/*` : null;+++}++++++/**+++ * Mount the HTTP metrics middleware and GET /metrics. Call before any other middleware.+++ * @returns {{ registry }}+++ */+++function mountMetrics(app, { release }) {+++ const m = metrics.instrument(app, {+++ service: 'live',+++ release: release && release.release,+++ normalize,+++ skip: (req) => STREAMING_PATHS.some((re) => re.test(req.path)),+++ });+++ return m;+++}++++++/**+++ * Domain gauges, read at scrape time. A source that throws is left out of that scrape.+++ * liveStreams() -> number of live streams (the rows /api/streams lists)+++ * wsServers -> { chat, broadcast, control, call } objects with a `wss` (ws.Server) once initialised+++ * outboxStatus() -> server/events/stream-events.js status()+++ */+++function registerDomainGauges(registry, { liveStreams, wsServers = {}, outboxStatus }) {+++ registry.gauge({ name: 'live_streams_live', help: 'Streams currently marked live (the list /api/streams serves)', collect: () => liveStreams() });+++ registry.gauge({+++ name: 'live_ws_connections', help: 'Open WebSocket connections per WebSocket server', labelNames: ['server'],+++ collect: () => Object.entries(wsServers)+++ .filter(([, s]) => s && s.wss && s.wss.clients) // a server that never initialised reports nothing+++ .map(([server, s]) => ({ labels: { server }, value: s.wss.clients.size })),+++ });+++ registry.gauge({+++ name: 'live_events_outbox', help: 'Stream lifecycle events in the transactional outbox, by status (absent while the outbox is disabled)', labelNames: ['status'],+++ collect: () => {+++ const s = outboxStatus();+++ if (!s || !s.enabled) return null;+++ return [{ labels: { status: 'pending' }, value: Number(s.pending) || 0 }, { labels: { status: 'rejected' }, value: Number(s.rejected) || 0 }];+++ },+++ });+++}++++++/**+++ * The /api/ready handler.+++ * bootComplete() -> boolean required+++ * dbQuery() -> row required (a real query; throws when the database cannot answer)+++ * sfuReady() -> boolean optional (WebRTC broadcasting/viewing through mediasoup)+++ * mediaUrl optional (OpenVibe.Media: VODs, clips, thumbnails), checked by+++ * GET <mediaUrl>/healthz, cached 30 s+++ * networkKey() -> PEM or null optional (sign-in: RS256 key from OpenVibe.Network)+++ */+++function createLiveReadiness({ release, bootComplete, dbQuery, sfuReady, mediaUrl, networkKey, fetchImpl = globalThis.fetch }) {+++ return createReadiness({+++ service: 'live',+++ release: release && release.release,+++ checks: [+++ { name: 'boot', required: true, description: 'boot finished', check: () => (bootComplete() ? true : 'boot not finished') },+++ { name: 'db', required: true, description: 'SQLite answers a query', check: () => { const row = dbQuery(); return row ? true : 'no row'; } },+++ { name: 'sfu', required: false, description: 'WebRTC SFU (mediasoup worker)', check: () => (sfuReady() ? true : 'mediasoup worker not running') },+++ {+++ name: 'media', required: false, cacheMs: 30000, timeoutMs: 2000, description: 'OpenVibe.Media (VODs, clips, thumbnails)',+++ check: async () => {+++ const res = await fetchImpl(`${mediaUrl}/healthz`, { signal: AbortSignal.timeout(2000) });+++ return res.ok ? true : `Media answered ${res.status}`;+++ },+++ },+++ { name: 'network_key', required: false, description: 'OpenVibe.Network RS256 key (sign-in)', check: () => (/BEGIN [A-Z ]*PUBLIC KEY/.test(String(networkKey() || '')) ? true : 'Network public key not loaded: sign-in cannot be verified') },+++ ],+++ // Fields the previous /api/ready served, so nothing that read them breaks.+++ details: (body) => ({+++ uptime: Math.round(process.uptime()),+++ optional: {+++ sfu: body.checks.sfu.status === 'ok',+++ media: body.checks.media.status === 'ok',+++ network_key: body.checks.network_key.status === 'ok',+++ },+++ }),+++ });+++}++++++module.exports = { mountMetrics, registerDomainGauges, createLiveReadiness, normalize };++diff --git a/test/observability.test.js b/test/observability.test.js++new file mode 100644++index 0000000..a052903++--- /dev/null++@@ -0,0 +1,175 @@+++'use strict';+++/**+++ * Metrics and readiness (server/web/observability.js, roadmap Track O):+++ * - /metrics answers direct loopback callers only; through a proxy (X-Forwarded-For) it is a 404+++ * - HTTP metrics are labelled by route template, never by raw URL, id or query+++ * - release_info and the domain gauges (live streams, WebSocket connections, outbox)+++ * - /api/ready: boot + db required (503 when either fails); SFU, Media and the Network key optional+++ * (200 "degraded", naming what is missing)+++ * A temp SQLite database stands in for Live's; no Media, Network or mediasoup is needed.+++ *+++ * Run: node test/observability.test.js+++ */+++const assert = require('assert');+++const fs = require('fs');+++const http = require('http');+++const os = require('os');+++const path = require('path');+++const express = require('express');+++const Database = require('better-sqlite3');+++const observability = require('../server/web/observability');++++++const tmpDb = path.join(os.tmpdir(), `openvibe-observability-test-${process.pid}-${Date.now()}.db`);++++++(async () => {+++ const sqlite = new Database(tmpDb);+++ sqlite.exec('CREATE TABLE streams (id INTEGER PRIMARY KEY, is_live INTEGER)');+++ sqlite.prepare('INSERT INTO streams (is_live) VALUES (1), (1), (0)').run();+++ let dbBroken = false;+++ const dbQuery = () => { if (dbBroken) throw new Error('SQLITE_IOERR: disk I/O error'); return sqlite.prepare('SELECT 1 AS ok').get(); };++++++ // Fake Media: /healthz answers 200 or 502.+++ let mediaUp = true; let mediaHits = 0;+++ const media = http.createServer((req, res) => { mediaHits++; res.statusCode = req.url === '/healthz' && mediaUp ? 200 : 502; res.end('{}'); });+++ await new Promise((r) => media.listen(0, '127.0.0.1', r));+++ const mediaUrl = `http://127.0.0.1:${media.address().port}`;++++++ const release = { release: 'abc123def456' };+++ let booted = false; let sfu = true; let key = '-----BEGIN PUBLIC KEY-----\nMII…\n-----END PUBLIC KEY-----';+++ const fakeWss = (n) => ({ wss: { clients: { size: n } } });+++ let outbox = { enabled: true, pending: 4, rejected: 1 };++++++ const app = express();+++ const { registry } = observability.mountMetrics(app, { release });+++ const readiness = observability.createLiveReadiness({+++ release, bootComplete: () => booted, dbQuery, sfuReady: () => sfu, mediaUrl, networkKey: () => key,+++ });+++ app.get('/api/ready', readiness.handler);+++ observability.registerDomainGauges(registry, {+++ liveStreams: () => sqlite.prepare('SELECT COUNT(*) AS n FROM streams WHERE is_live = 1').get().n,+++ wsServers: { chat: fakeWss(12), broadcast: fakeWss(2), control: fakeWss(0), call: { wss: null } },+++ outboxStatus: () => outbox,+++ });+++ const vods = express.Router();+++ vods.get('/:id', (req, res) => res.json({ id: req.params.id }));+++ app.use('/api/vods', vods);+++ app.get('/api/live-events', (_req, res) => { res.setHeader('Content-Type', 'text/event-stream'); res.end(); });+++ app.use('/js', express.static(path.join(__dirname, '..', 'public', 'js')));+++ app.get('*', (_req, res) => res.send('spa'));++++++ const server = await new Promise((r) => { const s = app.listen(0, '127.0.0.1', () => r(s)); });+++ const base = `http://127.0.0.1:${server.address().port}`;+++ const get = (p, headers = {}) => new Promise((resolve, reject) => {+++ http.get(base + p, { headers }, (res) => { let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b })); }).on('error', reject);+++ });++++++ // ── readiness ──+++ let r = await get('/api/ready');+++ assert.strictEqual(r.status, 503, 'not ready before boot completes');+++ let body = JSON.parse(r.body);+++ assert.strictEqual(body.ready, false);+++ assert.deepStrictEqual(body.failed, ['boot']);++++++ booted = true;+++ r = await get('/api/ready');+++ body = JSON.parse(r.body);+++ assert.strictEqual(r.status, 200);+++ assert.strictEqual(body.status, 'ready');+++ assert.strictEqual(body.service, 'live');+++ assert.strictEqual(body.release, 'abc123def456');+++ for (const name of ['boot', 'db', 'sfu', 'media', 'network_key']) {+++ const c = body.checks[name];+++ assert.strictEqual(c.status, 'ok', name);+++ assert.strictEqual(typeof c.latency_ms, 'number');+++ assert.ok(Date.parse(c.checked_at), `${name} checked_at`);+++ }+++ assert.strictEqual(body.checks.db.required, true);+++ assert.strictEqual(body.checks.media.required, false);+++ assert.deepStrictEqual(body.optional, { sfu: true, media: true, network_key: true }, 'the previous optional booleans are still served');+++ assert.strictEqual(typeof body.uptime, 'number');++++++ // Optional dependencies down: still 200, but degraded and named.+++ sfu = false; key = null;+++ r = await get('/api/ready');+++ body = JSON.parse(r.body);+++ assert.strictEqual(r.status, 200);+++ assert.strictEqual(body.status, 'degraded');+++ assert.deepStrictEqual(body.degraded.sort(), ['network_key', 'sfu']);+++ assert.match(body.checks.sfu.error, /mediasoup/);+++ assert.strictEqual(body.optional.sfu, false);++++++ // Media is cached for 30 s: its checked_at is when it really ran, and a down Media is not re-asked.+++ const hitsBefore = mediaHits;+++ mediaUp = false;+++ r = await get('/api/ready');+++ assert.strictEqual(mediaHits, hitsBefore, 'Media check is cached');+++ assert.strictEqual(JSON.parse(r.body).checks.media.status, 'ok', 'cached result, with its own checked_at');++++++ // The database failing is not ready (503), whatever else is up.+++ sfu = true; key = '-----BEGIN PUBLIC KEY-----x';+++ dbBroken = true;+++ r = await get('/api/ready');+++ body = JSON.parse(r.body);+++ assert.strictEqual(r.status, 503);+++ assert.deepStrictEqual(body.failed, ['db']);+++ assert.match(body.checks.db.error, /SQLITE_IOERR/);+++ dbBroken = false;++++++ // A fresh readiness with Media down reports it degraded.+++ const r2 = observability.createLiveReadiness({ release, bootComplete: () => true, dbQuery, sfuReady: () => true, mediaUrl, networkKey: () => key });+++ const b2 = await r2.run();+++ assert.strictEqual(b2.ready, true);+++ assert.deepStrictEqual(b2.degraded, ['media']);+++ assert.strictEqual(b2.checks.media.error, 'Media answered 502');+++ const r3 = observability.createLiveReadiness({ release, bootComplete: () => true, dbQuery, sfuReady: () => true, mediaUrl: 'http://127.0.0.1:1', networkKey: () => key });+++ const b3 = await r3.run();+++ assert.deepStrictEqual(b3.degraded, ['media'], 'Media unreachable is degraded, not down');++++++ // ── metrics ──+++ await get('/api/vods/12345?token=secret');+++ await get('/api/vods/67890');+++ await get('/some/channel-name-xyz');+++ await get('/api/live-events');+++ await get('/js/app.js?v=deadbeef');+++ const m = await get('/metrics');+++ assert.strictEqual(m.status, 200);+++ const text = m.body;+++ assert.ok(text.includes('http_requests_total{method="GET",route="/api/vods/:id",status_class="2xx"} 2\n'), text);+++ assert.ok(text.includes('route="*"'), 'the SPA fallback is its template');+++ assert.ok(text.includes('route="/js/*"'), 'static mounts are labelled by mount');+++ const labels = text.split('\n').filter((l) => l.startsWith('http_')).map((l) => (l.match(/\{[^}]*\}/) || [''])[0]).join('\n');+++ assert.ok(!/12345|67890|secret|channel-name|app\.js|deadbeef/.test(labels), 'no ids, queries or raw paths in labels');+++ assert.ok(!labels.includes('/api/live-events'), 'the SSE stream is not a request');+++ assert.ok(text.includes('release_info{service="live",release="abc123def456"} 1\n'));+++ assert.ok(text.includes('live_streams_live 2\n'));+++ assert.ok(text.includes('live_ws_connections{server="chat"} 12\n'));+++ assert.ok(text.includes('live_ws_connections{server="broadcast"} 2\n'));+++ assert.ok(text.includes('live_ws_connections{server="control"} 0\n'));+++ assert.ok(!text.includes('live_ws_connections{server="call"}'), 'an uninitialised server reports nothing');+++ assert.ok(text.includes('live_events_outbox{status="pending"} 4\n'));+++ assert.ok(text.includes('live_events_outbox{status="rejected"} 1\n'));+++ assert.ok(text.includes('\nprocess_resident_memory_bytes '));+++ assert.ok(text.includes('\nnodejs_eventloop_lag_seconds{stat="p99"} '));++++++ outbox = { enabled: false };+++ assert.ok(!(await get('/metrics')).body.includes('live_events_outbox{'), 'a disabled outbox reports nothing, not zero');++++++ // Through nginx (X-Forwarded-For on a loopback connection) /metrics does not exist.+++ assert.strictEqual((await get('/metrics', { 'X-Forwarded-For': '203.0.113.7' })).status, 404);+++ assert.strictEqual((await get('/metrics', { 'X-Real-IP': '203.0.113.7' })).status, 404);++++++ // server/index.js mounts metrics before anything else and serves /api/ready from this module.+++ const src = fs.readFileSync(path.join(__dirname, '..', 'server', 'index.js'), 'utf8');+++ const mountAt = src.indexOf('observability.mountMetrics(app');+++ assert.ok(mountAt > 0 && mountAt < src.indexOf('app.use(helmet('), 'metrics middleware is mounted before the rest of the middleware');+++ assert.ok(/app\.get\('\/api\/ready', readiness\.handler\)/.test(src));++++++ server.close(); media.close(); sqlite.close();+++ for (const f of [tmpDb, `${tmpDb}-wal`, `${tmpDb}-shm`]) { try { fs.unlinkSync(f); } catch { /* */ } }+++ console.log('observability: all checks passed');+++ process.exit(0);+++})().catch((err) => {+++ console.error(err);+++ for (const f of [tmpDb, `${tmpDb}-wal`, `${tmpDb}-shm`]) { try { fs.unlinkSync(f); } catch { /* */ } }+++ process.exit(1);+++});++--++2.43.0++nnew file mode 100644@@@ -0,0 +1,105 @@++{++ "version": 1,++ "status": "proposal",++ "source": "OpenVibe realignment roadmap §15.19 (Observability and reliability)",++ "note": "Every target below is a PROPOSAL for discussion. None is agreed, none is enforced, and no alerting exists for any of them. 'measured_by' names what can be measured today; 'not instrumented' means nothing measures it yet.",++ "binding": "Correctness-sensitive systems (billing, authentication) prioritise integrity over availability; live transport prioritises continuity, with eventual control-plane convergence.",++ "categories": [++ {++ "id": "identity-login-token-exchange",++ "name": "Identity login and token exchange",++ "services": ["network"],++ "priority": "integrity",++ "sli": "Share of POST /oauth/token and sign-in requests that answer without a 5xx; p95 latency of token issuance.",++ "proposed_target": { "availability": "99.9% of requests over 30 days", "latency_p95_seconds": 0.5 },++ "measured_by": ["network: http_requests_total{route=\"/oauth/token\"}", "network: http_request_duration_seconds{route=\"/oauth/token\"}", "network: network_tokens_issued_total{grant_type}", "network: network_token_failures_total{grant_type,error}"],++ "integrity_rule": "A token is never issued with a capability that was not granted; refusing is always preferred to issuing wrongly."++ },++ {++ "id": "public-page-availability",++ "name": "Public page availability",++ "services": ["live", "community", "tools", "media", "network"],++ "priority": "availability",++ "sli": "Share of GET requests to server-rendered public pages that answer without a 5xx.",++ "proposed_target": { "availability": "99.5% of requests over 30 days" },++ "measured_by": ["each service: http_requests_total{method=\"GET\",status_class}"]++ },++ {++ "id": "control-apis",++ "name": "Control APIs",++ "services": ["live", "media", "events", "community", "tools"],++ "priority": "availability",++ "sli": "Share of authenticated write/control API requests (stream settings, object writes, subscriptions, jobs) that answer without a 5xx; p95 latency.",++ "proposed_target": { "availability": "99.5% over 30 days", "latency_p95_seconds": 1.0 },++ "measured_by": ["each service: http_requests_total and http_request_duration_seconds by route template"]++ },++ {++ "id": "chat-persistence-delivery",++ "name": "Chat message persistence and delivery",++ "services": ["live", "chat"],++ "priority": "integrity",++ "sli": "Share of accepted chat messages that are persisted and delivered to connected viewers; delivery delay.",++ "proposed_target": { "persisted": "99.99% of accepted messages", "delivery_p95_seconds": 1.0 },++ "measured_by": [],++ "measurement_status": "not instrumented: Live exposes live_ws_connections only; no per-message persistence or delivery counter exists yet"++ },++ {++ "id": "ingest-acceptance",++ "name": "Ingest acceptance",++ "services": ["live", "openre"],++ "priority": "continuity",++ "sli": "Share of publish attempts (RTMP, WHIP, WebRTC, SRT) with a valid key that are accepted.",++ "proposed_target": { "accepted": "99.5% of valid publish attempts over 30 days" },++ "measured_by": [],++ "measurement_status": "not instrumented: no publish-attempt counter exists yet"++ },++ {++ "id": "active-stream-continuity",++ "name": "Active stream continuity",++ "services": ["live", "openre", "media"],++ "priority": "continuity",++ "sli": "Share of live streams that end by the broadcaster's action rather than by a server restart, deploy or crash.",++ "proposed_target": { "unplanned_interruptions": "fewer than 1 per 1000 stream-hours" },++ "measured_by": ["live: live_streams_live (count only)"],++ "measurement_status": "partly instrumented: the number of live streams is measured; the cause of each stream end is not recorded"++ },++ {++ "id": "media-upload-processing",++ "name": "Media upload and processing",++ "services": ["media"],++ "priority": "integrity",++ "sli": "Share of uploads and recordings that reach ready (never silently lost); time from upload or stream end to ready.",++ "proposed_target": { "reach_ready_or_explicit_failure": "100%", "processing_p95_seconds": 600 },++ "measured_by": ["media: media_recordings_in_progress", "media: media_objects{lifecycle_status}", "media: media_object_locations{provider,state}"],++ "measurement_status": "partly instrumented: states are counted; time-to-ready is not measured"++ },++ {++ "id": "ledger-settlement-correctness",++ "name": "Ledger settlement correctness",++ "services": ["billing"],++ "priority": "integrity",++ "sli": "Share of settlements whose journal balances and reconciles with the provider; duplicate monetary effects.",++ "proposed_target": { "unbalanced_or_duplicate_settlements": 0 },++ "measured_by": [],++ "measurement_status": "not instrumented by this change (Billing is owned by another track)"++ },++ {++ "id": "event-delivery-latency",++ "name": "Event delivery latency",++ "services": ["events"],++ "priority": "integrity",++ "sli": "Time from an event's acceptance to its successful delivery to each subscriber; dead-letter depth.",++ "proposed_target": { "delivery_p95_seconds": 30, "dead_letters_unreviewed_after_24h": 0 },++ "measured_by": ["events: events_delivery_latency_seconds", "events: events_deliveries{status}", "events: events_dlq_depth"]++ },++ {++ "id": "developer-api-availability",++ "name": "Developer API availability",++ "services": ["network", "events", "media"],++ "priority": "availability",++ "sli": "Share of public developer API requests (registry, JWKS, events pull, object API) that answer without a 5xx.",++ "proposed_target": { "availability": "99.9% of requests over 30 days" },++ "measured_by": ["each service: http_requests_total by route template"]++ }++ ]++}nnew file mode 100644@@@ -0,0 +1,48 @@++# SLO categories (proposal)++++Status: **proposal**. None of these targets is agreed or enforced, and nothing alerts on them.++They exist so that each service's metrics can be read against a stated intent. The categories come++from the realignment roadmap §15.19. The machine-readable copy is [`slo.json`](slo.json), served at++`GET /api/v1/status/slo`. The operator page `GET /status` lists them under the live service table.++++Binding rule from the roadmap: correctness-sensitive systems (billing, authentication) prioritise++**integrity over availability**. Live transport prioritises **continuity**, and the control plane++converges afterwards.++++| Category | Services | Priority | Proposed target | Measured today by |++|---|---|---|---|---|++| Identity login and token exchange | network | integrity | 99.9% of token requests without a 5xx over 30 days; p95 under 0.5 s | `http_requests_total{route="/oauth/token"}`, `network_tokens_issued_total{grant_type}`, `network_token_failures_total{grant_type,error}` |++| Public page availability | live, community, tools, media, network | availability | 99.5% of GETs without a 5xx over 30 days | `http_requests_total` per service |++| Control APIs | live, media, events, community, tools | availability | 99.5% without a 5xx; p95 under 1 s | `http_requests_total`, `http_request_duration_seconds` by route template |++| Chat message persistence and delivery | live, chat | integrity | 99.99% of accepted messages persisted; delivery p95 under 1 s | **not instrumented**: only `live_ws_connections` exists |++| Ingest acceptance | live, openre | continuity | 99.5% of valid publish attempts accepted | **not instrumented** |++| Active stream continuity | live, openre, media | continuity | fewer than 1 unplanned interruption per 1000 stream-hours | partly: `live_streams_live` counts streams; the cause of a stream ending is not recorded |++| Media upload and processing | media | integrity | every upload or recording ends ready or explicitly failed; p95 under 10 min | partly: `media_recordings_in_progress`, `media_objects{lifecycle_status}`, `media_object_locations{provider,state}`; time-to-ready is not measured |++| Ledger settlement correctness | billing | integrity | zero unbalanced or duplicate settlements | **not instrumented** by this change (Billing is another track) |++| Event delivery latency | events | integrity | p95 under 30 s; no dead letter unreviewed after 24 h | `events_delivery_latency_seconds`, `events_deliveries{status}`, `events_dlq_depth` |++| Developer API availability | network, events, media | availability | 99.9% without a 5xx over 30 days | `http_requests_total` by route template |++++## Where the numbers come from++++Every service (Network, Community, Media, Events, the Tools gateway and satellites, and Live once++`docs/patches/live-observability.diff` is applied) uses `openvibe-shared/metrics` ≥ 1.3.0:++++- `GET /metrics` returns Prometheus text to a direct loopback caller only. Through nginx it is a 404,++ both in the app (any `X-Forwarded-For`/`X-Real-IP`/`Forwarded` header) and in the nginx config.++- The HTTP golden signals are `http_requests_total{method,route,status_class}`,++ `http_request_duration_seconds{method,route}` and `http_requests_in_flight`. `route` is the route++ template, never a raw URL.++- The process metrics are `process_resident_memory_bytes`, `nodejs_heap_*`,++ `nodejs_eventloop_lag_seconds{stat}`, `process_cpu_seconds_total` and `process_uptime_seconds`.++- `release_info{service,release}` names the deployed release.++++No Prometheus server scrapes these yet. Setting one up on the host, with a loopback scrape of each++port, is the next operational step.++++## Readiness++++`/api/ready` on each service follows `openvibe-shared/ready`. It lists named checks, each with++`status`, `required`, `latency_ms` and `checked_at`. `ready` is false (HTTP 503) only when a++required check fails. A failed optional dependency is listed in `degraded` (HTTP 200,++`status: "degraded"`), so losing one capability never reads as healthy or as fully down.++`GET /status` shows each row as up, degraded, down, not running or unknown, using those bodies.@@@ -21,7 +21,7 @@"jsonwebtoken": "^9.0.0","multer": "^1.4.5-lts.1","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0","uuid": "^11.1.0","web-push": "^3.6.7"}@@@ -1480,9 +1480,8 @@}},"node_modules/openvibe-shared": {-- "version": "1.2.1",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",-- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",++ "version": "1.3.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0","license": "MIT","dependencies": {"jsonwebtoken": "^9.0.0"@@@ -6,7 +6,7 @@"scripts": {"start": "node server/index.js","dev": "NODE_ENV=development node server/index.js",-- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js",++ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/observability.test.js && node test/status.test.js","reset-db": "node server/reset-db.js","grant-admin": "node server/grant-admin.js"},@@@ -24,7 +24,7 @@"jsonwebtoken": "^9.0.0","multer": "^1.4.5-lts.1","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0","uuid": "^11.1.0","web-push": "^3.6.7"}@@@ -115,6 +115,8 @@ const DEFAULT_SERVICE_MAP = {gzip: true,// WebSocket URLs carry session tokens in the query string; keep them out of access logs.logWithoutQuery: true,++ // Wave 6: OpenVibe.Chat serves /ws/chat, /api/chat/, /api/dm/, /api/tts/ and /api/sounds.++ includes: ['/opt/openvibe.chat/deploy/nginx/*.locations.conf'],locations: [{match: '= /api/auth/register',@@@ -574,6 +576,13 @@ function generateServiceConfig(serviceId, svc, opts = {}) {lines.push('');}++ // Location files other services ship for this host (e.g. OpenVibe.Chat's chat paths on++ // openvibe.live). A glob include is a no-op when the service is not installed.++ for (const inc of svc.includes || []) {++ lines.push(` include ${inc};`);++ lines.push('');++ }++// Location blocksif (svc.locations && svc.locations.length > 0) {for (const loc of svc.locations) {@@@ -167,7 +167,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {legacy: legacy ? (r) => r.internalKeyOk === true : undefined,namespace,// Denials are final here; an allow is recorded below, after the ownership check.-- onDecision: (d) => { if (!d.allowed) record(d); },++ onDecision: (d) => { if (!d.allowed) { record(d); require('../observability').principalDenied(d); } },});}check(req, res, () => {@@@ -177,6 +177,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {const self = String(principal.sub).replace(/^svc:/, '');if (app !== undefined && app !== self) {record({ req, capability, principal, allowed: false, code: 'capability.owner_denied' });++ require('../observability').principalDenied({ req, code: 'capability.owner_denied' });return http.sendProblem(res, 403, 'capability.owner_denied', { detail: `${principal.sub} may only act for app_id '${self}'` });}}@@@ -34,6 +34,18 @@ const { signToken } = require('./auth/routes');const app = express();++// What this server is running (ADR-016, registry.release-manifest@1); open tabs poll it through++// /shared/release-watch.js and are prompted, or reloaded when safe, after a deploy.++const release = require('openvibe-shared/release').createRelease({ service: 'network', root: path.join(__dirname, '..') });++++// Metrics first, so every request is measured: HTTP golden signals by route template, process++// metrics, release_info, and GET /metrics for direct loopback callers only (server/observability.js).++const observability = require('./observability');++require('openvibe-shared/metrics').instrument(app, {++ service: 'network', release: release.release, registry: observability.registry,++ normalize: (req) => (req.route ? null : /^\/shared\//.test(req.originalUrl) ? '/shared/*' : /^\/data\/avatars\//.test(req.originalUrl) ? '/data/avatars/*' : null),++});++function getRequestHost(req) {return String(req.headers.host || '').split(':')[0].toLowerCase();}@@@ -357,6 +369,8 @@ const requireAuth = require('./auth/session').makeRequireAuth(() => ({ db, publiconst ecosystem = require('./registry/ecosystem').createEcosystemRegistry({ issuer: config.jwt.issuer });app.use(ecosystem.router());ecosystem.start();++// Operator status: GET /status (server-rendered, noindex), /api/v1/status, /api/v1/status/slo.++app.use(require('./status/routes').createStatusRoutes({ ecosystem }));app.get('/api/.well-known/jwks', (_req, res) => {const out = { public_key: publicKey, algorithm: privateKey === publicKey ? 'HS256' : 'RS256' };@@@ -374,6 +388,15 @@ app.get('/api/health', (_req, res) => {res.json({ status: 'ok', service: 'openvibe-network', version: '1.0.0' });});++// Readiness: named checks with status, latency and checked_at; 503 only when a required one fails.++{++ const ready = observability.createNetworkReadiness({++ db, release: release.release, ecosystem, discordService,++ getKeys: () => ({ privateKey: app.locals.privateKey, publicKey: app.locals.publicKey }),++ });++ app.get('/api/ready', ready.handler);++}++// Brand info (used by all frontends for consistent URLs/names)app.get('/api/brand', (_req, res) => res.json(BRAND));@@@ -386,7 +409,8 @@ app.use('/api/auth', require('./auth/email-verify').routes(requireAuth));// Discord account linking (OAuth2 flow)app.use('/api/auth/discord', requireAuth, require('./auth/discord-link'));--// OAuth2 authorization endpoints++// OAuth2 authorization endpoints (token issuance counted by grant type: server/observability.js)++app.use('/oauth/token', observability.tokenEndpointMetrics);app.use('/oauth', require('./auth/oauth-routes'));// Theme API@@@ -630,9 +654,6 @@ function serveShared(req, res, next) {app.use('/shared/v1', serveShared);app.use('/shared', serveShared);--// What this server is running (ADR-016, registry.release-manifest@1); open tabs poll it through--// /shared/release-watch.js and are prompted, or reloaded when safe, after a deploy.--const release = require('openvibe-shared/release').createRelease({ service: 'network', root: path.join(__dirname, '..') });app.get('/release.json', release.handler);// Avatar servingnnew file mode 100644@@@ -0,0 +1,98 @@++'use strict';++/**++ * Network's metrics and readiness (roadmap Track O, §15.19), on openvibe-shared/metrics and /ready.++ *++ * GET /metrics Prometheus text, direct loopback callers only (404 through nginx)++ * GET /api/ready named checks; 503 only when a required one fails++ *++ * Domain metrics: token issuance by grant type at /oauth/token (and failures by OAuth error code),++ * and service-token (principal) failures at Network's capability-guarded internal routes.++ * Labels are fixed vocabularies; a client id, subject or token never becomes a label.++ */++const jwt = require('jsonwebtoken');++const metrics = require('openvibe-shared/metrics');++const { createReadiness } = require('openvibe-shared/ready');++++const registry = metrics.createRegistry();++++const GRANTS = new Set(['authorization_code', 'refresh_token', 'client_credentials', 'urn:ietf:params:oauth:grant-type:jwt-bearer']);++const OAUTH_ERRORS = new Set(['invalid_request', 'invalid_client', 'invalid_grant', 'unauthorized_client', 'unsupported_grant_type', 'invalid_scope', 'access_denied', 'server_error', 'temporarily_unavailable']);++const grantLabel = (g) => (g === 'urn:ietf:params:oauth:grant-type:jwt-bearer' ? 'jwt_bearer' : GRANTS.has(g) ? g : 'other');++++const tokensIssued = registry.counter({ name: 'network_tokens_issued_total', help: 'Tokens issued by POST /oauth/token, by grant type', labelNames: ['grant_type'] });++const tokenFailures = registry.counter({ name: 'network_token_failures_total', help: 'POST /oauth/token requests refused, by grant type and OAuth error code', labelNames: ['grant_type', 'error'] });++const principalFailures = registry.counter({ name: 'network_principal_token_failures_total', help: 'Service (principal) tokens refused at capability-guarded Network routes, by problem code', labelNames: ['code'], maxSeries: 50 });++++/** Middleware for POST /oauth/token: counts the outcome without reading or logging any secret. */++function tokenEndpointMetrics(req, res, next) {++ if (req.method !== 'POST') return next();++ const grant = grantLabel(req.body && req.body.grant_type);++ let error = null;++ const json = res.json.bind(res);++ res.json = (body) => {++ if (body && typeof body === 'object' && typeof body.error === 'string') error = OAUTH_ERRORS.has(body.error) ? body.error : 'other';++ return json(body);++ };++ res.once('finish', () => {++ if (res.statusCode === 200 && !error) tokensIssued.inc({ grant_type: grant });++ else tokenFailures.inc({ grant_type: grant, error: error || `http_${res.statusCode}` });++ });++ next();++}++++/** Called by identity/principals.js guard() for every denial; counts only bearer (service-token) callers. */++function principalDenied({ req, code }) {++ if (!String((req && req.headers && req.headers.authorization) || '').startsWith('Bearer ')) return;++ const c = /^[a-z0-9_.-]{1,64}$/i.test(String(code || '')) ? String(code) : 'other';++ principalFailures.inc({ code: c });++}++++/**++ * Readiness with Network's real dependencies:++ * db (required) a query against the identity database++ * signing_key (required in production) an RS256 keypair is loaded and signs a token its own public++ * key verifies — the thing every service's offline verification depends on++ * registry_poll (optional) the ecosystem health poll has run recently (status page freshness)++ * discord_bot (optional, only when a bot token is configured) the bot is connected++ */++function createNetworkReadiness({ db, getKeys, release, ecosystem = null, discordService = null, production = process.env.NODE_ENV === 'production', pollMs = 60000 }) {++ const checks = [++ { name: 'db', required: true, check: () => { const r = db.prepare('SELECT COUNT(*) AS n FROM oauth_clients').get(); return { detail: { oauth_clients: r.n } }; } },++ {++ name: 'signing_key', required: production, cacheMs: 60000,++ check: () => {++ const { privateKey, publicKey } = getKeys();++ if (!privateKey || !publicKey) return 'no signing key loaded';++ if (privateKey === publicKey || !String(publicKey).includes('BEGIN')) return 'ephemeral HS256 development key: other services cannot verify tokens';++ const probe = jwt.sign({ probe: true }, privateKey, { algorithm: 'RS256', expiresIn: 30 });++ jwt.verify(probe, publicKey, { algorithms: ['RS256'] });++ return { detail: { algorithm: 'RS256', kid: 'ov-network-1' } };++ },++ },++ ];++ if (ecosystem) {++ checks.push({++ name: 'registry_poll', required: false,++ check: () => {++ const at = ecosystem.lastPollAt();++ if (!at) return 'the ecosystem health poll has not completed yet';++ const age = Date.now() - at;++ return age > 3 * pollMs ? `last completed ${Math.round(age / 1000)}s ago` : { detail: { last_poll_at: new Date(at).toISOString() } };++ },++ });++ }++ if (discordService && typeof discordService.isReady === 'function') {++ checks.push({++ name: 'discord_bot', required: false,++ check: () => {++ let configured = false;++ try { configured = !!discordService._getSetting('discord_bot_token'); } catch { configured = false; }++ if (!configured) return { detail: 'not configured' };++ return discordService.isReady() || 'bot token configured but the bot is not connected';++ },++ });++ }++ return createReadiness({ service: 'network', release, checks });++}++++module.exports = { registry, tokenEndpointMetrics, principalDenied, createNetworkReadiness, grantLabel };@@@ -29,28 +29,91 @@ const INTERNAL = {events: 'http://127.0.0.1:4300', billing: 'http://127.0.0.1:4600',};const POLL_MS = 60 * 1000;++// Readiness paths of services whose manifest (openvibe-contracts) does not carry `ready` yet. Each++// answers with openvibe-shared/ready's shape; until a service deploys it, its health path is used++// and the row says so (basis: 'health').++const READY_PATHS = { network: '/api/ready', media: '/api/ready', tools: '/api/ready' };--function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS } = {}) {++/** Row status from a readiness body (openvibe-shared/ready shape, or an older ad-hoc one). */++function statusFromReady(res, body) {++ const shared = body && typeof body === 'object' && typeof body.ready === 'boolean' && body.checks && typeof body.checks === 'object';++ if (!shared) return { status: res.ok ? 'up' : 'down', basis: 'ready-legacy' };++ if (!body.ready) return { status: 'down', basis: 'ready' };++ const degraded = Array.isArray(body.degraded) ? body.degraded : [];++ return { status: body.status === 'degraded' || degraded.length ? 'degraded' : 'up', basis: 'ready' };++}++++/** What a status row may repeat from a service's readiness body: names, states and times, no free-form detail. */++function readySummary(body) {++ if (!body || typeof body !== 'object' || typeof body.ready !== 'boolean') return null;++ const checks = {};++ for (const [name, c] of Object.entries(body.checks || {}).slice(0, 40)) {++ if (!c || typeof c !== 'object') continue;++ checks[String(name).slice(0, 64)] = {++ status: c.status === 'ok' ? 'ok' : 'fail', required: c.required !== false,++ latency_ms: Number.isFinite(c.latency_ms) ? c.latency_ms : null,++ checked_at: typeof c.checked_at === 'string' ? c.checked_at.slice(0, 40) : null,++ ...(c.error ? { error: String(c.error).slice(0, 200) } : {}),++ };++ }++ const names = (a) => (Array.isArray(a) ? a.map(x => String(x).slice(0, 64)).slice(0, 40) : []);++ return { ready: body.ready, status: String(body.status || (body.ready ? 'ready' : 'not_ready')).slice(0, 20), checked_at: typeof body.checked_at === 'string' ? body.checked_at.slice(0, 40) : null, failed: names(body.failed), degraded: names(body.degraded), checks };++}++++function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, now = () => Date.now() } = {}) {const internal = { ...INTERNAL, ...internalOverrides };-- const health = new Map(); // id -> { status: up|down|unknown, http_status, latency_ms, checked_at }++ // id -> { status: up|degraded|down|not-running|unknown, basis, reason, http_status, latency_ms, checked_at, ready, release }++ const health = new Map();let timer = null;++ let lastPollAt = null;++++ async function getJson(url) {++ const t0 = now();++ const res = await fetchImpl(url, { signal: AbortSignal.timeout(3000), headers: { accept: 'application/json' } });++ let body = null;++ try { body = await res.json(); } catch { body = null; }++ return { res, body, latency: now() - t0 };++ }++++ async function releaseOf(base) {++ try {++ const { res, body } = await getJson(`${base}/release.json`);++ if (!res.ok || !body || typeof body.release !== 'string') return { release: null, error: `release.json answered ${res.status}` };++ return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null };++ } catch (err) {++ return { release: null, error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };++ }++ }async function checkOne(m) {++ const at = () => new Date(now()).toISOString();const base = internal[m.id];-- const path = m.ready || m.health;-- if (m.status === 'placeholder' || !base || !path) {-- health.set(m.id, { status: m.status === 'placeholder' ? 'not-running' : 'unknown', checked_at: new Date().toISOString() });-- return;-- }-- const t0 = Date.now();++ if (m.status === 'placeholder') { health.set(m.id, { status: 'not-running', reason: 'placeholder', checked_at: at() }); return; }++ const readyPath = m.ready || readyPaths[m.id] || null;++ if (!m.health && !readyPath && !(m.domains || []).length) { health.set(m.id, { status: 'not-running', reason: 'no runtime', checked_at: at() }); return; }++ if (!base) { health.set(m.id, { status: 'unknown', reason: 'no internal address known to Network', checked_at: at() }); return; }++ if (!readyPath && !m.health) { health.set(m.id, { status: 'unknown', reason: 'no health or readiness path in its manifest', checked_at: at() }); return; }++ const releaseP = releaseOf(base);++ let entry;try {-- const res = await fetchImpl(`${base}${path}`, { signal: AbortSignal.timeout(3000) });-- health.set(m.id, { status: res.ok ? 'up' : 'down', http_status: res.status, latency_ms: Date.now() - t0, checked_at: new Date().toISOString() });++ let got = readyPath ? await getJson(`${base}${readyPath}`) : null;++ if (got && got.res.status !== 404) {++ entry = { ...statusFromReady(got.res, got.body), ready: readySummary(got.body) };++ } else {++ // No readiness endpoint (yet): liveness is all that can be said.++ got = await getJson(`${base}${m.health}`);++ entry = { status: got.res.ok ? 'up' : 'down', basis: 'health', reason: 'liveness only: no readiness endpoint answered', ready: null };++ }++ entry.http_status = got.res.status;++ entry.latency_ms = got.latency;} catch (err) {-- health.set(m.id, { status: 'down', error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable', checked_at: new Date().toISOString() });++ entry = { status: 'down', basis: 'unreachable', error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable', ready: null };}++ entry.release = await releaseP;++ entry.checked_at = at();++ health.set(m.id, entry);}-- const pollAll = () => Promise.all(contracts.services.manifests.map(checkOne));++ const pollAll = async () => { await Promise.all(contracts.services.manifests.map(checkOne)); lastPollAt = now(); };function start() {if (timer) return;const loop = async () => { try { await pollAll(); } catch { /* one bad poll never stops the loop */ } timer = setTimeout(loop, pollMs); if (timer.unref) timer.unref(); };@@@ -60,7 +123,16 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g}function stop() { if (timer) clearTimeout(timer); timer = null; }-- const withHealth = (m) => ({ ...m, runtime: health.get(m.id) || { status: 'unknown', checked_at: null } });++ /** The row as it stands now: a result older than three poll intervals is 'unknown', never a stale 'up'. */++ function current(id) {++ const h = health.get(id);++ if (!h) return { status: 'unknown', reason: 'not checked yet', checked_at: null };++ const age = now() - Date.parse(h.checked_at);++ if (age > 3 * pollMs) return { ...h, status: 'unknown', reason: `last check is ${Math.round(age / 1000)}s old`, last_status: h.status, stale: true };++ return h;++ }++++ const withHealth = (m) => ({ ...m, runtime: current(m.id) });function descriptor() {return {@@@ -120,7 +192,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = greturn r;}-- return { router, start, stop, pollAll, health, descriptor };++ return { router, start, stop, pollAll, health, current, descriptor, lastPollAt: () => lastPollAt, pollMs, internal };}--module.exports = { createEcosystemRegistry, INTERNAL };++module.exports = { createEcosystemRegistry, INTERNAL, READY_PATHS, statusFromReady, readySummary };nnew file mode 100644@@@ -0,0 +1,157 @@++'use strict';++/**++ * Operator status (roadmap Track O): what each OpenVibe service is doing right now, stated as++ * observed rather than hoped.++ *++ * GET /status server-rendered page (works without JavaScript; noindex)++ * GET /api/v1/status the same rows as JSON++ * GET /api/v1/status/slo SLO categories (docs/slo.json) — proposals, not commitments++ *++ * Rows come from the ecosystem registry's health poll (server/registry/ecosystem.js): each running++ * service's readiness (openvibe-shared/ready shape) and /release.json. Every row is one of++ * up | degraded | down | not-running | unknown and carries checked_at; a service not checked yet, or++ * whose last check is stale, is 'unknown' — never an optimistic default.++ */++const fs = require('fs');++const path = require('path');++const express = require('express');++const contracts = require('openvibe-contracts');++const seo = require('openvibe-shared/seo');++++const SLO_FILE = path.join(__dirname, '..', '..', 'docs', 'slo.json');++const esc = seo.esc;++const STATES = ['up', 'degraded', 'down', 'not-running', 'unknown'];++const LABEL = { up: 'Up', degraded: 'Degraded', down: 'Down', 'not-running': 'Not running', unknown: 'Unknown' };++++function loadSlo() { return JSON.parse(fs.readFileSync(SLO_FILE, 'utf8')); }++++function rows(ecosystem) {++ return contracts.services.manifests.map((m) => {++ const r = ecosystem.current(m.id);++ const status = STATES.includes(r.status) ? r.status : 'unknown';++ return {++ id: m.id,++ name: m.name,++ manifest_status: m.status,++ origin: m.publicOrigin || null,++ status,++ label: status === 'not-running' && r.reason ? `not running (${r.reason})` : status,++ basis: r.basis || null,++ reason: r.reason || r.error || null,++ checked_at: r.checked_at || null,++ http_status: r.http_status ?? null,++ latency_ms: r.latency_ms ?? null,++ release: r.release && r.release.release ? { release: r.release.release, released_at: r.release.released_at || null, booted_at: r.release.booted_at || null } : null,++ release_error: r.release && !r.release.release ? r.release.error || 'unknown' : null,++ ready: r.ready || null,++ ...(r.stale ? { stale: true, last_status: r.last_status } : {}),++ };++ });++}++++function summary(list) {++ const counts = Object.fromEntries(STATES.map((s) => [s, 0]));++ for (const r of list) counts[r.status]++;++ return counts;++}++++const CSS = `++.st{max-width:1080px;margin:32px auto 64px;padding:0 20px}.st h1{font-size:clamp(24px,3vw,32px);letter-spacing:-.02em;margin:0 0 6px}++.st p.lede{color:var(--text-secondary,#96a7c2);margin:0 0 18px;max-width:780px;line-height:1.5}++.st-sum{display:flex;flex-wrap:wrap;gap:8px;margin:0 0 18px;padding:0;list-style:none}.st-sum li{padding:6px 12px;border-radius:999px;border:1px solid var(--border,#1f2d47);font-size:13.5px}++.st table{width:100%;border-collapse:collapse;font-size:14px}.st th,.st td{text-align:left;padding:10px 8px;border-bottom:1px solid var(--border,#1f2d47);vertical-align:top}++.st th{font-size:12px;text-transform:uppercase;letter-spacing:.5px;color:var(--text-muted,#7386a3)}++.st td small{display:block;color:var(--text-muted,#7386a3);font-size:12.5px;margin-top:2px}++.st code{font-size:12.5px}.st-b{display:inline-block;padding:2px 9px;border-radius:999px;font-weight:700;font-size:12.5px;border:1px solid currentColor}++.st-up{color:var(--success,#22c55e)}.st-degraded{color:#f59e0b}.st-down{color:var(--live-red,#ef4444)}.st-not-running,.st-unknown{color:var(--text-muted,#7386a3)}++.st ul.chk{margin:4px 0 0;padding:0;list-style:none;font-size:12.5px}.st ul.chk li{margin:1px 0}++.st section{margin-top:40px}.st dl{display:grid;grid-template-columns:minmax(160px,240px) 1fr;gap:6px 16px;font-size:14px}.st dt{font-weight:600}.st dd{margin:0;color:var(--text-secondary,#96a7c2)}++@media (max-width:720px){.st table,.st tbody,.st tr,.st td{display:block}.st thead{display:none}.st tr{border-bottom:1px solid var(--border,#1f2d47);padding:8px 0}.st td{border:0;padding:4px 0}.st dl{grid-template-columns:1fr}}`;++++function checksList(ready) {++ if (!ready || !ready.checks) return '';++ const items = Object.entries(ready.checks).map(([n, c]) => `<li>${c.status === 'ok' ? 'ok' : 'FAIL'} · <code>${esc(n)}</code>${c.required ? '' : ' (optional)'}${c.error ? ` — ${esc(c.error)}` : ''}</li>`);++ return items.length ? `<ul class="chk">${items.join('')}</ul>` : '';++}++++function renderPage(list, slo, generatedAt) {++ const counts = summary(list);++ const tr = list.map((r) => `<tr id="svc-${esc(r.id)}">++<td><b>${esc(r.name)}</b><small>${esc(r.id)} · manifest: ${esc(r.manifest_status)}</small></td>++<td><span class="st-b st-${esc(r.status)}">${esc(r.status === 'not-running' ? r.label : LABEL[r.status])}</span>${r.stale ? `<small>last seen ${esc(r.last_status)}</small>` : ''}${r.basis === 'health' ? '<small>liveness only (no readiness endpoint)</small>' : ''}${r.reason && r.status !== 'not-running' ? `<small>${esc(r.reason)}</small>` : ''}${checksList(r.ready)}</td>++<td>${r.release ? `<code>${esc(r.release.release)}</code>${r.release.booted_at ? `<small>booted ${esc(r.release.booted_at)}</small>` : ''}` : `<small>${r.status === 'not-running' ? '—' : esc(r.release_error ? `unknown (${r.release_error})` : 'unknown')}</small>`}</td>++<td>${r.checked_at ? `<time datetime="${esc(r.checked_at)}">${esc(r.checked_at)}</time>` : '<small>not checked yet</small>'}${r.latency_ms != null ? `<small>${esc(r.latency_ms)} ms</small>` : ''}</td>++</tr>`).join('\n');++ const sloRows = slo.categories.map((c) => `<dt>${esc(c.name)}</dt><dd>${esc(c.sli)} <br><small>Proposed: ${esc(Object.entries(c.proposed_target).map(([k, v]) => `${k.replace(/_/g, ' ')} ${v}`).join('; '))} · priority: ${esc(c.priority)} · ${esc(c.measurement_status || (c.measured_by.length ? 'measured' : 'not instrumented'))}</small></dd>`).join('\n');++ return `<!DOCTYPE html>++<html lang="en">++<head>++<meta charset="utf-8">++<meta name="viewport" content="width=device-width, initial-scale=1">++<title>Status · OpenVibe.Network</title>++<meta name="description" content="Observed status of each OpenVibe service: readiness, release and when it was last checked.">++<meta name="robots" content="noindex, nofollow">++${require('openvibe-shared/app-icon').headTags({ site: 'network', iconBase: '/assets' })}++<meta name="color-scheme" content="dark light">++<script>(function(){try{var raw=localStorage.getItem('ov_theme');if(!raw)return;var t=JSON.parse(raw),v=t&&t.variables;if(!v)return;var el=document.documentElement;for(var k in v)if(k.charAt(0)==='-')el.style.setProperty(k,v[k]);if(t.id)el.setAttribute('data-theme',t.id);}catch(_){}})();</script>++<script src="/shared/theme-loader.js" defer></script>++<style>++*,*::before,*::after{box-sizing:border-box}++:root{--bg-primary:#0a0f1c;--bg-secondary:#101828;--border:#1f2d47;--accent:#3b82f6;--text-primary:#e6edf7;--text-secondary:#96a7c2;--text-muted:#7386a3;--live-red:#ef4444;--success:#22c55e}++html,body{margin:0;background:var(--bg-primary);color:var(--text-primary);font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif}++a{color:var(--accent)}++${CSS}++</style>++</head>++<body>++<div id="navbar-mount"></div>++${require('openvibe-shared/chrome-ssr').noscriptNav({ name: 'OpenVibe.Network', links: [{ label: 'Status', href: '/status' }] })}++<main class="st" id="main">++<h1>Service status</h1>++<p class="lede">What Network observed when it last checked each service: its readiness endpoint (named checks, required or optional), its deployed release, and when. A service that has not been checked, or whose last check is out of date, shows as <b>unknown</b>. Placeholders show as not running. Page generated <time datetime="${esc(generatedAt)}">${esc(generatedAt)}</time>; checks run about every ${esc(Math.round(slo.pollSeconds || 60))} seconds. JSON: <a href="/api/v1/status"><code>/api/v1/status</code></a>.</p>++<ul class="st-sum">${STATES.map((s) => `<li class="st-${s}">${esc(LABEL[s])}: ${counts[s]}</li>`).join('')}</ul>++<table>++<thead><tr><th>Service</th><th>Status</th><th>Release</th><th>Checked</th></tr></thead>++<tbody>++${tr}++</tbody>++</table>++<section aria-labelledby="h-slo"><h2 id="h-slo">SLO categories (proposals)</h2>++<p class="lede">${esc(slo.note)} ${esc(slo.binding)} JSON: <a href="/api/v1/status/slo"><code>/api/v1/status/slo</code></a>.</p>++<dl>++${sloRows}++</dl></section>++</main>++${require('openvibe-shared/footer').ssr({ service: 'network', variant: 'compact' })}++<script src="/shared/navbar.js" defer></script>++<script>document.addEventListener('DOMContentLoaded',function(){try{if(window.OpenVibeNavbar)OpenVibeNavbar.init({service:'network',apiBase:location.origin});}catch(e){}});</script>++</body>++</html>`;++}++++function createStatusRoutes({ ecosystem, now = () => new Date() }) {++ const r = express.Router();++ r.get('/api/v1/status', (_req, res) => {++ const list = rows(ecosystem);++ const last = ecosystem.lastPollAt();++ res.set('Cache-Control', 'no-cache, max-age=0').set('Access-Control-Allow-Origin', '*').json({++ generated_at: now().toISOString(),++ last_poll_at: last ? new Date(last).toISOString() : null,++ poll_seconds: Math.round(ecosystem.pollMs / 1000),++ states: STATES,++ summary: summary(list),++ services: list,++ });++ });++ r.get('/api/v1/status/slo', (_req, res) => {++ res.set('Cache-Control', 'public, max-age=300').set('Access-Control-Allow-Origin', '*').json(loadSlo());++ });++ r.get('/status', (_req, res) => {++ const slo = { ...loadSlo(), pollSeconds: ecosystem.pollMs / 1000 };++ res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', 'no-cache, max-age=0').set('X-Robots-Tag', 'noindex, nofollow');++ res.send(renderPage(rows(ecosystem), slo, now().toISOString()));++ });++ return r;++}++++module.exports = { createStatusRoutes, rows, STATES };@@@ -23,6 +23,7 @@ const must = [['Cloudflare client IP forwarded', /proxy_set_header CF-Connecting-IP \$http_cf_connecting_ip;/],['access log without query strings (tokens)', /log_format live_noquery [^;]*\$uri[^;]*;[\s\S]*access_log [^;]* live_noquery;/],['capture permissions header', /Permissions-Policy "camera=\*, microphone=\*, display-capture=\*"/],++ ['OpenVibe.Chat locations (glob, before /api/)', /include \/opt\/openvibe\.chat\/deploy\/nginx\/\*\.locations\.conf;[\s\S]*location \/api\/ \{/],];for (const [name, re] of must) assert.ok(re.test(conf), `generated live config is missing: ${name}`);// The asset regex must not swallow /api/ or /ws/ (a regex location beats prefix locations).nnew file mode 100644@@@ -0,0 +1,128 @@++'use strict';++// Metrics and readiness (server/observability.js): token issuance by grant type, principal-token++// failures, route-template labels, loopback-only /metrics, and a /api/ready that fails only on a++// required dependency. Roadmap Track O.++// node test/observability.test.js++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const os = require('os');++const path = require('path');++const http = require('http');++const express = require('express');++const metrics = require('openvibe-shared/metrics');++const { initDb } = require('../server/db/database');++const observability = require('../server/observability');++++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-observability-'));++const log = console.log; console.log = () => {};++const db = initDb(path.join(dir, 'network.db'));++console.log = log;++db.prepare("UPDATE oauth_clients SET client_secret = 'live-secret' WHERE client_id = 'live'").run();++++const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });++const ISSUER = 'https://openvibe.network';++const app = express();++metrics.instrument(app, { service: 'network', release: 'abc123def456', registry: observability.registry });++app.use(express.json());++app.use(express.urlencoded({ extended: true }));++app.locals.db = db;++app.locals.config = { internalKey: 'legacy-key', jwt: { issuer: ISSUER, accessTokenExpiry: '1h' } };++app.locals.privateKey = keys.privateKey;++app.locals.publicKey = keys.publicKey;++app.use('/oauth/token', observability.tokenEndpointMetrics);++app.use('/oauth', require('../server/auth/oauth-routes'));++app.use('/internal', require('../server/internal/routes'));++++let keysNow = { privateKey: keys.privateKey, publicKey: keys.publicKey };++let lastPoll = null;++let discordReady = false;++const ready = observability.createNetworkReadiness({++ db, release: 'abc123def456', production: true,++ getKeys: () => keysNow,++ ecosystem: { lastPollAt: () => lastPoll },++ discordService: { isReady: () => discordReady, _getSetting: () => 'configured-token' },++});++app.get('/api/ready', ready.handler);++const server = http.createServer(app);++++(async () => {++ await new Promise(r => server.listen(0, '127.0.0.1', r));++ const base = `http://127.0.0.1:${server.address().port}`;++ const form = (body) => fetch(`${base}/oauth/token`, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams(body) }).then(async r => ({ status: r.status, body: await r.json() }));++ const get = (p, headers = {}) => fetch(base + p, { headers }).then(async r => ({ status: r.status, text: await r.text() }));++++ // ── Token issuance and failures by grant type ──++ let t = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'live-secret', audience: 'openvibe.network' });++ assert.strictEqual(t.status, 200);++ const svcToken = t.body.access_token;++ t = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'wrong', audience: 'openvibe.network' });++ assert.strictEqual(t.status, 401);++ t = await form({ grant_type: 'refresh_token', client_id: 'live', client_secret: 'live-secret', refresh_token: 'nope' });++ assert.ok(t.status >= 400);++ t = await form({ grant_type: 'password', client_id: 'live', client_secret: 'live-secret' });++ assert.strictEqual(t.status, 400);++++ // ── Principal (service-token) failures at guarded routes ──++ const narrow = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'live-secret', audience: 'openvibe.network', scope: 'network.coins.credit' });++ let r = await fetch(`${base}/internal/coins/debit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${narrow.body.access_token}` }, body: JSON.stringify({ user_id: 1, app_id: 'live', amount: 1, reason: 't', idempotency_key: 'k1' }) });++ assert.strictEqual(r.status, 403);++ r = await fetch(`${base}/internal/coins/credit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${svcToken}` }, body: JSON.stringify({ user_id: 1, app_id: 'games', amount: 1, reason: 't', idempotency_key: 'k2' }) });++ assert.strictEqual(r.status, 403);++ r = await fetch(`${base}/internal/coins/credit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: 'Bearer not-a-token' }, body: '{}' });++ assert.strictEqual(r.status, 401);++++ const m = await get('/metrics');++ assert.strictEqual(m.status, 200);++ assert.ok(m.text.includes('network_tokens_issued_total{grant_type="client_credentials"} 2\n'), m.text);++ assert.ok(m.text.includes('network_token_failures_total{grant_type="client_credentials",error="invalid_client"} 1\n'));++ assert.ok(/network_token_failures_total\{grant_type="refresh_token",error="[a-z_]+"\} 1\n/.test(m.text));++ assert.ok(m.text.includes('network_token_failures_total{grant_type="other",error="unsupported_grant_type"} 1\n'), 'unknown grant types never become labels');++ assert.ok(m.text.includes('network_principal_token_failures_total{code="capability.denied"} 1\n'));++ assert.ok(m.text.includes('network_principal_token_failures_total{code="capability.owner_denied"} 1\n'));++ assert.ok(m.text.includes('http_requests_total{method="POST",route="/oauth/token",status_class="2xx"} 2\n'));++ assert.ok(m.text.includes('release_info{service="network",release="abc123def456"} 1\n'));++ assert.ok(!/secret|live-secret|not-a-token/.test(m.text), 'no credentials in metrics');++ assert.strictEqual((await get('/metrics', { 'x-forwarded-for': '198.51.100.7' })).status, 404, '/metrics through a proxy is 404');++++ // ── Readiness ──++ let rd = await get('/api/ready');++ let body = JSON.parse(rd.text);++ assert.strictEqual(rd.status, 200, rd.text);++ assert.strictEqual(body.status, 'degraded', 'poll not run yet and discord configured but down: optional only');++ assert.deepStrictEqual(body.degraded.sort(), ['discord_bot', 'registry_poll']);++ assert.strictEqual(body.checks.db.status, 'ok');++ assert.strictEqual(body.checks.signing_key.status, 'ok');++ assert.strictEqual(body.checks.signing_key.required, true);++ for (const c of Object.values(body.checks)) { assert.ok(c.checked_at && typeof c.latency_ms === 'number'); }++++ lastPoll = Date.now(); discordReady = true;++ rd = await get('/api/ready'); body = JSON.parse(rd.text);++ assert.strictEqual(body.status, 'ready');++++ // An ephemeral HS256 key in production is not ready: nothing else can verify its tokens.++ const eph = crypto.randomBytes(32).toString('hex');++ keysNow = { privateKey: eph, publicKey: eph };++ const ready2 = observability.createNetworkReadiness({ db, release: 'x', production: true, getKeys: () => keysNow });++ body = await ready2.run();++ assert.strictEqual(body.ready, false);++ assert.deepStrictEqual(body.failed, ['signing_key']);++ const devReady = observability.createNetworkReadiness({ db, release: 'x', production: false, getKeys: () => keysNow });++ body = await devReady.run();++ assert.strictEqual(body.ready, true, 'outside production the dev key only degrades');++ assert.deepStrictEqual(body.degraded, ['signing_key']);++++ // The database gone: not ready.++ console.log = () => {};++ const db2 = initDb(path.join(dir, 'other.db'));++ console.log = log;++ const ready3 = observability.createNetworkReadiness({ db: db2, release: 'x', production: true, getKeys: () => ({ privateKey: keys.privateKey, publicKey: keys.publicKey }) });++ db2.close();++ body = await ready3.run();++ assert.strictEqual(body.ready, false);++ assert.deepStrictEqual(body.failed, ['db']);++++ server.close();++ db.close();++ console.log('observability: all checks passed');++})().catch(err => { console.error(err); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,129 @@++'use strict';++// Operator status (server/status/routes.js over server/registry/ecosystem.js): every row is++// up | degraded | down | not-running | unknown with checked_at, built from each service's readiness++// and /release.json — never an optimistic default. Roadmap Track O.++// node test/status.test.js++const assert = require('assert');++const http = require('http');++const express = require('express');++const contracts = require('openvibe-contracts');++const { createEcosystemRegistry } = require('../server/registry/ecosystem');++const { createStatusRoutes } = require('../server/status/routes');++++function fakeService(routes) {++ const srv = http.createServer((req, res) => {++ const h = routes[req.url];++ if (!h) { res.statusCode = 404; return res.end('{"error":"Not found"}'); }++ const [status, body] = typeof h === 'function' ? h() : h;++ res.statusCode = status;++ res.setHeader('content-type', 'application/json');++ res.end(typeof body === 'string' ? body : JSON.stringify(body));++ });++ return new Promise(r => srv.listen(0, '127.0.0.1', () => r({ srv, url: `http://127.0.0.1:${srv.address().port}` })));++}++const check = (status, required, extra = {}) => ({ status, required, latency_ms: 1.2, checked_at: '2026-09-23T00:00:00.000Z', ...extra });++const rel = (release) => [200, { service: 'x', release, released_at: '2026-09-22T20:00:00.000Z', booted_at: '2026-09-23T00:00:00.000Z' }];++++(async () => {++ let communityDown = false;++ const network = await fakeService({ '/api/ready': [200, { ready: true, status: 'ready', failed: [], degraded: [], checks: { db: check('ok', true) } }], '/release.json': rel('aaaaaaaaaaaa') });++ const media = await fakeService({ '/api/ready': [200, { ready: true, status: 'degraded', failed: [], degraded: ['remote_b2'], checks: { db: check('ok', true), remote_b2: check('fail', false, { error: 'timeout' }) } }], '/release.json': rel('bbbbbbbbbbbb') });++ const community = await fakeService({ '/api/ready': () => (communityDown ? [503, { ready: false, status: 'not_ready', failed: ['db'], degraded: [], checks: { db: check('fail', true, { error: 'SQLITE_CANTOPEN' }) } }] : [200, { ready: true, status: 'ready', failed: [], degraded: [], checks: { db: check('ok', true) } }]), '/release.json': rel('cccccccccccc') });++ const tools = await fakeService({ '/api/health': [200, { status: 'ok' }] }); // no /api/ready yet, no release.json++ const events = await fakeService({ '/api/ready': [200, { status: 'ready', checks: { db: true } }] }); // an older ad-hoc shape++++ let clock = Date.parse('2026-09-23T01:00:00Z');++ const eco = createEcosystemRegistry({++ issuer: 'https://openvibe.network', pollMs: 60000, now: () => clock,++ internalOverrides: { network: network.url, media: media.url, community: community.url, tools: tools.url, events: events.url, live: 'http://127.0.0.1:1', games: 'http://127.0.0.1:1', billing: 'http://127.0.0.1:1' },++ });++ const app = express();++ app.use(createStatusRoutes({ ecosystem: eco, now: () => new Date(clock) }));++ app.use(eco.router());++ const srv = http.createServer(app);++ await new Promise(r => srv.listen(0, '127.0.0.1', r));++ const base = `http://127.0.0.1:${srv.address().port}`;++ const json = (p) => fetch(base + p).then(async r => ({ status: r.status, headers: r.headers, body: await r.json() }));++++ // Before any poll: everything that runs is 'unknown' with no checked_at — never 'up'.++ let r = await json('/api/v1/status');++ let by = Object.fromEntries(r.body.services.map(s => [s.id, s]));++ assert.strictEqual(by.network.status, 'unknown');++ assert.strictEqual(by.network.checked_at, null);++ assert.strictEqual(r.body.last_poll_at, null);++ assert.ok(r.body.services.every(s => r.body.states.includes(s.status)));++++ await eco.pollAll();++ r = await json('/api/v1/status');++ by = Object.fromEntries(r.body.services.map(s => [s.id, s]));++ assert.strictEqual(r.status, 200);++ assert.strictEqual(r.headers.get('access-control-allow-origin'), '*');++ assert.strictEqual(by.network.status, 'up');++ assert.strictEqual(by.network.basis, 'ready');++ assert.strictEqual(by.network.release.release, 'aaaaaaaaaaaa');++ assert.strictEqual(by.network.release.booted_at, '2026-09-23T00:00:00.000Z');++ assert.strictEqual(by.media.status, 'degraded', 'an optional failure degrades, it does not read as healthy or down');++ assert.deepStrictEqual(by.media.ready.degraded, ['remote_b2']);++ assert.strictEqual(by.media.ready.checks.remote_b2.error, 'timeout');++ assert.strictEqual(by.community.status, 'up');++ assert.strictEqual(by.tools.status, 'up');++ assert.strictEqual(by.tools.basis, 'health', 'no readiness endpoint: liveness only, and the row says so');++ assert.strictEqual(by.tools.release, null);++ assert.ok(by.tools.release_error);++ assert.strictEqual(by.events.basis, 'ready-legacy');++ assert.strictEqual(by.live.status, 'down', 'unreachable is down');++ assert.strictEqual(by.wiki.status, 'not-running');++ assert.strictEqual(by.wiki.label, 'not running (placeholder)');++ assert.strictEqual(by.contracts.status, 'not-running');++ assert.strictEqual(by.contracts.label, 'not running (no runtime)');++ for (const s of r.body.services) assert.ok(s.checked_at, `${s.id} says when it was checked`);++ assert.strictEqual(r.body.summary.up + r.body.summary.degraded + r.body.summary.down + r.body.summary['not-running'] + r.body.summary.unknown, contracts.services.manifests.length);++++ // A required failure on the service is 'down', with its failed checks.++ communityDown = true;++ await eco.pollAll();++ by = Object.fromEntries((await json('/api/v1/status')).body.services.map(s => [s.id, s]));++ assert.strictEqual(by.community.status, 'down');++ assert.deepStrictEqual(by.community.ready.failed, ['db']);++++ // The registry's runtime field says the same thing.++ r = await json('/api/v1/registry/services/community');++ assert.strictEqual(r.body.runtime.status, 'down');++++ // Stale: older than three poll intervals is 'unknown', never the last good state.++ clock += 4 * 60000;++ by = Object.fromEntries((await json('/api/v1/status')).body.services.map(s => [s.id, s]));++ assert.strictEqual(by.network.status, 'unknown');++ assert.strictEqual(by.network.stale, true);++ assert.strictEqual(by.network.last_status, 'up');++ clock -= 4 * 60000;++++ // The page: server-rendered, noindex, no JavaScript needed to read it.++ const page = await fetch(base + '/status');++ const html = await page.text();++ assert.strictEqual(page.status, 200);++ assert.match(page.headers.get('x-robots-tag'), /noindex/);++ assert.ok(html.includes('<meta name="robots" content="noindex, nofollow">'));++ assert.ok(html.includes('id="svc-media"') && /id="svc-media"[\s\S]*?Degraded/.test(html));++ assert.ok(/id="svc-community"[\s\S]*?Down[\s\S]*?SQLITE_CANTOPEN/.test(html));++ assert.ok(/id="svc-wiki"[\s\S]*?not running \(placeholder\)/.test(html));++ assert.ok(/id="svc-tools"[\s\S]*?liveness only/.test(html));++ assert.ok(html.includes('<noscript>'), 'navigation without JavaScript');++ assert.ok(html.includes('SLO categories (proposals)'));++ const noScript = html.replace(/<script[\s\S]*?<\/script>/g, '');++ assert.ok(noScript.includes('aaaaaaaaaaaa'), 'the rows are in the HTML itself');++++ // SLO categories: the ten from §15.19, every target labelled a proposal.++ r = await json('/api/v1/status/slo');++ assert.strictEqual(r.body.status, 'proposal');++ assert.strictEqual(r.body.categories.length, 10);++ for (const c of r.body.categories) {++ assert.ok(c.id && c.name && c.sli && c.proposed_target && ['integrity', 'availability', 'continuity'].includes(c.priority), c.id);++ assert.ok(Array.isArray(c.measured_by));++ if (!c.measured_by.length) assert.ok(/not instrumented/.test(c.measurement_status), `${c.id} says it is not measured`);++ }++++ for (const s of [network, media, community, tools, events]) s.srv.close();++ srv.close();++ console.log('status: all checks passed');++})().catch(err => { console.error(err); process.exit(1); });
@@ -53,6 +53,9 @@ server {
@@ -53,6 +53,9 @@ server {
add_header X-Frame-Options SAMEORIGIN always;
add_header X-Frame-Options SAMEORIGIN always;
add_header Referrer-Policy strict-origin-when-cross-origin always;
add_header Referrer-Policy strict-origin-when-cross-origin always;
+ # Prometheus metrics are for loopback scrapers only (the app also 404s any proxied request).
+ location = /metrics { return 404; }
+
# ── Auth endpoints (strict rate limit) ──────────────────
# ── Auth endpoints (strict rate limit) ──────────────────
location ~ ^/(api/auth/(register|login|anon-session)|oauth/(token|authorize)) {
location ~ ^/(api/auth/(register|login|anon-session)|oauth/(token|authorize)) {
limit_req zone=openvibenetwork_auth burst=5 nodelay;
limit_req zone=openvibenetwork_auth burst=5 nodelay;
new file mode 100644
new file mode 100644
@@ -0,0 +1,463 @@
@@ -0,0 +1,463 @@
+From a4186199f8ae7cf519307669f36e5864aca5f2f1 Mon Sep 17 00:00:00 2001
+From: OpenVibers <[email protected]>
+Date: Tue, 22 Sep 2026 19:03:59 -0700
+Subject: [PATCH] Metrics and truthful readiness (Track O): GET /metrics
+ (loopback only) with golden signals by route template, release_info, live
+ streams, WebSocket connections per server and outbox depth; /api/ready checks
+ boot + db (required) and SFU, Media, Network key (optional, reported as
+ degraded); openvibe-shared v1.3.0
+
+---
+ deploy/nginx/openvibe.live.conf | 3 +
+ docs/deploy.md | 6 ++
+ package-lock.json | 7 +-
+ package.json | 2 +-
+ server/index.js | 56 +++++-----
+ server/web/observability.js | 108 ++++++++++++++++++++
+ test/observability.test.js | 175 ++++++++++++++++++++++++++++++++
+ 7 files changed, 323 insertions(+), 34 deletions(-)
+ create mode 100644 server/web/observability.js
+ create mode 100644 test/observability.test.js
+
+diff --git a/deploy/nginx/openvibe.live.conf b/deploy/nginx/openvibe.live.conf
+index 07b075d..90a2414 100644
+--- a/deploy/nginx/openvibe.live.conf
+@@ -72,6 +72,9 @@ server {
+ # ceiling ordinary traffic cannot reach.
+ limit_conn addr_limit 256;
+
++ # Prometheus metrics are for local scrapers only; Node also 404s any proxied request to it.
++ location = /metrics { return 404; }
++
+ location = /api/auth/register {
+ limit_req zone=login_limit burst=5 nodelay;
+ proxy_pass http://127.0.0.1:3000;
+diff --git a/docs/deploy.md b/docs/deploy.md
+index 5af9ee8..65ad04d 100644
+--- a/docs/deploy.md
+@@ -47,6 +47,12 @@ restart. A rollback resets the checkout but cannot restore previous `node_module
+ `OV_APP_ROOT=/opt/openvibe.live/current`, so static files and docs are read through the symlink and
+ a static-only switch needs no restart.
+ - If the restarted release never becomes ready, `current` is switched back and restarted (exit 3).
++ "Ready" is `GET /api/ready` → 200: boot finished and the database answers. The WebRTC SFU,
++ OpenVibe.Media and the Network signing key are optional checks: when one is missing the answer is
++ still 200 with `"status": "degraded"` and the check named in `degraded`, never a silent pass.
++- `GET /metrics` (Prometheus text: requests by route template, latency, in-flight, process,
++ `release_info`, live streams, WebSocket connections per server, outbox) answers only
++ `curl http://127.0.0.1:3000/metrics` on the host; through nginx it is a 404.
+ - `--rollback` selects the previous release with **its own** `node_modules`.
+ - The last 5 releases are kept.
+ - Content-hashed assets from the previous release are still served under their old hashes, so a page
+diff --git a/package-lock.json b/package-lock.json
+index 7a0b166..0c980ff 100644
+--- a/package-lock.json
+@@ -27,7 +27,7 @@
+ "onnxruntime-node": "^1.20.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.2.2",
+- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "sanitize-html": "^2.17.7",
+ "sdp-transform": "^3.0.0",
+ "sharp": "^0.33.0",
+@@ -2839,9 +2839,8 @@
+ }
+ },
+ "node_modules/openvibe-shared": {
+- "version": "1.2.1",
+- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
+- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",
++ "version": "1.3.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "license": "MIT",
+ "dependencies": {
+ "jsonwebtoken": "^9.0.0"
+diff --git a/package.json b/package.json
+index 48e0cb1..d2c3c21 100644
+--- a/package.json
+@@ -45,7 +45,7 @@
+ "onnxruntime-node": "^1.20.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.2.2",
+- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "sanitize-html": "^2.17.7",
+ "sdp-transform": "^3.0.0",
+ "sharp": "^0.33.0",
+diff --git a/server/index.js b/server/index.js
+index 7a4c359..4bba504 100644
+--- a/server/index.js
+@@ -131,6 +131,13 @@ const whipHandler = require('./streaming/whip-handler');
+ // ── Express App ──────────────────────────────────────────────
+ const app = express();
+ const server = http.createServer(app);
++
++// What this server runs (ADR-016): GET /release.json below, and release_info in /metrics.
++const release = require('openvibe-shared/release').createRelease({ service: 'live', root: path.join(__dirname, '..') });
++// Metrics first, so every request is counted (by route template, never by raw URL). GET /metrics
++// answers direct loopback callers only; through nginx it is a 404 (server/web/observability.js).
++const observability = require('./web/observability');
++const { registry: metricsRegistry } = observability.mountMetrics(app, { release });
+ const vibeCodingPublishServer = new VibeCodingPublishServer(chatServer, db);
+
+ function normalizeOrigin(origin) {
+@@ -723,36 +730,27 @@ app.get('/api/health', (req, res) => {
+ * genuine gate.
+ */
+ let _bootComplete = false;
+-// What this server runs (ADR-016). The shared navbar loads /shared/release-watch.js, which polls
+-// this and prompts open tabs after a deploy; it never reloads a tab that is watching, broadcasting
+-// or typing.
+-{
+- const release = require('openvibe-shared/release').createRelease({ service: 'live', root: path.join(__dirname, '..') });
+- app.get('/release.json', release.handler);
+-}
++// The shared navbar loads /shared/release-watch.js, which polls this and prompts open tabs after a
++// deploy; it never reloads a tab that is watching, broadcasting or typing.
++app.get('/release.json', release.handler);
++
++// Required: boot finished and the database answers. Optional (a failure is "degraded", still 200):
++// the WebRTC SFU, OpenVibe.Media and the Network signing key — Live serves channels, chat and RTMP
++// without them, so they must neither pass silently nor take the whole service out of rotation.
++const readiness = observability.createLiveReadiness({
++ release,
++ bootComplete: () => _bootComplete,
++ dbQuery: () => db.get('SELECT 1 AS ok'),
++ sfuReady: () => webrtcSFU.ready === true && !!webrtcSFU.worker,
++ mediaUrl: mediaClient.MEDIA_URL,
++ networkKey: () => require('./auth/auth').getNetworkPublicKey(),
++});
++app.get('/api/ready', readiness.handler);
+
+-app.get('/api/ready', (req, res) => {
+- res.set('Cache-Control', 'no-store');
+- const checks = { boot: _bootComplete, db: false };
+- try {
+- // Cheapest possible proof that the connection is open and the schema exists.
+- db.get('SELECT 1 AS ok');
+- checks.db = true;
+- } catch (err) {
+- checks.dbError = String(err && err.message || err).slice(0, 200);
+- }
+- const ready = checks.boot && checks.db;
+- res.status(ready ? 200 : 503).json({
+- ready,
+- checks,
+- uptime: Math.round(process.uptime()),
+- // Advisory only — a deploy gate should not wait on optional subsystems, which can be
+- // legitimately unavailable (no mediasoup worker on a box that only relays RTMP, say).
+- optional: {
+- sfu: (() => { try { return !!webrtcSFU.isReady?.(); } catch { return null; } })(),
+- media: (() => { try { return require('./media-client').lastOk === true; } catch { return null; } })(),
+- },
+- });
++observability.registerDomainGauges(metricsRegistry, {
++ liveStreams: () => db.getLiveStreams().length,
++ wsServers: { chat: chatServer, broadcast: broadcastServer, control: controlServer, call: callServer },
++ outboxStatus: () => require('./events/stream-events').status(),
+ });
+
+ // ── Updates / Changelog ──────────────────────────────────────
+diff --git a/server/web/observability.js b/server/web/observability.js
+new file mode 100644
+index 0000000..f1170c4
+--- /dev/null
+@@ -0,0 +1,108 @@
++'use strict';
++/**
++ * Metrics and readiness for Live (roadmap Track O, §15.19), on openvibe-shared/metrics and
++ * openvibe-shared/ready.
++ *
++ * GET /metrics Prometheus text, direct loopback callers only (404 through nginx or any proxy):
++ * http_requests_total{method,route,status_class}, http_request_duration_seconds,
++ * http_requests_in_flight, process/event-loop metrics, release_info{service,release},
++ * live_streams_live, live_ws_connections{server}, live_events_outbox{status}
++ * GET /api/ready 200 while Live can serve (boot finished, the database answers); 503 otherwise.
++ * Media, the WebRTC SFU and the Network signing key are optional: when one is
++ * missing the answer stays 200 with status "degraded" and names it, because Live
++ * still serves channels, chat and RTMP without them.
++ *
++ * Everything the checks and gauges read is injected, so the test drives this module without booting
++ * the whole server.
++ */
++const metrics = require('openvibe-shared/metrics');
++const { createReadiness } = require('openvibe-shared/ready');
++
++// Long-lived responses whose "duration" is a viewing session, not a request.
++const STREAMING_PATHS = [/^\/api\/live-events$/, /^\/api\/streams\/rtmp-proxy\//];
++// Static mounts: served by express.static (no route), labelled by mount instead of 'unmatched'.
++const STATIC_MOUNT = /^\/(js|css|fragments|shared|assets|data|media|docs-assets|images|img|fonts|sounds)\//;
++
++function normalize(req) {
++ if (req.route) return null; // the matched route's template (openvibe-shared/metrics routeLabel)
++ const m = STATIC_MOUNT.exec(String(req.originalUrl || req.url || '').split('?')[0]);
++ return m ? `/${m[1]}/*` : null;
++}
++
++/**
++ * Mount the HTTP metrics middleware and GET /metrics. Call before any other middleware.
++ * @returns {{ registry }}
++ */
++function mountMetrics(app, { release }) {
++ const m = metrics.instrument(app, {
++ service: 'live',
++ release: release && release.release,
++ normalize,
++ skip: (req) => STREAMING_PATHS.some((re) => re.test(req.path)),
++ });
++ return m;
++}
++
++/**
++ * Domain gauges, read at scrape time. A source that throws is left out of that scrape.
++ * liveStreams() -> number of live streams (the rows /api/streams lists)
++ * wsServers -> { chat, broadcast, control, call } objects with a `wss` (ws.Server) once initialised
++ * outboxStatus() -> server/events/stream-events.js status()
++ */
++function registerDomainGauges(registry, { liveStreams, wsServers = {}, outboxStatus }) {
++ registry.gauge({ name: 'live_streams_live', help: 'Streams currently marked live (the list /api/streams serves)', collect: () => liveStreams() });
++ registry.gauge({
++ name: 'live_ws_connections', help: 'Open WebSocket connections per WebSocket server', labelNames: ['server'],
++ collect: () => Object.entries(wsServers)
++ .filter(([, s]) => s && s.wss && s.wss.clients) // a server that never initialised reports nothing
++ .map(([server, s]) => ({ labels: { server }, value: s.wss.clients.size })),
++ });
++ registry.gauge({
++ name: 'live_events_outbox', help: 'Stream lifecycle events in the transactional outbox, by status (absent while the outbox is disabled)', labelNames: ['status'],
++ collect: () => {
++ const s = outboxStatus();
++ if (!s || !s.enabled) return null;
++ return [{ labels: { status: 'pending' }, value: Number(s.pending) || 0 }, { labels: { status: 'rejected' }, value: Number(s.rejected) || 0 }];
++ },
++ });
++}
++
++/**
++ * The /api/ready handler.
++ * bootComplete() -> boolean required
++ * dbQuery() -> row required (a real query; throws when the database cannot answer)
++ * sfuReady() -> boolean optional (WebRTC broadcasting/viewing through mediasoup)
++ * mediaUrl optional (OpenVibe.Media: VODs, clips, thumbnails), checked by
++ * GET <mediaUrl>/healthz, cached 30 s
++ * networkKey() -> PEM or null optional (sign-in: RS256 key from OpenVibe.Network)
++ */
++function createLiveReadiness({ release, bootComplete, dbQuery, sfuReady, mediaUrl, networkKey, fetchImpl = globalThis.fetch }) {
++ return createReadiness({
++ service: 'live',
++ release: release && release.release,
++ checks: [
++ { name: 'boot', required: true, description: 'boot finished', check: () => (bootComplete() ? true : 'boot not finished') },
++ { name: 'db', required: true, description: 'SQLite answers a query', check: () => { const row = dbQuery(); return row ? true : 'no row'; } },
++ { name: 'sfu', required: false, description: 'WebRTC SFU (mediasoup worker)', check: () => (sfuReady() ? true : 'mediasoup worker not running') },
++ {
++ name: 'media', required: false, cacheMs: 30000, timeoutMs: 2000, description: 'OpenVibe.Media (VODs, clips, thumbnails)',
++ check: async () => {
++ const res = await fetchImpl(`${mediaUrl}/healthz`, { signal: AbortSignal.timeout(2000) });
++ return res.ok ? true : `Media answered ${res.status}`;
++ },
++ },
++ { name: 'network_key', required: false, description: 'OpenVibe.Network RS256 key (sign-in)', check: () => (/BEGIN [A-Z ]*PUBLIC KEY/.test(String(networkKey() || '')) ? true : 'Network public key not loaded: sign-in cannot be verified') },
++ ],
++ // Fields the previous /api/ready served, so nothing that read them breaks.
++ details: (body) => ({
++ uptime: Math.round(process.uptime()),
++ optional: {
++ sfu: body.checks.sfu.status === 'ok',
++ media: body.checks.media.status === 'ok',
++ network_key: body.checks.network_key.status === 'ok',
++ },
++ }),
++ });
++}
++
++module.exports = { mountMetrics, registerDomainGauges, createLiveReadiness, normalize };
+diff --git a/test/observability.test.js b/test/observability.test.js
+new file mode 100644
+index 0000000..a052903
+--- /dev/null
+@@ -0,0 +1,175 @@
++'use strict';
++/**
++ * Metrics and readiness (server/web/observability.js, roadmap Track O):
++ * - /metrics answers direct loopback callers only; through a proxy (X-Forwarded-For) it is a 404
++ * - HTTP metrics are labelled by route template, never by raw URL, id or query
++ * - release_info and the domain gauges (live streams, WebSocket connections, outbox)
++ * - /api/ready: boot + db required (503 when either fails); SFU, Media and the Network key optional
++ * (200 "degraded", naming what is missing)
++ * A temp SQLite database stands in for Live's; no Media, Network or mediasoup is needed.
++ *
++ * Run: node test/observability.test.js
++ */
++const assert = require('assert');
++const fs = require('fs');
++const http = require('http');
++const os = require('os');
++const path = require('path');
++const express = require('express');
++const Database = require('better-sqlite3');
++const observability = require('../server/web/observability');
++
++const tmpDb = path.join(os.tmpdir(), `openvibe-observability-test-${process.pid}-${Date.now()}.db`);
++
++(async () => {
++ const sqlite = new Database(tmpDb);
++ sqlite.exec('CREATE TABLE streams (id INTEGER PRIMARY KEY, is_live INTEGER)');
++ sqlite.prepare('INSERT INTO streams (is_live) VALUES (1), (1), (0)').run();
++ let dbBroken = false;
++ const dbQuery = () => { if (dbBroken) throw new Error('SQLITE_IOERR: disk I/O error'); return sqlite.prepare('SELECT 1 AS ok').get(); };
++
++ // Fake Media: /healthz answers 200 or 502.
++ let mediaUp = true; let mediaHits = 0;
++ const media = http.createServer((req, res) => { mediaHits++; res.statusCode = req.url === '/healthz' && mediaUp ? 200 : 502; res.end('{}'); });
++ await new Promise((r) => media.listen(0, '127.0.0.1', r));
++ const mediaUrl = `http://127.0.0.1:${media.address().port}`;
++
++ const release = { release: 'abc123def456' };
++ let booted = false; let sfu = true; let key = '-----BEGIN PUBLIC KEY-----\nMII…\n-----END PUBLIC KEY-----';
++ const fakeWss = (n) => ({ wss: { clients: { size: n } } });
++ let outbox = { enabled: true, pending: 4, rejected: 1 };
++
++ const app = express();
++ const { registry } = observability.mountMetrics(app, { release });
++ const readiness = observability.createLiveReadiness({
++ release, bootComplete: () => booted, dbQuery, sfuReady: () => sfu, mediaUrl, networkKey: () => key,
++ });
++ app.get('/api/ready', readiness.handler);
++ observability.registerDomainGauges(registry, {
++ liveStreams: () => sqlite.prepare('SELECT COUNT(*) AS n FROM streams WHERE is_live = 1').get().n,
++ wsServers: { chat: fakeWss(12), broadcast: fakeWss(2), control: fakeWss(0), call: { wss: null } },
++ outboxStatus: () => outbox,
++ });
++ const vods = express.Router();
++ vods.get('/:id', (req, res) => res.json({ id: req.params.id }));
++ app.use('/api/vods', vods);
++ app.get('/api/live-events', (_req, res) => { res.setHeader('Content-Type', 'text/event-stream'); res.end(); });
++ app.use('/js', express.static(path.join(__dirname, '..', 'public', 'js')));
++ app.get('*', (_req, res) => res.send('spa'));
++
++ const server = await new Promise((r) => { const s = app.listen(0, '127.0.0.1', () => r(s)); });
++ const base = `http://127.0.0.1:${server.address().port}`;
++ const get = (p, headers = {}) => new Promise((resolve, reject) => {
++ http.get(base + p, { headers }, (res) => { let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b })); }).on('error', reject);
++ });
++
++ // ── readiness ──
++ let r = await get('/api/ready');
++ assert.strictEqual(r.status, 503, 'not ready before boot completes');
++ let body = JSON.parse(r.body);
++ assert.strictEqual(body.ready, false);
++ assert.deepStrictEqual(body.failed, ['boot']);
++
++ booted = true;
++ r = await get('/api/ready');
++ body = JSON.parse(r.body);
++ assert.strictEqual(r.status, 200);
++ assert.strictEqual(body.status, 'ready');
++ assert.strictEqual(body.service, 'live');
++ assert.strictEqual(body.release, 'abc123def456');
++ for (const name of ['boot', 'db', 'sfu', 'media', 'network_key']) {
++ const c = body.checks[name];
++ assert.strictEqual(c.status, 'ok', name);
++ assert.strictEqual(typeof c.latency_ms, 'number');
++ assert.ok(Date.parse(c.checked_at), `${name} checked_at`);
++ }
++ assert.strictEqual(body.checks.db.required, true);
++ assert.strictEqual(body.checks.media.required, false);
++ assert.deepStrictEqual(body.optional, { sfu: true, media: true, network_key: true }, 'the previous optional booleans are still served');
++ assert.strictEqual(typeof body.uptime, 'number');
++
++ // Optional dependencies down: still 200, but degraded and named.
++ sfu = false; key = null;
++ r = await get('/api/ready');
++ body = JSON.parse(r.body);
++ assert.strictEqual(r.status, 200);
++ assert.strictEqual(body.status, 'degraded');
++ assert.deepStrictEqual(body.degraded.sort(), ['network_key', 'sfu']);
++ assert.match(body.checks.sfu.error, /mediasoup/);
++ assert.strictEqual(body.optional.sfu, false);
++
++ // Media is cached for 30 s: its checked_at is when it really ran, and a down Media is not re-asked.
++ const hitsBefore = mediaHits;
++ mediaUp = false;
++ r = await get('/api/ready');
++ assert.strictEqual(mediaHits, hitsBefore, 'Media check is cached');
++ assert.strictEqual(JSON.parse(r.body).checks.media.status, 'ok', 'cached result, with its own checked_at');
++
++ // The database failing is not ready (503), whatever else is up.
++ sfu = true; key = '-----BEGIN PUBLIC KEY-----x';
++ dbBroken = true;
++ r = await get('/api/ready');
++ body = JSON.parse(r.body);
++ assert.strictEqual(r.status, 503);
++ assert.deepStrictEqual(body.failed, ['db']);
++ assert.match(body.checks.db.error, /SQLITE_IOERR/);
++ dbBroken = false;
++
++ // A fresh readiness with Media down reports it degraded.
++ const r2 = observability.createLiveReadiness({ release, bootComplete: () => true, dbQuery, sfuReady: () => true, mediaUrl, networkKey: () => key });
++ const b2 = await r2.run();
++ assert.strictEqual(b2.ready, true);
++ assert.deepStrictEqual(b2.degraded, ['media']);
++ assert.strictEqual(b2.checks.media.error, 'Media answered 502');
++ const r3 = observability.createLiveReadiness({ release, bootComplete: () => true, dbQuery, sfuReady: () => true, mediaUrl: 'http://127.0.0.1:1', networkKey: () => key });
++ const b3 = await r3.run();
++ assert.deepStrictEqual(b3.degraded, ['media'], 'Media unreachable is degraded, not down');
++
++ // ── metrics ──
++ await get('/api/vods/12345?token=secret');
++ await get('/api/vods/67890');
++ await get('/some/channel-name-xyz');
++ await get('/api/live-events');
++ await get('/js/app.js?v=deadbeef');
++ const m = await get('/metrics');
++ assert.strictEqual(m.status, 200);
++ const text = m.body;
++ assert.ok(text.includes('http_requests_total{method="GET",route="/api/vods/:id",status_class="2xx"} 2\n'), text);
++ assert.ok(text.includes('route="*"'), 'the SPA fallback is its template');
++ assert.ok(text.includes('route="/js/*"'), 'static mounts are labelled by mount');
++ const labels = text.split('\n').filter((l) => l.startsWith('http_')).map((l) => (l.match(/\{[^}]*\}/) || [''])[0]).join('\n');
++ assert.ok(!/12345|67890|secret|channel-name|app\.js|deadbeef/.test(labels), 'no ids, queries or raw paths in labels');
++ assert.ok(!labels.includes('/api/live-events'), 'the SSE stream is not a request');
++ assert.ok(text.includes('release_info{service="live",release="abc123def456"} 1\n'));
++ assert.ok(text.includes('live_streams_live 2\n'));
++ assert.ok(text.includes('live_ws_connections{server="chat"} 12\n'));
++ assert.ok(text.includes('live_ws_connections{server="broadcast"} 2\n'));
++ assert.ok(text.includes('live_ws_connections{server="control"} 0\n'));
++ assert.ok(!text.includes('live_ws_connections{server="call"}'), 'an uninitialised server reports nothing');
++ assert.ok(text.includes('live_events_outbox{status="pending"} 4\n'));
++ assert.ok(text.includes('live_events_outbox{status="rejected"} 1\n'));
++ assert.ok(text.includes('\nprocess_resident_memory_bytes '));
++ assert.ok(text.includes('\nnodejs_eventloop_lag_seconds{stat="p99"} '));
++
++ outbox = { enabled: false };
++ assert.ok(!(await get('/metrics')).body.includes('live_events_outbox{'), 'a disabled outbox reports nothing, not zero');
++
++ // Through nginx (X-Forwarded-For on a loopback connection) /metrics does not exist.
++ assert.strictEqual((await get('/metrics', { 'X-Forwarded-For': '203.0.113.7' })).status, 404);
++ assert.strictEqual((await get('/metrics', { 'X-Real-IP': '203.0.113.7' })).status, 404);
++
++ // server/index.js mounts metrics before anything else and serves /api/ready from this module.
++ const src = fs.readFileSync(path.join(__dirname, '..', 'server', 'index.js'), 'utf8');
++ const mountAt = src.indexOf('observability.mountMetrics(app');
++ assert.ok(mountAt > 0 && mountAt < src.indexOf('app.use(helmet('), 'metrics middleware is mounted before the rest of the middleware');
++ assert.ok(/app\.get\('\/api\/ready', readiness\.handler\)/.test(src));
++
++ server.close(); media.close(); sqlite.close();
++ for (const f of [tmpDb, `${tmpDb}-wal`, `${tmpDb}-shm`]) { try { fs.unlinkSync(f); } catch { /* */ } }
++ console.log('observability: all checks passed');
++ process.exit(0);
++})().catch((err) => {
++ console.error(err);
++ for (const f of [tmpDb, `${tmpDb}-wal`, `${tmpDb}-shm`]) { try { fs.unlinkSync(f); } catch { /* */ } }
++ process.exit(1);
++});
+--
+2.43.0
+
new file mode 100644
new file mode 100644
@@ -0,0 +1,105 @@
@@ -0,0 +1,105 @@
+{
+ "version": 1,
+ "status": "proposal",
+ "source": "OpenVibe realignment roadmap §15.19 (Observability and reliability)",
+ "note": "Every target below is a PROPOSAL for discussion. None is agreed, none is enforced, and no alerting exists for any of them. 'measured_by' names what can be measured today; 'not instrumented' means nothing measures it yet.",
+ "binding": "Correctness-sensitive systems (billing, authentication) prioritise integrity over availability; live transport prioritises continuity, with eventual control-plane convergence.",
+ "categories": [
+ {
+ "id": "identity-login-token-exchange",
+ "name": "Identity login and token exchange",
+ "services": ["network"],
+ "priority": "integrity",
+ "sli": "Share of POST /oauth/token and sign-in requests that answer without a 5xx; p95 latency of token issuance.",
+ "proposed_target": { "availability": "99.9% of requests over 30 days", "latency_p95_seconds": 0.5 },
+ "measured_by": ["network: http_requests_total{route=\"/oauth/token\"}", "network: http_request_duration_seconds{route=\"/oauth/token\"}", "network: network_tokens_issued_total{grant_type}", "network: network_token_failures_total{grant_type,error}"],
+ "integrity_rule": "A token is never issued with a capability that was not granted; refusing is always preferred to issuing wrongly."
+ },
+ {
+ "id": "public-page-availability",
+ "name": "Public page availability",
+ "services": ["live", "community", "tools", "media", "network"],
+ "priority": "availability",
+ "sli": "Share of GET requests to server-rendered public pages that answer without a 5xx.",
+ "proposed_target": { "availability": "99.5% of requests over 30 days" },
+ "measured_by": ["each service: http_requests_total{method=\"GET\",status_class}"]
+ },
+ {
+ "id": "control-apis",
+ "name": "Control APIs",
+ "services": ["live", "media", "events", "community", "tools"],
+ "priority": "availability",
+ "sli": "Share of authenticated write/control API requests (stream settings, object writes, subscriptions, jobs) that answer without a 5xx; p95 latency.",
+ "proposed_target": { "availability": "99.5% over 30 days", "latency_p95_seconds": 1.0 },
+ "measured_by": ["each service: http_requests_total and http_request_duration_seconds by route template"]
+ },
+ {
+ "id": "chat-persistence-delivery",
+ "name": "Chat message persistence and delivery",
+ "services": ["live", "chat"],
+ "priority": "integrity",
+ "sli": "Share of accepted chat messages that are persisted and delivered to connected viewers; delivery delay.",
+ "proposed_target": { "persisted": "99.99% of accepted messages", "delivery_p95_seconds": 1.0 },
+ "measured_by": [],
+ "measurement_status": "not instrumented: Live exposes live_ws_connections only; no per-message persistence or delivery counter exists yet"
+ },
+ {
+ "id": "ingest-acceptance",
+ "name": "Ingest acceptance",
+ "services": ["live", "openre"],
+ "priority": "continuity",
+ "sli": "Share of publish attempts (RTMP, WHIP, WebRTC, SRT) with a valid key that are accepted.",
+ "proposed_target": { "accepted": "99.5% of valid publish attempts over 30 days" },
+ "measured_by": [],
+ "measurement_status": "not instrumented: no publish-attempt counter exists yet"
+ },
+ {
+ "id": "active-stream-continuity",
+ "name": "Active stream continuity",
+ "services": ["live", "openre", "media"],
+ "priority": "continuity",
+ "sli": "Share of live streams that end by the broadcaster's action rather than by a server restart, deploy or crash.",
+ "proposed_target": { "unplanned_interruptions": "fewer than 1 per 1000 stream-hours" },
+ "measured_by": ["live: live_streams_live (count only)"],
+ "measurement_status": "partly instrumented: the number of live streams is measured; the cause of each stream end is not recorded"
+ },
+ {
+ "id": "media-upload-processing",
+ "name": "Media upload and processing",
+ "services": ["media"],
+ "priority": "integrity",
+ "sli": "Share of uploads and recordings that reach ready (never silently lost); time from upload or stream end to ready.",
+ "proposed_target": { "reach_ready_or_explicit_failure": "100%", "processing_p95_seconds": 600 },
+ "measured_by": ["media: media_recordings_in_progress", "media: media_objects{lifecycle_status}", "media: media_object_locations{provider,state}"],
+ "measurement_status": "partly instrumented: states are counted; time-to-ready is not measured"
+ },
+ {
+ "id": "ledger-settlement-correctness",
+ "name": "Ledger settlement correctness",
+ "services": ["billing"],
+ "priority": "integrity",
+ "sli": "Share of settlements whose journal balances and reconciles with the provider; duplicate monetary effects.",
+ "proposed_target": { "unbalanced_or_duplicate_settlements": 0 },
+ "measured_by": [],
+ "measurement_status": "not instrumented by this change (Billing is owned by another track)"
+ },
+ {
+ "id": "event-delivery-latency",
+ "name": "Event delivery latency",
+ "services": ["events"],
+ "priority": "integrity",
+ "sli": "Time from an event's acceptance to its successful delivery to each subscriber; dead-letter depth.",
+ "proposed_target": { "delivery_p95_seconds": 30, "dead_letters_unreviewed_after_24h": 0 },
+ "measured_by": ["events: events_delivery_latency_seconds", "events: events_deliveries{status}", "events: events_dlq_depth"]
+ },
+ {
+ "id": "developer-api-availability",
+ "name": "Developer API availability",
+ "services": ["network", "events", "media"],
+ "priority": "availability",
+ "sli": "Share of public developer API requests (registry, JWKS, events pull, object API) that answer without a 5xx.",
+ "proposed_target": { "availability": "99.9% of requests over 30 days" },
+ "measured_by": ["each service: http_requests_total by route template"]
+ }
+ ]
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,48 @@
@@ -0,0 +1,48 @@
+# SLO categories (proposal)
+
+Status: **proposal**. None of these targets is agreed or enforced, and nothing alerts on them.
+They exist so that each service's metrics can be read against a stated intent. The categories come
+from the realignment roadmap §15.19. The machine-readable copy is [`slo.json`](slo.json), served at
+`GET /api/v1/status/slo`. The operator page `GET /status` lists them under the live service table.
+
+Binding rule from the roadmap: correctness-sensitive systems (billing, authentication) prioritise
+**integrity over availability**. Live transport prioritises **continuity**, and the control plane
+converges afterwards.
+
+| Category | Services | Priority | Proposed target | Measured today by |
+|---|---|---|---|---|
+| Identity login and token exchange | network | integrity | 99.9% of token requests without a 5xx over 30 days; p95 under 0.5 s | `http_requests_total{route="/oauth/token"}`, `network_tokens_issued_total{grant_type}`, `network_token_failures_total{grant_type,error}` |
+| Public page availability | live, community, tools, media, network | availability | 99.5% of GETs without a 5xx over 30 days | `http_requests_total` per service |
+| Control APIs | live, media, events, community, tools | availability | 99.5% without a 5xx; p95 under 1 s | `http_requests_total`, `http_request_duration_seconds` by route template |
+| Chat message persistence and delivery | live, chat | integrity | 99.99% of accepted messages persisted; delivery p95 under 1 s | **not instrumented**: only `live_ws_connections` exists |
+| Ingest acceptance | live, openre | continuity | 99.5% of valid publish attempts accepted | **not instrumented** |
+| Active stream continuity | live, openre, media | continuity | fewer than 1 unplanned interruption per 1000 stream-hours | partly: `live_streams_live` counts streams; the cause of a stream ending is not recorded |
+| Media upload and processing | media | integrity | every upload or recording ends ready or explicitly failed; p95 under 10 min | partly: `media_recordings_in_progress`, `media_objects{lifecycle_status}`, `media_object_locations{provider,state}`; time-to-ready is not measured |
+| Ledger settlement correctness | billing | integrity | zero unbalanced or duplicate settlements | **not instrumented** by this change (Billing is another track) |
+| Event delivery latency | events | integrity | p95 under 30 s; no dead letter unreviewed after 24 h | `events_delivery_latency_seconds`, `events_deliveries{status}`, `events_dlq_depth` |
+| Developer API availability | network, events, media | availability | 99.9% without a 5xx over 30 days | `http_requests_total` by route template |
+
+## Where the numbers come from
+
+Every service (Network, Community, Media, Events, the Tools gateway and satellites, and Live once
+`docs/patches/live-observability.diff` is applied) uses `openvibe-shared/metrics` ≥ 1.3.0:
+
+- `GET /metrics` returns Prometheus text to a direct loopback caller only. Through nginx it is a 404,
+ both in the app (any `X-Forwarded-For`/`X-Real-IP`/`Forwarded` header) and in the nginx config.
+- The HTTP golden signals are `http_requests_total{method,route,status_class}`,
+ `http_request_duration_seconds{method,route}` and `http_requests_in_flight`. `route` is the route
+ template, never a raw URL.
+- The process metrics are `process_resident_memory_bytes`, `nodejs_heap_*`,
+ `nodejs_eventloop_lag_seconds{stat}`, `process_cpu_seconds_total` and `process_uptime_seconds`.
+- `release_info{service,release}` names the deployed release.
+
+No Prometheus server scrapes these yet. Setting one up on the host, with a loopback scrape of each
+port, is the next operational step.
+
+## Readiness
+
+`/api/ready` on each service follows `openvibe-shared/ready`. It lists named checks, each with
+`status`, `required`, `latency_ms` and `checked_at`. `ready` is false (HTTP 503) only when a
+required check fails. A failed optional dependency is listed in `degraded` (HTTP 200,
+`status: "degraded"`), so losing one capability never reads as healthy or as fully down.
+`GET /status` shows each row as up, degraded, down, not running or unknown, using those bodies.
@@ -21,7 +21,7 @@
@@ -21,7 +21,7 @@
"jsonwebtoken": "^9.0.0",
"jsonwebtoken": "^9.0.0",
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
"uuid": "^11.1.0",
"uuid": "^11.1.0",
"web-push": "^3.6.7"
"web-push": "^3.6.7"
}
}
@@ -1480,9 +1480,8 @@
@@ -1480,9 +1480,8 @@
}
}
},
},
"node_modules/openvibe-shared": {
"node_modules/openvibe-shared": {
- "version": "1.2.1",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",
+ "version": "1.3.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
@@ -6,7 +6,7 @@
@@ -6,7 +6,7 @@
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js",
+ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/chrome.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/registry-ecosystem.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/observability.test.js && node test/status.test.js",
"reset-db": "node server/reset-db.js",
"reset-db": "node server/reset-db.js",
"grant-admin": "node server/grant-admin.js"
"grant-admin": "node server/grant-admin.js"
},
},
@@ -24,7 +24,7 @@
@@ -24,7 +24,7 @@
"jsonwebtoken": "^9.0.0",
"jsonwebtoken": "^9.0.0",
"multer": "^1.4.5-lts.1",
"multer": "^1.4.5-lts.1",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
"uuid": "^11.1.0",
"uuid": "^11.1.0",
"web-push": "^3.6.7"
"web-push": "^3.6.7"
}
}
@@ -115,6 +115,8 @@ const DEFAULT_SERVICE_MAP = {
@@ -115,6 +115,8 @@ const DEFAULT_SERVICE_MAP = {
gzip: true,
gzip: true,
// WebSocket URLs carry session tokens in the query string; keep them out of access logs.
// WebSocket URLs carry session tokens in the query string; keep them out of access logs.
logWithoutQuery: true,
logWithoutQuery: true,
+ // Wave 6: OpenVibe.Chat serves /ws/chat, /api/chat/, /api/dm/, /api/tts/ and /api/sounds.
+ includes: ['/opt/openvibe.chat/deploy/nginx/*.locations.conf'],
locations: [
locations: [
{
{
match: '= /api/auth/register',
match: '= /api/auth/register',
@@ -574,6 +576,13 @@ function generateServiceConfig(serviceId, svc, opts = {}) {
@@ -574,6 +576,13 @@ function generateServiceConfig(serviceId, svc, opts = {}) {
lines.push('');
lines.push('');
}
}
+ // Location files other services ship for this host (e.g. OpenVibe.Chat's chat paths on
+ // openvibe.live). A glob include is a no-op when the service is not installed.
+ for (const inc of svc.includes || []) {
+ lines.push(` include ${inc};`);
+ lines.push('');
+ }
+
// Location blocks
// Location blocks
if (svc.locations && svc.locations.length > 0) {
if (svc.locations && svc.locations.length > 0) {
for (const loc of svc.locations) {
for (const loc of svc.locations) {
@@ -167,7 +167,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {
@@ -167,7 +167,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {
legacy: legacy ? (r) => r.internalKeyOk === true : undefined,
legacy: legacy ? (r) => r.internalKeyOk === true : undefined,
namespace,
namespace,
// Denials are final here; an allow is recorded below, after the ownership check.
// Denials are final here; an allow is recorded below, after the ownership check.
- onDecision: (d) => { if (!d.allowed) record(d); },
+ onDecision: (d) => { if (!d.allowed) { record(d); require('../observability').principalDenied(d); } },
});
});
}
}
check(req, res, () => {
check(req, res, () => {
@@ -177,6 +177,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {
@@ -177,6 +177,7 @@ function guard(capability, { ownApp, namespace, legacy = true } = {}) {
const self = String(principal.sub).replace(/^svc:/, '');
const self = String(principal.sub).replace(/^svc:/, '');
if (app !== undefined && app !== self) {
if (app !== undefined && app !== self) {
record({ req, capability, principal, allowed: false, code: 'capability.owner_denied' });
record({ req, capability, principal, allowed: false, code: 'capability.owner_denied' });
+ require('../observability').principalDenied({ req, code: 'capability.owner_denied' });
return http.sendProblem(res, 403, 'capability.owner_denied', { detail: `${principal.sub} may only act for app_id '${self}'` });
return http.sendProblem(res, 403, 'capability.owner_denied', { detail: `${principal.sub} may only act for app_id '${self}'` });
}
}
}
}
@@ -34,6 +34,18 @@ const { signToken } = require('./auth/routes');
@@ -34,6 +34,18 @@ const { signToken } = require('./auth/routes');
const app = express();
const app = express();
+// What this server is running (ADR-016, registry.release-manifest@1); open tabs poll it through
+// /shared/release-watch.js and are prompted, or reloaded when safe, after a deploy.
+const release = require('openvibe-shared/release').createRelease({ service: 'network', root: path.join(__dirname, '..') });
+
+// Metrics first, so every request is measured: HTTP golden signals by route template, process
+// metrics, release_info, and GET /metrics for direct loopback callers only (server/observability.js).
+const observability = require('./observability');
+require('openvibe-shared/metrics').instrument(app, {
+ service: 'network', release: release.release, registry: observability.registry,
+ normalize: (req) => (req.route ? null : /^\/shared\//.test(req.originalUrl) ? '/shared/*' : /^\/data\/avatars\//.test(req.originalUrl) ? '/data/avatars/*' : null),
+});
+
function getRequestHost(req) {
function getRequestHost(req) {
return String(req.headers.host || '').split(':')[0].toLowerCase();
return String(req.headers.host || '').split(':')[0].toLowerCase();
}
}
@@ -357,6 +369,8 @@ const requireAuth = require('./auth/session').makeRequireAuth(() => ({ db, publi
@@ -357,6 +369,8 @@ const requireAuth = require('./auth/session').makeRequireAuth(() => ({ db, publi
const ecosystem = require('./registry/ecosystem').createEcosystemRegistry({ issuer: config.jwt.issuer });
const ecosystem = require('./registry/ecosystem').createEcosystemRegistry({ issuer: config.jwt.issuer });
app.use(ecosystem.router());
app.use(ecosystem.router());
ecosystem.start();
ecosystem.start();
+// Operator status: GET /status (server-rendered, noindex), /api/v1/status, /api/v1/status/slo.
+app.use(require('./status/routes').createStatusRoutes({ ecosystem }));
app.get('/api/.well-known/jwks', (_req, res) => {
app.get('/api/.well-known/jwks', (_req, res) => {
const out = { public_key: publicKey, algorithm: privateKey === publicKey ? 'HS256' : 'RS256' };
const out = { public_key: publicKey, algorithm: privateKey === publicKey ? 'HS256' : 'RS256' };
@@ -374,6 +388,15 @@ app.get('/api/health', (_req, res) => {
@@ -374,6 +388,15 @@ app.get('/api/health', (_req, res) => {
res.json({ status: 'ok', service: 'openvibe-network', version: '1.0.0' });
res.json({ status: 'ok', service: 'openvibe-network', version: '1.0.0' });
});
});
+// Readiness: named checks with status, latency and checked_at; 503 only when a required one fails.
+{
+ const ready = observability.createNetworkReadiness({
+ db, release: release.release, ecosystem, discordService,
+ getKeys: () => ({ privateKey: app.locals.privateKey, publicKey: app.locals.publicKey }),
+ });
+ app.get('/api/ready', ready.handler);
+}
+
// Brand info (used by all frontends for consistent URLs/names)
// Brand info (used by all frontends for consistent URLs/names)
app.get('/api/brand', (_req, res) => res.json(BRAND));
app.get('/api/brand', (_req, res) => res.json(BRAND));
@@ -386,7 +409,8 @@ app.use('/api/auth', require('./auth/email-verify').routes(requireAuth));
@@ -386,7 +409,8 @@ app.use('/api/auth', require('./auth/email-verify').routes(requireAuth));
// Discord account linking (OAuth2 flow)
// Discord account linking (OAuth2 flow)
app.use('/api/auth/discord', requireAuth, require('./auth/discord-link'));
app.use('/api/auth/discord', requireAuth, require('./auth/discord-link'));
-// OAuth2 authorization endpoints
+// OAuth2 authorization endpoints (token issuance counted by grant type: server/observability.js)
+app.use('/oauth/token', observability.tokenEndpointMetrics);
app.use('/oauth', require('./auth/oauth-routes'));
app.use('/oauth', require('./auth/oauth-routes'));
// Theme API
// Theme API
@@ -630,9 +654,6 @@ function serveShared(req, res, next) {
@@ -630,9 +654,6 @@ function serveShared(req, res, next) {
app.use('/shared/v1', serveShared);
app.use('/shared/v1', serveShared);
app.use('/shared', serveShared);
app.use('/shared', serveShared);
-// What this server is running (ADR-016, registry.release-manifest@1); open tabs poll it through
-// /shared/release-watch.js and are prompted, or reloaded when safe, after a deploy.
-const release = require('openvibe-shared/release').createRelease({ service: 'network', root: path.join(__dirname, '..') });
app.get('/release.json', release.handler);
app.get('/release.json', release.handler);
// Avatar serving
// Avatar serving
new file mode 100644
new file mode 100644
@@ -0,0 +1,98 @@
@@ -0,0 +1,98 @@
+'use strict';
+/**
+ * Network's metrics and readiness (roadmap Track O, §15.19), on openvibe-shared/metrics and /ready.
+ *
+ * GET /metrics Prometheus text, direct loopback callers only (404 through nginx)
+ * GET /api/ready named checks; 503 only when a required one fails
+ *
+ * Domain metrics: token issuance by grant type at /oauth/token (and failures by OAuth error code),
+ * and service-token (principal) failures at Network's capability-guarded internal routes.
+ * Labels are fixed vocabularies; a client id, subject or token never becomes a label.
+ */
+const jwt = require('jsonwebtoken');
+const metrics = require('openvibe-shared/metrics');
+const { createReadiness } = require('openvibe-shared/ready');
+
+const registry = metrics.createRegistry();
+
+const GRANTS = new Set(['authorization_code', 'refresh_token', 'client_credentials', 'urn:ietf:params:oauth:grant-type:jwt-bearer']);
+const OAUTH_ERRORS = new Set(['invalid_request', 'invalid_client', 'invalid_grant', 'unauthorized_client', 'unsupported_grant_type', 'invalid_scope', 'access_denied', 'server_error', 'temporarily_unavailable']);
+const grantLabel = (g) => (g === 'urn:ietf:params:oauth:grant-type:jwt-bearer' ? 'jwt_bearer' : GRANTS.has(g) ? g : 'other');
+
+const tokensIssued = registry.counter({ name: 'network_tokens_issued_total', help: 'Tokens issued by POST /oauth/token, by grant type', labelNames: ['grant_type'] });
+const tokenFailures = registry.counter({ name: 'network_token_failures_total', help: 'POST /oauth/token requests refused, by grant type and OAuth error code', labelNames: ['grant_type', 'error'] });
+const principalFailures = registry.counter({ name: 'network_principal_token_failures_total', help: 'Service (principal) tokens refused at capability-guarded Network routes, by problem code', labelNames: ['code'], maxSeries: 50 });
+
+/** Middleware for POST /oauth/token: counts the outcome without reading or logging any secret. */
+function tokenEndpointMetrics(req, res, next) {
+ if (req.method !== 'POST') return next();
+ const grant = grantLabel(req.body && req.body.grant_type);
+ let error = null;
+ const json = res.json.bind(res);
+ res.json = (body) => {
+ if (body && typeof body === 'object' && typeof body.error === 'string') error = OAUTH_ERRORS.has(body.error) ? body.error : 'other';
+ return json(body);
+ };
+ res.once('finish', () => {
+ if (res.statusCode === 200 && !error) tokensIssued.inc({ grant_type: grant });
+ else tokenFailures.inc({ grant_type: grant, error: error || `http_${res.statusCode}` });
+ });
+ next();
+}
+
+/** Called by identity/principals.js guard() for every denial; counts only bearer (service-token) callers. */
+function principalDenied({ req, code }) {
+ if (!String((req && req.headers && req.headers.authorization) || '').startsWith('Bearer ')) return;
+ const c = /^[a-z0-9_.-]{1,64}$/i.test(String(code || '')) ? String(code) : 'other';
+ principalFailures.inc({ code: c });
+}
+
+/**
+ * Readiness with Network's real dependencies:
+ * db (required) a query against the identity database
+ * signing_key (required in production) an RS256 keypair is loaded and signs a token its own public
+ * key verifies — the thing every service's offline verification depends on
+ * registry_poll (optional) the ecosystem health poll has run recently (status page freshness)
+ * discord_bot (optional, only when a bot token is configured) the bot is connected
+ */
+function createNetworkReadiness({ db, getKeys, release, ecosystem = null, discordService = null, production = process.env.NODE_ENV === 'production', pollMs = 60000 }) {
+ const checks = [
+ { name: 'db', required: true, check: () => { const r = db.prepare('SELECT COUNT(*) AS n FROM oauth_clients').get(); return { detail: { oauth_clients: r.n } }; } },
+ {
+ name: 'signing_key', required: production, cacheMs: 60000,
+ check: () => {
+ const { privateKey, publicKey } = getKeys();
+ if (!privateKey || !publicKey) return 'no signing key loaded';
+ if (privateKey === publicKey || !String(publicKey).includes('BEGIN')) return 'ephemeral HS256 development key: other services cannot verify tokens';
+ const probe = jwt.sign({ probe: true }, privateKey, { algorithm: 'RS256', expiresIn: 30 });
+ jwt.verify(probe, publicKey, { algorithms: ['RS256'] });
+ return { detail: { algorithm: 'RS256', kid: 'ov-network-1' } };
+ },
+ },
+ ];
+ if (ecosystem) {
+ checks.push({
+ name: 'registry_poll', required: false,
+ check: () => {
+ const at = ecosystem.lastPollAt();
+ if (!at) return 'the ecosystem health poll has not completed yet';
+ const age = Date.now() - at;
+ return age > 3 * pollMs ? `last completed ${Math.round(age / 1000)}s ago` : { detail: { last_poll_at: new Date(at).toISOString() } };
+ },
+ });
+ }
+ if (discordService && typeof discordService.isReady === 'function') {
+ checks.push({
+ name: 'discord_bot', required: false,
+ check: () => {
+ let configured = false;
+ try { configured = !!discordService._getSetting('discord_bot_token'); } catch { configured = false; }
+ if (!configured) return { detail: 'not configured' };
+ return discordService.isReady() || 'bot token configured but the bot is not connected';
+ },
+ });
+ }
+ return createReadiness({ service: 'network', release, checks });
+}
+
+module.exports = { registry, tokenEndpointMetrics, principalDenied, createNetworkReadiness, grantLabel };
@@ -29,28 +29,91 @@ const INTERNAL = {
@@ -29,28 +29,91 @@ const INTERNAL = {
events: 'http://127.0.0.1:4300', billing: 'http://127.0.0.1:4600',
events: 'http://127.0.0.1:4300', billing: 'http://127.0.0.1:4600',
};
};
const POLL_MS = 60 * 1000;
const POLL_MS = 60 * 1000;
+// Readiness paths of services whose manifest (openvibe-contracts) does not carry `ready` yet. Each
+// answers with openvibe-shared/ready's shape; until a service deploys it, its health path is used
+// and the row says so (basis: 'health').
+const READY_PATHS = { network: '/api/ready', media: '/api/ready', tools: '/api/ready' };
-function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS } = {}) {
+/** Row status from a readiness body (openvibe-shared/ready shape, or an older ad-hoc one). */
+function statusFromReady(res, body) {
+ const shared = body && typeof body === 'object' && typeof body.ready === 'boolean' && body.checks && typeof body.checks === 'object';
+ if (!shared) return { status: res.ok ? 'up' : 'down', basis: 'ready-legacy' };
+ if (!body.ready) return { status: 'down', basis: 'ready' };
+ const degraded = Array.isArray(body.degraded) ? body.degraded : [];
+ return { status: body.status === 'degraded' || degraded.length ? 'degraded' : 'up', basis: 'ready' };
+}
+
+/** What a status row may repeat from a service's readiness body: names, states and times, no free-form detail. */
+function readySummary(body) {
+ if (!body || typeof body !== 'object' || typeof body.ready !== 'boolean') return null;
+ const checks = {};
+ for (const [name, c] of Object.entries(body.checks || {}).slice(0, 40)) {
+ if (!c || typeof c !== 'object') continue;
+ checks[String(name).slice(0, 64)] = {
+ status: c.status === 'ok' ? 'ok' : 'fail', required: c.required !== false,
+ latency_ms: Number.isFinite(c.latency_ms) ? c.latency_ms : null,
+ checked_at: typeof c.checked_at === 'string' ? c.checked_at.slice(0, 40) : null,
+ ...(c.error ? { error: String(c.error).slice(0, 200) } : {}),
+ };
+ }
+ const names = (a) => (Array.isArray(a) ? a.map(x => String(x).slice(0, 64)).slice(0, 40) : []);
+ return { ready: body.ready, status: String(body.status || (body.ready ? 'ready' : 'not_ready')).slice(0, 20), checked_at: typeof body.checked_at === 'string' ? body.checked_at.slice(0, 40) : null, failed: names(body.failed), degraded: names(body.degraded), checks };
+}
+
+function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, now = () => Date.now() } = {}) {
const internal = { ...INTERNAL, ...internalOverrides };
const internal = { ...INTERNAL, ...internalOverrides };
- const health = new Map(); // id -> { status: up|down|unknown, http_status, latency_ms, checked_at }
+ // id -> { status: up|degraded|down|not-running|unknown, basis, reason, http_status, latency_ms, checked_at, ready, release }
+ const health = new Map();
let timer = null;
let timer = null;
+ let lastPollAt = null;
+
+ async function getJson(url) {
+ const t0 = now();
+ const res = await fetchImpl(url, { signal: AbortSignal.timeout(3000), headers: { accept: 'application/json' } });
+ let body = null;
+ try { body = await res.json(); } catch { body = null; }
+ return { res, body, latency: now() - t0 };
+ }
+
+ async function releaseOf(base) {
+ try {
+ const { res, body } = await getJson(`${base}/release.json`);
+ if (!res.ok || !body || typeof body.release !== 'string') return { release: null, error: `release.json answered ${res.status}` };
+ return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null };
+ } catch (err) {
+ return { release: null, error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };
+ }
+ }
async function checkOne(m) {
async function checkOne(m) {
+ const at = () => new Date(now()).toISOString();
const base = internal[m.id];
const base = internal[m.id];
- const path = m.ready || m.health;
- if (m.status === 'placeholder' || !base || !path) {
- health.set(m.id, { status: m.status === 'placeholder' ? 'not-running' : 'unknown', checked_at: new Date().toISOString() });
- return;
- }
- const t0 = Date.now();
+ if (m.status === 'placeholder') { health.set(m.id, { status: 'not-running', reason: 'placeholder', checked_at: at() }); return; }
+ const readyPath = m.ready || readyPaths[m.id] || null;
+ if (!m.health && !readyPath && !(m.domains || []).length) { health.set(m.id, { status: 'not-running', reason: 'no runtime', checked_at: at() }); return; }
+ if (!base) { health.set(m.id, { status: 'unknown', reason: 'no internal address known to Network', checked_at: at() }); return; }
+ if (!readyPath && !m.health) { health.set(m.id, { status: 'unknown', reason: 'no health or readiness path in its manifest', checked_at: at() }); return; }
+ const releaseP = releaseOf(base);
+ let entry;
try {
try {
- const res = await fetchImpl(`${base}${path}`, { signal: AbortSignal.timeout(3000) });
- health.set(m.id, { status: res.ok ? 'up' : 'down', http_status: res.status, latency_ms: Date.now() - t0, checked_at: new Date().toISOString() });
+ let got = readyPath ? await getJson(`${base}${readyPath}`) : null;
+ if (got && got.res.status !== 404) {
+ entry = { ...statusFromReady(got.res, got.body), ready: readySummary(got.body) };
+ } else {
+ // No readiness endpoint (yet): liveness is all that can be said.
+ got = await getJson(`${base}${m.health}`);
+ entry = { status: got.res.ok ? 'up' : 'down', basis: 'health', reason: 'liveness only: no readiness endpoint answered', ready: null };
+ }
+ entry.http_status = got.res.status;
+ entry.latency_ms = got.latency;
} catch (err) {
} catch (err) {
- health.set(m.id, { status: 'down', error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable', checked_at: new Date().toISOString() });
+ entry = { status: 'down', basis: 'unreachable', error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable', ready: null };
}
}
+ entry.release = await releaseP;
+ entry.checked_at = at();
+ health.set(m.id, entry);
}
}
- const pollAll = () => Promise.all(contracts.services.manifests.map(checkOne));
+ const pollAll = async () => { await Promise.all(contracts.services.manifests.map(checkOne)); lastPollAt = now(); };
function start() {
function start() {
if (timer) return;
if (timer) return;
const loop = async () => { try { await pollAll(); } catch { /* one bad poll never stops the loop */ } timer = setTimeout(loop, pollMs); if (timer.unref) timer.unref(); };
const loop = async () => { try { await pollAll(); } catch { /* one bad poll never stops the loop */ } timer = setTimeout(loop, pollMs); if (timer.unref) timer.unref(); };
@@ -60,7 +123,16 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -60,7 +123,16 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
}
}
function stop() { if (timer) clearTimeout(timer); timer = null; }
function stop() { if (timer) clearTimeout(timer); timer = null; }
- const withHealth = (m) => ({ ...m, runtime: health.get(m.id) || { status: 'unknown', checked_at: null } });
+ /** The row as it stands now: a result older than three poll intervals is 'unknown', never a stale 'up'. */
+ function current(id) {
+ const h = health.get(id);
+ if (!h) return { status: 'unknown', reason: 'not checked yet', checked_at: null };
+ const age = now() - Date.parse(h.checked_at);
+ if (age > 3 * pollMs) return { ...h, status: 'unknown', reason: `last check is ${Math.round(age / 1000)}s old`, last_status: h.status, stale: true };
+ return h;
+ }
+
+ const withHealth = (m) => ({ ...m, runtime: current(m.id) });
function descriptor() {
function descriptor() {
return {
return {
@@ -120,7 +192,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -120,7 +192,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
return r;
return r;
}
}
- return { router, start, stop, pollAll, health, descriptor };
+ return { router, start, stop, pollAll, health, current, descriptor, lastPollAt: () => lastPollAt, pollMs, internal };
}
}
-module.exports = { createEcosystemRegistry, INTERNAL };
+module.exports = { createEcosystemRegistry, INTERNAL, READY_PATHS, statusFromReady, readySummary };
new file mode 100644
new file mode 100644
@@ -0,0 +1,157 @@
@@ -0,0 +1,157 @@
+'use strict';
+/**
+ * Operator status (roadmap Track O): what each OpenVibe service is doing right now, stated as
+ * observed rather than hoped.
+ *
+ * GET /status server-rendered page (works without JavaScript; noindex)
+ * GET /api/v1/status the same rows as JSON
+ * GET /api/v1/status/slo SLO categories (docs/slo.json) — proposals, not commitments
+ *
+ * Rows come from the ecosystem registry's health poll (server/registry/ecosystem.js): each running
+ * service's readiness (openvibe-shared/ready shape) and /release.json. Every row is one of
+ * up | degraded | down | not-running | unknown and carries checked_at; a service not checked yet, or
+ * whose last check is stale, is 'unknown' — never an optimistic default.
+ */
+const fs = require('fs');
+const path = require('path');
+const express = require('express');
+const contracts = require('openvibe-contracts');
+const seo = require('openvibe-shared/seo');
+
+const SLO_FILE = path.join(__dirname, '..', '..', 'docs', 'slo.json');
+const esc = seo.esc;
+const STATES = ['up', 'degraded', 'down', 'not-running', 'unknown'];
+const LABEL = { up: 'Up', degraded: 'Degraded', down: 'Down', 'not-running': 'Not running', unknown: 'Unknown' };
+
+function loadSlo() { return JSON.parse(fs.readFileSync(SLO_FILE, 'utf8')); }
+
+function rows(ecosystem) {
+ return contracts.services.manifests.map((m) => {
+ const r = ecosystem.current(m.id);
+ const status = STATES.includes(r.status) ? r.status : 'unknown';
+ return {
+ id: m.id,
+ name: m.name,
+ manifest_status: m.status,
+ origin: m.publicOrigin || null,
+ status,
+ label: status === 'not-running' && r.reason ? `not running (${r.reason})` : status,
+ basis: r.basis || null,
+ reason: r.reason || r.error || null,
+ checked_at: r.checked_at || null,
+ http_status: r.http_status ?? null,
+ latency_ms: r.latency_ms ?? null,
+ release: r.release && r.release.release ? { release: r.release.release, released_at: r.release.released_at || null, booted_at: r.release.booted_at || null } : null,
+ release_error: r.release && !r.release.release ? r.release.error || 'unknown' : null,
+ ready: r.ready || null,
+ ...(r.stale ? { stale: true, last_status: r.last_status } : {}),
+ };
+ });
+}
+
+function summary(list) {
+ const counts = Object.fromEntries(STATES.map((s) => [s, 0]));
+ for (const r of list) counts[r.status]++;
+ return counts;
+}
+
+const CSS = `
+.st{max-width:1080px;margin:32px auto 64px;padding:0 20px}.st h1{font-size:clamp(24px,3vw,32px);letter-spacing:-.02em;margin:0 0 6px}
+.st p.lede{color:var(--text-secondary,#96a7c2);margin:0 0 18px;max-width:780px;line-height:1.5}
+.st-sum{display:flex;flex-wrap:wrap;gap:8px;margin:0 0 18px;padding:0;list-style:none}.st-sum li{padding:6px 12px;border-radius:999px;border:1px solid var(--border,#1f2d47);font-size:13.5px}
+.st table{width:100%;border-collapse:collapse;font-size:14px}.st th,.st td{text-align:left;padding:10px 8px;border-bottom:1px solid var(--border,#1f2d47);vertical-align:top}
+.st th{font-size:12px;text-transform:uppercase;letter-spacing:.5px;color:var(--text-muted,#7386a3)}
+.st td small{display:block;color:var(--text-muted,#7386a3);font-size:12.5px;margin-top:2px}
+.st code{font-size:12.5px}.st-b{display:inline-block;padding:2px 9px;border-radius:999px;font-weight:700;font-size:12.5px;border:1px solid currentColor}
+.st-up{color:var(--success,#22c55e)}.st-degraded{color:#f59e0b}.st-down{color:var(--live-red,#ef4444)}.st-not-running,.st-unknown{color:var(--text-muted,#7386a3)}
+.st ul.chk{margin:4px 0 0;padding:0;list-style:none;font-size:12.5px}.st ul.chk li{margin:1px 0}
+.st section{margin-top:40px}.st dl{display:grid;grid-template-columns:minmax(160px,240px) 1fr;gap:6px 16px;font-size:14px}.st dt{font-weight:600}.st dd{margin:0;color:var(--text-secondary,#96a7c2)}
+@media (max-width:720px){.st table,.st tbody,.st tr,.st td{display:block}.st thead{display:none}.st tr{border-bottom:1px solid var(--border,#1f2d47);padding:8px 0}.st td{border:0;padding:4px 0}.st dl{grid-template-columns:1fr}}`;
+
+function checksList(ready) {
+ if (!ready || !ready.checks) return '';
+ const items = Object.entries(ready.checks).map(([n, c]) => `<li>${c.status === 'ok' ? 'ok' : 'FAIL'} · <code>${esc(n)}</code>${c.required ? '' : ' (optional)'}${c.error ? ` — ${esc(c.error)}` : ''}</li>`);
+ return items.length ? `<ul class="chk">${items.join('')}</ul>` : '';
+}
+
+function renderPage(list, slo, generatedAt) {
+ const counts = summary(list);
+ const tr = list.map((r) => `<tr id="svc-${esc(r.id)}">
+<td><b>${esc(r.name)}</b><small>${esc(r.id)} · manifest: ${esc(r.manifest_status)}</small></td>
+<td><span class="st-b st-${esc(r.status)}">${esc(r.status === 'not-running' ? r.label : LABEL[r.status])}</span>${r.stale ? `<small>last seen ${esc(r.last_status)}</small>` : ''}${r.basis === 'health' ? '<small>liveness only (no readiness endpoint)</small>' : ''}${r.reason && r.status !== 'not-running' ? `<small>${esc(r.reason)}</small>` : ''}${checksList(r.ready)}</td>
+<td>${r.release ? `<code>${esc(r.release.release)}</code>${r.release.booted_at ? `<small>booted ${esc(r.release.booted_at)}</small>` : ''}` : `<small>${r.status === 'not-running' ? '—' : esc(r.release_error ? `unknown (${r.release_error})` : 'unknown')}</small>`}</td>
+<td>${r.checked_at ? `<time datetime="${esc(r.checked_at)}">${esc(r.checked_at)}</time>` : '<small>not checked yet</small>'}${r.latency_ms != null ? `<small>${esc(r.latency_ms)} ms</small>` : ''}</td>
+</tr>`).join('\n');
+ const sloRows = slo.categories.map((c) => `<dt>${esc(c.name)}</dt><dd>${esc(c.sli)} <br><small>Proposed: ${esc(Object.entries(c.proposed_target).map(([k, v]) => `${k.replace(/_/g, ' ')} ${v}`).join('; '))} · priority: ${esc(c.priority)} · ${esc(c.measurement_status || (c.measured_by.length ? 'measured' : 'not instrumented'))}</small></dd>`).join('\n');
+ return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Status · OpenVibe.Network</title>
+<meta name="description" content="Observed status of each OpenVibe service: readiness, release and when it was last checked.">
+<meta name="robots" content="noindex, nofollow">
+${require('openvibe-shared/app-icon').headTags({ site: 'network', iconBase: '/assets' })}
+<meta name="color-scheme" content="dark light">
+<script>(function(){try{var raw=localStorage.getItem('ov_theme');if(!raw)return;var t=JSON.parse(raw),v=t&&t.variables;if(!v)return;var el=document.documentElement;for(var k in v)if(k.charAt(0)==='-')el.style.setProperty(k,v[k]);if(t.id)el.setAttribute('data-theme',t.id);}catch(_){}})();</script>
+<script src="/shared/theme-loader.js" defer></script>
+<style>
+*,*::before,*::after{box-sizing:border-box}
+:root{--bg-primary:#0a0f1c;--bg-secondary:#101828;--border:#1f2d47;--accent:#3b82f6;--text-primary:#e6edf7;--text-secondary:#96a7c2;--text-muted:#7386a3;--live-red:#ef4444;--success:#22c55e}
+html,body{margin:0;background:var(--bg-primary);color:var(--text-primary);font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif}
+a{color:var(--accent)}
+${CSS}
+</style>
+</head>
+<body>
+<div id="navbar-mount"></div>
+${require('openvibe-shared/chrome-ssr').noscriptNav({ name: 'OpenVibe.Network', links: [{ label: 'Status', href: '/status' }] })}
+<main class="st" id="main">
+<h1>Service status</h1>
+<p class="lede">What Network observed when it last checked each service: its readiness endpoint (named checks, required or optional), its deployed release, and when. A service that has not been checked, or whose last check is out of date, shows as <b>unknown</b>. Placeholders show as not running. Page generated <time datetime="${esc(generatedAt)}">${esc(generatedAt)}</time>; checks run about every ${esc(Math.round(slo.pollSeconds || 60))} seconds. JSON: <a href="/api/v1/status"><code>/api/v1/status</code></a>.</p>
+<ul class="st-sum">${STATES.map((s) => `<li class="st-${s}">${esc(LABEL[s])}: ${counts[s]}</li>`).join('')}</ul>
+<table>
+<thead><tr><th>Service</th><th>Status</th><th>Release</th><th>Checked</th></tr></thead>
+<tbody>
+${tr}
+</tbody>
+</table>
+<section aria-labelledby="h-slo"><h2 id="h-slo">SLO categories (proposals)</h2>
+<p class="lede">${esc(slo.note)} ${esc(slo.binding)} JSON: <a href="/api/v1/status/slo"><code>/api/v1/status/slo</code></a>.</p>
+<dl>
+${sloRows}
+</dl></section>
+</main>
+${require('openvibe-shared/footer').ssr({ service: 'network', variant: 'compact' })}
+<script src="/shared/navbar.js" defer></script>
+<script>document.addEventListener('DOMContentLoaded',function(){try{if(window.OpenVibeNavbar)OpenVibeNavbar.init({service:'network',apiBase:location.origin});}catch(e){}});</script>
+</body>
+</html>`;
+}
+
+function createStatusRoutes({ ecosystem, now = () => new Date() }) {
+ const r = express.Router();
+ r.get('/api/v1/status', (_req, res) => {
+ const list = rows(ecosystem);
+ const last = ecosystem.lastPollAt();
+ res.set('Cache-Control', 'no-cache, max-age=0').set('Access-Control-Allow-Origin', '*').json({
+ generated_at: now().toISOString(),
+ last_poll_at: last ? new Date(last).toISOString() : null,
+ poll_seconds: Math.round(ecosystem.pollMs / 1000),
+ states: STATES,
+ summary: summary(list),
+ services: list,
+ });
+ });
+ r.get('/api/v1/status/slo', (_req, res) => {
+ res.set('Cache-Control', 'public, max-age=300').set('Access-Control-Allow-Origin', '*').json(loadSlo());
+ });
+ r.get('/status', (_req, res) => {
+ const slo = { ...loadSlo(), pollSeconds: ecosystem.pollMs / 1000 };
+ res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', 'no-cache, max-age=0').set('X-Robots-Tag', 'noindex, nofollow');
+ res.send(renderPage(rows(ecosystem), slo, now().toISOString()));
+ });
+ return r;
+}
+
+module.exports = { createStatusRoutes, rows, STATES };
@@ -23,6 +23,7 @@ const must = [
@@ -23,6 +23,7 @@ const must = [
['Cloudflare client IP forwarded', /proxy_set_header CF-Connecting-IP \$http_cf_connecting_ip;/],
['Cloudflare client IP forwarded', /proxy_set_header CF-Connecting-IP \$http_cf_connecting_ip;/],
['access log without query strings (tokens)', /log_format live_noquery [^;]*\$uri[^;]*;[\s\S]*access_log [^;]* live_noquery;/],
['access log without query strings (tokens)', /log_format live_noquery [^;]*\$uri[^;]*;[\s\S]*access_log [^;]* live_noquery;/],
['capture permissions header', /Permissions-Policy "camera=\*, microphone=\*, display-capture=\*"/],
['capture permissions header', /Permissions-Policy "camera=\*, microphone=\*, display-capture=\*"/],
+ ['OpenVibe.Chat locations (glob, before /api/)', /include \/opt\/openvibe\.chat\/deploy\/nginx\/\*\.locations\.conf;[\s\S]*location \/api\/ \{/],
];
];
for (const [name, re] of must) assert.ok(re.test(conf), `generated live config is missing: ${name}`);
for (const [name, re] of must) assert.ok(re.test(conf), `generated live config is missing: ${name}`);
// The asset regex must not swallow /api/ or /ws/ (a regex location beats prefix locations).
// The asset regex must not swallow /api/ or /ws/ (a regex location beats prefix locations).
new file mode 100644
new file mode 100644
@@ -0,0 +1,128 @@
@@ -0,0 +1,128 @@
+'use strict';
+// Metrics and readiness (server/observability.js): token issuance by grant type, principal-token
+// failures, route-template labels, loopback-only /metrics, and a /api/ready that fails only on a
+// required dependency. Roadmap Track O.
+// node test/observability.test.js
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+const express = require('express');
+const metrics = require('openvibe-shared/metrics');
+const { initDb } = require('../server/db/database');
+const observability = require('../server/observability');
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-observability-'));
+const log = console.log; console.log = () => {};
+const db = initDb(path.join(dir, 'network.db'));
+console.log = log;
+db.prepare("UPDATE oauth_clients SET client_secret = 'live-secret' WHERE client_id = 'live'").run();
+
+const keys = crypto.generateKeyPairSync('rsa', { modulusLength: 2048, publicKeyEncoding: { type: 'spki', format: 'pem' }, privateKeyEncoding: { type: 'pkcs8', format: 'pem' } });
+const ISSUER = 'https://openvibe.network';
+const app = express();
+metrics.instrument(app, { service: 'network', release: 'abc123def456', registry: observability.registry });
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+app.locals.db = db;
+app.locals.config = { internalKey: 'legacy-key', jwt: { issuer: ISSUER, accessTokenExpiry: '1h' } };
+app.locals.privateKey = keys.privateKey;
+app.locals.publicKey = keys.publicKey;
+app.use('/oauth/token', observability.tokenEndpointMetrics);
+app.use('/oauth', require('../server/auth/oauth-routes'));
+app.use('/internal', require('../server/internal/routes'));
+
+let keysNow = { privateKey: keys.privateKey, publicKey: keys.publicKey };
+let lastPoll = null;
+let discordReady = false;
+const ready = observability.createNetworkReadiness({
+ db, release: 'abc123def456', production: true,
+ getKeys: () => keysNow,
+ ecosystem: { lastPollAt: () => lastPoll },
+ discordService: { isReady: () => discordReady, _getSetting: () => 'configured-token' },
+});
+app.get('/api/ready', ready.handler);
+const server = http.createServer(app);
+
+(async () => {
+ await new Promise(r => server.listen(0, '127.0.0.1', r));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const form = (body) => fetch(`${base}/oauth/token`, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams(body) }).then(async r => ({ status: r.status, body: await r.json() }));
+ const get = (p, headers = {}) => fetch(base + p, { headers }).then(async r => ({ status: r.status, text: await r.text() }));
+
+ // ── Token issuance and failures by grant type ──
+ let t = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'live-secret', audience: 'openvibe.network' });
+ assert.strictEqual(t.status, 200);
+ const svcToken = t.body.access_token;
+ t = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'wrong', audience: 'openvibe.network' });
+ assert.strictEqual(t.status, 401);
+ t = await form({ grant_type: 'refresh_token', client_id: 'live', client_secret: 'live-secret', refresh_token: 'nope' });
+ assert.ok(t.status >= 400);
+ t = await form({ grant_type: 'password', client_id: 'live', client_secret: 'live-secret' });
+ assert.strictEqual(t.status, 400);
+
+ // ── Principal (service-token) failures at guarded routes ──
+ const narrow = await form({ grant_type: 'client_credentials', client_id: 'live', client_secret: 'live-secret', audience: 'openvibe.network', scope: 'network.coins.credit' });
+ let r = await fetch(`${base}/internal/coins/debit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${narrow.body.access_token}` }, body: JSON.stringify({ user_id: 1, app_id: 'live', amount: 1, reason: 't', idempotency_key: 'k1' }) });
+ assert.strictEqual(r.status, 403);
+ r = await fetch(`${base}/internal/coins/credit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${svcToken}` }, body: JSON.stringify({ user_id: 1, app_id: 'games', amount: 1, reason: 't', idempotency_key: 'k2' }) });
+ assert.strictEqual(r.status, 403);
+ r = await fetch(`${base}/internal/coins/credit`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: 'Bearer not-a-token' }, body: '{}' });
+ assert.strictEqual(r.status, 401);
+
+ const m = await get('/metrics');
+ assert.strictEqual(m.status, 200);
+ assert.ok(m.text.includes('network_tokens_issued_total{grant_type="client_credentials"} 2\n'), m.text);
+ assert.ok(m.text.includes('network_token_failures_total{grant_type="client_credentials",error="invalid_client"} 1\n'));
+ assert.ok(/network_token_failures_total\{grant_type="refresh_token",error="[a-z_]+"\} 1\n/.test(m.text));
+ assert.ok(m.text.includes('network_token_failures_total{grant_type="other",error="unsupported_grant_type"} 1\n'), 'unknown grant types never become labels');
+ assert.ok(m.text.includes('network_principal_token_failures_total{code="capability.denied"} 1\n'));
+ assert.ok(m.text.includes('network_principal_token_failures_total{code="capability.owner_denied"} 1\n'));
+ assert.ok(m.text.includes('http_requests_total{method="POST",route="/oauth/token",status_class="2xx"} 2\n'));
+ assert.ok(m.text.includes('release_info{service="network",release="abc123def456"} 1\n'));
+ assert.ok(!/secret|live-secret|not-a-token/.test(m.text), 'no credentials in metrics');
+ assert.strictEqual((await get('/metrics', { 'x-forwarded-for': '198.51.100.7' })).status, 404, '/metrics through a proxy is 404');
+
+ // ── Readiness ──
+ let rd = await get('/api/ready');
+ let body = JSON.parse(rd.text);
+ assert.strictEqual(rd.status, 200, rd.text);
+ assert.strictEqual(body.status, 'degraded', 'poll not run yet and discord configured but down: optional only');
+ assert.deepStrictEqual(body.degraded.sort(), ['discord_bot', 'registry_poll']);
+ assert.strictEqual(body.checks.db.status, 'ok');
+ assert.strictEqual(body.checks.signing_key.status, 'ok');
+ assert.strictEqual(body.checks.signing_key.required, true);
+ for (const c of Object.values(body.checks)) { assert.ok(c.checked_at && typeof c.latency_ms === 'number'); }
+
+ lastPoll = Date.now(); discordReady = true;
+ rd = await get('/api/ready'); body = JSON.parse(rd.text);
+ assert.strictEqual(body.status, 'ready');
+
+ // An ephemeral HS256 key in production is not ready: nothing else can verify its tokens.
+ const eph = crypto.randomBytes(32).toString('hex');
+ keysNow = { privateKey: eph, publicKey: eph };
+ const ready2 = observability.createNetworkReadiness({ db, release: 'x', production: true, getKeys: () => keysNow });
+ body = await ready2.run();
+ assert.strictEqual(body.ready, false);
+ assert.deepStrictEqual(body.failed, ['signing_key']);
+ const devReady = observability.createNetworkReadiness({ db, release: 'x', production: false, getKeys: () => keysNow });
+ body = await devReady.run();
+ assert.strictEqual(body.ready, true, 'outside production the dev key only degrades');
+ assert.deepStrictEqual(body.degraded, ['signing_key']);
+
+ // The database gone: not ready.
+ console.log = () => {};
+ const db2 = initDb(path.join(dir, 'other.db'));
+ console.log = log;
+ const ready3 = observability.createNetworkReadiness({ db: db2, release: 'x', production: true, getKeys: () => ({ privateKey: keys.privateKey, publicKey: keys.publicKey }) });
+ db2.close();
+ body = await ready3.run();
+ assert.strictEqual(body.ready, false);
+ assert.deepStrictEqual(body.failed, ['db']);
+
+ server.close();
+ db.close();
+ console.log('observability: all checks passed');
+})().catch(err => { console.error(err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,129 @@
@@ -0,0 +1,129 @@
+'use strict';
+// Operator status (server/status/routes.js over server/registry/ecosystem.js): every row is
+// up | degraded | down | not-running | unknown with checked_at, built from each service's readiness
+// and /release.json — never an optimistic default. Roadmap Track O.
+// node test/status.test.js
+const assert = require('assert');
+const http = require('http');
+const express = require('express');
+const contracts = require('openvibe-contracts');
+const { createEcosystemRegistry } = require('../server/registry/ecosystem');
+const { createStatusRoutes } = require('../server/status/routes');
+
+function fakeService(routes) {
+ const srv = http.createServer((req, res) => {
+ const h = routes[req.url];
+ if (!h) { res.statusCode = 404; return res.end('{"error":"Not found"}'); }
+ const [status, body] = typeof h === 'function' ? h() : h;
+ res.statusCode = status;
+ res.setHeader('content-type', 'application/json');
+ res.end(typeof body === 'string' ? body : JSON.stringify(body));
+ });
+ return new Promise(r => srv.listen(0, '127.0.0.1', () => r({ srv, url: `http://127.0.0.1:${srv.address().port}` })));
+}
+const check = (status, required, extra = {}) => ({ status, required, latency_ms: 1.2, checked_at: '2026-09-23T00:00:00.000Z', ...extra });
+const rel = (release) => [200, { service: 'x', release, released_at: '2026-09-22T20:00:00.000Z', booted_at: '2026-09-23T00:00:00.000Z' }];
+
+(async () => {
+ let communityDown = false;
+ const network = await fakeService({ '/api/ready': [200, { ready: true, status: 'ready', failed: [], degraded: [], checks: { db: check('ok', true) } }], '/release.json': rel('aaaaaaaaaaaa') });
+ const media = await fakeService({ '/api/ready': [200, { ready: true, status: 'degraded', failed: [], degraded: ['remote_b2'], checks: { db: check('ok', true), remote_b2: check('fail', false, { error: 'timeout' }) } }], '/release.json': rel('bbbbbbbbbbbb') });
+ const community = await fakeService({ '/api/ready': () => (communityDown ? [503, { ready: false, status: 'not_ready', failed: ['db'], degraded: [], checks: { db: check('fail', true, { error: 'SQLITE_CANTOPEN' }) } }] : [200, { ready: true, status: 'ready', failed: [], degraded: [], checks: { db: check('ok', true) } }]), '/release.json': rel('cccccccccccc') });
+ const tools = await fakeService({ '/api/health': [200, { status: 'ok' }] }); // no /api/ready yet, no release.json
+ const events = await fakeService({ '/api/ready': [200, { status: 'ready', checks: { db: true } }] }); // an older ad-hoc shape
+
+ let clock = Date.parse('2026-09-23T01:00:00Z');
+ const eco = createEcosystemRegistry({
+ issuer: 'https://openvibe.network', pollMs: 60000, now: () => clock,
+ internalOverrides: { network: network.url, media: media.url, community: community.url, tools: tools.url, events: events.url, live: 'http://127.0.0.1:1', games: 'http://127.0.0.1:1', billing: 'http://127.0.0.1:1' },
+ });
+ const app = express();
+ app.use(createStatusRoutes({ ecosystem: eco, now: () => new Date(clock) }));
+ app.use(eco.router());
+ const srv = http.createServer(app);
+ await new Promise(r => srv.listen(0, '127.0.0.1', r));
+ const base = `http://127.0.0.1:${srv.address().port}`;
+ const json = (p) => fetch(base + p).then(async r => ({ status: r.status, headers: r.headers, body: await r.json() }));
+
+ // Before any poll: everything that runs is 'unknown' with no checked_at — never 'up'.
+ let r = await json('/api/v1/status');
+ let by = Object.fromEntries(r.body.services.map(s => [s.id, s]));
+ assert.strictEqual(by.network.status, 'unknown');
+ assert.strictEqual(by.network.checked_at, null);
+ assert.strictEqual(r.body.last_poll_at, null);
+ assert.ok(r.body.services.every(s => r.body.states.includes(s.status)));
+
+ await eco.pollAll();
+ r = await json('/api/v1/status');
+ by = Object.fromEntries(r.body.services.map(s => [s.id, s]));
+ assert.strictEqual(r.status, 200);
+ assert.strictEqual(r.headers.get('access-control-allow-origin'), '*');
+ assert.strictEqual(by.network.status, 'up');
+ assert.strictEqual(by.network.basis, 'ready');
+ assert.strictEqual(by.network.release.release, 'aaaaaaaaaaaa');
+ assert.strictEqual(by.network.release.booted_at, '2026-09-23T00:00:00.000Z');
+ assert.strictEqual(by.media.status, 'degraded', 'an optional failure degrades, it does not read as healthy or down');
+ assert.deepStrictEqual(by.media.ready.degraded, ['remote_b2']);
+ assert.strictEqual(by.media.ready.checks.remote_b2.error, 'timeout');
+ assert.strictEqual(by.community.status, 'up');
+ assert.strictEqual(by.tools.status, 'up');
+ assert.strictEqual(by.tools.basis, 'health', 'no readiness endpoint: liveness only, and the row says so');
+ assert.strictEqual(by.tools.release, null);
+ assert.ok(by.tools.release_error);
+ assert.strictEqual(by.events.basis, 'ready-legacy');
+ assert.strictEqual(by.live.status, 'down', 'unreachable is down');
+ assert.strictEqual(by.wiki.status, 'not-running');
+ assert.strictEqual(by.wiki.label, 'not running (placeholder)');
+ assert.strictEqual(by.contracts.status, 'not-running');
+ assert.strictEqual(by.contracts.label, 'not running (no runtime)');
+ for (const s of r.body.services) assert.ok(s.checked_at, `${s.id} says when it was checked`);
+ assert.strictEqual(r.body.summary.up + r.body.summary.degraded + r.body.summary.down + r.body.summary['not-running'] + r.body.summary.unknown, contracts.services.manifests.length);
+
+ // A required failure on the service is 'down', with its failed checks.
+ communityDown = true;
+ await eco.pollAll();
+ by = Object.fromEntries((await json('/api/v1/status')).body.services.map(s => [s.id, s]));
+ assert.strictEqual(by.community.status, 'down');
+ assert.deepStrictEqual(by.community.ready.failed, ['db']);
+
+ // The registry's runtime field says the same thing.
+ r = await json('/api/v1/registry/services/community');
+ assert.strictEqual(r.body.runtime.status, 'down');
+
+ // Stale: older than three poll intervals is 'unknown', never the last good state.
+ clock += 4 * 60000;
+ by = Object.fromEntries((await json('/api/v1/status')).body.services.map(s => [s.id, s]));
+ assert.strictEqual(by.network.status, 'unknown');
+ assert.strictEqual(by.network.stale, true);
+ assert.strictEqual(by.network.last_status, 'up');
+ clock -= 4 * 60000;
+
+ // The page: server-rendered, noindex, no JavaScript needed to read it.
+ const page = await fetch(base + '/status');
+ const html = await page.text();
+ assert.strictEqual(page.status, 200);
+ assert.match(page.headers.get('x-robots-tag'), /noindex/);
+ assert.ok(html.includes('<meta name="robots" content="noindex, nofollow">'));
+ assert.ok(html.includes('id="svc-media"') && /id="svc-media"[\s\S]*?Degraded/.test(html));
+ assert.ok(/id="svc-community"[\s\S]*?Down[\s\S]*?SQLITE_CANTOPEN/.test(html));
+ assert.ok(/id="svc-wiki"[\s\S]*?not running \(placeholder\)/.test(html));
+ assert.ok(/id="svc-tools"[\s\S]*?liveness only/.test(html));
+ assert.ok(html.includes('<noscript>'), 'navigation without JavaScript');
+ assert.ok(html.includes('SLO categories (proposals)'));
+ const noScript = html.replace(/<script[\s\S]*?<\/script>/g, '');
+ assert.ok(noScript.includes('aaaaaaaaaaaa'), 'the rows are in the HTML itself');
+
+ // SLO categories: the ten from §15.19, every target labelled a proposal.
+ r = await json('/api/v1/status/slo');
+ assert.strictEqual(r.body.status, 'proposal');
+ assert.strictEqual(r.body.categories.length, 10);
+ for (const c of r.body.categories) {
+ assert.ok(c.id && c.name && c.sli && c.proposed_target && ['integrity', 'availability', 'continuity'].includes(c.priority), c.id);
+ assert.ok(Array.isArray(c.measured_by));
+ if (!c.measured_by.length) assert.ok(/not instrumented/.test(c.measurement_status), `${c.id} says it is not measured`);
+ }
+
+ for (const s of [network, media, community, tools, events]) s.srv.close();
+ srv.close();
+ console.log('status: all checks passed');
+})().catch(err => { console.error(err); process.exit(1); });