openvibers/openvibe.network Commits
Release health on /status (roadmap WS-P task 15): the ecosystem poll reads release_client_sessions and release_client_updates_total from the /metrics of services whose /release.json names a metrics_url (loopback), and /status and /api/v1/status show, per service, open tabs on the current and older releases, prompts, in-place updates, reloads, deferrals and failures with their reasons, and how long the last release took to drain. Drain reads true only once a service's process has been up 6 minutes (tabs beat every 5); a release first seen after that has no drain time; the last 10 releases are kept in memory
Changes
@@@ -6,7 +6,7 @@"scripts": {"start": "node server/index.js","dev": "NODE_ENV=development node server/index.js",-- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/frame.test.js && node test/github-integration.test.js && node test/revocation.test.js && node test/usernames.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/module-events.test.js && node test/registry-ecosystem.test.js && node test/registry-exposure.test.js && node test/registry-manifest-fixture.test.js && node test/staff-claims.test.js && node test/profile-events.test.js && node test/grants-admin.test.js && node test/events-ops.test.js && node test/deploy-drift.test.js && node test/registry-topics.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/events-consumer.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js && node test/data-ownership.test.js && node test/unknown-paths.test.js && node test/oidc-discovery.test.js && node test/refresh-tokens.test.js && node test/provider-secrets.test.js && node test/first-party-origins.test.js && node test/updates.test.js && node test/static-headers.test.js && node test/library-tags.test.js && node test/moderation-audit.test.js && node test/blocks.test.js && node test/site-config.test.js && node test/perf-budget.test.js && node test/shutdown.test.js && node test/loyalty-policy.test.js && node test/themes-review.test.js && node test/page-checks.test.js && node test/operator-checklist.test.js",++ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/frame.test.js && node test/github-integration.test.js && node test/revocation.test.js && node test/usernames.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/module-events.test.js && node test/registry-ecosystem.test.js && node test/registry-exposure.test.js && node test/registry-manifest-fixture.test.js && node test/staff-claims.test.js && node test/profile-events.test.js && node test/grants-admin.test.js && node test/events-ops.test.js && node test/deploy-drift.test.js && node test/registry-topics.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/events-consumer.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js && node test/data-ownership.test.js && node test/unknown-paths.test.js && node test/oidc-discovery.test.js && node test/refresh-tokens.test.js && node test/provider-secrets.test.js && node test/first-party-origins.test.js && node test/updates.test.js && node test/static-headers.test.js && node test/library-tags.test.js && node test/moderation-audit.test.js && node test/blocks.test.js && node test/site-config.test.js && node test/perf-budget.test.js && node test/shutdown.test.js && node test/loyalty-policy.test.js && node test/themes-review.test.js && node test/page-checks.test.js && node test/operator-checklist.test.js && node test/release-health.test.js","reset-db": "node server/reset-db.js","grant-admin": "node server/grant-admin.js"},@@@ -37,6 +37,7 @@ const { exposureOf, publicOriginOf, libraries } = require('./exposure');const { buildTopics, eventsOf } = require('./topics');const { driftOf } = require('./versions');const { SITES } = require('../frame/sites');++const { createReleaseHealth, parseClientMetrics } = require('./release-health');// Where each running service answers health checks from this host: its manifest's `internalOrigin`// (openvibe-contracts ≥ 0.42.0, WS-C task 1). OV_<ID>_INTERNAL_URL overrides any of them (internalFromEnv@@@ -110,7 +111,7 @@ function categoryOf(m) {return SITES.some(s => s.service === m.id) ? 'site' : 'platform';}--function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, healthPaths = HEALTH_PATHS, now = () => Date.now(), ranking = null } = {}) {++function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, healthPaths = HEALTH_PATHS, now = () => Date.now(), ranking = null, releaseHealth = createReleaseHealth({ now }) } = {}) {const internal = { ...INTERNAL, ...internalOverrides };let rankingOf = ranking;// id -> { status: up|degraded|down|not-running|unknown, basis, reason, http_status, latency_ms, checked_at, ready, release }@@@ -126,6 +127,15 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = greturn { res, body, latency: now() - t0 };}++ /** The release-client series of a service's /metrics (loopback, no forwarding headers), or null. */++ async function clientMetricsOf(base) {++ try {++ const res = await fetchImpl(`${base}/metrics`, { signal: AbortSignal.timeout(3000), headers: { accept: 'text/plain' } });++ if (!res.ok || typeof res.text !== 'function') return null;++ return parseClientMetrics(await res.text());++ } catch { return null; }++ }++async function releaseOf(base) {try {const { res, body } = await getJson(`${base}/release.json`);@@@ -137,7 +147,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = gfor (const [k, v] of Object.entries(body.packages).slice(0, 10)) if (/^openvibe-[a-z-]{1,30}$/.test(k) && /^\d+\.\d+\.\d+/.test(String(v))) packages[k] = String(v).slice(0, 20);}const ver = (v) => (typeof v === 'string' && /^\d+\.\d+\.\d+/.test(v) ? v.slice(0, 20) : null);-- return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null, contracts_version: ver(body.contracts_version), packages };++ return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null, contracts_version: ver(body.contracts_version), packages, collects: typeof body.metrics_url === 'string' };} catch (err) {return { release: null, error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };}@@@ -179,6 +189,11 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = gentry.release = await releaseP;entry.checked_at = at();health.set(m.id, entry);++ // Release health (WS-P task 15): a service that collects release-watch reports has them in its /metrics.++ if (entry.release && entry.release.release) {++ const metrics = entry.release.collects ? await clientMetricsOf(base) : null;++ releaseHealth.observe(m.id, entry.release, metrics);++ }}const pollAll = async () => { await Promise.all(contracts.services.manifests.map(checkOne)); lastPollAt = now(); };function start() {@@@ -380,7 +395,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = greturn r;}-- return { router, start, stop, pollAll, health, current, descriptor, categories, featured, releases, setRanking: (fn) => { rankingOf = fn; }, lastPollAt: () => lastPollAt, pollMs, internal };++ return { router, start, stop, pollAll, health, current, descriptor, categories, featured, releases, setRanking: (fn) => { rankingOf = fn; }, lastPollAt: () => lastPollAt, pollMs, internal, releaseHealth: (id) => releaseHealth.view(id), releaseHealthSince: () => releaseHealth.startedAt() };}module.exports = { createEcosystemRegistry, INTERNAL, READY_PATHS, HEALTH_PATHS, CATEGORIES, categoryOf, internalFromEnv, statusFromReady, readySummary };nnew file mode 100644@@@ -0,0 +1,104 @@++'use strict';++/**++ * Release health (roadmap WS-P task 15): what open tabs say about each service's releases, read from the++ * service's own /metrics on loopback during the ecosystem health poll (only for services whose++ * /release.json names a metrics_url, i.e. that collect release-watch reports; openvibe-shared ≥ 1.18.0).++ *++ * sessions release_client_sessions{generation}: tabs heard from in the last 12 minutes, on the++ * release the service serves (current) or an older one++ * updates release_client_updates_total{outcome,reason}: prompts, in-place updates, reloads,++ * deferrals and failures since the service's process started (so, for this release)++ * drain how long after a release went live the last older tab left. A tab beats every 5++ * minutes and the counts live in the service's process, which a deploy restarts, so++ * "no older tab" means something only once the process has been up WARM_MS; a release++ * whose first such reading is already 0 drained "within" that time.++ *++ * In memory: Network's own restart starts the history again (the status page says since when).++ */++const WARM_MS = 6 * 60 * 1000;++const KEEP = 10;++const SESSIONS_RE = /^release_client_sessions\{generation="(current|older)"\} (\d+(?:\.\d+)?)$/;++const UPDATES_RE = /^release_client_updates_total\{outcome="([a-z]+)",reason="([a-z+-]+)"\} (\d+(?:\.\d+)?)$/;++const OUTCOMES = ['prompted', 'applied', 'reloaded', 'deferred', 'failed'];++++/** The release-client series of a Prometheus text body: { sessions: { current, older } | null, updates: { outcome: { reason: n } } }. */++function parseClientMetrics(text) {++ let sessions = null;++ const updates = {};++ for (const line of String(text || '').split('\n')) {++ if (!line.startsWith('release_client_')) continue;++ let m = SESSIONS_RE.exec(line);++ if (m) { sessions = sessions || { current: 0, older: 0 }; sessions[m[1]] = Math.round(Number(m[2])); continue; }++ m = UPDATES_RE.exec(line);++ if (m && OUTCOMES.includes(m[1])) (updates[m[1]] = updates[m[1]] || {})[m[2]] = Math.round(Number(m[3]));++ }++ return { sessions, updates };++}++++const total = (byReason) => Object.values(byReason || {}).reduce((a, b) => a + b, 0);++++function createReleaseHealth({ now = () => Date.now(), warmMs = WARM_MS, keep = KEEP } = {}) {++ const state = new Map();++ const startedAt = new Date(now()).toISOString();++ const iso = (t) => new Date(t).toISOString();++++ /** One poll's reading for a service: its release (id, released_at, booted_at) and parsed metrics, or null metrics. */++ function observe(id, release, metrics) {++ if (!release || !release.release) return;++ let s = state.get(id);++ if (!s || s.release !== release.release) {++ const history = s ? [summaryOf(s, 'superseded'), ...s.history].slice(0, keep) : [];++ const since = release.booted_at || release.released_at || iso(now());++ s = { release: release.release, since, booted_at: release.booted_at || null, sessions: null, updates: {}, measured_at: null, peak_older: 0, drained_at: null, drain_s: null, within: false, history };++ // First seen after its warm-up (Network restarted later): an older tab that left before now left unseen.++ s.late = now() - Date.parse(since) >= warmMs;++ state.set(id, s);++ } else if (release.booted_at && release.booted_at !== s.booted_at) {++ s.booted_at = release.booted_at; // restarted on the same release: its counts start again, and so does the warm-up++ }++ if (!metrics) return;++ s.measured_at = iso(now());++ s.updates = metrics.updates || {};++ if (!metrics.sessions) return;++ s.sessions = metrics.sessions;++ s.peak_older = Math.max(s.peak_older, metrics.sessions.older);++ const up = now() - Date.parse(s.booted_at || s.since);++ if (!s.drained_at && up >= warmMs && metrics.sessions.older === 0) {++ s.drained_at = iso(now());++ const unseen = s.peak_older === 0; // no older tab at the first warm reading: it drained within the warm-up++ s.drain_s = unseen && s.late ? null : Math.max(0, Math.round((now() - Date.parse(s.since)) / 1000));++ s.within = unseen && !s.late;++ }++ }++++ function summaryOf(s, end) {++ return { release: s.release, since: s.since, drained_at: s.drained_at, drain_s: s.drain_s, within: s.within, peak_older: s.peak_older, end: s.drained_at ? 'drained' : end };++ }++++ /** The view for one service, or null when nothing was read for it. */++ function view(id) {++ const s = state.get(id);++ if (!s) return null;++ const up = now() - Date.parse(s.booted_at || s.since);++ const drain = !s.sessions ? { state: 'unknown' }++ : s.drained_at ? { state: 'drained', drained_at: s.drained_at, seconds: s.drain_s, within: s.within, ...(s.drain_s === null ? { seen_late: true } : {}) }++ : up < warmMs ? { state: 'warming', ready_in_s: Math.round((warmMs - up) / 1000), older: s.sessions.older }++ : { state: 'draining', older: s.sessions.older, for_s: Math.round((now() - Date.parse(s.since)) / 1000) };++ const u = s.updates;++ return {++ release: s.release,++ since: s.since,++ measured_at: s.measured_at,++ sessions: s.sessions,++ counts: Object.fromEntries(OUTCOMES.map((o) => [o, total(u[o])])),++ deferred: u.deferred || {},++ failed: u.failed || {},++ drain,++ history: s.history,++ };++ }++++ return { observe, view, startedAt: () => startedAt, WARM_MS: warmMs };++}++++module.exports = { createReleaseHealth, parseClientMetrics, WARM_MS };@@@ -12,6 +12,10 @@* up | degraded | down | not-running | unknown and carries checked_at; a service not checked yet, or* whose last check is stale, is 'unknown' — never an optimistic default.*++ * Each row also carries release_health (server/registry/release-health.js, WS-P task 15): what open tabs++ * report for services that collect release-watch reports: tabs on the current and older releases, prompts,++ * updates, deferrals and failures, and how long the last release took to drain.++ ** Each row also carries its exposure (server/registry/exposure.js): live (public), internal (loopback* only), library, repository or placeholder. A service that is up on loopback while its public domain* serves a placeholder reads "up (loopback only)" and has no public origin, never a bare "up".@@@ -77,11 +81,38 @@ function rows(ecosystem) {main: (() => { const d = require('../registry/deploy-drift').current(m.id); return d ? { state: d.state, behind_by: d.behind_by ?? null, since: d.since || null } : null; })(),release_error: r.release && !r.release.release ? r.release.error || 'unknown' : null,ready: r.ready || null,++ release_health: typeof ecosystem.releaseHealth === 'function' ? ecosystem.releaseHealth(m.id) : null,...(r.stale ? { stale: true, last_status: r.last_status } : {}),};});}++const dur = (s) => (s == null ? '—' : s < 90 ? `${s} s` : s < 5400 ? `${Math.round(s / 60)} min` : `${(s / 3600).toFixed(1)} h`);++const reasons = (o) => Object.entries(o || {}).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([k, v]) => `${k} ${v}`).join(', ');++function drainCell(d) {++ if (!d || d.state === 'unknown') return '<small>no reports yet</small>';++ if (d.state === 'warming') return `warming up<small>${esc(d.older)} older tab(s) so far; reliable in ${esc(dur(d.ready_in_s))}</small>`;++ if (d.state === 'draining') return `<b>${esc(d.older)}</b> older tab(s)<small>for ${esc(dur(d.for_s))}</small>`;++ if (d.seen_late) return 'drained<small>before Network watched</small>';++ return `drained ${d.within ? 'within' : 'in'} ${esc(dur(d.seconds))}`;++}++/** The release-health section: services whose tabs report (WS-P task 15). */++function releaseHealthSection(list, since) {++ const rowsOf = list.filter((r) => r.release_health);++ if (!rowsOf.length) return '<p class="lede">No service has reported release health yet.</p>';++ const tr = rowsOf.map((r) => {++ const h = r.release_health; const s = h.sessions; const c = h.counts;++ const mix = s ? (s.current + s.older ? `${s.current} / ${s.older}<small>${Math.round((100 * s.current) / (s.current + s.older))}% on current</small>` : '0 / 0') : '<small>—</small>';++ return `<tr><td><b>${esc(r.name)}</b><small><code>${esc(h.release)}</code></small></td><td>${mix}</td><td>${drainCell(h.drain)}</td>++<td>${esc(c.prompted)} · ${esc(c.applied)} · ${esc(c.reloaded)}</td><td>${esc(c.deferred)}${c.deferred ? `<small>${esc(reasons(h.deferred))}</small>` : ''}</td><td>${esc(c.failed)}${c.failed ? `<small>${esc(reasons(h.failed))}</small>` : ''}</td></tr>`;++ }).join('\n');++ return `<p class="lede">What open tabs report through release-watch: tabs heard from in the last 12 minutes on the release each service serves and on older ones, and since that release started, how many were prompted, updated in place or reloaded, deferred (and why) or failed. Drain is how long after a release went live the last older tab left; tabs report every 5 minutes, so it reads true only 6 minutes after a start. Watched since ${esc(since)}.</p>++<table><thead><tr><th>Service</th><th>Tabs current / older</th><th>Drain</th><th>Prompted · applied · reloaded</th><th>Deferred</th><th>Failed</th></tr></thead>++<tbody>++${tr}++</tbody></table>`;++}++function summary(list) {const counts = Object.fromEntries(STATES.map((s) => [s, 0]));for (const r of list) counts[r.status]++;@@@ -126,7 +157,7 @@ function proofSteps(rec) {}const lastRun = (rec) => `Last run <time datetime="${esc(rec.finished_at)}">${esc(rec.finished_at)}</time>: <b>${rec.ok ? 'passed' : 'failed'}</b> (${rec.steps.filter((st) => st.ok).length}/${rec.steps.length} steps).`;--function renderPage(list, slo, generatedAt, devPath = null, toolsJob = null) {++function renderPage(list, slo, generatedAt, devPath = null, toolsJob = null, healthSince = null) {const counts = summary(list);const tr = list.map((r) => `<tr id="svc-${esc(r.id)}"><td><b>${esc(r.name)}</b><small>${esc(r.id)} · manifest: ${esc(r.manifest_status)}</small></td>@@@ -168,6 +199,9 @@ ${require('openvibe-shared/frame').noscriptNav({ name: 'OpenVibe.Network', links${tr}</tbody></table>++<section aria-labelledby="h-release-health"><h2 id="h-release-health">Release health</h2>++${releaseHealthSection(list, healthSince || generatedAt)}++</section><section aria-labelledby="h-devpath"><h2 id="h-devpath">Developer path</h2>${devPath ? `<p class="lede">Once a day a test developer account goes from sign-in to a sandbox project, a Media upload, an Events publish and pull, a credential rotation and revocation, and cleanup, through the public API and the SDK. ${lastRun(devPath)}</p>${proofSteps(devPath)}` : '<p class="lede">The daily developer path has not reported here yet.</p>'}@@@ -202,6 +236,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {summary: summary(list),exposure_states: exposure.STATES,exposure_summary: exposureSummary(list),++ release_health_since: typeof ecosystem.releaseHealthSince === 'function' ? ecosystem.releaseHealthSince() : null,developer_path: loadDevPath(),tools_job_proof: loadDevPath(TOOLSJOB_FILE),services: list,@@@ -213,7 +248,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {r.get('/status', (_req, res) => {const slo = { ...loadSlo(), pollSeconds: ecosystem.pollMs / 1000 };res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', 'no-cache, max-age=0').set('X-Robots-Tag', 'noindex, nofollow');-- res.send(renderPage(rows(ecosystem), slo, now().toISOString(), loadDevPath(), loadDevPath(TOOLSJOB_FILE)));++ res.send(renderPage(rows(ecosystem), slo, now().toISOString(), loadDevPath(), loadDevPath(TOOLSJOB_FILE), typeof ecosystem.releaseHealthSince === 'function' ? ecosystem.releaseHealthSince() : null));});return r;}nnew file mode 100644@@@ -0,0 +1,100 @@++'use strict';++// Release health (roadmap WS-P task 15): the ecosystem poll reads the release-client series from the++// /metrics of services that collect release-watch reports, and /status shows tabs by generation,++// prompts, updates, deferrals, failures and drain times. Drain reads true only after the warm-up (tabs++// beat every 5 minutes into a process a deploy restarted); a release first seen late has no drain time.++// node test/release-health.test.js++const assert = require('assert');++const http = require('http');++const express = require('express');++const { createReleaseHealth, parseClientMetrics, WARM_MS } = require('../server/registry/release-health');++++const METRICS = `# HELP release_client_sessions Open tabs++# TYPE release_client_sessions gauge++release_client_sessions{generation="current"} 7++release_client_sessions{generation="older"} 2++release_client_updates_total{outcome="prompted",reason="optional"} 3++release_client_updates_total{outcome="applied",reason="content+style"} 4++release_client_updates_total{outcome="deferred",reason="typing"} 2++release_client_updates_total{outcome="deferred",reason="media"} 1++release_client_updates_total{outcome="failed",reason="style"} 1++release_client_updates_total{outcome="bogus",reason="x"} 9++http_requests_total{route="/"} 12++`;++++(async () => {++ // ── Parsing ──++ const p = parseClientMetrics(METRICS);++ assert.deepStrictEqual(p.sessions, { current: 7, older: 2 });++ assert.deepStrictEqual(p.updates, { prompted: { optional: 3 }, applied: { 'content+style': 4 }, deferred: { typing: 2, media: 1 }, failed: { style: 1 } }, 'unknown outcomes and other series are ignored');++ assert.deepStrictEqual(parseClientMetrics('http_requests_total 1\n'), { sessions: null, updates: {} });++++ // ── Drain ──++ const t0 = Date.parse('2026-09-26T10:00:00Z');++ let t = t0;++ const iso = (x) => new Date(x).toISOString();++ const rh = createReleaseHealth({ now: () => t });++ const r1 = { release: 'aaaaaaa', booted_at: iso(t0) };++ const reading = (current, older) => ({ sessions: { current, older }, updates: {} });++ t = t0 + 60e3; rh.observe('live', r1, reading(5, 3));++ assert.strictEqual(rh.view('live').drain.state, 'warming', 'a fresh process has not heard from every tab');++ t = t0 + WARM_MS + 60e3; rh.observe('live', r1, reading(8, 2));++ assert.deepStrictEqual([rh.view('live').drain.state, rh.view('live').drain.older], ['draining', 2]);++ t = t0 + 9 * 60e3; rh.observe('live', r1, reading(10, 0));++ assert.deepStrictEqual(rh.view('live').drain, { state: 'drained', drained_at: iso(t), seconds: 540, within: false });++ t = t0 + 10 * 60e3; rh.observe('live', r1, reading(10, 1));++ assert.strictEqual(rh.view('live').drain.seconds, 540, 'drained once, not again');++++ const r2 = { release: 'bbbbbbb', booted_at: iso(t0 + 10 * 60e3) };++ t = t0 + 11 * 60e3; rh.observe('live', r2, reading(0, 0));++ assert.strictEqual(rh.view('live').history[0].release, 'aaaaaaa', 'the last release goes to history');++ assert.strictEqual(rh.view('live').history[0].end, 'drained');++ t = t0 + 17 * 60e3; rh.observe('live', r2, reading(9, 0));++ assert.deepStrictEqual(rh.view('live').drain, { state: 'drained', drained_at: iso(t), seconds: 420, within: true }, 'no older tab by the first warm reading: within the warm-up');++++ // Restarted on the same release: the counts start again, so does the warm-up.++ t = t0 + 20 * 60e3; rh.observe('chat', { release: 'ccccccc', booted_at: iso(t0) }, reading(3, 1));++ assert.strictEqual(rh.view('chat').drain.state, 'draining');++ rh.observe('chat', { release: 'ccccccc', booted_at: iso(t) }, reading(0, 0));++ assert.strictEqual(rh.view('chat').drain.state, 'warming');++++ // First seen long after its start (Network restarted): no drain time is made up.++ const late = createReleaseHealth({ now: () => t0 + 3600e3 });++ late.observe('tools', { release: 'ddddddd', booted_at: iso(t0) }, reading(4, 0));++ assert.deepStrictEqual(late.view('tools').drain, { state: 'drained', drained_at: iso(t0 + 3600e3), seconds: null, within: false, seen_late: true });++ late.observe('wiki', { release: 'eeeeeee', booted_at: iso(t0) }, null);++ assert.deepStrictEqual([late.view('wiki').drain.state, late.view('wiki').sessions], ['unknown', null], 'a service that does not collect has no readings');++++ // ── The poll reads /metrics only where /release.json names a metrics_url; /status shows it ──++ const { createEcosystemRegistry } = require('../server/registry/ecosystem');++ const asked = [];++ const res = (status, body, text) => ({ ok: status < 400, status, json: async () => body, text: async () => text ?? JSON.stringify(body) });++ const fetchImpl = async (url) => {++ const u = new URL(url);++ asked.push(`${u.port}${u.pathname}`);++ if (u.pathname === '/release.json') return res(200, { release: `rel${u.port}`, booted_at: iso(Date.now() - 3600e3), ...(u.port === '3000' ? { metrics_url: '/release-metrics' } : {}) });++ if (u.pathname === '/metrics') return res(200, null, METRICS);++ return res(200, { ready: true, status: 'ready', checks: {} });++ };++ const eco = createEcosystemRegistry({ issuer: 'https://openvibe.network', fetchImpl, internalOverrides: { live: 'http://127.0.0.1:3000' } });++ await eco.pollAll();++ assert.ok(asked.includes('3000/metrics'), 'a collecting service is read');++ assert.ok(!asked.some((a) => a.endsWith('/metrics') && !a.startsWith('3000')), 'no /metrics for services without a metrics_url');++ const view = eco.releaseHealth('live');++ assert.deepStrictEqual([view.sessions, view.counts], [{ current: 7, older: 2 }, { prompted: 3, applied: 4, reloaded: 0, deferred: 3, failed: 1 }]);++++ const app = express();++ app.use(require('../server/status/routes').createStatusRoutes({ ecosystem: eco }));++ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });++ const base = `http://127.0.0.1:${server.address().port}`;++ const json = await (await fetch(`${base}/api/v1/status`)).json();++ assert.strictEqual(json.services.find((s) => s.id === 'live').release_health.sessions.current, 7);++ assert.ok(json.release_health_since);++ const html = await (await fetch(`${base}/status`)).text();++ assert.ok(html.includes('id="h-release-health"'), 'the page has the section');++ assert.match(html, /7 \/ 2<small>78% on current<\/small>/, 'tabs by generation');++ assert.match(html, /typing 2, media 1/, 'deferral reasons');++ assert.match(html, /drained<small>before Network watched<\/small>|older tab/, 'a drain state');++ server.close();++ console.log('release health: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });
@@ -6,7 +6,7 @@
@@ -6,7 +6,7 @@
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
- "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/frame.test.js && node test/github-integration.test.js && node test/revocation.test.js && node test/usernames.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/module-events.test.js && node test/registry-ecosystem.test.js && node test/registry-exposure.test.js && node test/registry-manifest-fixture.test.js && node test/staff-claims.test.js && node test/profile-events.test.js && node test/grants-admin.test.js && node test/events-ops.test.js && node test/deploy-drift.test.js && node test/registry-topics.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/events-consumer.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js && node test/data-ownership.test.js && node test/unknown-paths.test.js && node test/oidc-discovery.test.js && node test/refresh-tokens.test.js && node test/provider-secrets.test.js && node test/first-party-origins.test.js && node test/updates.test.js && node test/static-headers.test.js && node test/library-tags.test.js && node test/moderation-audit.test.js && node test/blocks.test.js && node test/site-config.test.js && node test/perf-budget.test.js && node test/shutdown.test.js && node test/loyalty-policy.test.js && node test/themes-review.test.js && node test/page-checks.test.js && node test/operator-checklist.test.js",
+ "test": "node test/registry-bootstrap.test.js && node test/nginx-generator-live.test.js && node test/admin-refresh.test.js && node test/history-sso.test.js && node test/sso-check.test.js && node test/fedcm.test.js && node test/domains.test.js && node test/frame.test.js && node test/github-integration.test.js && node test/revocation.test.js && node test/usernames.test.js && node test/avatar.test.js && node test/roadmap-baseline.test.js && node test/identity-subjects.test.js && node test/principals.test.js && node test/modules.test.js && node test/module-events.test.js && node test/registry-ecosystem.test.js && node test/registry-exposure.test.js && node test/registry-manifest-fixture.test.js && node test/staff-claims.test.js && node test/profile-events.test.js && node test/grants-admin.test.js && node test/events-ops.test.js && node test/deploy-drift.test.js && node test/registry-topics.test.js && node test/service-principal.test.js && node test/oauth-pkce.test.js && node test/developer-projects.test.js && node test/developer-defaults.test.js && node test/public-cors.test.js && node test/observability.test.js && node test/status.test.js && node test/events-consumer.test.js && node test/security-session.test.js && node test/security-redirects.test.js && node test/analytics-privacy.test.js && node test/data-ownership.test.js && node test/unknown-paths.test.js && node test/oidc-discovery.test.js && node test/refresh-tokens.test.js && node test/provider-secrets.test.js && node test/first-party-origins.test.js && node test/updates.test.js && node test/static-headers.test.js && node test/library-tags.test.js && node test/moderation-audit.test.js && node test/blocks.test.js && node test/site-config.test.js && node test/perf-budget.test.js && node test/shutdown.test.js && node test/loyalty-policy.test.js && node test/themes-review.test.js && node test/page-checks.test.js && node test/operator-checklist.test.js && node test/release-health.test.js",
"reset-db": "node server/reset-db.js",
"reset-db": "node server/reset-db.js",
"grant-admin": "node server/grant-admin.js"
"grant-admin": "node server/grant-admin.js"
},
},
@@ -37,6 +37,7 @@ const { exposureOf, publicOriginOf, libraries } = require('./exposure');
@@ -37,6 +37,7 @@ const { exposureOf, publicOriginOf, libraries } = require('./exposure');
const { buildTopics, eventsOf } = require('./topics');
const { buildTopics, eventsOf } = require('./topics');
const { driftOf } = require('./versions');
const { driftOf } = require('./versions');
const { SITES } = require('../frame/sites');
const { SITES } = require('../frame/sites');
+const { createReleaseHealth, parseClientMetrics } = require('./release-health');
// Where each running service answers health checks from this host: its manifest's `internalOrigin`
// Where each running service answers health checks from this host: its manifest's `internalOrigin`
// (openvibe-contracts ≥ 0.42.0, WS-C task 1). OV_<ID>_INTERNAL_URL overrides any of them (internalFromEnv
// (openvibe-contracts ≥ 0.42.0, WS-C task 1). OV_<ID>_INTERNAL_URL overrides any of them (internalFromEnv
@@ -110,7 +111,7 @@ function categoryOf(m) {
@@ -110,7 +111,7 @@ function categoryOf(m) {
return SITES.some(s => s.service === m.id) ? 'site' : 'platform';
return SITES.some(s => s.service === m.id) ? 'site' : 'platform';
}
}
-function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, healthPaths = HEALTH_PATHS, now = () => Date.now(), ranking = null } = {}) {
+function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = globalThis.fetch, pollMs = POLL_MS, readyPaths = READY_PATHS, healthPaths = HEALTH_PATHS, now = () => Date.now(), ranking = null, releaseHealth = createReleaseHealth({ now }) } = {}) {
const internal = { ...INTERNAL, ...internalOverrides };
const internal = { ...INTERNAL, ...internalOverrides };
let rankingOf = ranking;
let rankingOf = ranking;
// id -> { status: up|degraded|down|not-running|unknown, basis, reason, http_status, latency_ms, checked_at, ready, release }
// id -> { status: up|degraded|down|not-running|unknown, basis, reason, http_status, latency_ms, checked_at, ready, release }
@@ -126,6 +127,15 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -126,6 +127,15 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
return { res, body, latency: now() - t0 };
return { res, body, latency: now() - t0 };
}
}
+ /** The release-client series of a service's /metrics (loopback, no forwarding headers), or null. */
+ async function clientMetricsOf(base) {
+ try {
+ const res = await fetchImpl(`${base}/metrics`, { signal: AbortSignal.timeout(3000), headers: { accept: 'text/plain' } });
+ if (!res.ok || typeof res.text !== 'function') return null;
+ return parseClientMetrics(await res.text());
+ } catch { return null; }
+ }
+
async function releaseOf(base) {
async function releaseOf(base) {
try {
try {
const { res, body } = await getJson(`${base}/release.json`);
const { res, body } = await getJson(`${base}/release.json`);
@@ -137,7 +147,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -137,7 +147,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
for (const [k, v] of Object.entries(body.packages).slice(0, 10)) if (/^openvibe-[a-z-]{1,30}$/.test(k) && /^\d+\.\d+\.\d+/.test(String(v))) packages[k] = String(v).slice(0, 20);
for (const [k, v] of Object.entries(body.packages).slice(0, 10)) if (/^openvibe-[a-z-]{1,30}$/.test(k) && /^\d+\.\d+\.\d+/.test(String(v))) packages[k] = String(v).slice(0, 20);
}
}
const ver = (v) => (typeof v === 'string' && /^\d+\.\d+\.\d+/.test(v) ? v.slice(0, 20) : null);
const ver = (v) => (typeof v === 'string' && /^\d+\.\d+\.\d+/.test(v) ? v.slice(0, 20) : null);
- return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null, contracts_version: ver(body.contracts_version), packages };
+ return { release: body.release.slice(0, 40), released_at: body.released_at || null, booted_at: body.booted_at || null, contracts_version: ver(body.contracts_version), packages, collects: typeof body.metrics_url === 'string' };
} catch (err) {
} catch (err) {
return { release: null, error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };
return { release: null, error: err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };
}
}
@@ -179,6 +189,11 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -179,6 +189,11 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
entry.release = await releaseP;
entry.release = await releaseP;
entry.checked_at = at();
entry.checked_at = at();
health.set(m.id, entry);
health.set(m.id, entry);
+ // Release health (WS-P task 15): a service that collects release-watch reports has them in its /metrics.
+ if (entry.release && entry.release.release) {
+ const metrics = entry.release.collects ? await clientMetricsOf(base) : null;
+ releaseHealth.observe(m.id, entry.release, metrics);
+ }
}
}
const pollAll = async () => { await Promise.all(contracts.services.manifests.map(checkOne)); lastPollAt = now(); };
const pollAll = async () => { await Promise.all(contracts.services.manifests.map(checkOne)); lastPollAt = now(); };
function start() {
function start() {
@@ -380,7 +395,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
@@ -380,7 +395,7 @@ function createEcosystemRegistry({ issuer, internalOverrides = {}, fetchImpl = g
return r;
return r;
}
}
- return { router, start, stop, pollAll, health, current, descriptor, categories, featured, releases, setRanking: (fn) => { rankingOf = fn; }, lastPollAt: () => lastPollAt, pollMs, internal };
+ return { router, start, stop, pollAll, health, current, descriptor, categories, featured, releases, setRanking: (fn) => { rankingOf = fn; }, lastPollAt: () => lastPollAt, pollMs, internal, releaseHealth: (id) => releaseHealth.view(id), releaseHealthSince: () => releaseHealth.startedAt() };
}
}
module.exports = { createEcosystemRegistry, INTERNAL, READY_PATHS, HEALTH_PATHS, CATEGORIES, categoryOf, internalFromEnv, statusFromReady, readySummary };
module.exports = { createEcosystemRegistry, INTERNAL, READY_PATHS, HEALTH_PATHS, CATEGORIES, categoryOf, internalFromEnv, statusFromReady, readySummary };
new file mode 100644
new file mode 100644
@@ -0,0 +1,104 @@
@@ -0,0 +1,104 @@
+'use strict';
+/**
+ * Release health (roadmap WS-P task 15): what open tabs say about each service's releases, read from the
+ * service's own /metrics on loopback during the ecosystem health poll (only for services whose
+ * /release.json names a metrics_url, i.e. that collect release-watch reports; openvibe-shared ≥ 1.18.0).
+ *
+ * sessions release_client_sessions{generation}: tabs heard from in the last 12 minutes, on the
+ * release the service serves (current) or an older one
+ * updates release_client_updates_total{outcome,reason}: prompts, in-place updates, reloads,
+ * deferrals and failures since the service's process started (so, for this release)
+ * drain how long after a release went live the last older tab left. A tab beats every 5
+ * minutes and the counts live in the service's process, which a deploy restarts, so
+ * "no older tab" means something only once the process has been up WARM_MS; a release
+ * whose first such reading is already 0 drained "within" that time.
+ *
+ * In memory: Network's own restart starts the history again (the status page says since when).
+ */
+const WARM_MS = 6 * 60 * 1000;
+const KEEP = 10;
+const SESSIONS_RE = /^release_client_sessions\{generation="(current|older)"\} (\d+(?:\.\d+)?)$/;
+const UPDATES_RE = /^release_client_updates_total\{outcome="([a-z]+)",reason="([a-z+-]+)"\} (\d+(?:\.\d+)?)$/;
+const OUTCOMES = ['prompted', 'applied', 'reloaded', 'deferred', 'failed'];
+
+/** The release-client series of a Prometheus text body: { sessions: { current, older } | null, updates: { outcome: { reason: n } } }. */
+function parseClientMetrics(text) {
+ let sessions = null;
+ const updates = {};
+ for (const line of String(text || '').split('\n')) {
+ if (!line.startsWith('release_client_')) continue;
+ let m = SESSIONS_RE.exec(line);
+ if (m) { sessions = sessions || { current: 0, older: 0 }; sessions[m[1]] = Math.round(Number(m[2])); continue; }
+ m = UPDATES_RE.exec(line);
+ if (m && OUTCOMES.includes(m[1])) (updates[m[1]] = updates[m[1]] || {})[m[2]] = Math.round(Number(m[3]));
+ }
+ return { sessions, updates };
+}
+
+const total = (byReason) => Object.values(byReason || {}).reduce((a, b) => a + b, 0);
+
+function createReleaseHealth({ now = () => Date.now(), warmMs = WARM_MS, keep = KEEP } = {}) {
+ const state = new Map();
+ const startedAt = new Date(now()).toISOString();
+ const iso = (t) => new Date(t).toISOString();
+
+ /** One poll's reading for a service: its release (id, released_at, booted_at) and parsed metrics, or null metrics. */
+ function observe(id, release, metrics) {
+ if (!release || !release.release) return;
+ let s = state.get(id);
+ if (!s || s.release !== release.release) {
+ const history = s ? [summaryOf(s, 'superseded'), ...s.history].slice(0, keep) : [];
+ const since = release.booted_at || release.released_at || iso(now());
+ s = { release: release.release, since, booted_at: release.booted_at || null, sessions: null, updates: {}, measured_at: null, peak_older: 0, drained_at: null, drain_s: null, within: false, history };
+ // First seen after its warm-up (Network restarted later): an older tab that left before now left unseen.
+ s.late = now() - Date.parse(since) >= warmMs;
+ state.set(id, s);
+ } else if (release.booted_at && release.booted_at !== s.booted_at) {
+ s.booted_at = release.booted_at; // restarted on the same release: its counts start again, and so does the warm-up
+ }
+ if (!metrics) return;
+ s.measured_at = iso(now());
+ s.updates = metrics.updates || {};
+ if (!metrics.sessions) return;
+ s.sessions = metrics.sessions;
+ s.peak_older = Math.max(s.peak_older, metrics.sessions.older);
+ const up = now() - Date.parse(s.booted_at || s.since);
+ if (!s.drained_at && up >= warmMs && metrics.sessions.older === 0) {
+ s.drained_at = iso(now());
+ const unseen = s.peak_older === 0; // no older tab at the first warm reading: it drained within the warm-up
+ s.drain_s = unseen && s.late ? null : Math.max(0, Math.round((now() - Date.parse(s.since)) / 1000));
+ s.within = unseen && !s.late;
+ }
+ }
+
+ function summaryOf(s, end) {
+ return { release: s.release, since: s.since, drained_at: s.drained_at, drain_s: s.drain_s, within: s.within, peak_older: s.peak_older, end: s.drained_at ? 'drained' : end };
+ }
+
+ /** The view for one service, or null when nothing was read for it. */
+ function view(id) {
+ const s = state.get(id);
+ if (!s) return null;
+ const up = now() - Date.parse(s.booted_at || s.since);
+ const drain = !s.sessions ? { state: 'unknown' }
+ : s.drained_at ? { state: 'drained', drained_at: s.drained_at, seconds: s.drain_s, within: s.within, ...(s.drain_s === null ? { seen_late: true } : {}) }
+ : up < warmMs ? { state: 'warming', ready_in_s: Math.round((warmMs - up) / 1000), older: s.sessions.older }
+ : { state: 'draining', older: s.sessions.older, for_s: Math.round((now() - Date.parse(s.since)) / 1000) };
+ const u = s.updates;
+ return {
+ release: s.release,
+ since: s.since,
+ measured_at: s.measured_at,
+ sessions: s.sessions,
+ counts: Object.fromEntries(OUTCOMES.map((o) => [o, total(u[o])])),
+ deferred: u.deferred || {},
+ failed: u.failed || {},
+ drain,
+ history: s.history,
+ };
+ }
+
+ return { observe, view, startedAt: () => startedAt, WARM_MS: warmMs };
+}
+
+module.exports = { createReleaseHealth, parseClientMetrics, WARM_MS };
@@ -12,6 +12,10 @@
@@ -12,6 +12,10 @@
* up | degraded | down | not-running | unknown and carries checked_at; a service not checked yet, or
* up | degraded | down | not-running | unknown and carries checked_at; a service not checked yet, or
* whose last check is stale, is 'unknown' — never an optimistic default.
* whose last check is stale, is 'unknown' — never an optimistic default.
*
*
+ * Each row also carries release_health (server/registry/release-health.js, WS-P task 15): what open tabs
+ * report for services that collect release-watch reports: tabs on the current and older releases, prompts,
+ * updates, deferrals and failures, and how long the last release took to drain.
+ *
* Each row also carries its exposure (server/registry/exposure.js): live (public), internal (loopback
* Each row also carries its exposure (server/registry/exposure.js): live (public), internal (loopback
* only), library, repository or placeholder. A service that is up on loopback while its public domain
* only), library, repository or placeholder. A service that is up on loopback while its public domain
* serves a placeholder reads "up (loopback only)" and has no public origin, never a bare "up".
* serves a placeholder reads "up (loopback only)" and has no public origin, never a bare "up".
@@ -77,11 +81,38 @@ function rows(ecosystem) {
@@ -77,11 +81,38 @@ function rows(ecosystem) {
main: (() => { const d = require('../registry/deploy-drift').current(m.id); return d ? { state: d.state, behind_by: d.behind_by ?? null, since: d.since || null } : null; })(),
main: (() => { const d = require('../registry/deploy-drift').current(m.id); return d ? { state: d.state, behind_by: d.behind_by ?? null, since: d.since || null } : null; })(),
release_error: r.release && !r.release.release ? r.release.error || 'unknown' : null,
release_error: r.release && !r.release.release ? r.release.error || 'unknown' : null,
ready: r.ready || null,
ready: r.ready || null,
+ release_health: typeof ecosystem.releaseHealth === 'function' ? ecosystem.releaseHealth(m.id) : null,
...(r.stale ? { stale: true, last_status: r.last_status } : {}),
...(r.stale ? { stale: true, last_status: r.last_status } : {}),
};
};
});
});
}
}
+const dur = (s) => (s == null ? '—' : s < 90 ? `${s} s` : s < 5400 ? `${Math.round(s / 60)} min` : `${(s / 3600).toFixed(1)} h`);
+const reasons = (o) => Object.entries(o || {}).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([k, v]) => `${k} ${v}`).join(', ');
+function drainCell(d) {
+ if (!d || d.state === 'unknown') return '<small>no reports yet</small>';
+ if (d.state === 'warming') return `warming up<small>${esc(d.older)} older tab(s) so far; reliable in ${esc(dur(d.ready_in_s))}</small>`;
+ if (d.state === 'draining') return `<b>${esc(d.older)}</b> older tab(s)<small>for ${esc(dur(d.for_s))}</small>`;
+ if (d.seen_late) return 'drained<small>before Network watched</small>';
+ return `drained ${d.within ? 'within' : 'in'} ${esc(dur(d.seconds))}`;
+}
+/** The release-health section: services whose tabs report (WS-P task 15). */
+function releaseHealthSection(list, since) {
+ const rowsOf = list.filter((r) => r.release_health);
+ if (!rowsOf.length) return '<p class="lede">No service has reported release health yet.</p>';
+ const tr = rowsOf.map((r) => {
+ const h = r.release_health; const s = h.sessions; const c = h.counts;
+ const mix = s ? (s.current + s.older ? `${s.current} / ${s.older}<small>${Math.round((100 * s.current) / (s.current + s.older))}% on current</small>` : '0 / 0') : '<small>—</small>';
+ return `<tr><td><b>${esc(r.name)}</b><small><code>${esc(h.release)}</code></small></td><td>${mix}</td><td>${drainCell(h.drain)}</td>
+<td>${esc(c.prompted)} · ${esc(c.applied)} · ${esc(c.reloaded)}</td><td>${esc(c.deferred)}${c.deferred ? `<small>${esc(reasons(h.deferred))}</small>` : ''}</td><td>${esc(c.failed)}${c.failed ? `<small>${esc(reasons(h.failed))}</small>` : ''}</td></tr>`;
+ }).join('\n');
+ return `<p class="lede">What open tabs report through release-watch: tabs heard from in the last 12 minutes on the release each service serves and on older ones, and since that release started, how many were prompted, updated in place or reloaded, deferred (and why) or failed. Drain is how long after a release went live the last older tab left; tabs report every 5 minutes, so it reads true only 6 minutes after a start. Watched since ${esc(since)}.</p>
+<table><thead><tr><th>Service</th><th>Tabs current / older</th><th>Drain</th><th>Prompted · applied · reloaded</th><th>Deferred</th><th>Failed</th></tr></thead>
+<tbody>
+${tr}
+</tbody></table>`;
+}
+
function summary(list) {
function summary(list) {
const counts = Object.fromEntries(STATES.map((s) => [s, 0]));
const counts = Object.fromEntries(STATES.map((s) => [s, 0]));
for (const r of list) counts[r.status]++;
for (const r of list) counts[r.status]++;
@@ -126,7 +157,7 @@ function proofSteps(rec) {
@@ -126,7 +157,7 @@ function proofSteps(rec) {
}
}
const lastRun = (rec) => `Last run <time datetime="${esc(rec.finished_at)}">${esc(rec.finished_at)}</time>: <b>${rec.ok ? 'passed' : 'failed'}</b> (${rec.steps.filter((st) => st.ok).length}/${rec.steps.length} steps).`;
const lastRun = (rec) => `Last run <time datetime="${esc(rec.finished_at)}">${esc(rec.finished_at)}</time>: <b>${rec.ok ? 'passed' : 'failed'}</b> (${rec.steps.filter((st) => st.ok).length}/${rec.steps.length} steps).`;
-function renderPage(list, slo, generatedAt, devPath = null, toolsJob = null) {
+function renderPage(list, slo, generatedAt, devPath = null, toolsJob = null, healthSince = null) {
const counts = summary(list);
const counts = summary(list);
const tr = list.map((r) => `<tr id="svc-${esc(r.id)}">
const tr = list.map((r) => `<tr id="svc-${esc(r.id)}">
<td><b>${esc(r.name)}</b><small>${esc(r.id)} · manifest: ${esc(r.manifest_status)}</small></td>
<td><b>${esc(r.name)}</b><small>${esc(r.id)} · manifest: ${esc(r.manifest_status)}</small></td>
@@ -168,6 +199,9 @@ ${require('openvibe-shared/frame').noscriptNav({ name: 'OpenVibe.Network', links
@@ -168,6 +199,9 @@ ${require('openvibe-shared/frame').noscriptNav({ name: 'OpenVibe.Network', links
${tr}
${tr}
</tbody>
</tbody>
</table>
</table>
+<section aria-labelledby="h-release-health"><h2 id="h-release-health">Release health</h2>
+${releaseHealthSection(list, healthSince || generatedAt)}
+</section>
<section aria-labelledby="h-devpath"><h2 id="h-devpath">Developer path</h2>
<section aria-labelledby="h-devpath"><h2 id="h-devpath">Developer path</h2>
${devPath ? `<p class="lede">Once a day a test developer account goes from sign-in to a sandbox project, a Media upload, an Events publish and pull, a credential rotation and revocation, and cleanup, through the public API and the SDK. ${lastRun(devPath)}</p>
${devPath ? `<p class="lede">Once a day a test developer account goes from sign-in to a sandbox project, a Media upload, an Events publish and pull, a credential rotation and revocation, and cleanup, through the public API and the SDK. ${lastRun(devPath)}</p>
${proofSteps(devPath)}` : '<p class="lede">The daily developer path has not reported here yet.</p>'}
${proofSteps(devPath)}` : '<p class="lede">The daily developer path has not reported here yet.</p>'}
@@ -202,6 +236,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {
@@ -202,6 +236,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {
summary: summary(list),
summary: summary(list),
exposure_states: exposure.STATES,
exposure_states: exposure.STATES,
exposure_summary: exposureSummary(list),
exposure_summary: exposureSummary(list),
+ release_health_since: typeof ecosystem.releaseHealthSince === 'function' ? ecosystem.releaseHealthSince() : null,
developer_path: loadDevPath(),
developer_path: loadDevPath(),
tools_job_proof: loadDevPath(TOOLSJOB_FILE),
tools_job_proof: loadDevPath(TOOLSJOB_FILE),
services: list,
services: list,
@@ -213,7 +248,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {
@@ -213,7 +248,7 @@ function createStatusRoutes({ ecosystem, now = () => new Date() }) {
r.get('/status', (_req, res) => {
r.get('/status', (_req, res) => {
const slo = { ...loadSlo(), pollSeconds: ecosystem.pollMs / 1000 };
const slo = { ...loadSlo(), pollSeconds: ecosystem.pollMs / 1000 };
res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', 'no-cache, max-age=0').set('X-Robots-Tag', 'noindex, nofollow');
res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', 'no-cache, max-age=0').set('X-Robots-Tag', 'noindex, nofollow');
- res.send(renderPage(rows(ecosystem), slo, now().toISOString(), loadDevPath(), loadDevPath(TOOLSJOB_FILE)));
+ res.send(renderPage(rows(ecosystem), slo, now().toISOString(), loadDevPath(), loadDevPath(TOOLSJOB_FILE), typeof ecosystem.releaseHealthSince === 'function' ? ecosystem.releaseHealthSince() : null));
});
});
return r;
return r;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,100 @@
@@ -0,0 +1,100 @@
+'use strict';
+// Release health (roadmap WS-P task 15): the ecosystem poll reads the release-client series from the
+// /metrics of services that collect release-watch reports, and /status shows tabs by generation,
+// prompts, updates, deferrals, failures and drain times. Drain reads true only after the warm-up (tabs
+// beat every 5 minutes into a process a deploy restarted); a release first seen late has no drain time.
+// node test/release-health.test.js
+const assert = require('assert');
+const http = require('http');
+const express = require('express');
+const { createReleaseHealth, parseClientMetrics, WARM_MS } = require('../server/registry/release-health');
+
+const METRICS = `# HELP release_client_sessions Open tabs
+# TYPE release_client_sessions gauge
+release_client_sessions{generation="current"} 7
+release_client_sessions{generation="older"} 2
+release_client_updates_total{outcome="prompted",reason="optional"} 3
+release_client_updates_total{outcome="applied",reason="content+style"} 4
+release_client_updates_total{outcome="deferred",reason="typing"} 2
+release_client_updates_total{outcome="deferred",reason="media"} 1
+release_client_updates_total{outcome="failed",reason="style"} 1
+release_client_updates_total{outcome="bogus",reason="x"} 9
+http_requests_total{route="/"} 12
+`;
+
+(async () => {
+ // ── Parsing ──
+ const p = parseClientMetrics(METRICS);
+ assert.deepStrictEqual(p.sessions, { current: 7, older: 2 });
+ assert.deepStrictEqual(p.updates, { prompted: { optional: 3 }, applied: { 'content+style': 4 }, deferred: { typing: 2, media: 1 }, failed: { style: 1 } }, 'unknown outcomes and other series are ignored');
+ assert.deepStrictEqual(parseClientMetrics('http_requests_total 1\n'), { sessions: null, updates: {} });
+
+ // ── Drain ──
+ const t0 = Date.parse('2026-09-26T10:00:00Z');
+ let t = t0;
+ const iso = (x) => new Date(x).toISOString();
+ const rh = createReleaseHealth({ now: () => t });
+ const r1 = { release: 'aaaaaaa', booted_at: iso(t0) };
+ const reading = (current, older) => ({ sessions: { current, older }, updates: {} });
+ t = t0 + 60e3; rh.observe('live', r1, reading(5, 3));
+ assert.strictEqual(rh.view('live').drain.state, 'warming', 'a fresh process has not heard from every tab');
+ t = t0 + WARM_MS + 60e3; rh.observe('live', r1, reading(8, 2));
+ assert.deepStrictEqual([rh.view('live').drain.state, rh.view('live').drain.older], ['draining', 2]);
+ t = t0 + 9 * 60e3; rh.observe('live', r1, reading(10, 0));
+ assert.deepStrictEqual(rh.view('live').drain, { state: 'drained', drained_at: iso(t), seconds: 540, within: false });
+ t = t0 + 10 * 60e3; rh.observe('live', r1, reading(10, 1));
+ assert.strictEqual(rh.view('live').drain.seconds, 540, 'drained once, not again');
+
+ const r2 = { release: 'bbbbbbb', booted_at: iso(t0 + 10 * 60e3) };
+ t = t0 + 11 * 60e3; rh.observe('live', r2, reading(0, 0));
+ assert.strictEqual(rh.view('live').history[0].release, 'aaaaaaa', 'the last release goes to history');
+ assert.strictEqual(rh.view('live').history[0].end, 'drained');
+ t = t0 + 17 * 60e3; rh.observe('live', r2, reading(9, 0));
+ assert.deepStrictEqual(rh.view('live').drain, { state: 'drained', drained_at: iso(t), seconds: 420, within: true }, 'no older tab by the first warm reading: within the warm-up');
+
+ // Restarted on the same release: the counts start again, so does the warm-up.
+ t = t0 + 20 * 60e3; rh.observe('chat', { release: 'ccccccc', booted_at: iso(t0) }, reading(3, 1));
+ assert.strictEqual(rh.view('chat').drain.state, 'draining');
+ rh.observe('chat', { release: 'ccccccc', booted_at: iso(t) }, reading(0, 0));
+ assert.strictEqual(rh.view('chat').drain.state, 'warming');
+
+ // First seen long after its start (Network restarted): no drain time is made up.
+ const late = createReleaseHealth({ now: () => t0 + 3600e3 });
+ late.observe('tools', { release: 'ddddddd', booted_at: iso(t0) }, reading(4, 0));
+ assert.deepStrictEqual(late.view('tools').drain, { state: 'drained', drained_at: iso(t0 + 3600e3), seconds: null, within: false, seen_late: true });
+ late.observe('wiki', { release: 'eeeeeee', booted_at: iso(t0) }, null);
+ assert.deepStrictEqual([late.view('wiki').drain.state, late.view('wiki').sessions], ['unknown', null], 'a service that does not collect has no readings');
+
+ // ── The poll reads /metrics only where /release.json names a metrics_url; /status shows it ──
+ const { createEcosystemRegistry } = require('../server/registry/ecosystem');
+ const asked = [];
+ const res = (status, body, text) => ({ ok: status < 400, status, json: async () => body, text: async () => text ?? JSON.stringify(body) });
+ const fetchImpl = async (url) => {
+ const u = new URL(url);
+ asked.push(`${u.port}${u.pathname}`);
+ if (u.pathname === '/release.json') return res(200, { release: `rel${u.port}`, booted_at: iso(Date.now() - 3600e3), ...(u.port === '3000' ? { metrics_url: '/release-metrics' } : {}) });
+ if (u.pathname === '/metrics') return res(200, null, METRICS);
+ return res(200, { ready: true, status: 'ready', checks: {} });
+ };
+ const eco = createEcosystemRegistry({ issuer: 'https://openvibe.network', fetchImpl, internalOverrides: { live: 'http://127.0.0.1:3000' } });
+ await eco.pollAll();
+ assert.ok(asked.includes('3000/metrics'), 'a collecting service is read');
+ assert.ok(!asked.some((a) => a.endsWith('/metrics') && !a.startsWith('3000')), 'no /metrics for services without a metrics_url');
+ const view = eco.releaseHealth('live');
+ assert.deepStrictEqual([view.sessions, view.counts], [{ current: 7, older: 2 }, { prompted: 3, applied: 4, reloaded: 0, deferred: 3, failed: 1 }]);
+
+ const app = express();
+ app.use(require('../server/status/routes').createStatusRoutes({ ecosystem: eco }));
+ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const json = await (await fetch(`${base}/api/v1/status`)).json();
+ assert.strictEqual(json.services.find((s) => s.id === 'live').release_health.sessions.current, 7);
+ assert.ok(json.release_health_since);
+ const html = await (await fetch(`${base}/status`)).text();
+ assert.ok(html.includes('id="h-release-health"'), 'the page has the section');
+ assert.match(html, /7 \/ 2<small>78% on current<\/small>/, 'tabs by generation');
+ assert.match(html, /typing 2, media 1/, 'deferral reasons');
+ assert.match(html, /drained<small>before Network watched<\/small>|older tab/, 'a drain state');
+ server.close();
+ console.log('release health: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });