openvibers/openvibe.tips Commits
Track O: /metrics for direct loopback callers and a truthful /api/ready (openvibe-shared v1.3.0)
Changes
@@@ -232,7 +232,9 @@ This repository does not make the product real, and the domain keeps its placehoexist here (plan §12.12):1. an owning runtime with health/readiness endpoints and observability — ✔ `/api/health`, `/api/ready`-- (DB + Network key), outbox status in health; no `/metrics` yet;++ (openvibe-shared/ready: the database required; Network key, Billing and Events optional, so++ their loss degrades rather than fails), `/metrics` for direct loopback callers (golden signals by++ route template, pending effects and overlay deliveries, outbox backlog);2. canonical identity/auth integration (Network subjects, scoped service principals) — ✔; the `tips`principal is provisioned on the host;3. server-rendered public routes useful without JavaScript — ✔;@@@ -7,6 +7,6 @@"adr": "ADR-012","authoritative": false,"deployment": "Deployed internally, not launched: openvibe-ovh /opt/openvibe.tips (release 435f32b), unit openvibe-tips, 127.0.0.1:4610 loopback only; https://openvibe.tips still serves the OpenVibe.Sites placeholder. Principal tips; Events subscription billing.transaction.* -> /internal/events; TIPS_CHAT_ADAPTER=none. Pins openvibe-contracts v0.15.0, openvibe-sdk v0.4.0, openvibe-shared v1.2.1.",-- "notes": "Runtime, API, SSR pages, overlays, Billing settlement consumer, events outbox, Live import and reconciliation are built and tested against stubs. Capabilities (17 tips.*) and the service manifest are registered in OpenVibe.Contracts v0.15.0; no tips.* event payload schemas yet. Production database is empty (0 profiles, interactions, goals, overlay tokens); the Live import (scripts/import-live.js) has never been run; 0 billing events have ever arrived because Billing runs in shadow. Live's /internal/tips/deliveries route is deployed (Live f11f809) but never called. The restore drill passed on the host 2026-09-23 05:23 UTC. Not done: /metrics (404) and Billing/Events checks in /api/ready; privacy/moderation controls on the pages; an independent security review; the openvibe.tips launch. Live stays the place tips happen until the Billing cutover (blocked on the owner re-pointing the PowerChat webhook) and the launch rule in README.md holds.",++ "notes": "Runtime, API, SSR pages, overlays, Billing settlement consumer, events outbox, Live import and reconciliation are built and tested against stubs. Capabilities (17 tips.*) and the service manifest are registered in OpenVibe.Contracts v0.15.0; no tips.* event payload schemas yet. Production database is empty (0 profiles, interactions, goals, overlay tokens); the Live import (scripts/import-live.js) has never been run; 0 billing events have ever arrived because Billing runs in shadow. Live's /internal/tips/deliveries route is deployed (Live f11f809) but never called. The restore drill passed on the host 2026-09-23 05:23 UTC. Built since (not deployed yet): /metrics (loopback only) and the Billing/Events checks in /api/ready. Not done: privacy/moderation controls on the pages; an independent security review; the openvibe.tips launch. Live stays the place tips happen until the Billing cutover (blocked on the owner re-pointing the PowerChat webhook) and the launch rule in README.md holds.","updated": "2026-09-23"}@@@ -6,7 +6,8 @@## unit openvibe-tips) renders every page, serves the API and the overlay streams. nginx terminates## TLS, redirects www → apex, rate-limits auth and API, keeps overlay streams unbuffered, keeps## overlay tokens out of the access log, and never exposes /internal/ (OpenVibe.Events calls--## 127.0.0.1:4610 directly).++## 127.0.0.1:4610 directly) or /metrics (host-local scrapers only; Node also refuses any request++## carrying X-Forwarded-For).## ═══════════════════════════════════════════════════════════════limit_req_zone $binary_remote_addr zone=ovtips_api:10m rate=10r/s;@@@ -55,6 +56,8 @@ server {# Service-to-service only (Events → 127.0.0.1:4610/internal/events). Never public.location /internal/ { return 404; }++ # Prometheus text for host-local scrapers only (http://127.0.0.1:4610/metrics).++ location = /metrics { return 404; }location /auth/ {limit_req zone=ovtips_auth burst=10 nodelay;@@@ -16,7 +16,7 @@"jsonwebtoken": "^9.0.2","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1"++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"},"engines": {"node": ">=22.0.0"@@@ -1008,9 +1008,9 @@}},"node_modules/openvibe-shared": {-- "version": "1.2.1",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",-- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",++ "version": "1.3.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",++ "integrity": "sha512-w8yl/5+9eend8I2O/DzAJnEAL2N3GTUY7c7pzK+MLJuLAGD8FmJJQ9gwbgyrq70Snjt3ebrg3TlpORhRAp2kgA==","license": "MIT","dependencies": {"jsonwebtoken": "^9.0.0"@@@ -23,7 +23,7 @@"jsonwebtoken": "^9.0.2","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1"++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"},"engines": {"node": ">=22.0.0"@@@ -4,7 +4,7 @@* OpenVibe.Tips — creator support: tips, goals, paid messages, TTS and media requests, overlays.* Express app factory; server/index.js listens, tests build their own instance.*-- * GET /api/health, /api/ready, /release.json++ * GET /api/health, /api/ready, /release.json, /metrics (direct loopback callers only)* /api/v1/* API (service tokens + user tokens, see api/v1.js)* POST /internal/events Billing settlement events from OpenVibe.Events (signed webhook + inbox)* /auth/* Network SSO session for the pages@@@ -31,6 +31,7 @@ const { createSessionRoutes } = require('./web/session');const { createWebRoutes } = require('./web/routes');const { createLayout, assetVersion } = require('./web/layout');const pages = require('./web/pages');++const { createTipsReadiness, registerTipsGauges } = require('./observability');const VERSION = require('../package.json').version;const PUBLIC_DIR = path.join(__dirname, '..', 'public');@@@ -54,6 +55,12 @@ function createApp(opts = {}) {const app = express();app.disable('x-powered-by');app.set('trust proxy', config.trustProxy);++ // HTTP golden signals by route template, process metrics, release_info; GET /metrics answers++ // direct loopback callers only (Track O). An overlay's SSE stream is a session, not a request.++ const metrics = require('openvibe-shared/metrics').instrument(app, {++ service: 'tips', release: release.release, skip: (req) => /^\/overlay\/[^/]+\/events$/.test(req.path),++ });++ registerTipsGauges(metrics.registry, { db, outbox, now });app.use(http.middleware());app.use((req, res, next) => {res.setHeader('X-Content-Type-Options', 'nosniff');@@@ -71,13 +78,10 @@ function createApp(opts = {}) {app.get('/api/health', (req, res) => res.json({ok: true, service: 'tips', version: VERSION, chat_adapter: config.chat.adapter, events: outbox.status(),}));-- app.get('/api/ready', (req, res) => {-- const problems = [];-- try { db.prepare('SELECT 1 FROM settings WHERE id = 1').get(); } catch (e) { problems.push(`database: ${e.message}`); }-- if (!keys.get()) problems.push('Network public key not loaded');-- if (problems.length) return http.sendProblem(res, 503, 'service.not_ready', { detail: problems.join('; '), ctx: req.ov });-- return res.json({ ready: true });-- });++ // Readiness (openvibe-shared/ready): 503 only when the database fails; Network key, Billing and++ // Events are optional and degrade it (see observability.js).++ const readiness = createTipsReadiness({ db, keys, config, outbox, release: release.release, fetchImpl });++ app.get('/api/ready', readiness.handler);app.get('/release.json', release.handler);const consumer = consumerRouter({ domain, config, log });@@@ -113,7 +117,7 @@ function createApp(opts = {}) {return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));});-- Object.assign(app.locals, { config, db, domain, keys, outbox, adapters, billing, consumer });++ Object.assign(app.locals, { config, db, domain, keys, outbox, adapters, billing, consumer, metrics });return app;}@@@ -203,6 +203,7 @@ CREATE TABLE IF NOT EXISTS overlay_deliveries (created_at TEXT NOT NULL);CREATE INDEX IF NOT EXISTS idx_ovd_creator ON overlay_deliveries (creator_subject, seq);++CREATE INDEX IF NOT EXISTS idx_ovd_pending ON overlay_deliveries (seq) WHERE status = 'pending';CREATE TABLE IF NOT EXISTS interaction_effects (id INTEGER PRIMARY KEY AUTOINCREMENT,nnew file mode 100644@@@ -0,0 +1,81 @@++'use strict';++/**++ * Track O: truthful readiness for GET /api/ready and the Tips gauges on GET /metrics++ * (openvibe-shared/ready and openvibe-shared/metrics).++ *++ * db required a real query on Tips' SQLite (the settings row is there and answers)++ * network_jwks optional the Network signing key has loaded. Without it pages, overlays and the++ * signed Billing webhook still work, but no service or user token can be++ * verified (API calls and sign-in fail), so it degrades rather than fails++ * billing optional Billing answers /api/health. Without it checkouts and transfers wait++ * (due transfers retry once it is back); settled tips still deliver++ * events optional OpenVibe.Events answers /api/health. Without it (or with the relay off:++ * EVENTS_URL or the client secret unset) tips.* events wait in the outbox++ *++ * Gauges: pending deliveries (chat/TTS/media effects waiting for their adapter, overlay alerts no++ * overlay has shown yet) and the events outbox backlog. Counts only, never subjects.++ */++const { createReadiness } = require('openvibe-shared/ready');++++const PING_TTL_MS = 15_000;++// interaction_effects.effect (the CHECK constraint in db.js): each reported, 0 when none waits.++const EFFECTS = ['chat_line', 'paid_message', 'tts', 'media_request', 'overlay_alert'];++++function probe(url, fetchImpl) {++ return async () => {++ const res = await fetchImpl(url, { signal: AbortSignal.timeout(2000), headers: { Accept: 'application/json' } });++ try { await res.body?.cancel(); } catch { /* not needed */ }++ return res.ok ? { ok: true, detail: { http_status: res.status } } : { ok: false, error: `answered HTTP ${res.status}`, detail: { http_status: res.status } };++ };++}++++function createTipsReadiness({ db, keys, config, outbox, release = null, fetchImpl = globalThis.fetch }) {++ const events = outbox.enabled++ ? probe(`${config.events.url}/api/health`, fetchImpl)++ : () => 'events relay off (EVENTS_URL or OV_OAUTH_CLIENT_SECRET unset): tips.* events wait in the outbox';++ return createReadiness({++ service: 'tips',++ release,++ checks: [++ { name: 'db', required: true, check: () => (db.prepare('SELECT id FROM settings WHERE id = 1').get() ? true : 'database has no settings row') },++ { name: 'network_jwks', required: false, check: () => (keys.get() ? true : 'Network signing key not loaded yet: tokens cannot be verified') },++ { name: 'billing', required: false, cacheMs: PING_TTL_MS, timeoutMs: 2500, check: probe(`${config.billing.url}/api/health`, fetchImpl) },++ { name: 'events', required: false, cacheMs: outbox.enabled ? PING_TTL_MS : 0, timeoutMs: 2500, check: events },++ ],++ details: (body) => ({++ events_outbox: body.checks.db.status === 'ok' ? outbox.status() : null,++ billing_events_accepted: config.events.webhookSecrets.length > 0,++ pending_deliveries: body.checks.db.status === 'ok' ? pendingDeliveries(db) : null,++ }),++ });++}++++function pendingDeliveries(db) {++ const effects = {};++ for (const r of db.prepare("SELECT effect, COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' GROUP BY effect").all()) effects[r.effect] = r.n;++ const overlay = db.prepare("SELECT COUNT(*) AS n FROM overlay_deliveries WHERE status = 'pending'").get().n;++ return { effects, overlay };++}++++/** Tips gauges on the openvibe-shared/metrics registry. */++function registerTipsGauges(registry, { db, outbox, now = () => Date.now() }) {++ registry.gauge({++ name: 'tips_effects_pending', help: 'Chat, TTS, media and alert effects waiting for their delivery adapter, by effect', labelNames: ['effect'],++ collect: () => {++ const n = Object.fromEntries(EFFECTS.map((e) => [e, 0]));++ for (const r of db.prepare("SELECT effect, COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' GROUP BY effect").all()) n[r.effect] = r.n;++ return Object.entries(n).map(([effect, value]) => ({ labels: { effect }, value }));++ },++ });++ registry.gauge({++ name: 'tips_effects_due', help: 'Queued effects whose next attempt is due now (a growing number means the worker is behind)',++ collect: () => db.prepare("SELECT COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' AND next_attempt_at <= ?").get(now()).n,++ });++ registry.gauge({++ name: 'tips_overlay_deliveries_pending', help: 'Overlay alerts and goal updates no overlay has shown yet',++ collect: () => db.prepare("SELECT COUNT(*) AS n FROM overlay_deliveries WHERE status = 'pending'").get().n,++ });++ registry.gauge({ name: 'tips_outbox_pending', help: 'tips.* events waiting in the outbox', collect: () => outbox.status().pending });++}++++module.exports = { createTipsReadiness, registerTipsGauges };@@@ -7,6 +7,7 @@* matter here (idempotency keys, funds, self-dealing, capability + audience checks),* and the billing.transaction.* envelopes it would publish (billing.events)* startEvents() POST /api/v1/events recording what Tips' outbox relays++ * (Billing and Events also answer GET /api/health, which /api/ready probes)* startLive() POST /internal/tips/deliveries recording chat deliveries (the live-chat adapter)*/const http = require('http');@@@ -125,6 +126,7 @@ async function startBilling(network) {const server = http.createServer(async (req, res) => {const raw = await readBody(req);++ if (req.url === '/api/health') return send(res, state.down ? 503 : 200, state.down ? { code: 'billing.unavailable' } : { ok: true, service: 'billing' });const body = raw ? JSON.parse(raw) : {};calls.push({ method: req.method, url: req.url, body, key: req.headers['idempotency-key'] || null });if (state.down) return send(res, 503, { code: 'billing.unavailable' });@@@ -178,6 +180,7 @@ async function startEvents() {const tokens = [];const server = http.createServer(async (req, res) => {const raw = await readBody(req);++ if (req.url === '/api/health') return send(res, 200, { status: 'ok', service: 'openvibe-events' });tokens.push(req.headers.authorization);if (req.url === '/api/v1/events' && req.method === 'POST') {const b = JSON.parse(raw);nnew file mode 100644@@@ -0,0 +1,97 @@++'use strict';++// Track O: GET /metrics answers direct loopback callers only, labels requests by route template and++// carries the Tips gauges; /api/ready is 503 only when the database fails, and a missing Billing,++// Events or Network key degrades it instead.++const assert = require('assert');++const nodeHttp = require('http');++const { boot, check, done } = require('./helpers/app');++++function get(base, p, headers = {}) {++ return new Promise((resolve, reject) => nodeHttp.get(base + p, { headers }, (res) => {++ let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b }));++ }).on('error', reject));++}++++(async () => {++ const t = await boot();++ await t.creator('alex');++++ await check('/api/ready: every check reports; db is the only required one', async () => {++ const r = await t.call('GET', '/api/ready', { token: null });++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json.status, 'ready');++ assert.strictEqual(r.json.service, 'tips');++ assert.deepStrictEqual(Object.keys(r.json.checks), ['db', 'network_jwks', 'billing', 'events']);++ for (const [name, c] of Object.entries(r.json.checks)) {++ assert.strictEqual(c.status, 'ok', `${name}: ${c.error}`);++ assert.strictEqual(c.required, name === 'db', name);++ assert.strictEqual(typeof c.latency_ms, 'number');++ assert.ok(Date.parse(c.checked_at));++ }++ assert.ok(r.json.events_outbox && r.json.pending_deliveries);++ assert.strictEqual(r.headers.get('cache-control'), 'no-store');++ });++++ await check('/metrics: 404 through a proxy, golden signals by route template and Tips gauges direct', async () => {++ await t.call('GET', '/api/v1/profiles/alex', { token: null });++ await t.call('GET', '/api/no/such/12345', { token: null });++ for (const h of [{ 'X-Forwarded-For': '203.0.113.7' }, { 'X-Real-IP': '203.0.113.7' }, { 'CF-Connecting-IP': '203.0.113.7' }]) {++ const m = await get(t.base, '/metrics', h);++ assert.strictEqual(m.status, 404, JSON.stringify(h));++ assert.ok(!m.body.includes('http_requests_total'));++ }++ const m = await get(t.base, '/metrics');++ assert.strictEqual(m.status, 200);++ const text = m.body;++ assert.ok(/http_requests_total\{method="GET",route="\/api\/v1\/profiles\/:creator",status_class="2xx"\} 1\n/.test(text), 'route template, not the handle');++ assert.ok(/http_requests_total\{method="GET",route="unmatched",status_class="4xx"\} 1\n/.test(text));++ assert.ok(!/route="[^"]*(alex|12345)/.test(text), 'no concrete path in any label');++ assert.ok(/http_request_duration_seconds_bucket\{method="GET",route="\/api\/ready",le="\+Inf"\} 1\n/.test(text));++ assert.ok(/\nhttp_requests_in_flight \d+\n/.test(text));++ assert.ok(/\nprocess_resident_memory_bytes \d+\n/.test(text));++ assert.ok(/release_info\{service="tips",release="[^"]+"\} 1\n/.test(text));++ for (const e of ['chat_line', 'paid_message', 'tts', 'media_request', 'overlay_alert']) assert.ok(new RegExp(`tips_effects_pending\\{effect="${e}"\\} 0\\n`).test(text), e);++ assert.ok(/\ntips_effects_due 0\n/.test(text));++ assert.ok(/\ntips_overlay_deliveries_pending 0\n/.test(text));++ assert.ok(/\ntips_outbox_pending \d+\n/.test(text));++ });++++ await check('an overlay SSE stream is not a request sample', async () => {++ const reader = await t.sse('/overlay/not-a-token/events');++ reader.close();++ const text = (await get(t.base, '/metrics')).body;++ assert.ok(!text.includes('/overlay/:token/events') && !text.includes('not-a-token'));++ });++++ await t.close();++++ await check('Billing down and the events relay off: still ready (200), degraded, and says why', async () => {++ const d = await boot({ env: { EVENTS_URL: '' } });++ d.billing.state.down = true;++ const r = await d.call('GET', '/api/ready', { token: null });++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json.ready, true);++ assert.strictEqual(r.json.status, 'degraded');++ assert.deepStrictEqual(r.json.degraded, ['billing', 'events']);++ assert.strictEqual(r.json.checks.billing.error, 'answered HTTP 503');++ assert.match(r.json.checks.events.error, /events relay off/);++ await d.close();++ });++++ await check('a broken database makes the service unready (503); /metrics still answers', async () => {++ const d = await boot();++ d.db.close();++ const r = await d.call('GET', '/api/ready', { token: null });++ assert.strictEqual(r.status, 503, r.text);++ assert.strictEqual(r.json.ready, false);++ assert.strictEqual(r.json.status, 'not_ready');++ assert.deepStrictEqual(r.json.failed, ['db']);++ assert.strictEqual(r.json.pending_deliveries, null);++ const m = await get(d.base, '/metrics');++ assert.strictEqual(m.status, 200);++ assert.ok(!/\ntips_overlay_deliveries_pending \d/.test(m.body), 'a gauge that cannot be read is left out, not invented');++ try { await d.close(); } catch { /* the database is already closed */ }++ });++++ done();++})();
@@ -232,7 +232,9 @@ This repository does not make the product real, and the domain keeps its placeho
@@ -232,7 +232,9 @@ This repository does not make the product real, and the domain keeps its placeho
exist here (plan §12.12):
exist here (plan §12.12):
1. an owning runtime with health/readiness endpoints and observability — ✔ `/api/health`, `/api/ready`
1. an owning runtime with health/readiness endpoints and observability — ✔ `/api/health`, `/api/ready`
- (DB + Network key), outbox status in health; no `/metrics` yet;
+ (openvibe-shared/ready: the database required; Network key, Billing and Events optional, so
+ their loss degrades rather than fails), `/metrics` for direct loopback callers (golden signals by
+ route template, pending effects and overlay deliveries, outbox backlog);
2. canonical identity/auth integration (Network subjects, scoped service principals) — ✔; the `tips`
2. canonical identity/auth integration (Network subjects, scoped service principals) — ✔; the `tips`
principal is provisioned on the host;
principal is provisioned on the host;
3. server-rendered public routes useful without JavaScript — ✔;
3. server-rendered public routes useful without JavaScript — ✔;
@@ -7,6 +7,6 @@
@@ -7,6 +7,6 @@
"adr": "ADR-012",
"adr": "ADR-012",
"authoritative": false,
"authoritative": false,
"deployment": "Deployed internally, not launched: openvibe-ovh /opt/openvibe.tips (release 435f32b), unit openvibe-tips, 127.0.0.1:4610 loopback only; https://openvibe.tips still serves the OpenVibe.Sites placeholder. Principal tips; Events subscription billing.transaction.* -> /internal/events; TIPS_CHAT_ADAPTER=none. Pins openvibe-contracts v0.15.0, openvibe-sdk v0.4.0, openvibe-shared v1.2.1.",
"deployment": "Deployed internally, not launched: openvibe-ovh /opt/openvibe.tips (release 435f32b), unit openvibe-tips, 127.0.0.1:4610 loopback only; https://openvibe.tips still serves the OpenVibe.Sites placeholder. Principal tips; Events subscription billing.transaction.* -> /internal/events; TIPS_CHAT_ADAPTER=none. Pins openvibe-contracts v0.15.0, openvibe-sdk v0.4.0, openvibe-shared v1.2.1.",
- "notes": "Runtime, API, SSR pages, overlays, Billing settlement consumer, events outbox, Live import and reconciliation are built and tested against stubs. Capabilities (17 tips.*) and the service manifest are registered in OpenVibe.Contracts v0.15.0; no tips.* event payload schemas yet. Production database is empty (0 profiles, interactions, goals, overlay tokens); the Live import (scripts/import-live.js) has never been run; 0 billing events have ever arrived because Billing runs in shadow. Live's /internal/tips/deliveries route is deployed (Live f11f809) but never called. The restore drill passed on the host 2026-09-23 05:23 UTC. Not done: /metrics (404) and Billing/Events checks in /api/ready; privacy/moderation controls on the pages; an independent security review; the openvibe.tips launch. Live stays the place tips happen until the Billing cutover (blocked on the owner re-pointing the PowerChat webhook) and the launch rule in README.md holds.",
+ "notes": "Runtime, API, SSR pages, overlays, Billing settlement consumer, events outbox, Live import and reconciliation are built and tested against stubs. Capabilities (17 tips.*) and the service manifest are registered in OpenVibe.Contracts v0.15.0; no tips.* event payload schemas yet. Production database is empty (0 profiles, interactions, goals, overlay tokens); the Live import (scripts/import-live.js) has never been run; 0 billing events have ever arrived because Billing runs in shadow. Live's /internal/tips/deliveries route is deployed (Live f11f809) but never called. The restore drill passed on the host 2026-09-23 05:23 UTC. Built since (not deployed yet): /metrics (loopback only) and the Billing/Events checks in /api/ready. Not done: privacy/moderation controls on the pages; an independent security review; the openvibe.tips launch. Live stays the place tips happen until the Billing cutover (blocked on the owner re-pointing the PowerChat webhook) and the launch rule in README.md holds.",
"updated": "2026-09-23"
"updated": "2026-09-23"
}
}
@@ -6,7 +6,8 @@
@@ -6,7 +6,8 @@
## unit openvibe-tips) renders every page, serves the API and the overlay streams. nginx terminates
## unit openvibe-tips) renders every page, serves the API and the overlay streams. nginx terminates
## TLS, redirects www → apex, rate-limits auth and API, keeps overlay streams unbuffered, keeps
## TLS, redirects www → apex, rate-limits auth and API, keeps overlay streams unbuffered, keeps
## overlay tokens out of the access log, and never exposes /internal/ (OpenVibe.Events calls
## overlay tokens out of the access log, and never exposes /internal/ (OpenVibe.Events calls
-## 127.0.0.1:4610 directly).
+## 127.0.0.1:4610 directly) or /metrics (host-local scrapers only; Node also refuses any request
+## carrying X-Forwarded-For).
## ═══════════════════════════════════════════════════════════════
## ═══════════════════════════════════════════════════════════════
limit_req_zone $binary_remote_addr zone=ovtips_api:10m rate=10r/s;
limit_req_zone $binary_remote_addr zone=ovtips_api:10m rate=10r/s;
@@ -55,6 +56,8 @@ server {
@@ -55,6 +56,8 @@ server {
# Service-to-service only (Events → 127.0.0.1:4610/internal/events). Never public.
# Service-to-service only (Events → 127.0.0.1:4610/internal/events). Never public.
location /internal/ { return 404; }
location /internal/ { return 404; }
+ # Prometheus text for host-local scrapers only (http://127.0.0.1:4610/metrics).
+ location = /metrics { return 404; }
location /auth/ {
location /auth/ {
limit_req zone=ovtips_auth burst=10 nodelay;
limit_req zone=ovtips_auth burst=10 nodelay;
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "^9.0.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1"
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
},
},
"engines": {
"engines": {
"node": ">=22.0.0"
"node": ">=22.0.0"
@@ -1008,9 +1008,9 @@
@@ -1008,9 +1008,9 @@
}
}
},
},
"node_modules/openvibe-shared": {
"node_modules/openvibe-shared": {
- "version": "1.2.1",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1",
- "integrity": "sha512-t8EHgju5/H2VqkM89LzdCKxBogJ8xilxmvZVJWQIYaGPKe7vMe4yrITlJi/Sy98phsffeFRogpaukDuO6QlDUw==",
+ "version": "1.3.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
+ "integrity": "sha512-w8yl/5+9eend8I2O/DzAJnEAL2N3GTUY7c7pzK+MLJuLAGD8FmJJQ9gwbgyrq70Snjt3ebrg3TlpORhRAp2kgA==",
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
@@ -23,7 +23,7 @@
@@ -23,7 +23,7 @@
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "^9.0.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.15.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.4.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.2.1"
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
},
},
"engines": {
"engines": {
"node": ">=22.0.0"
"node": ">=22.0.0"
@@ -4,7 +4,7 @@
@@ -4,7 +4,7 @@
* OpenVibe.Tips — creator support: tips, goals, paid messages, TTS and media requests, overlays.
* OpenVibe.Tips — creator support: tips, goals, paid messages, TTS and media requests, overlays.
* Express app factory; server/index.js listens, tests build their own instance.
* Express app factory; server/index.js listens, tests build their own instance.
*
*
- * GET /api/health, /api/ready, /release.json
+ * GET /api/health, /api/ready, /release.json, /metrics (direct loopback callers only)
* /api/v1/* API (service tokens + user tokens, see api/v1.js)
* /api/v1/* API (service tokens + user tokens, see api/v1.js)
* POST /internal/events Billing settlement events from OpenVibe.Events (signed webhook + inbox)
* POST /internal/events Billing settlement events from OpenVibe.Events (signed webhook + inbox)
* /auth/* Network SSO session for the pages
* /auth/* Network SSO session for the pages
@@ -31,6 +31,7 @@ const { createSessionRoutes } = require('./web/session');
@@ -31,6 +31,7 @@ const { createSessionRoutes } = require('./web/session');
const { createWebRoutes } = require('./web/routes');
const { createWebRoutes } = require('./web/routes');
const { createLayout, assetVersion } = require('./web/layout');
const { createLayout, assetVersion } = require('./web/layout');
const pages = require('./web/pages');
const pages = require('./web/pages');
+const { createTipsReadiness, registerTipsGauges } = require('./observability');
const VERSION = require('../package.json').version;
const VERSION = require('../package.json').version;
const PUBLIC_DIR = path.join(__dirname, '..', 'public');
const PUBLIC_DIR = path.join(__dirname, '..', 'public');
@@ -54,6 +55,12 @@ function createApp(opts = {}) {
@@ -54,6 +55,12 @@ function createApp(opts = {}) {
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
app.set('trust proxy', config.trustProxy);
app.set('trust proxy', config.trustProxy);
+ // HTTP golden signals by route template, process metrics, release_info; GET /metrics answers
+ // direct loopback callers only (Track O). An overlay's SSE stream is a session, not a request.
+ const metrics = require('openvibe-shared/metrics').instrument(app, {
+ service: 'tips', release: release.release, skip: (req) => /^\/overlay\/[^/]+\/events$/.test(req.path),
+ });
+ registerTipsGauges(metrics.registry, { db, outbox, now });
app.use(http.middleware());
app.use(http.middleware());
app.use((req, res, next) => {
app.use((req, res, next) => {
res.setHeader('X-Content-Type-Options', 'nosniff');
res.setHeader('X-Content-Type-Options', 'nosniff');
@@ -71,13 +78,10 @@ function createApp(opts = {}) {
@@ -71,13 +78,10 @@ function createApp(opts = {}) {
app.get('/api/health', (req, res) => res.json({
app.get('/api/health', (req, res) => res.json({
ok: true, service: 'tips', version: VERSION, chat_adapter: config.chat.adapter, events: outbox.status(),
ok: true, service: 'tips', version: VERSION, chat_adapter: config.chat.adapter, events: outbox.status(),
}));
}));
- app.get('/api/ready', (req, res) => {
- const problems = [];
- try { db.prepare('SELECT 1 FROM settings WHERE id = 1').get(); } catch (e) { problems.push(`database: ${e.message}`); }
- if (!keys.get()) problems.push('Network public key not loaded');
- if (problems.length) return http.sendProblem(res, 503, 'service.not_ready', { detail: problems.join('; '), ctx: req.ov });
- return res.json({ ready: true });
- });
+ // Readiness (openvibe-shared/ready): 503 only when the database fails; Network key, Billing and
+ // Events are optional and degrade it (see observability.js).
+ const readiness = createTipsReadiness({ db, keys, config, outbox, release: release.release, fetchImpl });
+ app.get('/api/ready', readiness.handler);
app.get('/release.json', release.handler);
app.get('/release.json', release.handler);
const consumer = consumerRouter({ domain, config, log });
const consumer = consumerRouter({ domain, config, log });
@@ -113,7 +117,7 @@ function createApp(opts = {}) {
@@ -113,7 +117,7 @@ function createApp(opts = {}) {
return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));
return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));
});
});
- Object.assign(app.locals, { config, db, domain, keys, outbox, adapters, billing, consumer });
+ Object.assign(app.locals, { config, db, domain, keys, outbox, adapters, billing, consumer, metrics });
return app;
return app;
}
}
@@ -203,6 +203,7 @@ CREATE TABLE IF NOT EXISTS overlay_deliveries (
@@ -203,6 +203,7 @@ CREATE TABLE IF NOT EXISTS overlay_deliveries (
created_at TEXT NOT NULL
created_at TEXT NOT NULL
);
);
CREATE INDEX IF NOT EXISTS idx_ovd_creator ON overlay_deliveries (creator_subject, seq);
CREATE INDEX IF NOT EXISTS idx_ovd_creator ON overlay_deliveries (creator_subject, seq);
+CREATE INDEX IF NOT EXISTS idx_ovd_pending ON overlay_deliveries (seq) WHERE status = 'pending';
CREATE TABLE IF NOT EXISTS interaction_effects (
CREATE TABLE IF NOT EXISTS interaction_effects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER PRIMARY KEY AUTOINCREMENT,
new file mode 100644
new file mode 100644
@@ -0,0 +1,81 @@
@@ -0,0 +1,81 @@
+'use strict';
+/**
+ * Track O: truthful readiness for GET /api/ready and the Tips gauges on GET /metrics
+ * (openvibe-shared/ready and openvibe-shared/metrics).
+ *
+ * db required a real query on Tips' SQLite (the settings row is there and answers)
+ * network_jwks optional the Network signing key has loaded. Without it pages, overlays and the
+ * signed Billing webhook still work, but no service or user token can be
+ * verified (API calls and sign-in fail), so it degrades rather than fails
+ * billing optional Billing answers /api/health. Without it checkouts and transfers wait
+ * (due transfers retry once it is back); settled tips still deliver
+ * events optional OpenVibe.Events answers /api/health. Without it (or with the relay off:
+ * EVENTS_URL or the client secret unset) tips.* events wait in the outbox
+ *
+ * Gauges: pending deliveries (chat/TTS/media effects waiting for their adapter, overlay alerts no
+ * overlay has shown yet) and the events outbox backlog. Counts only, never subjects.
+ */
+const { createReadiness } = require('openvibe-shared/ready');
+
+const PING_TTL_MS = 15_000;
+// interaction_effects.effect (the CHECK constraint in db.js): each reported, 0 when none waits.
+const EFFECTS = ['chat_line', 'paid_message', 'tts', 'media_request', 'overlay_alert'];
+
+function probe(url, fetchImpl) {
+ return async () => {
+ const res = await fetchImpl(url, { signal: AbortSignal.timeout(2000), headers: { Accept: 'application/json' } });
+ try { await res.body?.cancel(); } catch { /* not needed */ }
+ return res.ok ? { ok: true, detail: { http_status: res.status } } : { ok: false, error: `answered HTTP ${res.status}`, detail: { http_status: res.status } };
+ };
+}
+
+function createTipsReadiness({ db, keys, config, outbox, release = null, fetchImpl = globalThis.fetch }) {
+ const events = outbox.enabled
+ ? probe(`${config.events.url}/api/health`, fetchImpl)
+ : () => 'events relay off (EVENTS_URL or OV_OAUTH_CLIENT_SECRET unset): tips.* events wait in the outbox';
+ return createReadiness({
+ service: 'tips',
+ release,
+ checks: [
+ { name: 'db', required: true, check: () => (db.prepare('SELECT id FROM settings WHERE id = 1').get() ? true : 'database has no settings row') },
+ { name: 'network_jwks', required: false, check: () => (keys.get() ? true : 'Network signing key not loaded yet: tokens cannot be verified') },
+ { name: 'billing', required: false, cacheMs: PING_TTL_MS, timeoutMs: 2500, check: probe(`${config.billing.url}/api/health`, fetchImpl) },
+ { name: 'events', required: false, cacheMs: outbox.enabled ? PING_TTL_MS : 0, timeoutMs: 2500, check: events },
+ ],
+ details: (body) => ({
+ events_outbox: body.checks.db.status === 'ok' ? outbox.status() : null,
+ billing_events_accepted: config.events.webhookSecrets.length > 0,
+ pending_deliveries: body.checks.db.status === 'ok' ? pendingDeliveries(db) : null,
+ }),
+ });
+}
+
+function pendingDeliveries(db) {
+ const effects = {};
+ for (const r of db.prepare("SELECT effect, COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' GROUP BY effect").all()) effects[r.effect] = r.n;
+ const overlay = db.prepare("SELECT COUNT(*) AS n FROM overlay_deliveries WHERE status = 'pending'").get().n;
+ return { effects, overlay };
+}
+
+/** Tips gauges on the openvibe-shared/metrics registry. */
+function registerTipsGauges(registry, { db, outbox, now = () => Date.now() }) {
+ registry.gauge({
+ name: 'tips_effects_pending', help: 'Chat, TTS, media and alert effects waiting for their delivery adapter, by effect', labelNames: ['effect'],
+ collect: () => {
+ const n = Object.fromEntries(EFFECTS.map((e) => [e, 0]));
+ for (const r of db.prepare("SELECT effect, COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' GROUP BY effect").all()) n[r.effect] = r.n;
+ return Object.entries(n).map(([effect, value]) => ({ labels: { effect }, value }));
+ },
+ });
+ registry.gauge({
+ name: 'tips_effects_due', help: 'Queued effects whose next attempt is due now (a growing number means the worker is behind)',
+ collect: () => db.prepare("SELECT COUNT(*) AS n FROM interaction_effects WHERE state = 'queued' AND next_attempt_at <= ?").get(now()).n,
+ });
+ registry.gauge({
+ name: 'tips_overlay_deliveries_pending', help: 'Overlay alerts and goal updates no overlay has shown yet',
+ collect: () => db.prepare("SELECT COUNT(*) AS n FROM overlay_deliveries WHERE status = 'pending'").get().n,
+ });
+ registry.gauge({ name: 'tips_outbox_pending', help: 'tips.* events waiting in the outbox', collect: () => outbox.status().pending });
+}
+
+module.exports = { createTipsReadiness, registerTipsGauges };
@@ -7,6 +7,7 @@
@@ -7,6 +7,7 @@
* matter here (idempotency keys, funds, self-dealing, capability + audience checks),
* matter here (idempotency keys, funds, self-dealing, capability + audience checks),
* and the billing.transaction.* envelopes it would publish (billing.events)
* and the billing.transaction.* envelopes it would publish (billing.events)
* startEvents() POST /api/v1/events recording what Tips' outbox relays
* startEvents() POST /api/v1/events recording what Tips' outbox relays
+ * (Billing and Events also answer GET /api/health, which /api/ready probes)
* startLive() POST /internal/tips/deliveries recording chat deliveries (the live-chat adapter)
* startLive() POST /internal/tips/deliveries recording chat deliveries (the live-chat adapter)
*/
*/
const http = require('http');
const http = require('http');
@@ -125,6 +126,7 @@ async function startBilling(network) {
@@ -125,6 +126,7 @@ async function startBilling(network) {
const server = http.createServer(async (req, res) => {
const server = http.createServer(async (req, res) => {
const raw = await readBody(req);
const raw = await readBody(req);
+ if (req.url === '/api/health') return send(res, state.down ? 503 : 200, state.down ? { code: 'billing.unavailable' } : { ok: true, service: 'billing' });
const body = raw ? JSON.parse(raw) : {};
const body = raw ? JSON.parse(raw) : {};
calls.push({ method: req.method, url: req.url, body, key: req.headers['idempotency-key'] || null });
calls.push({ method: req.method, url: req.url, body, key: req.headers['idempotency-key'] || null });
if (state.down) return send(res, 503, { code: 'billing.unavailable' });
if (state.down) return send(res, 503, { code: 'billing.unavailable' });
@@ -178,6 +180,7 @@ async function startEvents() {
@@ -178,6 +180,7 @@ async function startEvents() {
const tokens = [];
const tokens = [];
const server = http.createServer(async (req, res) => {
const server = http.createServer(async (req, res) => {
const raw = await readBody(req);
const raw = await readBody(req);
+ if (req.url === '/api/health') return send(res, 200, { status: 'ok', service: 'openvibe-events' });
tokens.push(req.headers.authorization);
tokens.push(req.headers.authorization);
if (req.url === '/api/v1/events' && req.method === 'POST') {
if (req.url === '/api/v1/events' && req.method === 'POST') {
const b = JSON.parse(raw);
const b = JSON.parse(raw);
new file mode 100644
new file mode 100644
@@ -0,0 +1,97 @@
@@ -0,0 +1,97 @@
+'use strict';
+// Track O: GET /metrics answers direct loopback callers only, labels requests by route template and
+// carries the Tips gauges; /api/ready is 503 only when the database fails, and a missing Billing,
+// Events or Network key degrades it instead.
+const assert = require('assert');
+const nodeHttp = require('http');
+const { boot, check, done } = require('./helpers/app');
+
+function get(base, p, headers = {}) {
+ return new Promise((resolve, reject) => nodeHttp.get(base + p, { headers }, (res) => {
+ let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b }));
+ }).on('error', reject));
+}
+
+(async () => {
+ const t = await boot();
+ await t.creator('alex');
+
+ await check('/api/ready: every check reports; db is the only required one', async () => {
+ const r = await t.call('GET', '/api/ready', { token: null });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json.status, 'ready');
+ assert.strictEqual(r.json.service, 'tips');
+ assert.deepStrictEqual(Object.keys(r.json.checks), ['db', 'network_jwks', 'billing', 'events']);
+ for (const [name, c] of Object.entries(r.json.checks)) {
+ assert.strictEqual(c.status, 'ok', `${name}: ${c.error}`);
+ assert.strictEqual(c.required, name === 'db', name);
+ assert.strictEqual(typeof c.latency_ms, 'number');
+ assert.ok(Date.parse(c.checked_at));
+ }
+ assert.ok(r.json.events_outbox && r.json.pending_deliveries);
+ assert.strictEqual(r.headers.get('cache-control'), 'no-store');
+ });
+
+ await check('/metrics: 404 through a proxy, golden signals by route template and Tips gauges direct', async () => {
+ await t.call('GET', '/api/v1/profiles/alex', { token: null });
+ await t.call('GET', '/api/no/such/12345', { token: null });
+ for (const h of [{ 'X-Forwarded-For': '203.0.113.7' }, { 'X-Real-IP': '203.0.113.7' }, { 'CF-Connecting-IP': '203.0.113.7' }]) {
+ const m = await get(t.base, '/metrics', h);
+ assert.strictEqual(m.status, 404, JSON.stringify(h));
+ assert.ok(!m.body.includes('http_requests_total'));
+ }
+ const m = await get(t.base, '/metrics');
+ assert.strictEqual(m.status, 200);
+ const text = m.body;
+ assert.ok(/http_requests_total\{method="GET",route="\/api\/v1\/profiles\/:creator",status_class="2xx"\} 1\n/.test(text), 'route template, not the handle');
+ assert.ok(/http_requests_total\{method="GET",route="unmatched",status_class="4xx"\} 1\n/.test(text));
+ assert.ok(!/route="[^"]*(alex|12345)/.test(text), 'no concrete path in any label');
+ assert.ok(/http_request_duration_seconds_bucket\{method="GET",route="\/api\/ready",le="\+Inf"\} 1\n/.test(text));
+ assert.ok(/\nhttp_requests_in_flight \d+\n/.test(text));
+ assert.ok(/\nprocess_resident_memory_bytes \d+\n/.test(text));
+ assert.ok(/release_info\{service="tips",release="[^"]+"\} 1\n/.test(text));
+ for (const e of ['chat_line', 'paid_message', 'tts', 'media_request', 'overlay_alert']) assert.ok(new RegExp(`tips_effects_pending\\{effect="${e}"\\} 0\\n`).test(text), e);
+ assert.ok(/\ntips_effects_due 0\n/.test(text));
+ assert.ok(/\ntips_overlay_deliveries_pending 0\n/.test(text));
+ assert.ok(/\ntips_outbox_pending \d+\n/.test(text));
+ });
+
+ await check('an overlay SSE stream is not a request sample', async () => {
+ const reader = await t.sse('/overlay/not-a-token/events');
+ reader.close();
+ const text = (await get(t.base, '/metrics')).body;
+ assert.ok(!text.includes('/overlay/:token/events') && !text.includes('not-a-token'));
+ });
+
+ await t.close();
+
+ await check('Billing down and the events relay off: still ready (200), degraded, and says why', async () => {
+ const d = await boot({ env: { EVENTS_URL: '' } });
+ d.billing.state.down = true;
+ const r = await d.call('GET', '/api/ready', { token: null });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json.ready, true);
+ assert.strictEqual(r.json.status, 'degraded');
+ assert.deepStrictEqual(r.json.degraded, ['billing', 'events']);
+ assert.strictEqual(r.json.checks.billing.error, 'answered HTTP 503');
+ assert.match(r.json.checks.events.error, /events relay off/);
+ await d.close();
+ });
+
+ await check('a broken database makes the service unready (503); /metrics still answers', async () => {
+ const d = await boot();
+ d.db.close();
+ const r = await d.call('GET', '/api/ready', { token: null });
+ assert.strictEqual(r.status, 503, r.text);
+ assert.strictEqual(r.json.ready, false);
+ assert.strictEqual(r.json.status, 'not_ready');
+ assert.deepStrictEqual(r.json.failed, ['db']);
+ assert.strictEqual(r.json.pending_deliveries, null);
+ const m = await get(d.base, '/metrics');
+ assert.strictEqual(m.status, 200);
+ assert.ok(!/\ntips_overlay_deliveries_pending \d/.test(m.body), 'a gauge that cannot be read is left out, not invented');
+ try { await d.close(); } catch { /* the database is already closed */ }
+ });
+
+ done();
+})();