openvibers/openvibe.search Commits

Track O: /metrics for direct loopback callers and a truthful /api/ready (openvibe-shared v1.3.0)

OP OpenVibers committed 23 Sept 2026 · 1be4b409c0 · 7 files, +341 −21

Adds openvibe-shared v1.3.0. /metrics (openvibe-shared/metrics): golden signals labelled by route
template, process metrics, release_info, search_documents{exposure}, search_documents_indexed{index}
(the full-text tables) and the outbox backlog; a request carrying X-Forwarded-For, X-Real-IP or
CF-Connecting-IP gets 404. /api/ready (openvibe-shared/ready): the database is required (a real
read of documents, fts_public and fts_restricted; 503 when it fails); the Network key is optional
(anonymous queries still answer) and so is the index check, which degrades the service when the
full-text tables disagree with the documents table. Engine, document counts, outbox and webhook
state stay in the body. GET /release.json. nginx answers /metrics with 404.
test/observability.test.js.

Changes

ddiff --git a/README.md b/README.md
iindex 2af1e2d..8f3c8db 100644
---- a/README.md
++++ b/README.md
@@@ -33,9 +33,13 @@ env `/etc/openvibe/search.env`, unit [deploy/systemd/openvibe-search.service](de
  index `/var/lib/openvibe-search/search.db`, nginx [deploy/nginx/search.openvibe.network.conf](deploy/nginx/search.openvibe.network.conf)
  (the public vhost exposes only the GET query API and health; owners and Events call `127.0.0.1:4710`).
  
--`GET /api/health` is liveness. `GET /api/ready` is 200 when the database answers and the Network
--signing key has loaded; it also reports document counts by exposure, the outbox backlog and
--whether the Events webhook is on.
++`GET /api/health` is liveness. `GET /api/ready` (openvibe-shared/ready) is 503 only when the
++database (the documents and both full-text tables) fails; a Network signing key that has not loaded
++(anonymous queries still answer) and a full-text index out of step with the documents table degrade
++it. It also reports document counts by exposure, the outbox backlog and whether the Events webhook
++is on. `GET /metrics` (openvibe-shared/metrics) answers direct loopback callers only: golden signals
++by route template, `search_documents{exposure}`, `search_documents_indexed{index}` and the outbox
++backlog.
  
  ## Query API
  
ddiff --git a/deploy/nginx/search.openvibe.network.conf b/deploy/nginx/search.openvibe.network.conf
iindex 847e81a..e9f139a 100644
---- a/deploy/nginx/search.openvibe.network.conf
++++ b/deploy/nginx/search.openvibe.network.conf
@@@ -13,6 +13,8 @@
  ##                                    writes; open them only for an off-host owner.
  ##   /internal/events                 OpenVibe.Events deliveries — host-local only.
  ##   /api/health, /api/ready
++##   /metrics                         Prometheus text, answered only to direct loopback callers
++##                                    (never proxied; Node refuses a request carrying X-Forwarded-For)
  ## ═══════════════════════════════════════════════════════════════
  
  limit_req_zone $binary_remote_addr zone=ovsearch_query:10m rate=10r/s;
@@@ -51,6 +53,9 @@ server {
      location = /api/health { proxy_pass http://127.0.0.1:4710; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      location = /api/ready  { proxy_pass http://127.0.0.1:4710; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
  
++    # Metrics are for host-local scrapers only (http://127.0.0.1:4710/metrics).
++    location = /metrics { return 404; }
++
      # ── Public query API (GET only) ─────────────────────────
      location ~ ^/api/v1/(search|suggest|documents/) {
          limit_except GET { deny all; }
ddiff --git a/package-lock.json b/package-lock.json
iindex f8b92fc..cc254bb 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -14,7 +14,8 @@
          "better-sqlite3": "^11.10.0",
          "dotenv": "^16.4.7",
          "express": "^4.21.2",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0"
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
++        "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
        },
        "engines": {
          "node": ">=22.0.0"
@@@ -171,6 +172,12 @@
          "ieee754": "^1.1.13"
        }
      },
++    "node_modules/buffer-equal-constant-time": {
++      "version": "1.0.1",
++      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
++      "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
++      "license": "BSD-3-Clause"
++    },
      "node_modules/bytes": {
        "version": "3.1.2",
        "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@@ -338,6 +345,15 @@
          "node": ">= 0.4"
        }
      },
++    "node_modules/ecdsa-sig-formatter": {
++      "version": "1.0.11",
++      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
++      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
++      "license": "Apache-2.0",
++      "dependencies": {
++        "safe-buffer": "^5.0.1"
++      }
++    },
      "node_modules/ee-first": {
        "version": "1.1.1",
        "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@@ -699,6 +715,97 @@
        "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
        "license": "MIT"
      },
++    "node_modules/jsonwebtoken": {
++      "version": "9.0.3",
++      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz",
++      "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==",
++      "license": "MIT",
++      "dependencies": {
++        "jws": "^4.0.1",
++        "lodash.includes": "^4.3.0",
++        "lodash.isboolean": "^3.0.3",
++        "lodash.isinteger": "^4.0.4",
++        "lodash.isnumber": "^3.0.3",
++        "lodash.isplainobject": "^4.0.6",
++        "lodash.isstring": "^4.0.1",
++        "lodash.once": "^4.0.0",
++        "ms": "^2.1.1",
++        "semver": "^7.5.4"
++      },
++      "engines": {
++        "node": ">=12",
++        "npm": ">=6"
++      }
++    },
++    "node_modules/jsonwebtoken/node_modules/ms": {
++      "version": "2.1.3",
++      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
++      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
++      "license": "MIT"
++    },
++    "node_modules/jwa": {
++      "version": "2.0.1",
++      "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",
++      "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==",
++      "license": "MIT",
++      "dependencies": {
++        "buffer-equal-constant-time": "^1.0.1",
++        "ecdsa-sig-formatter": "1.0.11",
++        "safe-buffer": "^5.0.1"
++      }
++    },
++    "node_modules/jws": {
++      "version": "4.0.1",
++      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
++      "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==",
++      "license": "MIT",
++      "dependencies": {
++        "jwa": "^2.0.1",
++        "safe-buffer": "^5.0.1"
++      }
++    },
++    "node_modules/lodash.includes": {
++      "version": "4.3.0",
++      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
++      "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isboolean": {
++      "version": "3.0.3",
++      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
++      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isinteger": {
++      "version": "4.0.4",
++      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
++      "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isnumber": {
++      "version": "3.0.3",
++      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
++      "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isplainobject": {
++      "version": "4.0.6",
++      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
++      "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.isstring": {
++      "version": "4.0.1",
++      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
++      "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
++      "license": "MIT"
++    },
++    "node_modules/lodash.once": {
++      "version": "4.1.1",
++      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
++      "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
++      "license": "MIT"
++    },
      "node_modules/math-intrinsics": {
        "version": "1.1.0",
        "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@@ -877,6 +984,15 @@
          "node": ">=20"
        }
      },
++    "node_modules/openvibe-shared": {
++      "version": "1.3.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0",
++      "integrity": "sha512-w8yl/5+9eend8I2O/DzAJnEAL2N3GTUY7c7pzK+MLJuLAGD8FmJJQ9gwbgyrq70Snjt3ebrg3TlpORhRAp2kgA==",
++      "license": "MIT",
++      "dependencies": {
++        "jsonwebtoken": "^9.0.0"
++      }
++    },
      "node_modules/parseurl": {
        "version": "1.3.3",
        "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
ddiff --git a/package.json b/package.json
iindex f12c282..acb622b 100644
---- a/package.json
++++ b/package.json
@@@ -20,7 +20,8 @@
      "better-sqlite3": "^11.10.0",
      "dotenv": "^16.4.7",
      "express": "^4.21.2",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0"
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.13.0",
++    "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.3.0"
    },
    "engines": {
      "node": ">=22.0.0"
ddiff --git a/server/app.js b/server/app.js
iindex d9e33ed..1212047 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -1,7 +1,11 @@
  'use strict';
--/** Express app: request context, the v1 API, the Events webhook, health/readiness. */
++/** Express app: request context, the v1 API, the Events webhook, health/readiness, metrics. */
++const path = require('path');
  const express = require('express');
  const { http } = require('openvibe-contracts');
++const { instrument } = require('openvibe-shared/metrics');
++const { createRelease } = require('openvibe-shared/release');
++const { createSearchReadiness, registerSearchGauges } = require('./observability');
  const { documentsRouter } = require('./api/documents');
  const { queryRouter } = require('./api/query');
  const { webhookRouter } = require('./api/webhook');
@@@ -12,6 +16,11 @@ function createApp({ config, db, store, engine, auth, keys, outbox, relay, log =
      app.disable('x-powered-by');
      app.set('trust proxy', 'loopback');
      app.set('query parser', 'extended');
++    const release = createRelease({ service: 'search', root: path.join(__dirname, '..') });
++    // HTTP golden signals by route template, process metrics, release_info and the Search gauges;
++    // GET /metrics answers direct loopback callers only (Track O).
++    const metrics = instrument(app, { service: 'search', release: release.release });
++    registerSearchGauges(metrics.registry, { db, outbox });
      app.use(http.middleware());
      app.use((req, res, next) => {
          res.setHeader('X-Content-Type-Options', 'nosniff');
@@@ -27,20 +36,11 @@ function createApp({ config, db, store, engine, auth, keys, outbox, relay, log =
          res.json({ status: 'ok', service: 'openvibe-search', version: pkg.version });
      });
  
--    app.get('/api/ready', (_req, res) => {
--        let dbOk = false;
--        try { dbOk = db.prepare('SELECT 1 AS ok').get().ok === 1; } catch { dbOk = false; }
--        const checks = { db: dbOk, key: keys.loaded() };
--        const ready = Object.values(checks).every(Boolean);
--        res.status(ready ? 200 : 503).json({
--            status: ready ? 'ready' : 'not_ready',
--            checks,
--            engine: engine.name,
--            documents: dbOk ? store.counts() : null,
--            outbox: dbOk ? { pending: outbox.pending(), rejected: outbox.rejected(), relay: config.events.url ? (relay.running() ? 'running' : 'stopped') : 'off (EVENTS_URL unset)' } : null,
--            webhook: config.events.webhookSecrets.length ? 'on' : 'off (SEARCH_EVENTS_SECRET unset)',
--        });
--    });
++    // Readiness (openvibe-shared/ready): 503 only when the database (documents and full-text tables)
++    // fails; the Network key and index consistency are optional and degrade it (see observability.js).
++    const readiness = createSearchReadiness({ db, keys, config, engine, store, outbox, relay, release: release.release });
++    app.get('/api/ready', readiness.handler);
++    app.get('/release.json', release.handler);
  
      app.use(documentsRouter({ store, auth, db, relay }));
      app.use(queryRouter({ config, store, engine, auth }));
@@@ -56,7 +56,7 @@ function createApp({ config, db, store, engine, auth, keys, outbox, relay, log =
              'DELETE /api/v1/documents/:owner/:type/:id?revision=           tombstone (owner)',
              'GET    /api/v1/owners/:owner/documents                        reconciliation (owner)',
              'POST   /internal/events                                       OpenVibe.Events delivery (signed)',
--            'GET    /api/health, /api/ready',
++            'GET    /api/health, /api/ready, /release.json',
              '',
              'Source: https://github.com/OpenVibers/OpenVibe.Search',
              '',
@@@ -74,6 +74,7 @@ function createApp({ config, db, store, engine, auth, keys, outbox, relay, log =
          return http.sendProblem(res, 500, 'search.internal', { detail: 'internal error', ctx: req.ov });
      });
  
++    app.locals.metrics = metrics;
      return app;
  }
  
ddiff --git a/server/observability.js b/server/observability.js
nnew file mode 100644
iindex 0000000..6cf8345
---- /dev/null
++++ b/server/observability.js
@@@ -0,0 +1,93 @@
++'use strict';
++/**
++ * Track O: truthful readiness for GET /api/ready and the Search gauges on GET /metrics
++ * (openvibe-shared/ready and openvibe-shared/metrics).
++ *
++ *   db            required  a real read of the documents table and of both full-text tables (they
++ *                           live in the same SQLite file): without them nothing can be indexed or found
++ *   network_jwks  optional  the Network signing key has loaded. Without it anonymous queries still
++ *                           answer (public documents only), but no token can be verified: owner
++ *                           writes and signed-in queries fail, so it degrades rather than fails
++ *   index         optional  the full-text index agrees with the documents table: fts_public holds
++ *                           exactly the public_listed documents and fts_restricted the restricted
++ *                           ones. A mismatch means queries miss (or would surface) documents
++ *
++ * Gauges: documents by exposure, full-text entries by table, and the events outbox backlog.
++ */
++const { createReadiness } = require('openvibe-shared/ready');
++const { EXPOSURE } = require('./document');
++
++const EXPOSURE_NAMES = Object.fromEntries(Object.entries(EXPOSURE).map(([name, n]) => [n, name]));
++
++function readers(db) {
++    const byExposure = db.prepare('SELECT exposure, COUNT(*) AS n FROM documents GROUP BY exposure');
++    const ftsPublic = db.prepare('SELECT COUNT(*) AS n FROM fts_public');
++    const ftsRestricted = db.prepare('SELECT COUNT(*) AS n FROM fts_restricted');
++    return {
++        /** { none, restricted, public_unlisted, public_listed }, every exposure present (0 when empty). */
++        documents() {
++            const out = Object.fromEntries(Object.keys(EXPOSURE).map((k) => [k, 0]));
++            for (const r of byExposure.all()) out[EXPOSURE_NAMES[r.exposure] || 'none'] += r.n;
++            return out;
++        },
++        indexed: () => ({ public: ftsPublic.get().n, restricted: ftsRestricted.get().n }),
++    };
++}
++
++function createSearchReadiness({ db, keys, config, engine, store, outbox, relay, release = null }) {
++    const read = readers(db);
++    return createReadiness({
++        service: 'search',
++        release,
++        checks: [
++            {
++                name: 'db', required: true,
++                check: () => {
++                    db.prepare('SELECT rid FROM documents LIMIT 1').all();
++                    db.prepare('SELECT rowid FROM fts_public LIMIT 1').all();
++                    db.prepare('SELECT rowid FROM fts_restricted LIMIT 1').all();
++                    return true;
++                },
++            },
++            { name: 'network_jwks', required: false, check: () => keys.loaded() || 'Network signing key not loaded yet: owner writes and signed-in queries cannot be verified' },
++            {
++                name: 'index', required: false, cacheMs: 10_000,
++                check: () => {
++                    const docs = read.documents();
++                    const fts = read.indexed();
++                    const detail = { indexed: fts, expected: { public: docs.public_listed, restricted: docs.restricted } };
++                    const off = [];
++                    if (fts.public !== docs.public_listed) off.push(`fts_public has ${fts.public} entries for ${docs.public_listed} public_listed documents`);
++                    if (fts.restricted !== docs.restricted) off.push(`fts_restricted has ${fts.restricted} entries for ${docs.restricted} restricted documents`);
++                    return off.length ? { ok: false, error: `index out of step: ${off.join('; ')}`, detail } : { ok: true, detail };
++                },
++            },
++        ],
++        details: (body) => {
++            const dbOk = body.checks.db.status === 'ok';
++            return {
++                engine: engine.name,
++                documents: dbOk ? store.counts() : null,
++                outbox: dbOk ? { pending: outbox.pending(), rejected: outbox.rejected(), relay: config.events.url ? (relay.running() ? 'running' : 'stopped') : 'off (EVENTS_URL unset)' } : null,
++                webhook: config.events.webhookSecrets.length ? 'on' : 'off (SEARCH_EVENTS_SECRET unset)',
++            };
++        },
++    });
++}
++
++/** Search gauges on the openvibe-shared/metrics registry. */
++function registerSearchGauges(registry, { db, outbox }) {
++    const read = readers(db);
++    registry.gauge({
++        name: 'search_documents', help: 'Documents held, by exposure (public_listed and restricted are searchable; tombstones and drafts are none)', labelNames: ['exposure'],
++        collect: () => Object.entries(read.documents()).map(([exposure, value]) => ({ labels: { exposure }, value })),
++    });
++    registry.gauge({
++        name: 'search_documents_indexed', help: 'Documents in the full-text index, by table', labelNames: ['index'],
++        collect: () => Object.entries(read.indexed()).map(([index, value]) => ({ labels: { index }, value })),
++    });
++    registry.gauge({ name: 'search_outbox_pending', help: 'Events waiting in the outbox', collect: () => outbox.pending() });
++    registry.gauge({ name: 'search_outbox_rejected', help: 'Events OpenVibe.Events refused for good', collect: () => outbox.rejected() });
++}
++
++module.exports = { createSearchReadiness, registerSearchGauges };
ddiff --git a/test/observability.test.js b/test/observability.test.js
nnew file mode 100644
iindex 0000000..3c0a462
---- /dev/null
++++ b/test/observability.test.js
@@@ -0,0 +1,100 @@
++'use strict';
++// Track O: GET /metrics answers direct loopback callers only, labels requests by route template and
++// carries the Search gauges (documents by exposure, full-text entries); /api/ready is 503 when the
++// database fails, and a full-text index out of step with the documents degrades it.
++const assert = require('assert');
++const nodeHttp = require('http');
++const { boot, request, serviceToken, doc, suite } = require('./helpers');
++
++const t = suite('observability');
++const WIKI = serviceToken('wiki', ['search.document.write']);
++let svc;
++
++function get(base, p, headers = {}) {
++    return new Promise((resolve, reject) => nodeHttp.get(base + p, { headers }, (res) => {
++        let b = ''; res.on('data', (d) => { b += d; }); res.on('end', () => resolve({ status: res.statusCode, body: b }));
++    }).on('error', reject));
++}
++const put = (d) => request(svc.base, 'PUT', `/api/v1/documents/${d.owner}/${d.type}/${d.id}`, { token: WIKI, body: d });
++
++t('boot and index two public documents, one members-only and one noindex', async () => {
++    svc = await boot();
++    await svc.keyLoaded;
++    for (const d of [doc(), doc(), doc({ visibility: 'members', acl: { groups: ['vip'] } }), doc({ indexability: { decision: 'noindex', reasons: ['thin'] } })]) {
++        const r = await put(d);
++        assert.ok(r.status === 200 || r.status === 201, r.text);
++    }
++});
++
++t('/api/ready: every check reports; db is the only required one; the index agrees', async () => {
++    const r = await request(svc.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 200, r.text);
++    assert.strictEqual(r.body.status, 'ready');
++    assert.strictEqual(r.body.service, 'search');
++    assert.deepStrictEqual(Object.keys(r.body.checks), ['db', 'network_jwks', 'index']);
++    for (const [name, c] of Object.entries(r.body.checks)) {
++        assert.strictEqual(c.status, 'ok', `${name}: ${c.error}`);
++        assert.strictEqual(c.required, name === 'db', name);
++        assert.strictEqual(typeof c.latency_ms, 'number');
++        assert.ok(Date.parse(c.checked_at));
++    }
++    assert.deepStrictEqual(r.body.checks.index.detail, { indexed: { public: 2, restricted: 1 }, expected: { public: 2, restricted: 1 } });
++    assert.strictEqual(r.body.engine, 'sqlite-fts5');
++    assert.strictEqual(r.body.documents.total, 4);
++});
++
++t('/metrics: 404 through a proxy; route templates and Search gauges direct', async () => {
++    const d = doc({ id: 'pg_metricsdoc0001' });
++    await put(d);
++    await request(svc.base, 'GET', `/api/v1/documents/wiki/page/${d.id}`);
++    await request(svc.base, 'GET', '/api/v1/search?q=example');
++    for (const hdr of [{ 'X-Forwarded-For': '203.0.113.7' }, { 'X-Real-IP': '203.0.113.7' }, { 'CF-Connecting-IP': '203.0.113.7' }]) {
++        const m = await get(svc.base, '/metrics', hdr);
++        assert.strictEqual(m.status, 404, JSON.stringify(hdr));
++        assert.ok(!m.body.includes('search_documents'));
++    }
++    const m = await get(svc.base, '/metrics');
++    assert.strictEqual(m.status, 200);
++    const text = m.body;
++    assert.ok(/http_requests_total\{method="PUT",route="\/api\/v1\/documents\/:owner\/:type\/:id",status_class="2xx"\} 5\n/.test(text), 'route template for writes');
++    assert.ok(/http_requests_total\{method="GET",route="\/api\/v1\/documents\/:owner\/:type\/:id",status_class="2xx"\} 1\n/.test(text));
++    assert.ok(/http_requests_total\{method="GET",route="\/api\/v1\/search",status_class="2xx"\} 1\n/.test(text));
++    assert.ok(!/route="[^"]*(pg_metricsdoc0001|example)/.test(text), 'no id or query in any label');
++    assert.ok(/\nprocess_resident_memory_bytes \d+\n/.test(text));
++    assert.ok(/release_info\{service="search",release="[^"]+"\} 1\n/.test(text));
++    assert.ok(/search_documents\{exposure="public_listed"\} 3\n/.test(text));
++    assert.ok(/search_documents\{exposure="public_unlisted"\} 1\n/.test(text));
++    assert.ok(/search_documents\{exposure="restricted"\} 1\n/.test(text));
++    assert.ok(/search_documents\{exposure="none"\} 0\n/.test(text));
++    assert.ok(/search_documents_indexed\{index="public"\} 3\n/.test(text));
++    assert.ok(/search_documents_indexed\{index="restricted"\} 1\n/.test(text));
++    assert.ok(/\nsearch_outbox_pending \d+\n/.test(text));
++});
++
++t('a full-text index out of step with the documents degrades /api/ready (still 200)', async () => {
++    await svc.stop();
++    svc = await boot();                              // fresh: the index check has no cached result
++    await put(doc());
++    await put(doc());
++    svc.db.prepare('DELETE FROM fts_public WHERE rowid = (SELECT MIN(rowid) FROM fts_public)').run();
++    const r = await request(svc.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 200, r.text);
++    assert.strictEqual(r.body.status, 'degraded');
++    assert.deepStrictEqual(r.body.degraded, ['index']);
++    assert.match(r.body.checks.index.error, /fts_public has 1 entries for 2 public_listed documents/);
++});
++
++t('a broken database makes the service unready (503); /metrics still answers', async () => {
++    svc.db.close();
++    const r = await request(svc.base, 'GET', '/api/ready');
++    assert.strictEqual(r.status, 503, r.text);
++    assert.strictEqual(r.body.ready, false);
++    assert.deepStrictEqual(r.body.failed, ['db']);
++    assert.strictEqual(r.body.documents, null);
++    const m = await get(svc.base, '/metrics');
++    assert.strictEqual(m.status, 200);
++    assert.ok(!/search_documents\{/.test(m.body), 'a gauge that cannot be read is left out, not invented');
++    try { await svc.stop(); } catch { /* the database is already closed */ }
++});
++
++t.run();