openvibers/openvibe.help Commits

Help: article pages in OpenVibe.Search (search-feed, help.index_document.*) (#13)

OP OpenVibers committed 10 Oct 2026 · 7d349e142b · 10 files, +221 −11

Plan T19 (Help = **Search** + Wiki + Actor + Chat + a ticket authority) and T9 ("Search: real documents from every product"). Help's article pages go into OpenVibe.Search, so a question asked anywhere on the network finds the answer page.

- `server/search-index.js` uses `openvibe-publishing/search-feed` (1.4.0): one `search.index-document@1` per article (`/a/<domain>/<slug>`, id `<domain>:<slug>`), sent as `help.index_document.upserted|deleted` (Contracts 0.128.0) through a new events outbox.
  - The document is the question, the answer verbatim and the site's name. Facets: site, kind, state.
  - Provenance names the manifest (the Contracts package and its repository).
- **When it runs:** the catalog is built at boot, so a sweep a minute after boot and every six hours is enough. A Contracts pin that adds, rewords or drops a question sends a document, a new revision or a tombstone (`help_index_revisions` is the sequencer); a quiet boot sends nothing.
- **Migration `0004_search_index.sql`** (expand): `event_outbox` + `help_index_revisions`.
- **Delivery:** the relay publishes with Help's token once `HELP_EVENTS_URL`/`EVENTS_URL` and `OV_OAUTH_CLIENT_SECRET` are set; Network#120 grants it and Events#42 makes Help a source.
- **Pins:** openvibe-contracts v0.128.0; openvibe-publishing v1.4.0 added.

Checked: openvibe.help/sites names no domain we don't own.

Tests: `test/search-index.test.js`, 3 checks:
- every article is sent once, valid against its contract, and the page it names answers 200;
- a second sweep sends nothing;
- an article the catalog dropped becomes a tombstone, once.

Full suite 18/18.

Changes

ddiff --git a/README.md b/README.md
iindex 947f105..22f262b 100644
---- a/README.md
++++ b/README.md
@@@ -278,7 +278,8 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md).
  Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
  
  <!-- versions:start -->
--- openvibe-contracts: v0.127.0
++- openvibe-contracts: v0.128.0
++- openvibe-publishing: v1.4.0
  - openvibe-sdk: v0.36.0
  - openvibe-shared: v2.21.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 4a627cb..4d2b034 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -7,13 +7,14 @@
    "runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9",
    "port": "5020",
    "domain": "openvibe.help",
--  "contracts": "openvibe-contracts v0.127.0",
++  "contracts": "openvibe-contracts v0.128.0",
    "sdk": "openvibe-sdk v0.36.0",
    "shared": "openvibe-shared v2.21.0",
    "packages": [
--    "openvibe-contracts v0.127.0",
++    "openvibe-contracts v0.128.0",
      "openvibe-sdk v0.36.0",
--    "openvibe-shared v2.21.0"
++    "openvibe-shared v2.21.0",
++    "openvibe-publishing v1.4.0"
    ],
    "summary": "The help and support centre for every OpenVibe site. The help centre is generated at boot from openvibe-contracts (every product manifest and every service manifest's site block): one page per site, one page per question, each text quoted verbatim with its manifest named. /search searches those articles with a fixed explainable score and the network through OpenVibe.Search's public API (cached 5 minutes, degrading to the articles alone when it fails). A signed-in person opens a support ticket that stays one conversation — site, subject, message, an optional page URL stored as text and never fetched — sees only their own, and replies; staff by their Network role claim read the queue, filter it and set the state. Tickets live in PostgreSQL (migrations/0002_tickets.sql), writes made with the session cookie must be same-origin, and opening and answering are limited per caller. Sign-in is OpenVibe.Network (OAuth 2 + PKCE), the pages are server-rendered through the OpenVibe Frame and work without JavaScript, and the crawl artifacts cover every site and every question.",
    "works": [
ddiff --git a/migrations/0004_search_index.sql b/migrations/0004_search_index.sql
nnew file mode 100644
iindex 0000000..3edbfff
---- /dev/null
++++ b/migrations/0004_search_index.sql
@@@ -0,0 +1,31 @@
++-- phase: expand
++-- OpenVibe.Help: article pages in OpenVibe.Search (server/search-index.js, openvibe-publishing/search-feed).
++-- event_outbox holds the help.index_document.* events until the relay delivers them to OpenVibe.Events (written in
++-- the same transaction as the revision that produced them); help_index_revisions is the index sequencer, the last
++-- revision and content hash Search was sent for each page, so an unchanged page is never sent twice.
++
++CREATE TABLE IF NOT EXISTS event_outbox (
++    id              bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
++    event_id        text NOT NULL UNIQUE,
++    envelope        jsonb NOT NULL,
++    traceparent     text,
++    created_at      bigint NOT NULL,
++    attempts        integer NOT NULL DEFAULT 0,
++    next_attempt_at bigint NOT NULL DEFAULT 0,
++    sent_at         bigint,
++    seq             bigint,
++    rejected_at     bigint,
++    last_error      text
++);
++CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;
++CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;
++
++CREATE TABLE IF NOT EXISTS help_index_revisions (
++    owner      text COLLATE "C" NOT NULL,
++    type       text COLLATE "C" NOT NULL,
++    id         text COLLATE "C" NOT NULL,
++    revision   integer NOT NULL,
++    hash       text NOT NULL,
++    updated_at bigint NOT NULL,
++    PRIMARY KEY (owner, type, id)
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex 2e0e864..632233a 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -15,7 +15,8 @@
          "express-rate-limit": "^7.5.0",
          "helmet": "^8.1.0",
          "iovalkey": "^0.4.0",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
++        "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.21.0",
          "pg": "^8.23.0"
@@@ -871,9 +872,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.127.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
--      "integrity": "sha512-ySC+Qh8I3OlN5MULCqv0vfcAGubZQzWM0Fw7EOpnOo76vVnrkwx+m33aTxDum2yQqPkU8Ot5UtMuMwm1Cc1k3g==",
++      "version": "0.128.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
++      "integrity": "sha512-r/WKk3NhdqxEsVxqsVPjg+HTD2So1amRx0BY6EOHhvmwjLuL9j5XJrNDZAfqf/4/P2xZCBzTLrtUqlWxC2EA6Q==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
@@@ -886,6 +887,24 @@
          "node": ">=20"
        }
      },
++    "node_modules/openvibe-publishing": {
++      "version": "1.4.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
++      "integrity": "sha512-9vByyVqpvy2zVQI4PrJWk5QqCym/YQ/fXZDgrcloKGBqvukKrUn7fg4ZERiJHcgEq+rgDbxbU4rNflZhw62K/Q==",
++      "license": "MIT",
++      "engines": {
++        "node": ">=20"
++      },
++      "peerDependencies": {
++        "openvibe-sdk": ">=0.15.0",
++        "openvibe-shared": ">=1.5.0"
++      },
++      "peerDependenciesMeta": {
++        "openvibe-shared": {
++          "optional": true
++        }
++      }
++    },
      "node_modules/openvibe-sdk": {
        "version": "0.36.0",
        "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
ddiff --git a/package.json b/package.json
iindex f2f174a..277caf4 100644
---- a/package.json
++++ b/package.json
@@@ -2,7 +2,7 @@
    "name": "openvibe-help",
    "version": "0.1.0",
    "private": true,
--  "description": "OpenVibe.Help — Answers, support and a way to reach people.",
++  "description": "OpenVibe.Help \u2014 Answers, support and a way to reach people.",
    "main": "server/index.js",
    "scripts": {
      "start": "node server/index.js",
@@@ -22,7 +22,8 @@
      "express-rate-limit": "^7.5.0",
      "helmet": "^8.1.0",
      "iovalkey": "^0.4.0",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
++    "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.21.0",
      "pg": "^8.23.0"
ddiff --git a/server/app.js b/server/app.js
iindex af86896..713ad31 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -26,6 +26,7 @@ const cache = require('openvibe-shared/cache-policy');
  const configLib = require('./config');
  const { openStore } = require('./db');
  const { createCatalog } = require('./catalog');
++const { createSearchIndex } = require('./search-index');
  const { createNetworkSearch } = require('./search');
  const { createKeyStore } = require('./auth/keys');
  const { createSso } = require('./auth/sso');
@@@ -60,6 +61,8 @@ async function createApp(opts = {}) {
      const catalog = createCatalog({ now: s.now });
      const search = createNetworkSearch({ config, fetchImpl, now: s.now, log });
      const ctx = { config, s, keys, sso, principal, catalog, search, log };
++    // The articles in OpenVibe.Search (./search-index.js), swept on its own timer once server/index.js starts it.
++    ctx.searchIndex = opts.searchIndex || createSearchIndex({ config, s, catalog, log });
  
      // Account export and deletion (ADR-033, ./identity/account-data.js): the two tables that hold a person's rows.
      // The sender posts to Network's internal export/deletion routes with this service's own client-credentials token;
ddiff --git a/server/index.js b/server/index.js
iindex 292f723..f7c5dc7 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -15,7 +15,7 @@ const { gracefulStop } = require('openvibe-sdk/service');
  function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }) {
      return gracefulStop({
          name: 'OpenVibe.Help', server, deadlineExitCode: 0, exit, signals, deadlineMs: 10_000,
--        close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.s.close(), ...extra],
++        close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.searchIndex.stop(), () => ctx.s.close(), ...extra],
      });
  }
  
@@@ -28,6 +28,7 @@ async function start() {
      });
      server.keepAliveTimeout = 65_000;
      ctx.keys.client.start();
++    ctx.searchIndex.start();
  
      // Subscribe to the two ADR-033 topics at OpenVibe.Events (idempotent; off without HELP_EVENTS_URL and
      // HELP_EVENTS_SECRET). The consumer itself is mounted in server/app.js.
ddiff --git a/server/search-index.js b/server/search-index.js
nnew file mode 100644
iindex 0000000..f9418d1
---- /dev/null
++++ b/server/search-index.js
@@@ -0,0 +1,89 @@
++'use strict';
++
++/**
++ * Help's article pages in OpenVibe.Search (openvibe-publishing/search-feed): one search.index-document@1 per article
++ * (/a/<domain>/<slug>, id <domain>:<slug>), sent as help.index_document.upserted|deleted through this service's events
++ * outbox. An article is one question from a product manifest's FAQ with its answer, quoted verbatim from
++ * OpenVibe.Contracts (server/catalog.js); the document names the manifest's package as provenance.
++ *
++ * The catalog is built at boot, so a sweep a minute after boot and every six hours is all it takes: a new Contracts
++ * pin that adds, rewords or drops a question is a document, a new revision or a tombstone (help_index_revisions is
++ * the sequencer), and a quiet boot sends nothing.
++ *
++ *   const search = createSearchIndex({ config, s, catalog, log });
++ *   search.start(); search.sweep(); search.stop();
++ */
++const { createServiceOutbox } = require('openvibe-sdk/events');
++const { createSearchFeed } = require('openvibe-publishing/search-feed');
++
++const EVENT_TYPES = ['help.index_document.upserted', 'help.index_document.deleted'];
++const START_DELAY_MS = 60_000;
++const INTERVAL_MS = 6 * 3_600_000;
++
++const articleId = (a) => `${a.site}:${a.slug}`;
++
++/** One catalog article → what its page shows, as search-feed's document description. */
++function describe(article, catalog) {
++    const site = catalog.get(article.site);
++    return {
++        id: articleId(article),
++        title: article.question,
++        summary: String(article.answer || '').slice(0, 300),
++        body: [article.question, article.answer, article.siteName].filter(Boolean).join('\n'),
++        facets: { site: article.site, kind: (site && site.kind) || null, state: article.siteState || null },
++        authorship: { mode: 'human' },
++        provenance: [{ service: 'help', type: 'manifest', id: article.site, label: catalog.source.name, url: catalog.source.url }],
++        updatedAt: catalog.generatedAt,
++    };
++}
++
++function createSearchIndex({ config, s, catalog, log = console, outbox = null }) {
++    const out = outbox || createServiceOutbox({
++        db: s.db, source: 'help', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
++        clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,
++    });
++    // The catalog in id order once: rows() pages through it like a table.
++    const ordered = catalog.articles.map((a) => ({ ...a, id: articleId(a) })).sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
++    const ids = new Set(ordered.map((a) => a.id));
++    const feed = createSearchFeed({
++        owner: 'help', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,
++        types: {
++            article: {
++                page: (a) => a.path,
++                document: (a) => describe(a, catalog),
++                rows: async (after, limit) => ordered.filter((a) => a.id > after).slice(0, limit),
++                exists: async (list) => list.filter((id) => ids.has(id)),
++            },
++        },
++    });
++    const timers = [];
++
++    async function sweep() {
++        const res = await feed.sweep();
++        const a = res.article;
++        if (a.sent || a.removed || a.failed) log.log(`[Search] articles: ${a.sent} sent, ${a.removed} removed, ${a.failed} failed of ${a.seen}`);
++        if (out.kick) out.kick().catch(() => {});
++        return res;
++    }
++    const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
++
++    function start() {
++        if (timers.length) return false;
++        out.start();
++        const kick = setTimeout(quietly, START_DELAY_MS);
++        if (kick.unref) kick.unref();
++        const tick = setInterval(quietly, INTERVAL_MS);
++        if (tick.unref) tick.unref();
++        timers.push(kick, tick);
++        return true;
++    }
++
++    function stop() {
++        for (const t of timers.splice(0)) { clearTimeout(t); clearInterval(t); }
++        out.stop();
++    }
++
++    return { feed, outbox: out, describe: (a) => describe(a, catalog), sweep, start, stop, status: () => out.status() };
++}
++
++module.exports = { createSearchIndex, describe, articleId, EVENT_TYPES };
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex 3ba6c09..0cf9f31 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -69,6 +69,7 @@ async function boot(opts = {}) {
          base, network, config, ctx: built.ctx, get, signIn, dbDump,
          logs: () => captured.join('\n'),
          async close() {
++            built.ctx.searchIndex.stop();
              await new Promise((r) => server.close(r));
              built.ctx.keys.client.stop();
              await testdb.close();
ddiff --git a/test/search-index.test.js b/test/search-index.test.js
nnew file mode 100644
iindex 0000000..c1d7e72
---- /dev/null
++++ b/test/search-index.test.js
@@@ -0,0 +1,63 @@
++'use strict';
++/**
++ * Help's articles in OpenVibe.Search (server/search-index.js): a sweep puts every catalog article in the outbox as one
++ * help.index_document.upserted, exactly the contract, with the article's own page as its URL and the Contracts package
++ * as provenance; a second sweep sends nothing; an article the catalog no longer has becomes a tombstone.
++ */
++const assert = require('assert');
++const contracts = require('openvibe-contracts');
++const { boot, check, done } = require('./helpers/boot');
++const { createSearchIndex, articleId } = require('../server/search-index');
++
++(async () => {
++    const t = await boot();
++    const { s, catalog, searchIndex } = t.ctx;
++    const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);
++    const valid = (env) => {
++        assert.ok(contracts.validate('events.event-envelope@1', env).valid, 'envelope');
++        const p = contracts.validate(env.event_type, env.payload);
++        assert.ok(p.valid, `${env.event_type}: ${JSON.stringify(p.errors)} ${JSON.stringify(env.payload).slice(0, 200)}`);
++        assert.strictEqual(env.source, 'help');
++        return env;
++    };
++
++    try {
++        await check('a sweep sends every article once, as the contract says, and the page it names answers', async () => {
++            assert.ok(catalog.articles.length > 10, 'the catalog has articles');
++            const res = await searchIndex.sweep();
++            assert.deepStrictEqual(res.article, { seen: catalog.articles.length, sent: catalog.articles.length, removed: 0, failed: 0 });
++            const envs = (await outbox()).map(valid);
++            assert.strictEqual(envs.length, catalog.articles.length);
++            const a = catalog.articles[0];
++            const doc = envs.find((e) => e.payload.id === articleId(a)).payload;
++            assert.strictEqual(doc.title, a.question);
++            assert.strictEqual(doc.canonical_url, `https://openvibe.help${a.path}`);
++            assert.ok(doc.body.includes(a.answer.slice(0, 40)), 'the answer, verbatim');
++            assert.strictEqual(doc.facets.site, a.site);
++            assert.strictEqual(doc.provenance[0].service, 'help');
++            assert.strictEqual(doc.provenance[0].type, 'manifest');
++            const page = await t.get(a.path);
++            assert.strictEqual(page.status, 200, a.path);
++        });
++
++        await check('a second sweep sends nothing', async () => {
++            const before = (await outbox()).length;
++            assert.strictEqual((await searchIndex.sweep()).article.sent, 0);
++            assert.strictEqual((await outbox()).length, before);
++        });
++
++        await check('an article the catalog dropped becomes a tombstone, once', async () => {
++            const dropped = catalog.articles[1];
++            const smaller = { ...catalog, articles: catalog.articles.filter((x) => x !== dropped) };
++            const other = createSearchIndex({ config: t.config, s, catalog: smaller, outbox: searchIndex.outbox, log: { log() {}, warn() {} } });
++            const res = await other.sweep();
++            assert.strictEqual(res.article.removed, 1);
++            const env = valid((await outbox()).pop());
++            assert.strictEqual(env.event_type, 'help.index_document.deleted');
++            assert.deepStrictEqual(env.payload, { type: 'article', id: articleId(dropped), revision: 2 });
++            assert.strictEqual((await other.sweep()).article.removed, 0);
++        });
++    } finally {
++        await done(t);
++    }
++})();