openvibers/openvibe.live Commits

Live's guides go to OpenVibe.Search, the headline one included (#562)

OP OpenVibers committed 11 days ago · cf6df36d86 · 3 files, +201 −0

Live's guides (/docs/<name>, such as go-live-in-your-browser) were on no
network search. Each is now a Search document of type doc (Contracts
0.278.0): its first heading as the title, its first paragraph as the
summary, its text without markup or code as the body, sent through the
outbox once after boot and daily, again only when it changed. Operator
notes and retired pages (deploy, performance-audit, arena, restream-branding,
vods-and-clips) stay on the site and off search.

Changes

ddiff --git a/server/events/search-documents.js b/server/events/search-documents.js
iindex cbdb589f..cae381bf 100644
---- a/server/events/search-documents.js
++++ b/server/events/search-documents.js
@@@ -113,6 +113,8 @@ function init() {
const jobs = require('../utils/jobs');
jobs.every('search-documents-scan', 5 * 60 * 1000, () => scan(), { initialDelayMs: 2 * 60 * 1000, jitterMs: 15 * 1000 });
jobs.every('search-documents-refresh', DAY_MS, () => refresh(), { initialDelayMs: 4 * 60 * 1000, jitterMs: 60 * 1000 });
++ // Live's guides (./search-guides.js): once soon after boot, then daily; a guide changes only with a release.
++ jobs.every('search-guides', DAY_MS, () => require('./search-guides').publishAll(), { initialDelayMs: 5 * 60 * 1000, jitterMs: 60 * 1000 });
return true;
}
ddiff --git a/server/events/search-guides.js b/server/events/search-guides.js
nnew file mode 100644
iindex 00000000..f1d02c79
---- /dev/null
++++ b/server/events/search-guides.js
@@@ -0,0 +1,121 @@
++'use strict';
++/**
++ * Live's guides in OpenVibe.Search (Contracts 0.278.0: live.index_document type `doc`): one document per Markdown file in
++ * ./docs that /docs/<name> serves (server/docs/routes.js), published through Live's transactional outbox like the
++ * channels (./search-documents.js). The headline guide, go-live-in-your-browser, and the rest were on no network search.
++ *
++ * documentFor(name, markdown) the document: the guide's first heading as its title, its first paragraph as the
++ * summary, its text (no markup, no code blocks) as the body
++ * publishAll() every guide whose document changed since this process last sent it
++ *
++ * The revision is the send time in seconds, so it only grows across restarts; a restart sends each guide once more,
++ * unchanged documents after that are not sent. The docs index (README) is the /docs page itself and is left out.
++ */
++const crypto = require('crypto');
++const fs = require('fs');
++const path = require('path');
++const db = require('../db/database');
++const streamEvents = require('./stream-events');
++
++const PUBLIC_BASE = (process.env.LIVE_PUBLIC_ORIGIN || 'https://openvibe.live').replace(/\/+$/, '');
++const DOCS_DIR = process.env.OV_APP_ROOT ? path.join(process.env.OV_APP_ROOT, 'docs') : path.join(__dirname, '../../docs');
++const NAME_RE = /^[a-z][a-z0-9-]{1,63}$/;
++// Operator notes and retired pages stay on the site but off network search: how Live is deployed, a performance log,
++// the retired Arena, the brand sheet for restream overlays, and the pre-split VOD pipeline (it opens on an env var).
++const NOT_A_GUIDE = new Set(['deploy', 'performance-audit', 'arena', 'restream-branding', 'vods-and-clips']);
++const sent = new Map(); // name → hash of the document last sent by this process
++const stats = { sent: 0, unchanged: 0, lastError: null };
++
++/** Markdown to plain text: no code blocks, no markup, links as their words, one space between words. */
++function plain(markdown) {
++ return String(markdown || '')
++ .replace(/```[\s\S]*?```/g, ' ')
++ .replace(/`([^`]*)`/g, '$1')
++ .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
++ .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
++ .replace(/^\s{0,3}#{1,6}\s+/gm, '')
++ .replace(/^\s*[-*+]\s+/gm, '')
++ .replace(/^\s*\d+\.\s+/gm, '')
++ .replace(/^\s*>\s?/gm, '')
++ .replace(/^\s*\|?\s*:?-{3,}.*$/gm, ' ')
++ .replace(/\|/g, ' ')
++ .replace(/(\*\*|__|\*|_)(\S[^*_]*?)\1/g, '$2')
++ .replace(/<[^>]+>/g, ' ')
++ .replace(/\s+/g, ' ')
++ .trim();
++}
++
++/** A summary that ends on a sentence when it can (the same rule the /docs pages use for their description). */
++function summaryOf(text, max = 300) {
++ if (text.length <= max) return text;
++ const cut = text.slice(0, max);
++ const end = Math.max(cut.lastIndexOf('. '), cut.lastIndexOf('! '), cut.lastIndexOf('? '));
++ return end >= 80 ? cut.slice(0, end + 1) : `${cut.slice(0, cut.lastIndexOf(' '))}…`;
++}
++
++/** The Search document for one guide, or null when it has no title. */
++function documentFor(name, markdown) {
++ if (!NAME_RE.test(name)) return null;
++ const md = String(markdown || '').replace(/\r\n/g, '\n');
++ const heading = /^#\s+(.+)$/m.exec(md);
++ if (!heading) return null;
++ const title = plain(heading[1]).slice(0, 200);
++ const afterTitle = md.slice(heading.index + heading[0].length);
++ // The first paragraph that is prose: not a heading, list, table, quote, code fence or badge line.
++ const first = afterTitle.split(/\n\s*\n/).map((p) => p.trim()).find((p) => p && !/^(#|[-*+]\s|\d+\.\s|\||>|```|!\[|<)/.test(p)) || '';
++ const summary = summaryOf(plain(first));
++ const body = plain(afterTitle).slice(0, 8000);
++ return {
++ owner: 'live', type: 'doc', id: name, deleted: false, visibility: 'public',
++ canonical_url: `${PUBLIC_BASE}/docs/${name}`,
++ title,
++ summary: summary || `${title}: a guide to OpenVibe.Live.`,
++ // The summary is read with the body, so the body leaves out what the summary already says.
++ body: summary && body.startsWith(summary.replace(/…$/, '')) ? body.slice(summary.replace(/…$/, '').length).trim() : body,
++ facets: { kind: 'guide' },
++ authorship: 'human', publication_state: 'published',
++ indexability: { decision: 'index' },
++ };
++}
++
++const hashOf = (doc) => crypto.createHash('sha256').update(JSON.stringify(doc)).digest('hex').slice(0, 32);
++
++/** The guides on disk: [{ name, markdown }]. */
++function guides(dir = DOCS_DIR) {
++ let files = [];
++ try { files = fs.readdirSync(dir); } catch { return []; }
++ return files.filter((f) => f.endsWith('.md') && f !== 'README.md').map((f) => f.slice(0, -3)).filter((n) => NAME_RE.test(n) && !NOT_A_GUIDE.has(n)).sort()
++ .map((name) => ({ name, markdown: fs.readFileSync(path.join(dir, `${name}.md`), 'utf8') }));
++}
++
++/** Send every guide whose document changed since this process last sent it. → how many were sent */
++async function publishAll({ now = Date.now(), dir = DOCS_DIR } = {}) {
++ if (!streamEvents.status().enabled) return 0;
++ let count = 0;
++ let revision = Math.floor(now / 1000);
++ for (const { name, markdown } of guides(dir)) {
++ try {
++ const doc = documentFor(name, markdown);
++ if (!doc) continue;
++ const hash = hashOf(doc);
++ if (sent.get(name) === hash) { stats.unchanged++; continue; }
++ revision += 1; // one more each, so two guides sent in the same second never share one
++ // The outbox takes events inside a transaction (as the channels' documents do).
++ await db.getDb().tx(async () => {
++ await streamEvents.enqueue({
++ event_type: 'live.index_document.upserted', actor: { type: 'service', id: 'live' }, subject: { type: 'doc', id: name, revision },
++ visibility: 'internal', priority: 'low', payload: { ...doc, revision, updated_at: new Date(now).toISOString() },
++ });
++ });
++ sent.set(name, hash);
++ stats.sent++;
++ count++;
++ } catch (err) { stats.lastError = err.message; }
++ }
++ if (count) streamEvents.kick();
++ return count;
++}
++
++function status() { return { ...stats }; }
++
++module.exports = { documentFor, publishAll, guides, plain, status };
ddiff --git a/test/search-guides.test.js b/test/search-guides.test.js
nnew file mode 100644
iindex 00000000..ff1bfef7
---- /dev/null
++++ b/test/search-guides.test.js
@@@ -0,0 +1,78 @@
++'use strict';
++// Live's guides in OpenVibe.Search (server/events/search-guides.js; Contracts 0.278.0 type `doc`): each guide in ./docs
++// goes through the outbox as live.index_document.upserted, a valid Search document whose id is the guide's name, again
++// only when it changed; operator notes and retired pages stay off; the headline guide reads as itself.
++const assert = require('assert');
++const http = require('http');
++const quiet = console.log;
++console.log = (...a) => { if (!/^\[/.test(String(a[0]))) quiet(...a); };
++console.warn = () => {};
++
++const published = [];
++const stub = http.createServer((req, res) => {
++ let body = '';
++ req.on('data', (c) => { body += c; });
++ req.on('end', () => {
++ res.setHeader('Content-Type', 'application/json');
++ if (req.url === '/oauth/token') return res.end(JSON.stringify({ access_token: 'tok', token_type: 'Bearer', expires_in: 300 }));
++ if (req.url === '/api/v1/events' && req.method === 'POST') {
++ const parsed = JSON.parse(body);
++ const list = parsed.events || [parsed];
++ const { validate } = require('openvibe-contracts');
++ const bad = list.find((e) => !validate('events.event-envelope@1', e).valid);
++ if (bad) { res.statusCode = 422; return res.end(JSON.stringify({ code: 'events.invalid_envelope' })); }
++ const results = list.map((e) => { published.push(e); return { event_id: e.event_id, seq: published.length, duplicate: false }; });
++ return res.end(JSON.stringify(parsed.events ? { results } : results[0]));
++ }
++ res.statusCode = 404; res.end('{}');
++ });
++});
++
++(async () => {
++ await new Promise((r) => stub.listen(0, '127.0.0.1', r));
++ const base = `http://127.0.0.1:${stub.address().port}`;
++ process.env.OV_NETWORK_INTERNAL_URL = base;
++ const { validate } = require('openvibe-contracts');
++ const db = require('../server/db/database');
++ await db.initDb();
++ const streamEvents = require('../server/events/stream-events');
++ const outbox = streamEvents.init({ eventsUrl: base, clientSecret: 's3cret', intervalMs: 50 });
++ const guides = require('../server/events/search-guides');
++
++ const names = guides.guides().map((g) => g.name);
++ assert.ok(names.includes('go-live-in-your-browser') && names.includes('whip') && names.includes('streams-api'), names.join(','));
++ for (const off of ['deploy', 'performance-audit', 'arena', 'restream-branding', 'vods-and-clips', 'README']) assert.ok(!names.includes(off), `${off} stays off search`);
++
++ const now = Date.parse('2026-11-02T18:00:00Z');
++ const n = await guides.publishAll({ now });
++ assert.strictEqual(n, names.length, 'every guide once');
++ await outbox.flush();
++ const sentDocs = published.filter((e) => e.event_type === 'live.index_document.upserted');
++ assert.strictEqual(sentDocs.length, names.length);
++ const head = sentDocs.find((e) => e.payload.id === 'go-live-in-your-browser');
++ assert.deepStrictEqual([head.subject.type, head.subject.id], ['doc', 'go-live-in-your-browser']);
++ // What Search validates (server/api/webhook.js: search.index-document@1), with the owner it takes from the event.
++ const v = validate('search.index-document@1', head.payload);
++ assert.ok(v.valid, JSON.stringify(v.errors));
++ assert.strictEqual(head.payload.canonical_url, 'https://openvibe.live/docs/go-live-in-your-browser');
++ assert.match(head.payload.title, /^Go live from your browser/);
++ assert.match(head.payload.summary, /nothing to install/);
++ assert.ok(!head.payload.body.includes(head.payload.summary.slice(0, 60)), 'the body does not repeat the summary');
++ assert.ok(!/[#`*|]{2}|\]\(/.test(head.payload.body), 'no Markdown left in the text');
++ const revs = sentDocs.map((e) => e.payload.revision);
++ assert.strictEqual(new Set(revs).size, revs.length, 'each its own revision');
++ assert.ok(Math.min(...revs) > Math.floor(now / 1000), 'revisions from the send time, so a restart only goes up');
++ assert.strictEqual(await guides.publishAll({ now: now + 60_000 }), 0, 'nothing changed: nothing sent');
++
++ // A guide's own shape: title from its heading, summary from its first prose paragraph.
++ const doc = guides.documentFor('sample-guide', '# Sample guide\n\n> A note.\n\nFirst **real** paragraph with a [link](https://x.y). More.\n\n## Part\n\n- a point\n\n```js\ncode();\n```\n');
++ assert.deepStrictEqual([doc.title, doc.summary], ['Sample guide', 'First real paragraph with a link. More.']);
++ assert.ok(!doc.body.includes('code()'), 'no code blocks in the text');
++ assert.strictEqual(guides.documentFor('Bad Name', '# x'), null);
++ assert.strictEqual(guides.documentFor('no-title', 'no heading here'), null);
++
++ await outbox.stop();
++ stub.close();
++ console.log('search guides: every guide once, valid for Search, the headline guide as itself, nothing again until it changes');
++ process.exit(0);
++})().catch((err) => { console.error(err); process.exit(1); });