openvibers/openvibe.bot Commits

Bot: the robot model pages are in OpenVibe.Search (owner bot, type model) (#130)

OP OpenVibers committed 28 days ago · 8945fd80ae · 5 files, +209 −0

Every model Bot drives (/models/<id>) is a search.index-document@1 through Search's owner API with
Bot's service token (search.document.write): name and vendor, description, how it connects, what it
does and its setup steps. A minute after boot and every six hours; re-sent one revision up only when it
changed; a model that leaves the catalogue is tombstoned. Off without BOT_SEARCH_URL.

Changes

ddiff --git a/README.md b/README.md
iindex 24777db..4997f29 100644
---- a/README.md
++++ b/README.md
@@@ -120,6 +120,8 @@ Production runs the Node process under systemd as `openvibe-bot.service` (unit r
To run the schema migrations before the service serves, start it once with the direct owner connection in `/etc/openvibe/bot.env` (`DATABASE_DIRECT_URL`); the boot applies `migrations/` and then serves on the pooled `DATABASE_URL`.
++Search: every robot model page (`/models/<id>`) is an OpenVibe.Search document (owner `bot`, type `model`) written through Search's owner API with Bot's service token (`search.document.write`; `server/search-index.js`, `BOT_SEARCH_URL`), a minute after boot and every six hours; a model that leaves the catalogue is tombstoned.
++
Request telemetry: each HTTP request is aggregated by route template, method and status class, never by raw URL, and `/shared` files, static assets, health, readiness and `/metrics` are not counted. Once a minute one `http.request` sample per key, plus the active-requests, p95 latency and event-loop-lag gauges, is written to Bot's own `telemetry_samples` table (migration `0018_telemetry.sql`; openvibe-sdk 0.46 telemetry, the `platform.telemetry-sample@1` shape), and rows older than 14 days are deleted daily. `server/app.js` mounts openvibe-sdk's `telemetryMiddleware` (a no-op until init); `server/index.js` calls `telemetry.init()` at production start, runs the retention and flushes on stop.
Set `BOT_PAIRING_AUTHORITY` to `bot` (the default) or `network`; any other value refuses to boot.
ddiff --git a/server/config.js b/server/config.js
iindex d256bec..b997f81 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -146,6 +146,8 @@ function loadConfig(env = process.env) {
scope: env.OV_OAUTH_SCOPE || 'profile',
},
cookies: { secure: env.COOKIE_SECURE != null ? bool(env.COOKIE_SECURE) : isProduction },
++ // OpenVibe.Search's owner API (server/search-index.js): the model pages as Search documents. Unset: off.
++ search: { url: trim(env.BOT_SEARCH_URL || '') },
events: {
url: trim(env.EVENTS_URL || ''),
ddiff --git a/server/index.js b/server/index.js
iindex 492b2be..13a0000 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -48,6 +48,8 @@ async function main() {
routines.start(); // scheduled routines (server/routines.js)
webhooks.start(); // robot webhook deliveries (server/webhooks.js)
offers.start(); // the worker offers of the machines connected here, to Network (server/jobs/offers.js)
++ // The model pages in OpenVibe.Search (server/search-index.js; off without BOT_SEARCH_URL).
++ require('./search-index').createModelSearchIndex({ config, db, log: console }).start();
}
const server = app.listen(config.port, config.host, () => {
ddiff --git a/server/search-index.js b/server/search-index.js
nnew file mode 100644
iindex 0000000..2dd05c6
---- /dev/null
++++ b/server/search-index.js
@@@ -0,0 +1,113 @@
++'use strict';
++/**
++ * Bot's model pages in OpenVibe.Search: every robot model Bot drives (/models/<id>: Cozmo, LOOI, the Adeept kits, a
++ * phone camera, ONVIF cameras, ROS 2 bases…) is one document (search.index-document@1, owner bot, type model, id = the
++ * profile id), written through Search's owner API with Bot's service token (search.document.write on
++ * openvibe.search). A model that leaves the catalogue is tombstoned.
++ *
++ * Runs a minute after boot and every six hours (the catalogue changes with a deploy or an approved community driver).
++ * The stored copy is read back for each model and re-sent, one revision up, only when its content differs.
++ *
++ * Off unless BOT_SEARCH_URL and the OAuth client secret are set.
++ */
++const { serviceAuth } = require('openvibe-contracts');
++const { listProfiles } = require('./profiles');
++const { modelOf, CONNECT, CAPABILITY_WORDS } = require('./web/directory');
++
++const OWNER = 'bot';
++const TYPE = 'model';
++const EVERY_MS = 6 * 60 * 60 * 1000;
++const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:~-]{0,127}$/;
++const COMPARED = ['canonical_url', 'title', 'summary', 'body'];
++
++/** A model (directory.modelOf) → the document body; identity and revision are added by the caller. */
++function documentFor(m, { baseUrl }) {
++ const how = CONNECT[m.connect] || null;
++ const caps = (m.capabilities || []).map((c) => CAPABILITY_WORDS[c]).filter(Boolean);
++ return {
++ visibility: 'public',
++ canonical_url: `${baseUrl}/models/${m.id}`,
++ title: `${m.name}${m.vendor ? ` (${m.vendor})` : ''}`.slice(0, 500),
++ summary: (m.description || `Drive a ${m.name} from your browser with OpenVibe.Bot.`).slice(0, 4000),
++ body: [m.name, m.vendor, m.description, how ? `${how.label}: ${how.text}` : '', caps.length ? `What it does: ${caps.join(', ')}.` : '',
++ ...(m.setup || []).map((step) => (typeof step === 'string' ? step : step && (step.text || step.title) ? String(step.text || step.title) : ''))]
++ .filter(Boolean).join('\n').slice(0, 48000),
++ facets: { kind: 'robot_model', connect: m.connect, ...(m.vendor ? { vendor: m.vendor } : {}), community: Boolean(m.community) },
++ authorship: 'human',
++ publication_state: 'published',
++ indexability: { decision: 'index', reasons: [] },
++ };
++}
++
++function createModelSearchIndex({ config, db, fetchImpl = globalThis.fetch, log = console, tokens = null, models = null }) {
++ const url = String((config.search && config.search.url) || '').replace(/\/+$/, '');
++ const enabled = Boolean(url && (tokens || config.oauth.clientSecret));
++ const auth = tokens || (enabled ? serviceAuth.createTokenClient({
++ tokenUrl: `${config.network.internalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
++ audience: 'openvibe.search', scope: 'search.document.write', fetchImpl,
++ }) : null);
++ const baseUrl = String(config.baseUrl).replace(/\/+$/, '');
++ const list = models || (async () => (await listProfiles(db)).map((row) => modelOf(row.profile)));
++ const stats = { runs: 0, upserted: 0, deleted: 0, unchanged: 0, failed: 0, last_error: null };
++
++ async function call(method, p, body) {
++ const res = await fetchImpl(`${url}${p}`, {
++ method,
++ headers: { Accept: 'application/json', ...(body ? { 'Content-Type': 'application/json' } : {}), ...(await auth.authHeaders()) },
++ body: body ? JSON.stringify(body) : undefined,
++ signal: AbortSignal.timeout(15000),
++ });
++ const json = await res.json().catch(() => ({}));
++ if (res.status === 401 && auth.invalidate) auth.invalidate();
++ return { status: res.status, ok: res.ok, body: json };
++ }
++
++ async function run() {
++ if (!enabled) return null;
++ stats.runs++;
++ const listing = await call('GET', `/api/v1/owners/${OWNER}/documents?type=${TYPE}&limit=1000`);
++ if (!listing.ok) throw new Error(`listing: ${listing.status} ${listing.body.code || ''}`);
++ const remote = new Map((listing.body.documents || []).map((d) => [d.id, d]));
++ const keep = new Set();
++ for (const m of await list()) {
++ if (!ID_RE.test(String(m.id))) continue;
++ keep.add(m.id);
++ const doc = documentFor(m, { baseUrl });
++ const r = remote.get(m.id);
++ if (r && !r.deleted) {
++ const stored = await call('GET', `/api/v1/owners/${OWNER}/documents/${TYPE}/${encodeURIComponent(m.id)}`);
++ const s = stored.ok ? stored.body.document : null;
++ if (s && COMPARED.every((k) => s[k] === doc[k])) { stats.unchanged++; continue; }
++ }
++ const res = await call('PUT', `/api/v1/documents/${OWNER}/${TYPE}/${encodeURIComponent(m.id)}`, { ...doc, revision: r ? Number(r.revision) + 1 : 1 });
++ if (res.ok) stats.upserted++;
++ else { stats.failed++; stats.last_error = `${m.id}: ${res.status} ${res.body.code || ''}`; }
++ }
++ for (const [id, r] of remote) {
++ if (keep.has(id) || r.deleted) continue;
++ const res = await call('DELETE', `/api/v1/documents/${OWNER}/${TYPE}/${encodeURIComponent(id)}?revision=${Number(r.revision) + 1}`);
++ if (res.ok) stats.deleted++;
++ else { stats.failed++; stats.last_error = `${id}: ${res.status} ${res.body.code || ''}`; }
++ }
++ return { ...stats };
++ }
++
++ const quietly = () => run().catch((err) => { stats.failed++; stats.last_error = err.message; log.warn(`[Bot] search index run failed: ${err.message}`); });
++ let timer = null;
++ let first = null;
++ return {
++ enabled,
++ run,
++ stats: () => ({ ...stats }),
++ start({ firstDelayMs = 60_000 } = {}) {
++ if (!enabled || timer) return;
++ first = setTimeout(quietly, firstDelayMs);
++ if (first.unref) first.unref();
++ timer = setInterval(quietly, EVERY_MS);
++ if (timer.unref) timer.unref();
++ },
++ stop() { clearTimeout(first); clearInterval(timer); timer = null; first = null; },
++ };
++}
++
++module.exports = { createModelSearchIndex, documentFor };
ddiff --git a/test/model-search-index.test.js b/test/model-search-index.test.js
nnew file mode 100644
iindex 0000000..e639c7d
---- /dev/null
++++ b/test/model-search-index.test.js
@@@ -0,0 +1,90 @@
++'use strict';
++/**
++ * Bot's model pages in OpenVibe.Search (server/search-index.js), against a stand-in Search owner API: every model of
++ * the catalogue is a valid search.index-document@1 (owner bot, type model, /models/<id>); unchanged ones are not sent
++ * again; a model that leaves the catalogue is tombstoned; off without BOT_SEARCH_URL.
++ */
++const assert = require('assert');
++const http = require('http');
++const { validate } = require('openvibe-contracts');
++const { modelOf } = require('../server/web/directory');
++const { createModelSearchIndex } = require('../server/search-index');
++
++let failures = 0;
++async function check(name, fn) {
++ try { await fn(); console.log(` ✓ ${name}`); } catch (err) { failures++; console.log(` ✗ ${name}\n ${err && err.stack}`); }
++}
++
++function startSearch() {
++ const docs = new Map();
++ const puts = [];
++ const server = http.createServer((req, res) => {
++ let raw = '';
++ req.on('data', (c) => { raw += c; });
++ req.on('end', () => {
++ const u = new URL(req.url, 'http://search.test');
++ const send = (status, body) => { res.writeHead(status, { 'content-type': 'application/json' }); res.end(JSON.stringify(body)); };
++ if (req.headers.authorization !== 'Bearer svc-bot') return send(401, { code: 'token.invalid' });
++ if (req.method === 'GET' && u.pathname === '/api/v1/owners/bot/documents') {
++ return send(200, { documents: [...docs.values()].map((d) => ({ type: 'model', id: d.id, revision: d.revision, deleted: Boolean(d.deleted) })) });
++ }
++ let m = /^\/api\/v1\/owners\/bot\/documents\/model\/([^/]+)$/.exec(u.pathname);
++ if (m && req.method === 'GET') {
++ const d = docs.get(decodeURIComponent(m[1]));
++ return d && !d.deleted ? send(200, { document: d }) : send(404, { code: 'search.not_found' });
++ }
++ m = /^\/api\/v1\/documents\/bot\/model\/([^/]+)$/.exec(u.pathname);
++ if (m && req.method === 'PUT') {
++ const doc = { ...JSON.parse(raw), owner: 'bot', type: 'model', id: decodeURIComponent(m[1]) };
++ const v = validate('search.index-document@1', doc);
++ if (!v.valid) return send(422, { code: 'search.bad_document', errors: v.errors });
++ docs.set(doc.id, doc);
++ puts.push(doc.id);
++ return send(200, { outcome: 'applied' });
++ }
++ if (m && req.method === 'DELETE') {
++ const id = decodeURIComponent(m[1]);
++ docs.set(id, { id, revision: Number(u.searchParams.get('revision')), deleted: true });
++ return send(200, { outcome: 'applied' });
++ }
++ return send(404, { code: 'route.not_found' });
++ });
++ });
++ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ docs, puts, url: `http://127.0.0.1:${server.address().port}`, close: () => new Promise((r) => server.close(r)) })));
++}
++
++(async () => {
++ const search = await startSearch();
++ const profile = (id, name, extra = {}) => ({ id, name, vendor: 'Acme', description: `${name} drives in your browser.`, kind: 'bridge', capabilities: ['drive', 'camera'], ...extra });
++ let catalogue = [modelOf(profile('cozmo', 'Cozmo', { vendor: 'Digital Dream Labs' })), modelOf(profile('looi', 'LOOI'))];
++ const config = { search: { url: search.url }, oauth: { clientSecret: 'x', clientId: 'bot' }, network: { internalUrl: 'http://network.test' }, baseUrl: 'https://openvibe.bot' };
++ const index = createModelSearchIndex({ config, tokens: { authHeaders: async () => ({ authorization: 'Bearer svc-bot' }) }, models: async () => catalogue, log: { warn() {} } });
++
++ await check('every model is a valid document at /models/<id>', async () => {
++ const r = await index.run();
++ assert.strictEqual(r.upserted, 2, JSON.stringify(r));
++ const doc = search.docs.get('cozmo');
++ assert.strictEqual(doc.canonical_url, 'https://openvibe.bot/models/cozmo');
++ assert.strictEqual(doc.title, 'Cozmo (Digital Dream Labs)');
++ assert.strictEqual(doc.facets.kind, 'robot_model');
++ });
++
++ await check('unchanged is not sent again; a model gone from the catalogue is tombstoned', async () => {
++ await index.run();
++ assert.strictEqual(search.puts.length, 2);
++ catalogue = catalogue.filter((m) => m.id !== 'looi');
++ const r = await index.run();
++ assert.strictEqual(r.deleted, 1);
++ assert.deepStrictEqual(search.docs.get('looi'), { id: 'looi', revision: 2, deleted: true });
++ });
++
++ await check('off without BOT_SEARCH_URL', async () => {
++ const off = createModelSearchIndex({ config: { ...config, search: { url: '' } }, models: async () => catalogue });
++ assert.strictEqual(off.enabled, false);
++ assert.strictEqual(await off.run(), null);
++ });
++
++ await search.close();
++ console.log(failures ? `\n${failures} failed` : '\nall passed');
++ process.exit(failures ? 1 : 0);
++})();