openvibers/openvibe.design Commits

Icon and theme pages in OpenVibe.Search (#130)

OP OpenVibers committed 11 days ago · d1d751e125 · 10 files, +267 −9

277 icon pages and 50 theme pages were on no network search. server/search-index.js sends one
design.index_document per glyph (its name, the Font Awesome names it stands in for, the formats)
and per public theme (name, mode, description), through a new event_outbox (migration 0010, with
design_index_revisions), a minute after boot and every six hours. A page that left the catalogue
gets a tombstone; a Network that does not answer the theme list stops the sweep. Off under
DESIGN_DRILL. The account-data guard exempts the sequencer: its owner column is the service.

Changes

ddiff --git a/README.md b/README.md
iindex 0abd205..8e534a0 100644
---- a/README.md
++++ b/README.md
@@@ -227,7 +227,8 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by
[OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
--- openvibe-contracts: v0.236.0
++- openvibe-contracts: v0.275.0
++- openvibe-publishing: v1.5.0
- openvibe-sdk: v0.81.0
- openvibe-shared: v4.103.0
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex bcfe788..6ae1faa 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,11 +8,12 @@
"runtime": "node 22, express 4, openvibe-shared/shell; PostgreSQL (or embedded PGlite in development), OpenVibe.AI generate jobs, OpenVibe.Events account data in, no Valkey",
"port": "5090",
"domain": "openvibe.design",
-- "contracts": "openvibe-contracts v0.236.0",
++ "contracts": "openvibe-contracts v0.275.0",
"sdk": "openvibe-sdk v0.81.0",
"shared": "openvibe-shared v4.103.0",
"packages": [
-- "openvibe-contracts v0.236.0",
++ "openvibe-contracts v0.275.0",
++ "openvibe-publishing v1.5.0",
"openvibe-sdk v0.81.0",
"openvibe-shared v4.103.0"
],
ddiff --git a/migrations/0010_search_index.sql b/migrations/0010_search_index.sql
nnew file mode 100644
iindex 0000000..791807d
---- /dev/null
++++ b/migrations/0010_search_index.sql
@@@ -0,0 +1,31 @@
++-- phase: expand
++-- OpenVibe.Design: the icon and theme pages in OpenVibe.Search (server/search-index.js, openvibe-publishing/search-feed).
++-- event_outbox holds the design.index_document.* events until the relay delivers them to OpenVibe.Events (written in
++-- the same transaction as the revision that produced them); design_index_revisions is the index sequencer, the last
++-- revision and content hash Search was sent for each page, so an unchanged page is never sent twice.
++
++CREATE TABLE IF NOT EXISTS event_outbox (
++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
++ event_id text NOT NULL UNIQUE,
++ envelope jsonb NOT NULL,
++ traceparent text,
++ created_at bigint NOT NULL,
++ attempts integer NOT NULL DEFAULT 0,
++ next_attempt_at bigint NOT NULL DEFAULT 0,
++ sent_at bigint,
++ seq bigint,
++ rejected_at bigint,
++ last_error text
++);
++CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;
++CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;
++
++CREATE TABLE IF NOT EXISTS design_index_revisions (
++ owner text COLLATE "C" NOT NULL,
++ type text COLLATE "C" NOT NULL,
++ id text COLLATE "C" NOT NULL,
++ revision integer NOT NULL,
++ hash text NOT NULL,
++ updated_at bigint NOT NULL,
++ PRIMARY KEY (owner, type, id)
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex fb6b659..3858e74 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -15,7 +15,8 @@
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.236.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.275.0",
++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.5.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.103.0",
"pg": "^8.23.1",
@@@ -1321,9 +1322,9 @@
}
},
"node_modules/openvibe-contracts": {
-- "version": "0.236.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.236.0",
-- "integrity": "sha512-SPq+/t6ohdwW2DPqK9nZmtbeFex7q6KN/eQVc19MTQ30Df/m7A/idixZGcfs4I3Z43pUpLN8I4ylfOLlFShqsA==",
++ "version": "0.275.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.275.0",
++ "integrity": "sha512-s8D3Upj5lfCGSnTVXKEWyzDEG5b7QnY/O2tAPieaZgFhzM4rnZyboGCZv3qthVOZ1ibxx/ZTEtS9nxZ/AVL+Hg==",
"license": "AGPL-3.0-only",
"dependencies": {
"ajv": "^8.17.1",
@@@ -1336,6 +1337,24 @@
"node": ">=20"
}
},
++ "node_modules/openvibe-publishing": {
++ "version": "1.5.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.5.0",
++ "integrity": "sha512-8cKVhJvpbDgWuGR+Dh1mpgqfWPJoC7EIcw7eTRc5/2ueDc8caCA7XKdcpAHNZYMdukIAL8o7FvxE8QV5/1uSLw==",
++ "license": "MIT",
++ "engines": {
++ "node": ">=20"
++ },
++ "peerDependencies": {
++ "openvibe-sdk": ">=0.15.0",
++ "openvibe-shared": ">=1.5.0"
++ },
++ "peerDependenciesMeta": {
++ "openvibe-shared": {
++ "optional": true
++ }
++ }
++ },
"node_modules/openvibe-sdk": {
"version": "0.81.0",
"resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
ddiff --git a/package.json b/package.json
iindex aba4fb9..cc6d491 100644
---- a/package.json
++++ b/package.json
@@@ -21,7 +21,8 @@
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.236.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.275.0",
++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.5.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.103.0",
"pg": "^8.23.1",
ddiff --git a/server/app.js b/server/app.js
iindex 425a35f..a0da3fe 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -32,6 +32,7 @@ const { openStore } = require('./db');
const { createKeyStore } = require('./auth/keys');
const { createSso } = require('./auth/sso');
const { createThemeCatalog } = require('./themes');
++const { createSearchIndex } = require('./search-index');
const { createApi } = require('./http/api');
const { createMcpRoutes } = require('./http/mcp');
const { createArenaApi } = require('./http/arena-api');
@@@ -65,6 +66,8 @@ async function createApp(opts = {}) {
const keys = createKeyStore({ config, fetchImpl, log });
const sso = createSso({ config, keys, fetchImpl, log, now });
const catalog = opts.catalog || createThemeCatalog({ config, fetchImpl, log, now });
++ // The icon and theme pages in OpenVibe.Search (started by server/index.js, never under DESIGN_DRILL).
++ const searchIndex = opts.searchIndex || createSearchIndex({ config, s, catalog, log });
const bench = opts.bench || createBenchClient({ config, fetchImpl, log });
const arena = opts.arena || createArenaStore({ s, config, bench, log });
const ai = opts.arenaAi || opts.ai || createArenaAi({ config, fetchImpl, log });
@@@ -78,7 +81,7 @@ async function createApp(opts = {}) {
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Design cannot answer account events'); });
const ctx = {
-- config, s, store: s, keys, sso, catalog, arena, bench, ai, jobs, designs, accountData, accountSend, log, picks,
++ config, s, store: s, keys, sso, catalog, searchIndex, arena, bench, ai, jobs, designs, accountData, accountSend, log, picks,
random: opts.random || Math.random,
};
ddiff --git a/server/index.js b/server/index.js
iindex 0f54cab..57c0ab9 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -22,6 +22,7 @@ function createLifecycle({ server, ctx, exit, signals } = {}) {
deadlineMs: 10_000,
close: [
() => ctx.jobs.stop(),
++ () => ctx.searchIndex.stop(),
() => ctx.keys.client.stop(),
() => ctx.s.close(),
],
@@@ -40,6 +41,8 @@ async function start() {
ctx.keys.ensure().catch(() => {});
// Generate queued Arena rounds. DESIGN_DRILL=1 keeps the loop off (a test or a drill drives tick()).
ctx.jobs.start();
++ // The icon and theme pages in OpenVibe.Search: a sweep a minute after boot and every six hours.
++ if (!config.drill) ctx.searchIndex.start();
ctx.designs.resume().catch((err) => console.warn('[OpenVibe.Design] resume designs:', err.message));
});
server.keepAliveTimeout = 65_000;
ddiff --git a/server/search-index.js b/server/search-index.js
nnew file mode 100644
iindex 0000000..51cba23
---- /dev/null
++++ b/server/search-index.js
@@@ -0,0 +1,124 @@
++'use strict';
++
++/**
++ * OpenVibe.Design's icon and theme pages in OpenVibe.Search (openvibe-publishing/search-feed, contracts 0.275.0): one
++ * search.index-document@1 per icon (/icons/<name>, the glyph name as the id) and per public theme (/t/<slug>), sent as
++ * design.index_document.upserted|deleted through this service's events outbox. Each document is what its page says:
++ * an icon's name, the Font Awesome names it stands in for and the formats it downloads in; a theme's name, mode and
++ * description. 280 icons and 50 themes were on no network search (2026-11-02).
++ *
++ * The icons are openvibe-shared's glyphs (they change with a Shared release, so with a deploy) and the themes are
++ * OpenVibe.Network's public catalogue (catalog.list(), cached five minutes): a sweep a minute after boot and every six
++ * hours. The sequencer (design_index_revisions) gives an unchanged page its old revision and a quiet sweep sends
++ * nothing; a page that left the catalogue gets a tombstone. When Network does not answer the theme list, the sweep
++ * stops rather than read an empty catalogue as "every theme is gone". Off under DESIGN_DRILL.
++ */
++const glyphs = require('openvibe-shared/glyphs');
++const { createServiceOutbox } = require('openvibe-sdk/events');
++const { createSearchFeed } = require('openvibe-publishing/search-feed');
++const { faOf, title: glyphTitle } = require('./views/icon-page');
++
++const EVENT_TYPES = ['design.index_document.upserted', 'design.index_document.deleted'];
++const START_DELAY_MS = 60_000;
++const INTERVAL_MS = 6 * 3_600_000;
++const ID_RE = /^[a-z0-9][a-z0-9-]{0,89}$/;
++const byId = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
++
++/** One glyph name → what its /icons/<name> page says. */
++function describeIcon(row) {
++ const name = row.id;
++ const fa = faOf(name);
++ const label = glyphTitle(name);
++ const solid = Boolean(glyphs.SOLID && Object.prototype.hasOwnProperty.call(glyphs.SOLID, name));
++ return {
++ listed: true,
++ title: `${label} icon`,
++ summary: `A free ${label.toLowerCase()} icon: download the SVG, make a PNG at any size and colour, or use the font.${fa.length ? ` A drop-in for Font Awesome's fa-${fa[0]}.` : ''} MIT.`,
++ body: [name, name.replace(/-/g, ' '), ...fa.map((f) => `fa-${f} ${f.replace(/-/g, ' ')}`),
++ `SVG, PNG, web font, React, Android vector drawable.${solid ? ' Outline and solid.' : ''}`].join('\n'),
++ facets: { kind: solid ? 'solid' : 'outline' },
++ authorship: { mode: 'human' },
++ };
++}
++
++/** One public theme from Network's catalogue → what its /t/<slug> page says. */
++function describeTheme(t) {
++ const name = String(t.name || t.slug);
++ const mode = t.mode ? String(t.mode) : '';
++ return {
++ listed: true,
++ title: `${name}: an OpenVibe theme`,
++ summary: [t.description ? String(t.description) : '', `A ${mode ? `${mode} ` : ''}theme for every OpenVibe site.`].filter(Boolean).join(' '),
++ body: [name, mode ? `${mode} theme` : 'theme', t.description || ''].filter(Boolean).join('\n'),
++ facets: mode ? { mode } : {},
++ authorship: { mode: 'human' },
++ publishedAt: t.created_at || null,
++ updatedAt: t.updated_at || null,
++ };
++}
++
++function createSearchIndex({ config, s, catalog, log = console, outbox = null }) {
++ const db = s.db;
++ const out = outbox || createServiceOutbox({
++ db, source: 'design', eventsUrl: (config.events && config.events.url) || null, networkInternalUrl: config.networkInternalUrl,
++ clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,
++ });
++
++ const icons = () => glyphs.NAMES.filter((n) => ID_RE.test(n)).map((n) => ({ id: n })).sort(byId);
++ // Throws when Network does not answer: an empty answer must never tombstone every theme.
++ async function themes() {
++ const list = await catalog.list();
++ return list.themes.filter((t) => t && ID_RE.test(String(t.slug || ''))).map((t) => ({ ...t, id: String(t.slug) })).sort(byId);
++ }
++ const after = (rows, id, limit) => rows.filter((r) => r.id > String(id || '')).slice(0, limit);
++ const present = (rows, ids) => { const have = new Set(rows.map((r) => r.id)); return ids.filter((id) => have.has(id)); };
++
++ const feed = createSearchFeed({
++ owner: 'design', db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,
++ types: {
++ icon: {
++ page: (row) => `/icons/${row.id}`,
++ document: describeIcon,
++ rows: async (id, limit) => after(icons(), id, limit),
++ exists: async (ids) => present(icons(), ids),
++ },
++ theme: {
++ page: (row) => `/t/${encodeURIComponent(row.id)}`,
++ document: describeTheme,
++ rows: async (id, limit) => after(await themes(), id, limit),
++ exists: async (ids) => present(await themes(), ids),
++ },
++ },
++ });
++ const timers = [];
++
++ async function sweep() {
++ const res = await feed.sweep();
++ for (const [type, r] of Object.entries(res)) {
++ if (r.sent || r.removed || r.failed) log.log(`[Search] ${type}: ${r.sent} sent, ${r.removed} removed, ${r.failed} failed of ${r.seen}`);
++ }
++ if (out.kick) out.kick().catch(() => {});
++ return res;
++ }
++ const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
++
++ function start() {
++ if (config.drill || timers.length) return false;
++ out.start();
++ const kick = setTimeout(quietly, START_DELAY_MS);
++ if (kick.unref) kick.unref();
++ const tick = setInterval(quietly, INTERVAL_MS);
++ if (tick.unref) tick.unref();
++ timers.push(kick, tick);
++ return true;
++ }
++
++ function stop() {
++ for (const t of timers.splice(0)) { clearTimeout(t); clearInterval(t); }
++ out.stop();
++ }
++
++ return { feed, outbox: out, sweep, start, stop, status: () => out.status() };
++}
++
++module.exports = { createSearchIndex, describeIcon, describeTheme, EVENT_TYPES };
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
iindex 16be21a..ebdc437 100644
---- a/test/account-data.test.js
++++ b/test/account-data.test.js
@@@ -16,6 +16,7 @@ const { testDb } = require('./helpers/db');
taskKeyed: ['design_versions'], // versions belong to the design and cascade with it
exempt: {
account_data_events: 'the ledger of requests itself: which export and deletion design applied',
++ design_index_revisions: 'the Search index sequencer: owner is the service (design), the rows are icon and theme pages, never a person',
},
});
ddiff --git a/test/search-index.test.js b/test/search-index.test.js
nnew file mode 100644
iindex 0000000..610d859
---- /dev/null
++++ b/test/search-index.test.js
@@@ -0,0 +1,74 @@
++'use strict';
++/**
++ * Icon and theme pages in OpenVibe.Search (server/search-index.js): a sweep puts every glyph (/icons/<name>) and every
++ * public theme (/t/<slug>) in the outbox as one design.index_document.upserted, exactly the contract (contracts
++ * 0.275.0), for pages that answer; a second sweep sends nothing; a theme that left the catalogue becomes a tombstone,
++ * once; a Network that does not answer the catalogue stops the sweep instead of tombstoning every theme.
++ */
++const assert = require('assert');
++const contracts = require('openvibe-contracts');
++const glyphs = require('openvibe-shared/glyphs');
++const { boot, check, done } = require('./helpers/boot');
++const { describeIcon, describeTheme } = require('../server/search-index');
++
++(async () => {
++ const t = await boot({ env: { DESIGN_THEME_CACHE_MS: '0' } });
++ const { s, searchIndex } = t.ctx;
++ const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);
++ const valid = (env) => {
++ assert.ok(contracts.validate('events.event-envelope@1', env).valid, 'envelope');
++ const p = contracts.validate(env.event_type, env.payload);
++ assert.ok(p.valid, `${env.event_type}: ${JSON.stringify(p.errors)}`);
++ assert.strictEqual(env.source, 'design');
++ return env;
++ };
++ try {
++ await check('a sweep sends every icon and every public theme once, as the contract says, and their pages answer', async () => {
++ const themes = t.network.state.themes.length;
++ const res = await searchIndex.sweep();
++ assert.deepStrictEqual(res.icon, { seen: glyphs.NAMES.length, sent: glyphs.NAMES.length, removed: 0, failed: 0 });
++ assert.deepStrictEqual(res.theme, { seen: themes, sent: themes, removed: 0, failed: 0 });
++ const docs = (await outbox()).map(valid).map((e) => e.payload);
++ const icon = docs.find((d) => d.type === 'icon' && d.id === 'arrow-down');
++ assert.strictEqual(icon.canonical_url, 'https://openvibe.design/icons/arrow-down');
++ assert.strictEqual(icon.title, 'Arrow down icon');
++ const theme = docs.find((d) => d.type === 'theme' && d.id === 'paper');
++ assert.strictEqual(theme.canonical_url, 'https://openvibe.design/t/paper');
++ assert.strictEqual(theme.facets.mode, 'light');
++ assert.strictEqual((await t.get('/icons/arrow-down')).status, 200);
++ assert.strictEqual((await t.get('/t/paper')).status, 200);
++ const again = await searchIndex.sweep();
++ assert.strictEqual(again.icon.sent + again.theme.sent, 0, 'a second sweep sends nothing');
++ });
++
++ await check('a theme that left the catalogue becomes a tombstone, once', async () => {
++ t.network.state.themes = t.network.state.themes.filter((x) => x.slug !== 'aurora');
++ const before = (await outbox()).length;
++ assert.strictEqual((await searchIndex.sweep()).theme.removed, 1);
++ const env = valid((await outbox()).slice(before).find((e) => e.event_type === 'design.index_document.deleted'));
++ assert.deepStrictEqual([env.payload.type, env.payload.id], ['theme', 'aurora']);
++ assert.strictEqual((await searchIndex.sweep()).theme.removed, 0);
++ });
++
++ await check('when Network does not answer the catalogue, the sweep stops and tombstones nothing', async () => {
++ t.network.state.catalogStatus = 503;
++ const before = (await outbox()).length;
++ await assert.rejects(() => searchIndex.sweep(), /theme catalogue/);
++ assert.strictEqual((await outbox()).length, before);
++ t.network.state.catalogStatus = 200;
++ });
++
++ await check('the words: an icon\'s Font Awesome names and formats; a theme\'s mode and description', async () => {
++ const name = glyphs.NAMES.find((n) => Object.values(glyphs.FA).includes(n));
++ const d = describeIcon({ id: name });
++ assert.match(d.summary, /A drop-in for Font Awesome's fa-/);
++ assert.match(d.body, /SVG, PNG, web font/);
++ const th = describeTheme({ slug: 'abyss', name: 'Abyss', mode: 'dark', description: 'Nearly black; the interface is the light.' });
++ assert.strictEqual(th.title, 'Abyss: an OpenVibe theme');
++ assert.strictEqual(th.summary, 'Nearly black; the interface is the light. A dark theme for every OpenVibe site.');
++ });
++ } finally {
++ await t.close();
++ done();
++ }
++})();