openvibers/openvibe.host Commits
Host: catalog plans and providers in OpenVibe.Search (host.index_document.*) (#78)
Changes
@@@ -24,6 +24,7 @@ OpenVibe's own static hosting (Stage B below) has its page at `/hosting`; it is- **Order.** It comes only from the visitor's sort: price, per vCPU, per GB of RAM, per GB of disk, or recently checked. Affiliate links carry `rel="sponsored nofollow noopener"` with the provider's own disclosure beside them, and never change the order (`test/catalog-public.test.js`).- **Staff.** `host.catalog.manage` (Network staff only) covers `POST`/`PATCH /api/v1/catalog/{providers,plans,promos}` and the no-JS forms at `/catalog/manage`.- **Public reads.** `GET /api/v1/catalog/plans` (filters `kind`, `min_vcpu`, `min_ram_mb`, `min_disk_gb`, `max_price`, `currency`, `location`, `provider`; `sort`; cursor), `/plans/:id` (with price history), `/providers`, `/providers/:slug` and `/promos`.++- **Search.** Every public plan and provider page is an OpenVibe.Search document (`host.index_document.upserted|deleted`, `server/catalog/search-index.js` through openvibe-publishing/search-feed): sent a minute after boot, hourly, after each catalog pull and shortly after a staff change; only changed pages are sent, and a hidden or gone entry becomes a tombstone.- **Pages.** `/catalog`, `/catalog/providers`, `/catalog/providers/:slug`, `/catalog/plans/:id` and `/catalog/promos`. They are server-rendered and work without JavaScript; the styles are `public/css/catalog.css`.- **Discussion.** Active public provider and plan pages lazily resolve Community threads with refs `{ service: 'host', type: 'provider', id: provider.id }` and `{ service: 'host', type: 'plan', id: plan.id }` (plus a display label), then render recent comments with a link to the thread on OpenVibe.Community. Host stores only the thread reference. `HOST_COMMUNITY_URL` enables the internal client; `OV_COMMUNITY_URL` sets the public link. With the internal URL unset, no Discussion block appears. Hidden providers and hidden or gone plans have no public page; status changes hide or reopen existing threads after commit. Host's Network service principal needs `community.comment.write` and `community.comment.moderate` for audience `openvibe.community`.@@@ -584,7 +585,7 @@ review is [docs/threat-review.md](docs/threat-review.md). The launch has not beePart of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).<!-- versions:start -->--- openvibe-contracts: v0.143.0++- openvibe-contracts: v0.145.0- openvibe-sdk: v0.44.0- openvibe-shared: v4.0.3- openvibe-publishing: v1.4.0@@@ -14,11 +14,11 @@"port": 4910,"domain": "openvibe.host","tenantDomain": "*.openvibe.host",-- "contracts": "openvibe-contracts v0.143.0",++ "contracts": "openvibe-contracts v0.145.0","sdk": "openvibe-sdk v0.44.0","shared": "openvibe-shared v4.0.3","packages": [-- "openvibe-contracts v0.143.0",++ "openvibe-contracts v0.145.0","openvibe-sdk v0.44.0","openvibe-shared v4.0.3"],@@@ -68,5 +68,5 @@"per-actor limits on Stage B writes (openvibe-sdk/limits)","catalog provider and plan discussion threads on OpenVibe.Community (optional HOST_COMMUNITY_URL; server-rendered comments, after-commit visibility sync)"],-- "updated": "2026-10-11"++ "updated": "2026-10-12"}nnew file mode 100644@@@ -0,0 +1,13 @@++-- phase: expand++-- The hosting catalog in OpenVibe.Search (server/catalog/search-index.js, openvibe-publishing/search-feed): the index++-- sequencer, the last revision and content hash Search was sent for each catalog page, so an unchanged page is never++-- sent twice. The host.index_document.* events themselves go through event_outbox like Host's other events.++CREATE TABLE host_index_revisions (++ owner text COLLATE "C" NOT NULL,++ type text COLLATE "C" NOT NULL,++ id text COLLATE "C" NOT NULL,++ revision integer NOT NULL,++ hash text NOT NULL,++ updated_at bigint NOT NULL,++ PRIMARY KEY (owner, type, id)++);@@@ -16,7 +16,7 @@"express": "^4.21.2","express-rate-limit": "^7.5.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",@@@ -1271,9 +1271,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.143.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",-- "integrity": "sha512-Cdbf+9RoPAUBibpM2zXQvtsiKMugb55dKubTt8pFZvAmxsLtGvX+zaC++yL1vOSub8FX3+GgqcGtoJS6tklrUg==",++ "version": "0.145.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0",++ "integrity": "sha512-MUe54oAc0ZVvzt3HpX/ybC6T9cjjysS1hjjSXCRUpQuBBwU28xuoBRwoKVzx8r+TZsxHOVf/OAK31H7EBzaRaA==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -32,7 +32,7 @@"express": "^4.21.2","express-rate-limit": "^7.5.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",@@@ -48,6 +48,7 @@ const { createWorker } = require('./worker');const { createCatalog } = require('./catalog/store');const { createCommunity } = require('./catalog/community');const { createCatalogIngest } = require('./catalog/ingest');++const { createCatalogSearch } = require('./catalog/search-index');const { createCatalogApi, createCatalogPages } = require('./http/catalog');const { renderPage, assetVersion, setRelease } = require('./render/layout');const pages = require('./render/pages');@@@ -158,9 +159,11 @@ async function createApp(opts = {}) {const community = createCommunity({ store, config, fetchImpl: opts.fetchImpl, log });const catalog = createCatalog({ store, community, log });const catalogIngest = createCatalogIngest({ config, store, catalog, fetchImpl: opts.fetchImpl });-- const worker = createWorker({ config, store, domains, blobs, outbox, catalogIngest, log });++ const catalogSearch = createCatalogSearch({ config, store, outbox, log });++ catalog.onChange(() => { if (config.worker.enabled) catalogSearch.sweepSoon(); }); // the worker's own switch: off in tests and drills++ const worker = createWorker({ config, store, domains, blobs, outbox, catalogIngest, catalogSearch, log });-- const ctx = { config, store, blobs, outbox, access, takedowns, projects, sites, deploys, domains, siteConfig, siteSources, catalog, community, catalogIngest, auth, viewers, tenant, worker, uploadGate, indexnow, log };++ const ctx = { config, store, blobs, outbox, access, takedowns, projects, sites, deploys, domains, siteConfig, siteSources, catalog, community, catalogIngest, catalogSearch, auth, viewers, tenant, worker, uploadGate, indexnow, log };const app = express();app.disable('x-powered-by');nnew file mode 100644@@@ -0,0 +1,110 @@++'use strict';++++/**++ * The hosting catalog in OpenVibe.Search (openvibe-publishing/search-feed): one search.index-document@1 per public++ * catalog page, a plan (/catalog/plans/<id>) or a provider (/catalog/providers/<slug>), sent as++ * host.index_document.upserted|deleted through Host's events outbox (contracts 0.145.0). A hidden or gone entry, or a++ * plan of a hidden provider, becomes a tombstone. The document says what the page shows: the plan's specifications,++ * its monthly price and currency and the day it was checked; a plan from OpenVibe.Sources names its source as++ * provenance (authorship imported), a staff entry its source URL.++ *++ * Search is brought level by a sweep (the sequencer, host_index_revisions, gives an unchanged document its old++ * revision, so a quiet sweep sends nothing): after each catalog pull, a minute after boot, hourly, and shortly after++ * a staff change.++ *++ * const search = createCatalogSearch({ config, store, outbox, log });++ * await search.sweep(); search.sweepSoon();++ */++const { createSearchFeed } = require('openvibe-publishing/search-feed');++const { money, bytesMb, sizeGb } = require('../render/catalog');++++const KINDS = { cloud: 'Cloud', vps: 'VPS', dedicated: 'Dedicated server', storage: 'Storage', domain: 'Domain' };++const day = (ms) => (Number.isFinite(Number(ms)) ? new Date(Number(ms)).toISOString().slice(0, 10) : null);++const iso = (ms) => (Number.isFinite(Number(ms)) && ms != null ? new Date(Number(ms)).toISOString() : null);++++function planDocument(row) {++ const price = money(row.price_monthly, row.currency);++ const transfer = row.unmetered ? 'unmetered transfer' : (sizeGb(row.bandwidth_gb) ? `${sizeGb(row.bandwidth_gb)} transfer` : null);++ const specs = [row.vcpu != null ? `${Number(row.vcpu)} vCPU` : null, bytesMb(row.ram_mb) ? `${bytesMb(row.ram_mb)} RAM` : null,++ sizeGb(row.disk_gb) ? `${sizeGb(row.disk_gb)}${row.disk_type ? ` ${row.disk_type.toUpperCase()}` : ''} disk` : null, transfer].filter(Boolean);++ const checked = day(row.checked_at);++ return {++ listed: row.status === 'active' && row.provider_status === 'active',++ title: `${row.provider_name} ${row.name}${price ? `: ${price} a month` : ''}`,++ summary: [...specs, checked ? `checked ${checked}` : null].filter(Boolean).join(' · ') || null,++ body: [`${row.provider_name} ${row.name}`, KINDS[row.kind] || row.kind, specs.join(', '),++ price ? `${price} a month (${row.currency})${checked ? `, checked ${checked}` : ''}` : null,++ Array.isArray(row.locations) && row.locations.length ? `Locations: ${row.locations.join(', ')}` : null].filter(Boolean).join('\n'),++ facets: { provider: row.provider_slug, kind: row.kind, currency: row.currency },++ authorship: { mode: row.origin === 'sources' ? 'imported' : 'human' },++ provenance: row.source_url ? [{++ service: row.origin === 'sources' ? 'sources' : 'host', type: row.origin === 'sources' ? 'item' : 'staff_entry',++ id: String(row.origin === 'sources' && row.source_item_id ? row.source_item_id : row.id).slice(0, 128),++ label: row.origin === 'sources' ? (row.source_key || 'OpenVibe.Sources') : `${row.provider_name} pricing`,++ url: row.source_url, retrievedAt: iso(row.checked_at),++ }] : [],++ publishedAt: iso(row.first_seen_at || row.created_at),++ updatedAt: iso(row.checked_at || row.updated_at),++ };++}++++function providerDocument(row) {++ const count = Number(row.plan_count) || 0;++ const from = row.from_price != null ? money(row.from_price, row.from_currency) : null;++ return {++ listed: row.status === 'active' && count > 0,++ title: `${row.name} hosting plans and prices`,++ summary: `${count} plan${count === 1 ? '' : 's'}${from ? ` from ${from} a month` : ''}`,++ body: [row.name, row.description, `${count} plan${count === 1 ? '' : 's'} in the catalog${from ? `, from ${from} a month` : ''}`].filter(Boolean).join('\n'),++ facets: { provider: row.slug },++ authorship: { mode: 'human' },++ publishedAt: iso(row.created_at),++ updatedAt: iso(row.updated_at),++ };++}++++function createCatalogSearch({ config, store, outbox, log = console }) {++ const db = store.db;++ const feed = createSearchFeed({++ owner: 'host', db, outbox, baseUrl: config.baseUrl, now: store.now, log,++ types: {++ plan: {++ page: (row) => `/catalog/plans/${row.id}`,++ document: planDocument,++ rows: (after, limit) => db.many(`SELECT p.*, v.slug AS provider_slug, v.name AS provider_name, v.status AS provider_status++ FROM catalog_plans p JOIN catalog_providers v ON v.id = p.provider_id WHERE p.id > $1 ORDER BY p.id LIMIT $2`, [after, limit]),++ exists: async (ids) => (await db.many('SELECT id FROM catalog_plans WHERE id = ANY($1)', [ids])).map((r) => r.id),++ },++ provider: {++ page: (row) => `/catalog/providers/${row.slug}`,++ document: providerDocument,++ // The cheapest active plan is named in its own currency (the first currency by price, USD first when tied).++ rows: (after, limit) => db.many(`SELECT v.*,++ (SELECT COUNT(*) FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active') AS plan_count,++ (SELECT p.price_monthly FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active' ORDER BY (p.currency <> 'USD'), p.price_monthly, p.id LIMIT 1) AS from_price,++ (SELECT p.currency FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active' ORDER BY (p.currency <> 'USD'), p.price_monthly, p.id LIMIT 1) AS from_currency++ FROM catalog_providers v WHERE v.id > $1 ORDER BY v.id LIMIT $2`, [after, limit]),++ exists: async (ids) => (await db.many('SELECT id FROM catalog_providers WHERE id = ANY($1)', [ids])).map((r) => r.id),++ },++ },++ });++++ let soon = null;++ async function sweep() {++ const res = await feed.sweep();++ for (const [type, r] of Object.entries(res)) {++ if (r.sent || r.removed || r.failed) log.log(`[Search] catalog ${type}: ${r.sent} sent, ${r.removed} removed, ${r.failed} failed of ${r.seen}`);++ }++ if (outbox && outbox.kick) outbox.kick().catch(() => {});++ return res;++ }++ /** A staff change: one sweep a few seconds later (several edits in a row share it). */++ function sweepSoon(delayMs = 5000) {++ if (soon) return;++ soon = setTimeout(() => { soon = null; sweep().catch((err) => log.warn(`[Search] catalog sweep failed: ${(err && err.message) || err}`)); }, delayMs);++ if (soon.unref) soon.unref();++ }++ return { feed, sweep, sweepSoon, planDocument, providerDocument };++}++++module.exports = { createCatalogSearch, planDocument, providerDocument };@@@ -48,6 +48,8 @@ const clean = (type, input, current, now) => {};function createCatalog({ store, community = null, log = console }) {const db = store.db;++ const listeners = [];++ const changed = (type) => { for (const fn of listeners) { try { fn(type); } catch { /* a listener never breaks a save */ } } };let visibilityQueue = Promise.resolve();const open = async (type, id) => {const sql = type === 'provider'@@@ -59,7 +61,8 @@ function createCatalog({ store, community = null, log = console }) {const providerOut = (p) => ({ ...p, affiliate_rel: p.affiliate_url ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate_url ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });const promoOut = (p) => ({ ...p, status: p.status === 'active' && p.ends_at != null && Number(p.ends_at) <= store.now() ? 'ended' : p.status, affiliate_rel: p.affiliate ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });const get = async (type, id) => { const row = await db.prepare(`SELECT * FROM catalog_${type} WHERE id = ?`).get(id); if (!row) missing(); return type === 'promos' ? promoOut(row) : row; };-- const save = async (type, input, id, actor) => await db.tx(async () => {++ const save = async (type, input, id, actor) => { const out = await saveTx(type, input, id, actor); changed(type); return out; };++ const saveTx = async (type, input, id, actor) => await db.tx(async () => {const current = id ? await get(type, id) : null;const watched = [];if (community && community.enabled && current && ['providers', 'plans'].includes(type)) {@@@ -174,6 +177,6 @@ function createCatalog({ store, community = null, log = console }) {return { plans: Number(r.plans) || 0, providers: Number(r.providers) || 0, lastChecked: r.last == null ? null : Number(r.last) };}async function sitemapPlans() { return await db.prepare("SELECT p.id FROM catalog_plans p JOIN catalog_providers v ON v.id = p.provider_id WHERE p.status = 'active' AND v.status = 'active' ORDER BY p.id").all(); }-- return { save, get, applyItem, plans, plan, providers, promos, currencies: currenciesInUse, stats, sitemapPlans, discussionIdle: idle, all: async (type) => { const rows = await db.prepare(`SELECT * FROM catalog_${type} ORDER BY created_at DESC, id DESC LIMIT 200`).all(); return type === 'promos' ? rows.map(promoOut) : rows; } };++ return { onChange: (fn) => listeners.push(fn), save, get, applyItem, plans, plan, providers, promos, currencies: currenciesInUse, stats, sitemapPlans, discussionIdle: idle, all: async (type) => { const rows = await db.prepare(`SELECT * FROM catalog_${type} ORDER BY created_at DESC, id DESC LIMIT 200`).all(); return type === 'promos' ? rows.map(promoOut) : rows; } };}module.exports = { createCatalog };@@@ -7,6 +7,7 @@* host.deploy.activated a site's active deploy changed (payload.rollback marks a rollback)* host.deploy.failed an upload was refused (payload.code, problem codes; no file contents)* host.domain.verified a custom domain's TXT record was found; the domain is now served++ * host.index_document.upserted|deleted a hosting catalog page in OpenVibe.Search (server/catalog/search-index.js)** The SDK writes each row inside the change's own transaction (emit joins the ambient openvibe-sdk/db* transaction), so an event exists if and only if its change committed. The relay publishes with@@@ -16,7 +17,8 @@*/const { createServiceOutbox } = require('openvibe-sdk/events');--const EVENT_TYPES = Object.freeze(['host.deploy.created', 'host.deploy.activated', 'host.deploy.failed', 'host.domain.verified']);++const EVENT_TYPES = Object.freeze(['host.deploy.created', 'host.deploy.activated', 'host.deploy.failed', 'host.domain.verified',++ 'host.index_document.upserted', 'host.index_document.deleted']);function createHostOutbox({ db, config, fetchImpl, now, log = console }) {return createServiceOutbox({@@@ -11,7 +11,7 @@*/const fs = require('fs');--function createWorker({ config, store, domains, blobs, outbox, catalogIngest, log = console }) {++function createWorker({ config, store, domains, blobs, outbox, catalogIngest, catalogSearch, log = console }) {const timers = [];let checking = false;@@@ -61,8 +61,11 @@ function createWorker({ config, store, domains, blobs, outbox, catalogIngest, loevery(3600 * 1000, async () => { try { await sweepObjects(); } catch (err) { log.warn('[Host] object sweep failed:', err.message); } });every(24 * 3600 * 1000, async () => { try { await outbox.outbox.prune(); } catch (err) { log.warn('[Host] outbox prune failed:', err.message); } });setTimeout(domainTick, 5000).unref();++ // The catalog in OpenVibe.Search: a minute after boot, hourly, and after each pull (only changes are sent).++ const searchSweep = async () => { if (!catalogSearch) return; try { await catalogSearch.sweep(); } catch (err) { log.warn('[Host] catalog search sweep failed:', err.message); } };++ if (catalogSearch) { setTimeout(searchSweep, 60_000).unref(); every(3600 * 1000, searchSweep); }if (catalogIngest && catalogIngest.enabled) {-- const pull = async () => { try { await catalogIngest.pull(); } catch (err) { log.warn('[Host] catalog pull failed:', err.message); } };++ const pull = async () => { try { await catalogIngest.pull(); } catch (err) { log.warn('[Host] catalog pull failed:', err.message); } await searchSweep(); };every(5 * 60_000, pull);setTimeout(pull, 5000).unref();}nnew file mode 100644@@@ -0,0 +1,72 @@++'use strict';++/**++ * The hosting catalog in OpenVibe.Search: a sweep sends one host.index_document.upserted per public plan and provider++ * page (a valid search.index-document@1 the contracts declare), sends nothing for an unchanged page, and tombstones a++ * hidden plan and the plans of a hidden provider.++ */++const assert = require('assert');++const contracts = require('openvibe-contracts');++const { boot, check, done } = require('./stageb/boot');++++(async () => {++ const t = await boot();++ const staff = t.user('catalog-search-editor', { role: 'admin' });++ const events = async () => (await t.ctx.store.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'host.index_document.%' ORDER BY id"))++ .map((r) => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope));++ try {++ const post = async (type, body) => { const r = await t.api('POST', `/api/v1/catalog/${type}`, { as: staff, json: body }); assert.strictEqual(r.status, 201, r.text); return r.json()[type.slice(0, -1)]; };++ const patch = async (type, id, body) => { const r = await t.api('PATCH', `/api/v1/catalog/${type}/${id}`, { as: staff, json: body }); assert.strictEqual(r.status, 200, r.text); return r.json()[type.slice(0, -1)]; };++ const v = await post('providers', { slug: 'hetzner', name: 'Hetzner', homepage_url: 'https://www.hetzner.com/', description: 'German cloud servers.' });++ const plan = await post('plans', { provider_id: v.id, external_id: 'cx22', name: 'CX22', kind: 'cloud', vcpu: 2, ram_mb: 4096, disk_gb: 40, disk_type: 'nvme', bandwidth_gb: 20000, currency: 'USD', price_monthly: 4.99, source_url: 'https://www.hetzner.com/cloud/' });++++ await check('a sweep sends a valid document for each public plan and provider page', async () => {++ const res = await t.ctx.catalogSearch.sweep();++ assert.deepStrictEqual([res.plan.sent, res.provider.sent], [1, 1]);++ const sent = await events();++ assert.strictEqual(sent.length, 2);++ for (const e of sent) {++ assert.deepStrictEqual(contracts.validate(`${e.event_type}@1`, e.payload).errors || [], [], e.event_type);++ assert.strictEqual(e.source, 'host');++ }++ const p = sent.find((e) => e.payload.type === 'plan').payload;++ assert.strictEqual(p.id, plan.id);++ assert.strictEqual(p.canonical_url, `${t.config.baseUrl}/catalog/plans/${plan.id}`);++ assert.match(p.title, /^Hetzner CX22: \$4\.99 a month$/);++ assert.match(p.summary, /2 vCPU · 4 GB RAM · 40 GB NVME disk · 20 TB transfer · checked \d{4}-\d{2}-\d{2}/);++ assert.deepStrictEqual(p.facets, { provider: 'hetzner', kind: 'cloud', currency: 'USD' });++ const pr = sent.find((e) => e.payload.type === 'provider').payload;++ assert.strictEqual(pr.canonical_url, `${t.config.baseUrl}/catalog/providers/hetzner`);++ assert.strictEqual(pr.summary, '1 plan from $4.99 a month');++ });++++ await check('an unchanged page is not sent again', async () => {++ const res = await t.ctx.catalogSearch.sweep();++ assert.deepStrictEqual([res.plan.sent, res.provider.sent], [0, 0]);++ assert.strictEqual((await events()).length, 2);++ });++++ await check('a price change re-sends the plan; hiding it sends a tombstone', async () => {++ await patch('plans', plan.id, { price_monthly: 5.49 });++ let res = await t.ctx.catalogSearch.sweep();++ assert.strictEqual(res.plan.sent, 1);++ await patch('plans', plan.id, { status: 'hidden' });++ res = await t.ctx.catalogSearch.sweep();++ const last = (await events()).filter((e) => e.payload.id === plan.id || e.subject && e.subject.id === plan.id).pop();++ assert.strictEqual(last.event_type, 'host.index_document.deleted', JSON.stringify(res));++ });++++ await check('a hidden provider takes its plans and its own page out of Search', async () => {++ const other = await post('plans', { provider_id: v.id, external_id: 'cpx31', name: 'CPX31', kind: 'cloud', vcpu: 4, ram_mb: 8192, currency: 'USD', price_monthly: 15.99, source_url: 'https://www.hetzner.com/cloud/' });++ await t.ctx.catalogSearch.sweep();++ await patch('providers', v.id, { status: 'hidden' });++ await t.ctx.catalogSearch.sweep();++ const sent = await events();++ const lastFor = (id) => sent.filter((e) => (e.subject && e.subject.id === id) || e.payload.id === id).pop();++ assert.strictEqual(lastFor(other.id).event_type, 'host.index_document.deleted');++ assert.strictEqual(lastFor(v.id).event_type, 'host.index_document.deleted');++ });++ } finally {++ await t.close();++ }++ done();++})();
@@ -24,6 +24,7 @@ OpenVibe's own static hosting (Stage B below) has its page at `/hosting`; it is
@@ -24,6 +24,7 @@ OpenVibe's own static hosting (Stage B below) has its page at `/hosting`; it is
- **Order.** It comes only from the visitor's sort: price, per vCPU, per GB of RAM, per GB of disk, or recently checked. Affiliate links carry `rel="sponsored nofollow noopener"` with the provider's own disclosure beside them, and never change the order (`test/catalog-public.test.js`).
- **Order.** It comes only from the visitor's sort: price, per vCPU, per GB of RAM, per GB of disk, or recently checked. Affiliate links carry `rel="sponsored nofollow noopener"` with the provider's own disclosure beside them, and never change the order (`test/catalog-public.test.js`).
- **Staff.** `host.catalog.manage` (Network staff only) covers `POST`/`PATCH /api/v1/catalog/{providers,plans,promos}` and the no-JS forms at `/catalog/manage`.
- **Staff.** `host.catalog.manage` (Network staff only) covers `POST`/`PATCH /api/v1/catalog/{providers,plans,promos}` and the no-JS forms at `/catalog/manage`.
- **Public reads.** `GET /api/v1/catalog/plans` (filters `kind`, `min_vcpu`, `min_ram_mb`, `min_disk_gb`, `max_price`, `currency`, `location`, `provider`; `sort`; cursor), `/plans/:id` (with price history), `/providers`, `/providers/:slug` and `/promos`.
- **Public reads.** `GET /api/v1/catalog/plans` (filters `kind`, `min_vcpu`, `min_ram_mb`, `min_disk_gb`, `max_price`, `currency`, `location`, `provider`; `sort`; cursor), `/plans/:id` (with price history), `/providers`, `/providers/:slug` and `/promos`.
+- **Search.** Every public plan and provider page is an OpenVibe.Search document (`host.index_document.upserted|deleted`, `server/catalog/search-index.js` through openvibe-publishing/search-feed): sent a minute after boot, hourly, after each catalog pull and shortly after a staff change; only changed pages are sent, and a hidden or gone entry becomes a tombstone.
- **Pages.** `/catalog`, `/catalog/providers`, `/catalog/providers/:slug`, `/catalog/plans/:id` and `/catalog/promos`. They are server-rendered and work without JavaScript; the styles are `public/css/catalog.css`.
- **Pages.** `/catalog`, `/catalog/providers`, `/catalog/providers/:slug`, `/catalog/plans/:id` and `/catalog/promos`. They are server-rendered and work without JavaScript; the styles are `public/css/catalog.css`.
- **Discussion.** Active public provider and plan pages lazily resolve Community threads with refs `{ service: 'host', type: 'provider', id: provider.id }` and `{ service: 'host', type: 'plan', id: plan.id }` (plus a display label), then render recent comments with a link to the thread on OpenVibe.Community. Host stores only the thread reference. `HOST_COMMUNITY_URL` enables the internal client; `OV_COMMUNITY_URL` sets the public link. With the internal URL unset, no Discussion block appears. Hidden providers and hidden or gone plans have no public page; status changes hide or reopen existing threads after commit. Host's Network service principal needs `community.comment.write` and `community.comment.moderate` for audience `openvibe.community`.
- **Discussion.** Active public provider and plan pages lazily resolve Community threads with refs `{ service: 'host', type: 'provider', id: provider.id }` and `{ service: 'host', type: 'plan', id: plan.id }` (plus a display label), then render recent comments with a link to the thread on OpenVibe.Community. Host stores only the thread reference. `HOST_COMMUNITY_URL` enables the internal client; `OV_COMMUNITY_URL` sets the public link. With the internal URL unset, no Discussion block appears. Hidden providers and hidden or gone plans have no public page; status changes hide or reopen existing threads after commit. Host's Network service principal needs `community.comment.write` and `community.comment.moderate` for audience `openvibe.community`.
@@ -584,7 +585,7 @@ review is [docs/threat-review.md](docs/threat-review.md). The launch has not bee
@@ -584,7 +585,7 @@ review is [docs/threat-review.md](docs/threat-review.md). The launch has not bee
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
<!-- versions:start -->
-- openvibe-contracts: v0.143.0
+- openvibe-contracts: v0.145.0
- openvibe-sdk: v0.44.0
- openvibe-sdk: v0.44.0
- openvibe-shared: v4.0.3
- openvibe-shared: v4.0.3
- openvibe-publishing: v1.4.0
- openvibe-publishing: v1.4.0
@@ -14,11 +14,11 @@
@@ -14,11 +14,11 @@
"port": 4910,
"port": 4910,
"domain": "openvibe.host",
"domain": "openvibe.host",
"tenantDomain": "*.openvibe.host",
"tenantDomain": "*.openvibe.host",
- "contracts": "openvibe-contracts v0.143.0",
+ "contracts": "openvibe-contracts v0.145.0",
"sdk": "openvibe-sdk v0.44.0",
"sdk": "openvibe-sdk v0.44.0",
"shared": "openvibe-shared v4.0.3",
"shared": "openvibe-shared v4.0.3",
"packages": [
"packages": [
- "openvibe-contracts v0.143.0",
+ "openvibe-contracts v0.145.0",
"openvibe-sdk v0.44.0",
"openvibe-sdk v0.44.0",
"openvibe-shared v4.0.3"
"openvibe-shared v4.0.3"
],
],
@@ -68,5 +68,5 @@
@@ -68,5 +68,5 @@
"per-actor limits on Stage B writes (openvibe-sdk/limits)",
"per-actor limits on Stage B writes (openvibe-sdk/limits)",
"catalog provider and plan discussion threads on OpenVibe.Community (optional HOST_COMMUNITY_URL; server-rendered comments, after-commit visibility sync)"
"catalog provider and plan discussion threads on OpenVibe.Community (optional HOST_COMMUNITY_URL; server-rendered comments, after-commit visibility sync)"
],
],
- "updated": "2026-10-11"
+ "updated": "2026-10-12"
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
+-- phase: expand
+-- The hosting catalog in OpenVibe.Search (server/catalog/search-index.js, openvibe-publishing/search-feed): the index
+-- sequencer, the last revision and content hash Search was sent for each catalog page, so an unchanged page is never
+-- sent twice. The host.index_document.* events themselves go through event_outbox like Host's other events.
+CREATE TABLE host_index_revisions (
+ owner text COLLATE "C" NOT NULL,
+ type text COLLATE "C" NOT NULL,
+ id text COLLATE "C" NOT NULL,
+ revision integer NOT NULL,
+ hash text NOT NULL,
+ updated_at bigint NOT NULL,
+ PRIMARY KEY (owner, type, id)
+);
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",
@@ -1271,9 +1271,9 @@
@@ -1271,9 +1271,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.143.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",
- "integrity": "sha512-Cdbf+9RoPAUBibpM2zXQvtsiKMugb55dKubTt8pFZvAmxsLtGvX+zaC++yL1vOSub8FX3+GgqcGtoJS6tklrUg==",
+ "version": "0.145.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0",
+ "integrity": "sha512-MUe54oAc0ZVvzt3HpX/ybC6T9cjjysS1hjjSXCRUpQuBBwU28xuoBRwoKVzx8r+TZsxHOVf/OAK31H7EBzaRaA==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -32,7 +32,7 @@
@@ -32,7 +32,7 @@
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.143.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.145.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.3",
@@ -48,6 +48,7 @@ const { createWorker } = require('./worker');
@@ -48,6 +48,7 @@ const { createWorker } = require('./worker');
const { createCatalog } = require('./catalog/store');
const { createCatalog } = require('./catalog/store');
const { createCommunity } = require('./catalog/community');
const { createCommunity } = require('./catalog/community');
const { createCatalogIngest } = require('./catalog/ingest');
const { createCatalogIngest } = require('./catalog/ingest');
+const { createCatalogSearch } = require('./catalog/search-index');
const { createCatalogApi, createCatalogPages } = require('./http/catalog');
const { createCatalogApi, createCatalogPages } = require('./http/catalog');
const { renderPage, assetVersion, setRelease } = require('./render/layout');
const { renderPage, assetVersion, setRelease } = require('./render/layout');
const pages = require('./render/pages');
const pages = require('./render/pages');
@@ -158,9 +159,11 @@ async function createApp(opts = {}) {
@@ -158,9 +159,11 @@ async function createApp(opts = {}) {
const community = createCommunity({ store, config, fetchImpl: opts.fetchImpl, log });
const community = createCommunity({ store, config, fetchImpl: opts.fetchImpl, log });
const catalog = createCatalog({ store, community, log });
const catalog = createCatalog({ store, community, log });
const catalogIngest = createCatalogIngest({ config, store, catalog, fetchImpl: opts.fetchImpl });
const catalogIngest = createCatalogIngest({ config, store, catalog, fetchImpl: opts.fetchImpl });
- const worker = createWorker({ config, store, domains, blobs, outbox, catalogIngest, log });
+ const catalogSearch = createCatalogSearch({ config, store, outbox, log });
+ catalog.onChange(() => { if (config.worker.enabled) catalogSearch.sweepSoon(); }); // the worker's own switch: off in tests and drills
+ const worker = createWorker({ config, store, domains, blobs, outbox, catalogIngest, catalogSearch, log });
- const ctx = { config, store, blobs, outbox, access, takedowns, projects, sites, deploys, domains, siteConfig, siteSources, catalog, community, catalogIngest, auth, viewers, tenant, worker, uploadGate, indexnow, log };
+ const ctx = { config, store, blobs, outbox, access, takedowns, projects, sites, deploys, domains, siteConfig, siteSources, catalog, community, catalogIngest, catalogSearch, auth, viewers, tenant, worker, uploadGate, indexnow, log };
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
new file mode 100644
new file mode 100644
@@ -0,0 +1,110 @@
@@ -0,0 +1,110 @@
+'use strict';
+
+/**
+ * The hosting catalog in OpenVibe.Search (openvibe-publishing/search-feed): one search.index-document@1 per public
+ * catalog page, a plan (/catalog/plans/<id>) or a provider (/catalog/providers/<slug>), sent as
+ * host.index_document.upserted|deleted through Host's events outbox (contracts 0.145.0). A hidden or gone entry, or a
+ * plan of a hidden provider, becomes a tombstone. The document says what the page shows: the plan's specifications,
+ * its monthly price and currency and the day it was checked; a plan from OpenVibe.Sources names its source as
+ * provenance (authorship imported), a staff entry its source URL.
+ *
+ * Search is brought level by a sweep (the sequencer, host_index_revisions, gives an unchanged document its old
+ * revision, so a quiet sweep sends nothing): after each catalog pull, a minute after boot, hourly, and shortly after
+ * a staff change.
+ *
+ * const search = createCatalogSearch({ config, store, outbox, log });
+ * await search.sweep(); search.sweepSoon();
+ */
+const { createSearchFeed } = require('openvibe-publishing/search-feed');
+const { money, bytesMb, sizeGb } = require('../render/catalog');
+
+const KINDS = { cloud: 'Cloud', vps: 'VPS', dedicated: 'Dedicated server', storage: 'Storage', domain: 'Domain' };
+const day = (ms) => (Number.isFinite(Number(ms)) ? new Date(Number(ms)).toISOString().slice(0, 10) : null);
+const iso = (ms) => (Number.isFinite(Number(ms)) && ms != null ? new Date(Number(ms)).toISOString() : null);
+
+function planDocument(row) {
+ const price = money(row.price_monthly, row.currency);
+ const transfer = row.unmetered ? 'unmetered transfer' : (sizeGb(row.bandwidth_gb) ? `${sizeGb(row.bandwidth_gb)} transfer` : null);
+ const specs = [row.vcpu != null ? `${Number(row.vcpu)} vCPU` : null, bytesMb(row.ram_mb) ? `${bytesMb(row.ram_mb)} RAM` : null,
+ sizeGb(row.disk_gb) ? `${sizeGb(row.disk_gb)}${row.disk_type ? ` ${row.disk_type.toUpperCase()}` : ''} disk` : null, transfer].filter(Boolean);
+ const checked = day(row.checked_at);
+ return {
+ listed: row.status === 'active' && row.provider_status === 'active',
+ title: `${row.provider_name} ${row.name}${price ? `: ${price} a month` : ''}`,
+ summary: [...specs, checked ? `checked ${checked}` : null].filter(Boolean).join(' · ') || null,
+ body: [`${row.provider_name} ${row.name}`, KINDS[row.kind] || row.kind, specs.join(', '),
+ price ? `${price} a month (${row.currency})${checked ? `, checked ${checked}` : ''}` : null,
+ Array.isArray(row.locations) && row.locations.length ? `Locations: ${row.locations.join(', ')}` : null].filter(Boolean).join('\n'),
+ facets: { provider: row.provider_slug, kind: row.kind, currency: row.currency },
+ authorship: { mode: row.origin === 'sources' ? 'imported' : 'human' },
+ provenance: row.source_url ? [{
+ service: row.origin === 'sources' ? 'sources' : 'host', type: row.origin === 'sources' ? 'item' : 'staff_entry',
+ id: String(row.origin === 'sources' && row.source_item_id ? row.source_item_id : row.id).slice(0, 128),
+ label: row.origin === 'sources' ? (row.source_key || 'OpenVibe.Sources') : `${row.provider_name} pricing`,
+ url: row.source_url, retrievedAt: iso(row.checked_at),
+ }] : [],
+ publishedAt: iso(row.first_seen_at || row.created_at),
+ updatedAt: iso(row.checked_at || row.updated_at),
+ };
+}
+
+function providerDocument(row) {
+ const count = Number(row.plan_count) || 0;
+ const from = row.from_price != null ? money(row.from_price, row.from_currency) : null;
+ return {
+ listed: row.status === 'active' && count > 0,
+ title: `${row.name} hosting plans and prices`,
+ summary: `${count} plan${count === 1 ? '' : 's'}${from ? ` from ${from} a month` : ''}`,
+ body: [row.name, row.description, `${count} plan${count === 1 ? '' : 's'} in the catalog${from ? `, from ${from} a month` : ''}`].filter(Boolean).join('\n'),
+ facets: { provider: row.slug },
+ authorship: { mode: 'human' },
+ publishedAt: iso(row.created_at),
+ updatedAt: iso(row.updated_at),
+ };
+}
+
+function createCatalogSearch({ config, store, outbox, log = console }) {
+ const db = store.db;
+ const feed = createSearchFeed({
+ owner: 'host', db, outbox, baseUrl: config.baseUrl, now: store.now, log,
+ types: {
+ plan: {
+ page: (row) => `/catalog/plans/${row.id}`,
+ document: planDocument,
+ rows: (after, limit) => db.many(`SELECT p.*, v.slug AS provider_slug, v.name AS provider_name, v.status AS provider_status
+ FROM catalog_plans p JOIN catalog_providers v ON v.id = p.provider_id WHERE p.id > $1 ORDER BY p.id LIMIT $2`, [after, limit]),
+ exists: async (ids) => (await db.many('SELECT id FROM catalog_plans WHERE id = ANY($1)', [ids])).map((r) => r.id),
+ },
+ provider: {
+ page: (row) => `/catalog/providers/${row.slug}`,
+ document: providerDocument,
+ // The cheapest active plan is named in its own currency (the first currency by price, USD first when tied).
+ rows: (after, limit) => db.many(`SELECT v.*,
+ (SELECT COUNT(*) FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active') AS plan_count,
+ (SELECT p.price_monthly FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active' ORDER BY (p.currency <> 'USD'), p.price_monthly, p.id LIMIT 1) AS from_price,
+ (SELECT p.currency FROM catalog_plans p WHERE p.provider_id = v.id AND p.status = 'active' ORDER BY (p.currency <> 'USD'), p.price_monthly, p.id LIMIT 1) AS from_currency
+ FROM catalog_providers v WHERE v.id > $1 ORDER BY v.id LIMIT $2`, [after, limit]),
+ exists: async (ids) => (await db.many('SELECT id FROM catalog_providers WHERE id = ANY($1)', [ids])).map((r) => r.id),
+ },
+ },
+ });
+
+ let soon = null;
+ async function sweep() {
+ const res = await feed.sweep();
+ for (const [type, r] of Object.entries(res)) {
+ if (r.sent || r.removed || r.failed) log.log(`[Search] catalog ${type}: ${r.sent} sent, ${r.removed} removed, ${r.failed} failed of ${r.seen}`);
+ }
+ if (outbox && outbox.kick) outbox.kick().catch(() => {});
+ return res;
+ }
+ /** A staff change: one sweep a few seconds later (several edits in a row share it). */
+ function sweepSoon(delayMs = 5000) {
+ if (soon) return;
+ soon = setTimeout(() => { soon = null; sweep().catch((err) => log.warn(`[Search] catalog sweep failed: ${(err && err.message) || err}`)); }, delayMs);
+ if (soon.unref) soon.unref();
+ }
+ return { feed, sweep, sweepSoon, planDocument, providerDocument };
+}
+
+module.exports = { createCatalogSearch, planDocument, providerDocument };
@@ -48,6 +48,8 @@ const clean = (type, input, current, now) => {
@@ -48,6 +48,8 @@ const clean = (type, input, current, now) => {
};
};
function createCatalog({ store, community = null, log = console }) {
function createCatalog({ store, community = null, log = console }) {
const db = store.db;
const db = store.db;
+ const listeners = [];
+ const changed = (type) => { for (const fn of listeners) { try { fn(type); } catch { /* a listener never breaks a save */ } } };
let visibilityQueue = Promise.resolve();
let visibilityQueue = Promise.resolve();
const open = async (type, id) => {
const open = async (type, id) => {
const sql = type === 'provider'
const sql = type === 'provider'
@@ -59,7 +61,8 @@ function createCatalog({ store, community = null, log = console }) {
@@ -59,7 +61,8 @@ function createCatalog({ store, community = null, log = console }) {
const providerOut = (p) => ({ ...p, affiliate_rel: p.affiliate_url ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate_url ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });
const providerOut = (p) => ({ ...p, affiliate_rel: p.affiliate_url ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate_url ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });
const promoOut = (p) => ({ ...p, status: p.status === 'active' && p.ends_at != null && Number(p.ends_at) <= store.now() ? 'ended' : p.status, affiliate_rel: p.affiliate ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });
const promoOut = (p) => ({ ...p, status: p.status === 'active' && p.ends_at != null && Number(p.ends_at) <= store.now() ? 'ended' : p.status, affiliate_rel: p.affiliate ? 'sponsored nofollow noopener' : null, affiliate_disclosure: p.affiliate ? 'OpenVibe may earn a commission from links marked as affiliate links; it never changes the order or the prices shown.' : null });
const get = async (type, id) => { const row = await db.prepare(`SELECT * FROM catalog_${type} WHERE id = ?`).get(id); if (!row) missing(); return type === 'promos' ? promoOut(row) : row; };
const get = async (type, id) => { const row = await db.prepare(`SELECT * FROM catalog_${type} WHERE id = ?`).get(id); if (!row) missing(); return type === 'promos' ? promoOut(row) : row; };
- const save = async (type, input, id, actor) => await db.tx(async () => {
+ const save = async (type, input, id, actor) => { const out = await saveTx(type, input, id, actor); changed(type); return out; };
+ const saveTx = async (type, input, id, actor) => await db.tx(async () => {
const current = id ? await get(type, id) : null;
const current = id ? await get(type, id) : null;
const watched = [];
const watched = [];
if (community && community.enabled && current && ['providers', 'plans'].includes(type)) {
if (community && community.enabled && current && ['providers', 'plans'].includes(type)) {
@@ -174,6 +177,6 @@ function createCatalog({ store, community = null, log = console }) {
@@ -174,6 +177,6 @@ function createCatalog({ store, community = null, log = console }) {
return { plans: Number(r.plans) || 0, providers: Number(r.providers) || 0, lastChecked: r.last == null ? null : Number(r.last) };
return { plans: Number(r.plans) || 0, providers: Number(r.providers) || 0, lastChecked: r.last == null ? null : Number(r.last) };
}
}
async function sitemapPlans() { return await db.prepare("SELECT p.id FROM catalog_plans p JOIN catalog_providers v ON v.id = p.provider_id WHERE p.status = 'active' AND v.status = 'active' ORDER BY p.id").all(); }
async function sitemapPlans() { return await db.prepare("SELECT p.id FROM catalog_plans p JOIN catalog_providers v ON v.id = p.provider_id WHERE p.status = 'active' AND v.status = 'active' ORDER BY p.id").all(); }
- return { save, get, applyItem, plans, plan, providers, promos, currencies: currenciesInUse, stats, sitemapPlans, discussionIdle: idle, all: async (type) => { const rows = await db.prepare(`SELECT * FROM catalog_${type} ORDER BY created_at DESC, id DESC LIMIT 200`).all(); return type === 'promos' ? rows.map(promoOut) : rows; } };
+ return { onChange: (fn) => listeners.push(fn), save, get, applyItem, plans, plan, providers, promos, currencies: currenciesInUse, stats, sitemapPlans, discussionIdle: idle, all: async (type) => { const rows = await db.prepare(`SELECT * FROM catalog_${type} ORDER BY created_at DESC, id DESC LIMIT 200`).all(); return type === 'promos' ? rows.map(promoOut) : rows; } };
}
}
module.exports = { createCatalog };
module.exports = { createCatalog };
@@ -7,6 +7,7 @@
@@ -7,6 +7,7 @@
* host.deploy.activated a site's active deploy changed (payload.rollback marks a rollback)
* host.deploy.activated a site's active deploy changed (payload.rollback marks a rollback)
* host.deploy.failed an upload was refused (payload.code, problem codes; no file contents)
* host.deploy.failed an upload was refused (payload.code, problem codes; no file contents)
* host.domain.verified a custom domain's TXT record was found; the domain is now served
* host.domain.verified a custom domain's TXT record was found; the domain is now served
+ * host.index_document.upserted|deleted a hosting catalog page in OpenVibe.Search (server/catalog/search-index.js)
*
*
* The SDK writes each row inside the change's own transaction (emit joins the ambient openvibe-sdk/db
* The SDK writes each row inside the change's own transaction (emit joins the ambient openvibe-sdk/db
* transaction), so an event exists if and only if its change committed. The relay publishes with
* transaction), so an event exists if and only if its change committed. The relay publishes with
@@ -16,7 +17,8 @@
@@ -16,7 +17,8 @@
*/
*/
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createServiceOutbox } = require('openvibe-sdk/events');
-const EVENT_TYPES = Object.freeze(['host.deploy.created', 'host.deploy.activated', 'host.deploy.failed', 'host.domain.verified']);
+const EVENT_TYPES = Object.freeze(['host.deploy.created', 'host.deploy.activated', 'host.deploy.failed', 'host.domain.verified',
+ 'host.index_document.upserted', 'host.index_document.deleted']);
function createHostOutbox({ db, config, fetchImpl, now, log = console }) {
function createHostOutbox({ db, config, fetchImpl, now, log = console }) {
return createServiceOutbox({
return createServiceOutbox({
@@ -11,7 +11,7 @@
@@ -11,7 +11,7 @@
*/
*/
const fs = require('fs');
const fs = require('fs');
-function createWorker({ config, store, domains, blobs, outbox, catalogIngest, log = console }) {
+function createWorker({ config, store, domains, blobs, outbox, catalogIngest, catalogSearch, log = console }) {
const timers = [];
const timers = [];
let checking = false;
let checking = false;
@@ -61,8 +61,11 @@ function createWorker({ config, store, domains, blobs, outbox, catalogIngest, lo
@@ -61,8 +61,11 @@ function createWorker({ config, store, domains, blobs, outbox, catalogIngest, lo
every(3600 * 1000, async () => { try { await sweepObjects(); } catch (err) { log.warn('[Host] object sweep failed:', err.message); } });
every(3600 * 1000, async () => { try { await sweepObjects(); } catch (err) { log.warn('[Host] object sweep failed:', err.message); } });
every(24 * 3600 * 1000, async () => { try { await outbox.outbox.prune(); } catch (err) { log.warn('[Host] outbox prune failed:', err.message); } });
every(24 * 3600 * 1000, async () => { try { await outbox.outbox.prune(); } catch (err) { log.warn('[Host] outbox prune failed:', err.message); } });
setTimeout(domainTick, 5000).unref();
setTimeout(domainTick, 5000).unref();
+ // The catalog in OpenVibe.Search: a minute after boot, hourly, and after each pull (only changes are sent).
+ const searchSweep = async () => { if (!catalogSearch) return; try { await catalogSearch.sweep(); } catch (err) { log.warn('[Host] catalog search sweep failed:', err.message); } };
+ if (catalogSearch) { setTimeout(searchSweep, 60_000).unref(); every(3600 * 1000, searchSweep); }
if (catalogIngest && catalogIngest.enabled) {
if (catalogIngest && catalogIngest.enabled) {
- const pull = async () => { try { await catalogIngest.pull(); } catch (err) { log.warn('[Host] catalog pull failed:', err.message); } };
+ const pull = async () => { try { await catalogIngest.pull(); } catch (err) { log.warn('[Host] catalog pull failed:', err.message); } await searchSweep(); };
every(5 * 60_000, pull);
every(5 * 60_000, pull);
setTimeout(pull, 5000).unref();
setTimeout(pull, 5000).unref();
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,72 @@
@@ -0,0 +1,72 @@
+'use strict';
+/**
+ * The hosting catalog in OpenVibe.Search: a sweep sends one host.index_document.upserted per public plan and provider
+ * page (a valid search.index-document@1 the contracts declare), sends nothing for an unchanged page, and tombstones a
+ * hidden plan and the plans of a hidden provider.
+ */
+const assert = require('assert');
+const contracts = require('openvibe-contracts');
+const { boot, check, done } = require('./stageb/boot');
+
+(async () => {
+ const t = await boot();
+ const staff = t.user('catalog-search-editor', { role: 'admin' });
+ const events = async () => (await t.ctx.store.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'host.index_document.%' ORDER BY id"))
+ .map((r) => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope));
+ try {
+ const post = async (type, body) => { const r = await t.api('POST', `/api/v1/catalog/${type}`, { as: staff, json: body }); assert.strictEqual(r.status, 201, r.text); return r.json()[type.slice(0, -1)]; };
+ const patch = async (type, id, body) => { const r = await t.api('PATCH', `/api/v1/catalog/${type}/${id}`, { as: staff, json: body }); assert.strictEqual(r.status, 200, r.text); return r.json()[type.slice(0, -1)]; };
+ const v = await post('providers', { slug: 'hetzner', name: 'Hetzner', homepage_url: 'https://www.hetzner.com/', description: 'German cloud servers.' });
+ const plan = await post('plans', { provider_id: v.id, external_id: 'cx22', name: 'CX22', kind: 'cloud', vcpu: 2, ram_mb: 4096, disk_gb: 40, disk_type: 'nvme', bandwidth_gb: 20000, currency: 'USD', price_monthly: 4.99, source_url: 'https://www.hetzner.com/cloud/' });
+
+ await check('a sweep sends a valid document for each public plan and provider page', async () => {
+ const res = await t.ctx.catalogSearch.sweep();
+ assert.deepStrictEqual([res.plan.sent, res.provider.sent], [1, 1]);
+ const sent = await events();
+ assert.strictEqual(sent.length, 2);
+ for (const e of sent) {
+ assert.deepStrictEqual(contracts.validate(`${e.event_type}@1`, e.payload).errors || [], [], e.event_type);
+ assert.strictEqual(e.source, 'host');
+ }
+ const p = sent.find((e) => e.payload.type === 'plan').payload;
+ assert.strictEqual(p.id, plan.id);
+ assert.strictEqual(p.canonical_url, `${t.config.baseUrl}/catalog/plans/${plan.id}`);
+ assert.match(p.title, /^Hetzner CX22: \$4\.99 a month$/);
+ assert.match(p.summary, /2 vCPU · 4 GB RAM · 40 GB NVME disk · 20 TB transfer · checked \d{4}-\d{2}-\d{2}/);
+ assert.deepStrictEqual(p.facets, { provider: 'hetzner', kind: 'cloud', currency: 'USD' });
+ const pr = sent.find((e) => e.payload.type === 'provider').payload;
+ assert.strictEqual(pr.canonical_url, `${t.config.baseUrl}/catalog/providers/hetzner`);
+ assert.strictEqual(pr.summary, '1 plan from $4.99 a month');
+ });
+
+ await check('an unchanged page is not sent again', async () => {
+ const res = await t.ctx.catalogSearch.sweep();
+ assert.deepStrictEqual([res.plan.sent, res.provider.sent], [0, 0]);
+ assert.strictEqual((await events()).length, 2);
+ });
+
+ await check('a price change re-sends the plan; hiding it sends a tombstone', async () => {
+ await patch('plans', plan.id, { price_monthly: 5.49 });
+ let res = await t.ctx.catalogSearch.sweep();
+ assert.strictEqual(res.plan.sent, 1);
+ await patch('plans', plan.id, { status: 'hidden' });
+ res = await t.ctx.catalogSearch.sweep();
+ const last = (await events()).filter((e) => e.payload.id === plan.id || e.subject && e.subject.id === plan.id).pop();
+ assert.strictEqual(last.event_type, 'host.index_document.deleted', JSON.stringify(res));
+ });
+
+ await check('a hidden provider takes its plans and its own page out of Search', async () => {
+ const other = await post('plans', { provider_id: v.id, external_id: 'cpx31', name: 'CPX31', kind: 'cloud', vcpu: 4, ram_mb: 8192, currency: 'USD', price_monthly: 15.99, source_url: 'https://www.hetzner.com/cloud/' });
+ await t.ctx.catalogSearch.sweep();
+ await patch('providers', v.id, { status: 'hidden' });
+ await t.ctx.catalogSearch.sweep();
+ const sent = await events();
+ const lastFor = (id) => sent.filter((e) => (e.subject && e.subject.id === id) || e.payload.id === id).pop();
+ assert.strictEqual(lastFor(other.id).event_type, 'host.index_document.deleted');
+ assert.strictEqual(lastFor(v.id).event_type, 'host.index_document.deleted');
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})();