openvibers/openvibe.tours Commits

Places, cities and walks in OpenVibe.Search (search-feed, contracts 0.274.0) (#40)

OP OpenVibers committed 11 days ago · f56cd9939f · 9 files, +292 −9

Changes

ddiff --git a/README.md b/README.md
iindex c6b8ace..ba5bb23 100644
---- a/README.md
++++ b/README.md
@@@ -175,7 +175,8 @@ Room id `tours` in `openvibe-shared/rooms`: dark `#F98942` on `#0B0D12` is **8.0
  AGPL-3.0-only. See [LICENSE](LICENSE).
  
  <!-- versions:start -->
--- openvibe-contracts: v0.265.0
++- openvibe-contracts: v0.274.0
++- openvibe-publishing: v1.4.1
  - openvibe-sdk: v0.81.0
  - openvibe-shared: v4.103.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 577b9ba..491be56 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,11 +8,12 @@
    "runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9",
    "port": "5170",
    "domain": "openvibe.tours",
--  "contracts": "openvibe-contracts v0.265.0",
++  "contracts": "openvibe-contracts v0.274.0",
    "sdk": "openvibe-sdk v0.81.0",
    "shared": "openvibe-shared v4.103.0",
    "packages": [
--    "openvibe-contracts v0.265.0",
++    "openvibe-contracts v0.274.0",
++    "openvibe-publishing v1.4.1",
      "openvibe-sdk v0.81.0",
      "openvibe-shared v4.103.0"
    ],
ddiff --git a/migrations/0003_search_index.sql b/migrations/0003_search_index.sql
nnew file mode 100644
iindex 0000000..bdcd15a
---- /dev/null
++++ b/migrations/0003_search_index.sql
@@@ -0,0 +1,31 @@
++-- phase: expand
++-- OpenVibe.Tours: places, cities and self-guided walks in OpenVibe.Search (server/search-index.js, openvibe-publishing/search-feed).
++-- event_outbox holds the tours.index_document.* events until the relay delivers them to OpenVibe.Events (written in
++-- the same transaction as the revision that produced them); tours_index_revisions is the index sequencer, the last
++-- revision and content hash Search was sent for each page, so an unchanged page is never sent twice.
++
++CREATE TABLE IF NOT EXISTS event_outbox (
++    id              bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
++    event_id        text NOT NULL UNIQUE,
++    envelope        jsonb NOT NULL,
++    traceparent     text,
++    created_at      bigint NOT NULL,
++    attempts        integer NOT NULL DEFAULT 0,
++    next_attempt_at bigint NOT NULL DEFAULT 0,
++    sent_at         bigint,
++    seq             bigint,
++    rejected_at     bigint,
++    last_error      text
++);
++CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;
++CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;
++
++CREATE TABLE IF NOT EXISTS tours_index_revisions (
++    owner      text COLLATE "C" NOT NULL,
++    type       text COLLATE "C" NOT NULL,
++    id         text COLLATE "C" NOT NULL,
++    revision   integer NOT NULL,
++    hash       text NOT NULL,
++    updated_at bigint NOT NULL,
++    PRIMARY KEY (owner, type, id)
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex 69a8f18..db4e092 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -15,7 +15,8 @@
          "express-rate-limit": "^7.5.0",
          "helmet": "^8.1.0",
          "iovalkey": "^0.4.0",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.265.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.274.0",
++        "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.103.0",
          "pg": "^8.23.0"
@@@ -1154,9 +1155,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.265.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.265.0",
--      "integrity": "sha512-0KnKj1Rig6acZcv4U2uwaM2msarGUY36pNqFbqmKdLeOPPozf7haEDTam27A/WzSU9ii2yEZ6SnuusPPQoGZaA==",
++      "version": "0.274.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.274.0",
++      "integrity": "sha512-klrHvepa4En2jAA7tU/YCpNAFJBVR1GDKXv1ohfA0I6ZwUtIgWO7ny7rQhSGBMI5oPUut21xgun+/MqM5Is4QA==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
@@@ -1169,6 +1170,24 @@
          "node": ">=20"
        }
      },
++    "node_modules/openvibe-publishing": {
++      "version": "1.4.1",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
++      "integrity": "sha512-YjrcxCnXbpDQwyYpgsRcDj0+ckqZBm/ErEmQjvf0mCFMImwErG+lzZKXnXWb6eWOPxXEVj9YYuOgZiWVL41Vfg==",
++      "license": "MIT",
++      "engines": {
++        "node": ">=20"
++      },
++      "peerDependencies": {
++        "openvibe-sdk": ">=0.15.0",
++        "openvibe-shared": ">=1.5.0"
++      },
++      "peerDependenciesMeta": {
++        "openvibe-shared": {
++          "optional": true
++        }
++      }
++    },
      "node_modules/openvibe-sdk": {
        "version": "0.81.0",
        "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
ddiff --git a/package.json b/package.json
iindex a0ba26f..7366cc0 100644
---- a/package.json
++++ b/package.json
@@@ -23,7 +23,8 @@
      "express-rate-limit": "^7.5.0",
      "helmet": "^8.1.0",
      "iovalkey": "^0.4.0",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.265.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.274.0",
++    "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.81.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.103.0",
      "pg": "^8.23.0"
ddiff --git a/server/app.js b/server/app.js
iindex d951c91..4f8c8fa 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -33,6 +33,7 @@ const { assetVersion, send } = require('./render/layout');
  const { raw } = require('./render/html');
  const { createStore: dataStore } = require('./store');
  const { createJobs } = require('./jobs');
++const { createSearchIndex } = require('./search-index');
  const accountDataLib = require('./identity/account-data');
  const { createNetworkSender } = require('openvibe-sdk/account-data');
  
@@@ -53,13 +54,15 @@ async function createApp(opts = {}) {
      const principal = createPrincipal({ config, keys });
      const store = dataStore(s, { config });
      const jobs = createJobs({ config, s, store, fetchImpl, log });
++    // The places, cities and walks in OpenVibe.Search (started by server/index.js, never under TOURS_DRILL).
++    const searchIndex = createSearchIndex({ config, s, log });
  
      const accountData = accountDataLib.create({ db: s.db, log });
      const accountSend = opts.accountSend || (config.oauth.clientSecret
          ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl })
          : async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Tours cannot answer account events'); });
  
--    const ctx = { config, s, keys, sso, principal, store, jobs, accountData, accountSend, log };
++    const ctx = { config, s, keys, sso, principal, store, jobs, searchIndex, accountData, accountSend, log };
  
      const app = express();
      app.disable('x-powered-by');
ddiff --git a/server/index.js b/server/index.js
iindex 8ada0de..720721f 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -15,6 +15,7 @@ function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }
          close: [
              () => { for (const t of timers) clearInterval(t); },
              () => ctx.jobs.stop(),
++            () => { if (ctx.searchIndex) ctx.searchIndex.stop(); },
              () => ctx.keys.client.stop(),
              async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); },
              () => ctx.s.close(),
@@@ -37,6 +38,8 @@ async function start() {
      ctx.telemetryTick.unref();
      ctx.keys.client.start();
      if (!config.drill) ctx.jobs.start();
++    // The places, cities and walks in OpenVibe.Search: a sweep a minute after boot and every six hours.
++    if (!config.drill) ctx.searchIndex.start();
  
      const subs = config.drill ? null : startSubscriptions({
          eventsUrl: config.events.url,
ddiff --git a/server/search-index.js b/server/search-index.js
nnew file mode 100644
iindex 0000000..7f2c423
---- /dev/null
++++ b/server/search-index.js
@@@ -0,0 +1,148 @@
++'use strict';
++
++/**
++ * OpenVibe.Tours' pages in OpenVibe.Search (openvibe-publishing/search-feed, contracts 0.274.0): one
++ * search.index-document@1 per place to see (/place/<plc_ id>), city (/city/<slug>, the slug as the id) and self-guided
++ * walk (/routes/<rte_ id>), sent as tours.index_document.upserted|deleted through this service's events outbox. Each
++ * document is what its page says: a place's kind, city and access facts (free, step-free) and its description; a
++ * city's Wikivoyage summary and how many places and walks it has; a walk's stops, distance and time. 2,754 places,
++ * 100 cities and 170 walks were on no network search (2026-11-02).
++ *
++ * The rows come from the source jobs (OpenStreetMap, Wikidata, Wikivoyage), which refresh daily, so a sweep a minute
++ * after boot and every six hours is enough: the sequencer (tours_index_revisions) gives an unchanged page its old
++ * revision and a quiet sweep sends nothing; a page whose row is gone gets a tombstone. Off under TOURS_DRILL.
++ */
++const { createServiceOutbox } = require('openvibe-sdk/events');
++const { createSearchFeed } = require('openvibe-publishing/search-feed');
++
++const EVENT_TYPES = ['tours.index_document.upserted', 'tours.index_document.deleted'];
++const START_DELAY_MS = 60_000;
++const INTERVAL_MS = 6 * 3_600_000;
++
++const iso = (t) => (t ? new Date(t).toISOString() : null);
++const clip = (text, n) => { const t = String(text || '').replace(/\s+/g, ' ').trim(); return t.length > n ? `${t.slice(0, n - 1).replace(/\s+\S*$/, '')}…` : t; };
++const plural = (n, one, many) => `${n} ${n === 1 ? one : many}`;
++const kindWord = (kind) => { const k = String(kind || 'place').replace(/_/g, ' '); return k.charAt(0).toUpperCase() + k.slice(1); };
++
++function describePlace(row) {
++    const where = [row.city_name, row.country].filter(Boolean).join(', ');
++    const facts = [row.fee === 'no' ? 'Free to visit.' : row.fee === 'yes' ? 'Entry is paid.' : '', row.wheelchair === 'yes' ? 'Step-free.' : row.wheelchair === 'limited' ? 'Partly step-free.' : ''].filter(Boolean);
++    const about = `${kindWord(row.kind)} in ${where}.`;
++    return {
++        listed: true,
++        title: row.name,
++        summary: [about, clip(row.description, 220), ...facts].filter(Boolean).join(' '),
++        body: [row.name, about, row.description, row.address, ...facts].filter(Boolean).join('\n'),
++        facets: { city: row.city_slug, country: row.country, kind: row.kind },
++        authorship: { mode: 'imported' },
++        publishedAt: iso(row.created_at),
++        updatedAt: iso(row.updated_at),
++    };
++}
++
++function describeCity(row) {
++    const places = Number(row.places) || 0;
++    const routes = Number(row.routes) || 0;
++    const what = `${plural(places, 'place to see', 'places to see')} and ${plural(routes, 'self-guided walk', 'self-guided walks')} on OpenVibe.Tours.`;
++    return {
++        listed: true,
++        noindex: !places && !routes,
++        title: `${row.name}, ${row.country}`,
++        summary: [clip(row.summary, 260), what].filter(Boolean).join(' '),
++        body: [`${row.name}, ${row.region ? `${row.region}, ` : ''}${row.country}`, row.summary, what, 'Things to do, places to see, walks, stays and flights.'].filter(Boolean).join('\n'),
++        facets: { city: row.id, country: row.country, places, routes },
++        authorship: { mode: 'imported' },
++        publishedAt: iso(row.created_at),
++        updatedAt: iso(row.updated_at),
++    };
++}
++
++function describeRoute(row) {
++    const km = (Number(row.distance_m) || 0) / 1000;
++    const shape = [plural(Number(row.stop_count) || 0, 'stop', 'stops'), km ? `${km.toFixed(1)} km` : '', row.duration_min ? `about ${row.duration_min} minutes` : ''].filter(Boolean).join(', ');
++    const about = `A self-guided walk in ${row.city_name}, ${row.country}: ${shape}.`;
++    return {
++        listed: true,
++        noindex: !Number(row.stop_count),
++        title: row.title,
++        summary: [about, clip(row.summary, 220)].filter(Boolean).join(' '),
++        body: [row.title, about, row.summary].filter(Boolean).join('\n'),
++        facets: { city: row.city_slug, country: row.country, stops: Number(row.stop_count) || 0, distance_m: Number(row.distance_m) || 0, duration_min: Number(row.duration_min) || 0 },
++        authorship: { mode: 'imported' },
++        publishedAt: iso(row.created_at),
++        updatedAt: iso(row.updated_at),
++    };
++}
++
++function createSearchIndex({ config, s, log = console, outbox = null }) {
++    const db = s.db;
++    const out = outbox || createServiceOutbox({
++        db, source: 'tours', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
++        clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,
++    });
++    const page = (sql) => async (after, limit) => db.many(sql, [String(after || ''), limit]);
++    const existing = (sql) => async (ids) => (ids.length ? (await db.many(sql, [ids])).map((r) => r.id) : []);
++    const feed = createSearchFeed({
++        owner: 'tours', db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,
++        types: {
++            place: {
++                page: (row) => `/place/${row.id}`,
++                document: describePlace,
++                rows: page(`SELECT p.id, p.name, p.kind, p.description, p.address, p.fee, p.wheelchair, p.created_at, p.updated_at,
++                                   c.name AS city_name, c.slug AS city_slug, c.country
++                              FROM tours_places p JOIN tours_cities c ON c.id = p.city_id
++                             WHERE p.id > $1 ORDER BY p.id LIMIT $2`),
++                exists: existing('SELECT id FROM tours_places WHERE id = ANY($1::text[])'),
++            },
++            city: {
++                page: (row) => `/city/${row.id}`,
++                document: describeCity,
++                rows: page(`SELECT c.slug AS id, c.name, c.country, c.region, c.summary, c.created_at, c.updated_at,
++                                   (SELECT COUNT(*)::int FROM tours_places p WHERE p.city_id = c.id) AS places,
++                                   (SELECT COUNT(*)::int FROM tours_routes r WHERE r.city_id = c.id) AS routes
++                              FROM tours_cities c WHERE c.slug > $1 ORDER BY c.slug LIMIT $2`),
++                exists: existing('SELECT slug AS id FROM tours_cities WHERE slug = ANY($1::text[])'),
++            },
++            route: {
++                page: (row) => `/routes/${row.id}`,
++                document: describeRoute,
++                rows: page(`SELECT r.id, r.title, r.summary, r.distance_m, r.duration_min, r.stop_count, r.created_at, r.updated_at,
++                                   c.name AS city_name, c.slug AS city_slug, c.country
++                              FROM tours_routes r JOIN tours_cities c ON c.id = r.city_id
++                             WHERE r.id > $1 ORDER BY r.id LIMIT $2`),
++                exists: existing('SELECT id FROM tours_routes WHERE id = ANY($1::text[])'),
++            },
++        },
++    });
++    const timers = [];
++
++    async function sweep() {
++        const res = await feed.sweep();
++        for (const [type, r] of Object.entries(res)) {
++            if (r.sent || r.removed || r.failed) log.log(`[Search] ${type}: ${r.sent} sent, ${r.removed} removed, ${r.failed} failed of ${r.seen}`);
++        }
++        if (out.kick) out.kick().catch(() => {});
++        return res;
++    }
++    const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
++
++    function start() {
++        if (config.drill || timers.length) return false;
++        out.start();
++        const kick = setTimeout(quietly, START_DELAY_MS);
++        if (kick.unref) kick.unref();
++        const tick = setInterval(quietly, INTERVAL_MS);
++        if (tick.unref) tick.unref();
++        timers.push(kick, tick);
++        return true;
++    }
++
++    function stop() {
++        for (const t of timers.splice(0)) { clearTimeout(t); clearInterval(t); }
++        out.stop();
++    }
++
++    return { feed, outbox: out, sweep, start, stop, status: () => out.status() };
++}
++
++module.exports = { createSearchIndex, describePlace, describeCity, describeRoute, EVENT_TYPES };
ddiff --git a/test/search-index.test.js b/test/search-index.test.js
nnew file mode 100644
iindex 0000000..e9530f5
---- /dev/null
++++ b/test/search-index.test.js
@@@ -0,0 +1,76 @@
++'use strict';
++/**
++ * Places, cities and walks in OpenVibe.Search (server/search-index.js): a sweep puts every page in the outbox as one
++ * tours.index_document.upserted, exactly the contract (contracts 0.274.0), with a page that answers; a second sweep
++ * sends nothing; a place whose row is gone becomes a tombstone, once. The documents say what the pages say.
++ */
++const assert = require('assert');
++const contracts = require('openvibe-contracts');
++const { boot, check, done } = require('./helpers/boot');
++const { seed } = require('./helpers/seed');
++const { describePlace, describeCity, describeRoute } = require('../server/search-index');
++
++(async () => {
++    const t = await boot();
++    const { s, searchIndex } = t.ctx;
++    const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);
++    const valid = (env) => {
++        assert.ok(contracts.validate('events.event-envelope@1', env).valid, 'envelope');
++        const p = contracts.validate(env.event_type, env.payload);
++        assert.ok(p.valid, `${env.event_type}: ${JSON.stringify(p.errors)}`);
++        assert.strictEqual(env.source, 'tours');
++        return env;
++    };
++    try {
++        await seed(t);
++        const counts = { place: 0, city: 0, route: 0 };
++        counts.place = (await s.db.many('SELECT id FROM tours_places')).length;
++        counts.city = (await s.db.many('SELECT id FROM tours_cities')).length;
++        counts.route = (await s.db.many('SELECT id FROM tours_routes')).length;
++
++        await check('a sweep sends every place, city and walk once, as the contract says, and their pages answer', async () => {
++            assert.ok(counts.place && counts.city, `the jobs made pages (${JSON.stringify(counts)})`);
++            const res = await searchIndex.sweep();
++            for (const type of ['place', 'city', 'route']) {
++                assert.deepStrictEqual(res[type], { seen: counts[type], sent: counts[type], removed: 0, failed: 0 }, type);
++            }
++            const envs = (await outbox()).map(valid);
++            for (const type of ['place', 'city', 'route']) {
++                const doc = envs.map((e) => e.payload).find((p) => p.type === type);
++                if (!doc) continue;
++                const path = new URL(doc.canonical_url).pathname;
++                assert.strictEqual((await t.get(path)).status, 200, `${type} page ${path}`);
++            }
++            const city = envs.map((e) => e.payload).find((p) => p.type === 'city');
++            assert.match(city.summary, /places? to see and \d+ self-guided walks?/);
++            assert.strictEqual((await searchIndex.sweep()).place.sent, 0, 'a second sweep sends nothing');
++        });
++
++        await check('a place whose row is gone becomes a tombstone, once', async () => {
++            const gone = (await s.db.many('SELECT id FROM tours_places ORDER BY id DESC LIMIT 1'))[0].id;
++            await s.db.many('DELETE FROM tours_places WHERE id = $1 RETURNING id', [gone]);
++            const before = (await outbox()).length;
++            const res = await searchIndex.sweep();
++            assert.strictEqual(res.place.removed, 1);
++            assert.strictEqual(res.city.sent, 1, 'its city counts one place fewer');
++            const env = valid((await outbox()).slice(before).find((e) => e.event_type === 'tours.index_document.deleted'));
++            assert.strictEqual(env.event_type, 'tours.index_document.deleted');
++            assert.deepStrictEqual([env.payload.type, env.payload.id], ['place', gone]);
++            assert.strictEqual((await searchIndex.sweep()).place.removed, 0);
++        });
++
++        await check('the words: a place\'s kind, city and access; a city\'s counts; a walk\'s shape', async () => {
++            const p = describePlace({ name: 'Hongya Cave', kind: 'theme_park', city_name: 'Chongqing', city_slug: 'chongqing', country: 'China', fee: 'no', wheelchair: 'yes', description: 'Stilted houses on the river.' });
++            assert.strictEqual(p.summary, 'Theme park in Chongqing, China. Stilted houses on the river. Free to visit. Step-free.');
++            assert.deepStrictEqual(p.facets, { city: 'chongqing', country: 'China', kind: 'theme_park' });
++            const c = describeCity({ id: 'chongqing', name: 'Chongqing', country: 'China', summary: 'A city on the Yangtze.', places: 1, routes: 0 });
++            assert.strictEqual(c.title, 'Chongqing, China');
++            assert.match(c.summary, /1 place to see and 0 self-guided walks/);
++            assert.strictEqual(describeCity({ id: 'x', name: 'X', country: 'Y', places: 0, routes: 0 }).noindex, true, 'an empty city is noindex');
++            const r = describeRoute({ title: 'Old town walk', city_name: 'Chongqing', city_slug: 'chongqing', country: 'China', stop_count: 6, distance_m: 3200, duration_min: 90 });
++            assert.match(r.summary, /6 stops, 3\.2 km, about 90 minutes/);
++        });
++    } finally {
++        done();
++    }
++})();