openvibers/openvibe.rent Commits
Rent: listing pages in OpenVibe.Search (search-feed, rent.index_document.*) (#10)
Changes
@@@ -247,7 +247,8 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md).Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).<!-- versions:start -->--- openvibe-contracts: v0.127.0++- openvibe-contracts: v0.128.0++- openvibe-publishing: v1.4.0- openvibe-sdk: v0.36.0- openvibe-shared: v2.20.4<!-- versions:end -->@@@ -7,13 +7,14 @@"runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9","port": "5010","domain": "openvibe.rent",-- "contracts": "openvibe-contracts v0.127.0",++ "contracts": "openvibe-contracts v0.128.0","sdk": "openvibe-sdk v0.36.0","shared": "openvibe-shared v2.20.4","packages": [-- "openvibe-contracts v0.127.0",++ "openvibe-contracts v0.128.0","openvibe-sdk v0.36.0",-- "openvibe-shared v2.20.4"++ "openvibe-shared v2.20.4",++ "openvibe-publishing v1.4.0"],"summary": "OpenVibe.Rent — Find a place, or rent yours out. One place to find and offer things to rent: apartments, rooms, houses, commercial space, parking and storage, and equipment. In this first version every listing is posted by a person on this site: no scraped listings and no partner feeds, no payments, no deposits and no messaging. Search with filters and facets, a listing page with a plain safety note, reports that hide a listing at 3 until staff look, saved searches that count what is new, and a 30-day expiry an owner can renew.","works": [nnew file mode 100644@@@ -0,0 +1,31 @@++-- phase: expand++-- OpenVibe.Rent: listing pages in OpenVibe.Search (server/search-index.js, openvibe-publishing/search-feed).++-- event_outbox holds the rent.index_document.* events until the relay delivers them to OpenVibe.Events (written in++-- the same transaction as the revision that produced them); rent_index_revisions is the index sequencer, the last++-- revision and content hash Search was sent for each page, so an unchanged page is never sent twice.++++CREATE TABLE IF NOT EXISTS event_outbox (++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,++ event_id text NOT NULL UNIQUE,++ envelope jsonb NOT NULL,++ traceparent text,++ created_at bigint NOT NULL,++ attempts integer NOT NULL DEFAULT 0,++ next_attempt_at bigint NOT NULL DEFAULT 0,++ sent_at bigint,++ seq bigint,++ rejected_at bigint,++ last_error text++);++CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;++CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;++++CREATE TABLE IF NOT EXISTS rent_index_revisions (++ owner text COLLATE "C" NOT NULL,++ type text COLLATE "C" NOT NULL,++ id text COLLATE "C" NOT NULL,++ revision integer NOT NULL,++ hash text NOT NULL,++ updated_at bigint NOT NULL,++ PRIMARY KEY (owner, type, id)++);@@@ -15,7 +15,8 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4","pg": "^8.23.0"@@@ -871,9 +872,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.127.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",-- "integrity": "sha512-ySC+Qh8I3OlN5MULCqv0vfcAGubZQzWM0Fw7EOpnOo76vVnrkwx+m33aTxDum2yQqPkU8Ot5UtMuMwm1Cc1k3g==",++ "version": "0.128.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",++ "integrity": "sha512-r/WKk3NhdqxEsVxqsVPjg+HTD2So1amRx0BY6EOHhvmwjLuL9j5XJrNDZAfqf/4/P2xZCBzTLrtUqlWxC2EA6Q==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -886,6 +887,24 @@"node": ">=20"}},++ "node_modules/openvibe-publishing": {++ "version": "1.4.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",++ "integrity": "sha512-9vByyVqpvy2zVQI4PrJWk5QqCym/YQ/fXZDgrcloKGBqvukKrUn7fg4ZERiJHcgEq+rgDbxbU4rNflZhw62K/Q==",++ "license": "MIT",++ "engines": {++ "node": ">=20"++ },++ "peerDependencies": {++ "openvibe-sdk": ">=0.15.0",++ "openvibe-shared": ">=1.5.0"++ },++ "peerDependenciesMeta": {++ "openvibe-shared": {++ "optional": true++ }++ }++ },"node_modules/openvibe-sdk": {"version": "0.36.0","resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",@@@ -2,7 +2,7 @@"name": "openvibe-rent","version": "0.1.0","private": true,-- "description": "OpenVibe.Rent — Find a place, or rent yours out.",++ "description": "OpenVibe.Rent \u2014 Find a place, or rent yours out.","main": "server/index.js","scripts": {"start": "node server/index.js",@@@ -22,7 +22,8 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4","pg": "^8.23.0"@@@ -21,6 +21,7 @@ const rateLimit = require('express-rate-limit');const contracts = require('openvibe-contracts');const cache = require('openvibe-shared/cache-policy');++const { createSearchIndex } = require('./search-index');const configLib = require('./config');const { openStore } = require('./db');const { createKeyStore } = require('./auth/keys');@@@ -52,6 +53,10 @@ async function createApp(opts = {}) {const sso = createSso({ config, keys, fetchImpl, now: s.now, log });const principal = createPrincipal({ config, keys });const ctx = { config, s, keys, sso, principal, log };++ // Listing pages in OpenVibe.Search (./search-index.js): every store write names the listing it touched (s.search),++ // and a sweep runs on its own timer once server/index.js starts it.++ ctx.search = opts.search || createSearchIndex({ config, s, log });++ s.search = ctx.search;// Account export and deletion (ADR-033, ./identity/account-data.js): the three tables that hold a person's rows.// The sender posts to Network's internal export/deletion routes with this service's own client-credentials token;@@@ -19,7 +19,7 @@ const { startExpiryTimer } = require('./listings/expiry');function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }) {return gracefulStop({name: 'OpenVibe.Rent', server, deadlineExitCode: 0, exit, signals, deadlineMs: 10_000,-- close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.s.close(), ...extra],++ close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.search.stop(), () => ctx.s.close(), ...extra],});}@@@ -32,6 +32,7 @@ async function start() {});server.keepAliveTimeout = 65_000;ctx.keys.client.start();++ ctx.search.start();// One sweep at boot (a restart must not leave a stale listing up a moment longer than it should), then a timer.const expiry = startExpiryTimer(ctx.s);@@@ -15,6 +15,9 @@ const listings = require('./listings');const DAY_MS = 86_400_000;++/** A write changed this listing: OpenVibe.Search syncs it shortly (server/search-index.js; absent in some tests). */++const touched = (s, id) => { if (s.search) s.search.touch(id); };++/** The WHERE clause for a filter set. `exclude` leaves one dimension out — how the kind and country facets count. */function buildWhere(f, { exclude = [], sinceISO = null } = {}) {const parts = ["state = 'active'"];@@@ -42,6 +45,7 @@ async function insert(s, row) {[row.id, row.owner, row.kind, row.title, row.description, row.price, row.currency, row.period, row.city,row.region, row.country, row.neighbourhood, row.bedrooms, row.available_from, row.contact_url,row.created_at, row.expires_at]);++ touched(s, row.id);return get(s, row.id);}@@@ -80,13 +84,16 @@ async function update(s, id, patch) {const cols = Object.keys(patch);const sets = cols.map((c, i) => `${c} = $${i + 3}`);await s.db.query(`UPDATE rent_listings SET ${sets.join(', ')}, updated_at = $2 WHERE id = $1`, [id, s.iso(), ...cols.map((c) => patch[c])]);++ touched(s, id);return get(s, id);}/** Every active listing whose 30 days are up becomes expired — the sweep a timer runs in production. */async function expireListings(s) {const now = s.iso();-- return Number(await s.db.exec("UPDATE rent_listings SET state = 'expired', updated_at = $1 WHERE state = 'active' AND expires_at <= $1", [now]));++ const n = Number(await s.db.exec("UPDATE rent_listings SET state = 'expired', updated_at = $1 WHERE state = 'active' AND expires_at <= $1", [now]));++ if (n && s.search) s.search.afterExpiry();++ return n;}/** The owner may renew: 30 days from now (not a top-up that could run far past it), and an expired listing returns. */@@@ -94,16 +101,22 @@ async function renew(s, id) {const now = s.now();const expires = new Date(now + listings.EXPIRY_DAYS * DAY_MS).toISOString();await s.db.query("UPDATE rent_listings SET expires_at = $2, updated_at = $3, state = CASE WHEN state = 'expired' THEN 'active' ELSE state END WHERE id = $1", [id, expires, s.iso()]);++ touched(s, id);return get(s, id);}/** Staff action: restore (active), hide or remove. */async function setState(s, id, state) {await s.db.query('UPDATE rent_listings SET state = $2, updated_at = $3 WHERE id = $1', [id, state, s.iso()]);++ touched(s, id);return get(s, id);}--const remove = (s, id) => s.db.exec('DELETE FROM rent_listings WHERE id = $1', [id]);++async function remove(s, id) {++ const n = await s.db.exec('DELETE FROM rent_listings WHERE id = $1', [id]);++ touched(s, id);++ return n;++}const countActiveByOwner = (s, owner) => s.db.value("SELECT count(*) FROM rent_listings WHERE owner = $1 AND state = 'active'", [owner]).then(Number);@@@ -124,7 +137,7 @@ const reportsFor = (s, listingId) => s.db.many('SELECT * FROM rent_reports WHERE* member of staff has restored is not hidden again by the same reports.*/async function addReport(s, listingId, { id, reporter, reason, note, createdAt }) {-- return s.tx(async () => {++ const out = await s.tx(async () => {const row = await s.db.maybe('SELECT * FROM rent_listings WHERE id = $1', [listingId]);if (!row) return { error: 'listing.not_found' };if (row.state === 'removed') return { error: 'listing.removed' };@@@ -144,6 +157,8 @@ async function addReport(s, listingId, { id, reporter, reason, note, createdAt }[listingId, count, s.iso(), hide]);return { row: after, report, hidden: hide };});++ if (out && out.hidden) touched(s, listingId);++ return out;}/**nnew file mode 100644@@@ -0,0 +1,138 @@++'use strict';++++/**++ * Rent's listing pages in OpenVibe.Search (openvibe-publishing/search-feed): one search.index-document@1 per active++ * listing page (/listings/<id>), sent as rent.index_document.upserted|deleted through this service's events outbox. A++ * listing that is hidden (reports or staff), removed, expired or deleted becomes a tombstone, as its page stops being++ * indexable. The document carries what the public page shows: title, description, kind, price and period, city,++ * region, country, neighbourhood, bedrooms and availability. Never the contact link (signed-in viewers only) and never++ * the poster.++ *++ * Every store write names the listing it touched (store.js calls s.search.touch(id)); touched listings are synced a++ * moment later, each in its own transaction. A sweep after every expiry tick that expired something, once a minute++ * after boot and every ten minutes brings Search level with the table whatever happened in between: it re-sends only++ * what changed (rent_index_revisions is the sequencer) and tombstones listings whose row is gone (an account deletion).++ *++ * const search = createSearchIndex({ config, s, log }); // outbox relay off until the events URL and the OAuth++ * search.touch(id); search.sweep(); search.start(); search.stop(); // client secret are set; rows wait meanwhile++ */++const { createServiceOutbox } = require('openvibe-sdk/events');++const { createSearchFeed } = require('openvibe-publishing/search-feed');++const listings = require('./listings/listings');++++const EVENT_TYPES = ['rent.index_document.upserted', 'rent.index_document.deleted'];++const FLUSH_MS = 250;++const START_DELAY_MS = 60_000;++const INTERVAL_MS = 10 * 60_000;++++/** One listing row → what its public page shows, as search-feed's document description. */++function describe(row, nowMs) {++ const kind = listings.KIND_TEXT[row.kind] || row.kind;++ const place = [row.neighbourhood, row.city, row.region, row.country].filter(Boolean).join(', ');++ const price = listings.priceText(row);++ const facts = [kind, price, place, row.bedrooms == null ? '' : `${row.bedrooms} bedroom${Number(row.bedrooms) === 1 ? '' : 's'}`,++ row.available_from ? `available from ${row.available_from}` : ''].filter(Boolean);++ return {++ listed: row.state === 'active' && Date.parse(row.expires_at) > nowMs,++ title: row.title,++ summary: facts.join(' · '),++ body: [row.title, facts.join(' · '), row.description].filter(Boolean).join('\n'),++ facets: {++ kind: row.kind, country: row.country, city: row.city, currency: row.currency, period: row.period,++ bedrooms: row.bedrooms == null ? null : Number(row.bedrooms),++ },++ authorship: { mode: 'human' },++ publishedAt: row.created_at,++ updatedAt: row.updated_at,++ };++}++++function createSearchIndex({ config, s, log = console, outbox = null }) {++ const out = outbox || createServiceOutbox({++ db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,++ clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,++ });++ const feed = createSearchFeed({++ owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,++ types: {++ listing: {++ page: (row) => `/listings/${row.id}`,++ document: (row) => describe(row, s.now()),++ rows: (after, limit) => s.db.many('SELECT * FROM rent_listings WHERE id > $1 ORDER BY id LIMIT $2', [after, limit]),++ exists: async (ids) => (await s.db.many('SELECT id FROM rent_listings WHERE id = ANY($1)', [ids])).map((r) => r.id),++ },++ },++ });++ const pending = new Set();++ const timers = [];++ let flushTimer = null;++ let flushing = null;++++ /** Sync one listing now: its document, or a tombstone when the row is gone. */++ async function syncId(id) {++ return await s.db.tx(async (t) => {++ const row = await t.maybe('SELECT * FROM rent_listings WHERE id = $1', [id]);++ return row ? await feed.sync(t, 'listing', row) : await feed.remove(t, 'listing', id);++ });++ }++++ async function flush() {++ flushTimer = null;++ if (flushing) { await flushing; }++ const ids = [...pending];++ pending.clear();++ flushing = (async () => {++ for (const id of ids) {++ try { await syncId(id); } catch (err) { log.warn(`[Search] listing ${id}: ${(err && err.message) || err}`); }++ }++ if (ids.length && out.kick) out.kick().catch(() => {});++ })();++ try { await flushing; } finally { flushing = null; }++ }++++ /** A store write changed this listing: sync it shortly (never awaited by the request, never fatal). */++ function touch(id) {++ if (!id) return;++ pending.add(String(id));++ if (!flushTimer) {++ flushTimer = setTimeout(() => { flush().catch(() => {}); }, FLUSH_MS);++ if (flushTimer.unref) flushTimer.unref();++ }++ }++++ async function sweep() {++ const res = await feed.sweep();++ const l = res.listing;++ if (l.sent || l.removed || l.failed) log.log(`[Search] listings: ${l.sent} sent, ${l.removed} removed, ${l.failed} failed of ${l.seen}`);++ if (out.kick) out.kick().catch(() => {});++ return res;++ }++ const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };++++ function start() {++ if (timers.length) return false;++ out.start();++ const kick = setTimeout(quietly, START_DELAY_MS);++ if (kick.unref) kick.unref();++ const tick = setInterval(quietly, INTERVAL_MS);++ if (tick.unref) tick.unref();++ timers.push(kick, tick);++ return true;++ }++++ function stop() {++ for (const t of timers.splice(0)) { clearTimeout(t); clearInterval(t); }++ if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }++ out.stop();++ }++++ /** Wait for touched listings to be synced (tests, and a graceful stop). */++ async function settle() {++ if (flushTimer) { clearTimeout(flushTimer); await flush(); }++ if (flushing) await flushing;++ }++++ return { feed, outbox: out, describe, touch, syncId, sweep, afterExpiry: quietly, settle, start, stop, status: () => out.status() };++}++++module.exports = { createSearchIndex, describe, EVENT_TYPES };@@@ -69,6 +69,7 @@ async function boot(opts = {}) {base, network, config, ctx: built.ctx, get, signIn, dbDump,logs: () => captured.join('\n'),async close() {++ built.ctx.search.stop();await new Promise((r) => server.close(r));built.ctx.keys.client.stop();await testdb.close();nnew file mode 100644@@@ -0,0 +1,110 @@++'use strict';++/**++ * Listing pages in OpenVibe.Search (server/search-index.js): a posted listing is one rent.index_document.upserted in++ * the outbox, exactly the contract, with what the public page shows and never the contact link or the poster; an edit++ * is the next revision; hiding, removing, expiring or deleting it is a tombstone (once); renewing an expired listing++ * brings it back; a sweep re-sends nothing that did not change.++ */++const assert = require('assert');++const contracts = require('openvibe-contracts');++const { boot, check, done } = require('./helpers/boot');++const { post } = require('./helpers/rent');++const store = require('../server/listings/store');++++const DAY = 86_400_000;++++(async () => {++ const clock = { t: Date.now() }; // real time: the test's sign-in tokens are issued now++ const t = await boot({ now: () => clock.t });++ const { s, search } = t.ctx;++ const kim = t.network.addUser('kim');++ const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);++ const valid = (env) => {++ const e = contracts.validate('events.event-envelope@1', env);++ assert.ok(e.valid, `envelope: ${JSON.stringify(e.errors)}`);++ const p = contracts.validate(env.event_type, env.payload);++ assert.ok(p.valid, `${env.event_type}: ${JSON.stringify(p.errors)}`);++ if (env.event_type.endsWith('.upserted')) assert.ok(contracts.validate('search.index-document@1', env.payload).valid);++ assert.strictEqual(env.source, 'rent');++ assert.strictEqual(env.visibility, 'internal');++ return env;++ };++ const last = async () => valid((await outbox()).pop());++ let id;++++ try {++ await check('a posted listing is one document with what its public page shows, and never the contact link', async () => {++ const made = await post(t, kim, { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', city: 'Porto', country: 'PT', neighbourhood: 'Ribeira', bedrooms: 1 });++ assert.strictEqual(made.status, 201, made.text);++ id = made.json().id;++ await search.settle();++ const envs = await outbox();++ assert.strictEqual(envs.length, 1);++ const doc = valid(envs[0]).payload;++ assert.strictEqual(envs[0].event_type, 'rent.index_document.upserted');++ assert.strictEqual(doc.id, id);++ assert.strictEqual(doc.canonical_url, `https://openvibe.rent/listings/${id}`);++ assert.strictEqual(doc.title, 'Bright room near the river');++ assert.match(doc.summary, /^Room · 650 EUR a month · Ribeira, Porto, PT · 1 bedroom/);++ assert.deepStrictEqual(doc.facets, { kind: 'room', country: 'PT', city: 'Porto', currency: 'EUR', period: 'month', bedrooms: 1 });++ assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });++ const text = JSON.stringify(envs[0]);++ assert.ok(!text.includes('example.com/rooms'), 'the contact link stays off Search');++ assert.ok(!text.includes('usr_'), 'the poster is not in the document');++ });++++ await check('an edit is the next revision; a sweep then sends nothing', async () => {++ await store.update(s, id, { title: 'Bright room near the river, bills included' });++ await search.settle();++ const env = await last();++ assert.strictEqual(env.payload.revision, 2);++ assert.match(env.payload.title, /bills included/);++ const before = (await outbox()).length;++ const res = await search.sweep();++ assert.strictEqual(res.listing.sent, 0);++ assert.strictEqual((await outbox()).length, before);++ });++++ await check('hidden by staff is a tombstone, restored is a document again', async () => {++ await store.setState(s, id, 'hidden');++ await search.settle();++ let env = await last();++ assert.strictEqual(env.event_type, 'rent.index_document.deleted');++ assert.deepStrictEqual(env.payload, { type: 'listing', id, revision: 3 });++ await store.setState(s, id, 'active');++ await search.settle();++ env = await last();++ assert.strictEqual(env.event_type, 'rent.index_document.upserted');++ assert.strictEqual(env.payload.revision, 4);++ });++++ await check('an expired listing leaves Search through the expiry sweep, and renewing brings it back', async () => {++ clock.t += 31 * DAY;++ assert.strictEqual(await store.expireListings(s), 1);++ for (let i = 0; i < 100 && (await last()).event_type !== 'rent.index_document.deleted'; i += 1) await new Promise((r) => setTimeout(r, 50));++ assert.strictEqual((await last()).event_type, 'rent.index_document.deleted', 'the expiry tick swept');++ await store.renew(s, id);++ await search.settle();++ assert.strictEqual((await last()).event_type, 'rent.index_document.upserted');++ });++++ await check('a deleted listing is a tombstone, sent once', async () => {++ await store.remove(s, id);++ await search.settle();++ const env = await last();++ assert.strictEqual(env.event_type, 'rent.index_document.deleted');++ assert.strictEqual(env.payload.id, id);++ const before = (await outbox()).length;++ assert.strictEqual((await search.sweep()).listing.removed, 0, 'already a tombstone');++ assert.strictEqual((await outbox()).length, before);++ });++++ await check('the relay stays off without the events URL and the client secret; rows wait', async () => {++ const st = await search.status();++ assert.strictEqual(st.enabled, false);++ assert.ok(st.pending > 0);++ });++ } finally {++ await done(t);++ }++})();
@@ -247,7 +247,8 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md).
@@ -247,7 +247,8 @@ Reporting a vulnerability: [SECURITY.md](SECURITY.md).
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
<!-- versions:start -->
-- openvibe-contracts: v0.127.0
+- openvibe-contracts: v0.128.0
+- openvibe-publishing: v1.4.0
- openvibe-sdk: v0.36.0
- openvibe-sdk: v0.36.0
- openvibe-shared: v2.20.4
- openvibe-shared: v2.20.4
<!-- versions:end -->
<!-- versions:end -->
@@ -7,13 +7,14 @@
@@ -7,13 +7,14 @@
"runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9",
"runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9",
"port": "5010",
"port": "5010",
"domain": "openvibe.rent",
"domain": "openvibe.rent",
- "contracts": "openvibe-contracts v0.127.0",
+ "contracts": "openvibe-contracts v0.128.0",
"sdk": "openvibe-sdk v0.36.0",
"sdk": "openvibe-sdk v0.36.0",
"shared": "openvibe-shared v2.20.4",
"shared": "openvibe-shared v2.20.4",
"packages": [
"packages": [
- "openvibe-contracts v0.127.0",
+ "openvibe-contracts v0.128.0",
"openvibe-sdk v0.36.0",
"openvibe-sdk v0.36.0",
- "openvibe-shared v2.20.4"
+ "openvibe-shared v2.20.4",
+ "openvibe-publishing v1.4.0"
],
],
"summary": "OpenVibe.Rent — Find a place, or rent yours out. One place to find and offer things to rent: apartments, rooms, houses, commercial space, parking and storage, and equipment. In this first version every listing is posted by a person on this site: no scraped listings and no partner feeds, no payments, no deposits and no messaging. Search with filters and facets, a listing page with a plain safety note, reports that hide a listing at 3 until staff look, saved searches that count what is new, and a 30-day expiry an owner can renew.",
"summary": "OpenVibe.Rent — Find a place, or rent yours out. One place to find and offer things to rent: apartments, rooms, houses, commercial space, parking and storage, and equipment. In this first version every listing is posted by a person on this site: no scraped listings and no partner feeds, no payments, no deposits and no messaging. Search with filters and facets, a listing page with a plain safety note, reports that hide a listing at 3 until staff look, saved searches that count what is new, and a 30-day expiry an owner can renew.",
"works": [
"works": [
new file mode 100644
new file mode 100644
@@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
+-- phase: expand
+-- OpenVibe.Rent: listing pages in OpenVibe.Search (server/search-index.js, openvibe-publishing/search-feed).
+-- event_outbox holds the rent.index_document.* events until the relay delivers them to OpenVibe.Events (written in
+-- the same transaction as the revision that produced them); rent_index_revisions is the index sequencer, the last
+-- revision and content hash Search was sent for each page, so an unchanged page is never sent twice.
+
+CREATE TABLE IF NOT EXISTS event_outbox (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ event_id text NOT NULL UNIQUE,
+ envelope jsonb NOT NULL,
+ traceparent text,
+ created_at bigint NOT NULL,
+ attempts integer NOT NULL DEFAULT 0,
+ next_attempt_at bigint NOT NULL DEFAULT 0,
+ sent_at bigint,
+ seq bigint,
+ rejected_at bigint,
+ last_error text
+);
+CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;
+CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;
+
+CREATE TABLE IF NOT EXISTS rent_index_revisions (
+ owner text COLLATE "C" NOT NULL,
+ type text COLLATE "C" NOT NULL,
+ id text COLLATE "C" NOT NULL,
+ revision integer NOT NULL,
+ hash text NOT NULL,
+ updated_at bigint NOT NULL,
+ PRIMARY KEY (owner, type, id)
+);
@@ -15,7 +15,8 @@
@@ -15,7 +15,8 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -871,9 +872,9 @@
@@ -871,9 +872,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.127.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
- "integrity": "sha512-ySC+Qh8I3OlN5MULCqv0vfcAGubZQzWM0Fw7EOpnOo76vVnrkwx+m33aTxDum2yQqPkU8Ot5UtMuMwm1Cc1k3g==",
+ "version": "0.128.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
+ "integrity": "sha512-r/WKk3NhdqxEsVxqsVPjg+HTD2So1amRx0BY6EOHhvmwjLuL9j5XJrNDZAfqf/4/P2xZCBzTLrtUqlWxC2EA6Q==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -886,6 +887,24 @@
@@ -886,6 +887,24 @@
"node": ">=20"
"node": ">=20"
}
}
},
},
+ "node_modules/openvibe-publishing": {
+ "version": "1.4.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
+ "integrity": "sha512-9vByyVqpvy2zVQI4PrJWk5QqCym/YQ/fXZDgrcloKGBqvukKrUn7fg4ZERiJHcgEq+rgDbxbU4rNflZhw62K/Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20"
+ },
+ "peerDependencies": {
+ "openvibe-sdk": ">=0.15.0",
+ "openvibe-shared": ">=1.5.0"
+ },
+ "peerDependenciesMeta": {
+ "openvibe-shared": {
+ "optional": true
+ }
+ }
+ },
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
"version": "0.36.0",
"version": "0.36.0",
"resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
"resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
@@ -2,7 +2,7 @@
@@ -2,7 +2,7 @@
"name": "openvibe-rent",
"name": "openvibe-rent",
"version": "0.1.0",
"version": "0.1.0",
"private": true,
"private": true,
- "description": "OpenVibe.Rent — Find a place, or rent yours out.",
+ "description": "OpenVibe.Rent \u2014 Find a place, or rent yours out.",
"main": "server/index.js",
"main": "server/index.js",
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
@@ -22,7 +22,8 @@
@@ -22,7 +22,8 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.127.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.128.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.36.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.20.4",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -21,6 +21,7 @@ const rateLimit = require('express-rate-limit');
@@ -21,6 +21,7 @@ const rateLimit = require('express-rate-limit');
const contracts = require('openvibe-contracts');
const contracts = require('openvibe-contracts');
const cache = require('openvibe-shared/cache-policy');
const cache = require('openvibe-shared/cache-policy');
+const { createSearchIndex } = require('./search-index');
const configLib = require('./config');
const configLib = require('./config');
const { openStore } = require('./db');
const { openStore } = require('./db');
const { createKeyStore } = require('./auth/keys');
const { createKeyStore } = require('./auth/keys');
@@ -52,6 +53,10 @@ async function createApp(opts = {}) {
@@ -52,6 +53,10 @@ async function createApp(opts = {}) {
const sso = createSso({ config, keys, fetchImpl, now: s.now, log });
const sso = createSso({ config, keys, fetchImpl, now: s.now, log });
const principal = createPrincipal({ config, keys });
const principal = createPrincipal({ config, keys });
const ctx = { config, s, keys, sso, principal, log };
const ctx = { config, s, keys, sso, principal, log };
+ // Listing pages in OpenVibe.Search (./search-index.js): every store write names the listing it touched (s.search),
+ // and a sweep runs on its own timer once server/index.js starts it.
+ ctx.search = opts.search || createSearchIndex({ config, s, log });
+ s.search = ctx.search;
// Account export and deletion (ADR-033, ./identity/account-data.js): the three tables that hold a person's rows.
// Account export and deletion (ADR-033, ./identity/account-data.js): the three tables that hold a person's rows.
// The sender posts to Network's internal export/deletion routes with this service's own client-credentials token;
// The sender posts to Network's internal export/deletion routes with this service's own client-credentials token;
@@ -19,7 +19,7 @@ const { startExpiryTimer } = require('./listings/expiry');
@@ -19,7 +19,7 @@ const { startExpiryTimer } = require('./listings/expiry');
function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }) {
function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }) {
return gracefulStop({
return gracefulStop({
name: 'OpenVibe.Rent', server, deadlineExitCode: 0, exit, signals, deadlineMs: 10_000,
name: 'OpenVibe.Rent', server, deadlineExitCode: 0, exit, signals, deadlineMs: 10_000,
- close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.s.close(), ...extra],
+ close: [() => { for (const t of timers) clearInterval(t); }, () => ctx.keys.client.stop(), () => ctx.search.stop(), () => ctx.s.close(), ...extra],
});
});
}
}
@@ -32,6 +32,7 @@ async function start() {
@@ -32,6 +32,7 @@ async function start() {
});
});
server.keepAliveTimeout = 65_000;
server.keepAliveTimeout = 65_000;
ctx.keys.client.start();
ctx.keys.client.start();
+ ctx.search.start();
// One sweep at boot (a restart must not leave a stale listing up a moment longer than it should), then a timer.
// One sweep at boot (a restart must not leave a stale listing up a moment longer than it should), then a timer.
const expiry = startExpiryTimer(ctx.s);
const expiry = startExpiryTimer(ctx.s);
@@ -15,6 +15,9 @@ const listings = require('./listings');
@@ -15,6 +15,9 @@ const listings = require('./listings');
const DAY_MS = 86_400_000;
const DAY_MS = 86_400_000;
+/** A write changed this listing: OpenVibe.Search syncs it shortly (server/search-index.js; absent in some tests). */
+const touched = (s, id) => { if (s.search) s.search.touch(id); };
+
/** The WHERE clause for a filter set. `exclude` leaves one dimension out — how the kind and country facets count. */
/** The WHERE clause for a filter set. `exclude` leaves one dimension out — how the kind and country facets count. */
function buildWhere(f, { exclude = [], sinceISO = null } = {}) {
function buildWhere(f, { exclude = [], sinceISO = null } = {}) {
const parts = ["state = 'active'"];
const parts = ["state = 'active'"];
@@ -42,6 +45,7 @@ async function insert(s, row) {
@@ -42,6 +45,7 @@ async function insert(s, row) {
[row.id, row.owner, row.kind, row.title, row.description, row.price, row.currency, row.period, row.city,
[row.id, row.owner, row.kind, row.title, row.description, row.price, row.currency, row.period, row.city,
row.region, row.country, row.neighbourhood, row.bedrooms, row.available_from, row.contact_url,
row.region, row.country, row.neighbourhood, row.bedrooms, row.available_from, row.contact_url,
row.created_at, row.expires_at]);
row.created_at, row.expires_at]);
+ touched(s, row.id);
return get(s, row.id);
return get(s, row.id);
}
}
@@ -80,13 +84,16 @@ async function update(s, id, patch) {
@@ -80,13 +84,16 @@ async function update(s, id, patch) {
const cols = Object.keys(patch);
const cols = Object.keys(patch);
const sets = cols.map((c, i) => `${c} = $${i + 3}`);
const sets = cols.map((c, i) => `${c} = $${i + 3}`);
await s.db.query(`UPDATE rent_listings SET ${sets.join(', ')}, updated_at = $2 WHERE id = $1`, [id, s.iso(), ...cols.map((c) => patch[c])]);
await s.db.query(`UPDATE rent_listings SET ${sets.join(', ')}, updated_at = $2 WHERE id = $1`, [id, s.iso(), ...cols.map((c) => patch[c])]);
+ touched(s, id);
return get(s, id);
return get(s, id);
}
}
/** Every active listing whose 30 days are up becomes expired — the sweep a timer runs in production. */
/** Every active listing whose 30 days are up becomes expired — the sweep a timer runs in production. */
async function expireListings(s) {
async function expireListings(s) {
const now = s.iso();
const now = s.iso();
- return Number(await s.db.exec("UPDATE rent_listings SET state = 'expired', updated_at = $1 WHERE state = 'active' AND expires_at <= $1", [now]));
+ const n = Number(await s.db.exec("UPDATE rent_listings SET state = 'expired', updated_at = $1 WHERE state = 'active' AND expires_at <= $1", [now]));
+ if (n && s.search) s.search.afterExpiry();
+ return n;
}
}
/** The owner may renew: 30 days from now (not a top-up that could run far past it), and an expired listing returns. */
/** The owner may renew: 30 days from now (not a top-up that could run far past it), and an expired listing returns. */
@@ -94,16 +101,22 @@ async function renew(s, id) {
@@ -94,16 +101,22 @@ async function renew(s, id) {
const now = s.now();
const now = s.now();
const expires = new Date(now + listings.EXPIRY_DAYS * DAY_MS).toISOString();
const expires = new Date(now + listings.EXPIRY_DAYS * DAY_MS).toISOString();
await s.db.query("UPDATE rent_listings SET expires_at = $2, updated_at = $3, state = CASE WHEN state = 'expired' THEN 'active' ELSE state END WHERE id = $1", [id, expires, s.iso()]);
await s.db.query("UPDATE rent_listings SET expires_at = $2, updated_at = $3, state = CASE WHEN state = 'expired' THEN 'active' ELSE state END WHERE id = $1", [id, expires, s.iso()]);
+ touched(s, id);
return get(s, id);
return get(s, id);
}
}
/** Staff action: restore (active), hide or remove. */
/** Staff action: restore (active), hide or remove. */
async function setState(s, id, state) {
async function setState(s, id, state) {
await s.db.query('UPDATE rent_listings SET state = $2, updated_at = $3 WHERE id = $1', [id, state, s.iso()]);
await s.db.query('UPDATE rent_listings SET state = $2, updated_at = $3 WHERE id = $1', [id, state, s.iso()]);
+ touched(s, id);
return get(s, id);
return get(s, id);
}
}
-const remove = (s, id) => s.db.exec('DELETE FROM rent_listings WHERE id = $1', [id]);
+async function remove(s, id) {
+ const n = await s.db.exec('DELETE FROM rent_listings WHERE id = $1', [id]);
+ touched(s, id);
+ return n;
+}
const countActiveByOwner = (s, owner) => s.db.value("SELECT count(*) FROM rent_listings WHERE owner = $1 AND state = 'active'", [owner]).then(Number);
const countActiveByOwner = (s, owner) => s.db.value("SELECT count(*) FROM rent_listings WHERE owner = $1 AND state = 'active'", [owner]).then(Number);
@@ -124,7 +137,7 @@ const reportsFor = (s, listingId) => s.db.many('SELECT * FROM rent_reports WHERE
@@ -124,7 +137,7 @@ const reportsFor = (s, listingId) => s.db.many('SELECT * FROM rent_reports WHERE
* member of staff has restored is not hidden again by the same reports.
* member of staff has restored is not hidden again by the same reports.
*/
*/
async function addReport(s, listingId, { id, reporter, reason, note, createdAt }) {
async function addReport(s, listingId, { id, reporter, reason, note, createdAt }) {
- return s.tx(async () => {
+ const out = await s.tx(async () => {
const row = await s.db.maybe('SELECT * FROM rent_listings WHERE id = $1', [listingId]);
const row = await s.db.maybe('SELECT * FROM rent_listings WHERE id = $1', [listingId]);
if (!row) return { error: 'listing.not_found' };
if (!row) return { error: 'listing.not_found' };
if (row.state === 'removed') return { error: 'listing.removed' };
if (row.state === 'removed') return { error: 'listing.removed' };
@@ -144,6 +157,8 @@ async function addReport(s, listingId, { id, reporter, reason, note, createdAt }
@@ -144,6 +157,8 @@ async function addReport(s, listingId, { id, reporter, reason, note, createdAt }
[listingId, count, s.iso(), hide]);
[listingId, count, s.iso(), hide]);
return { row: after, report, hidden: hide };
return { row: after, report, hidden: hide };
});
});
+ if (out && out.hidden) touched(s, listingId);
+ return out;
}
}
/**
/**
new file mode 100644
new file mode 100644
@@ -0,0 +1,138 @@
@@ -0,0 +1,138 @@
+'use strict';
+
+/**
+ * Rent's listing pages in OpenVibe.Search (openvibe-publishing/search-feed): one search.index-document@1 per active
+ * listing page (/listings/<id>), sent as rent.index_document.upserted|deleted through this service's events outbox. A
+ * listing that is hidden (reports or staff), removed, expired or deleted becomes a tombstone, as its page stops being
+ * indexable. The document carries what the public page shows: title, description, kind, price and period, city,
+ * region, country, neighbourhood, bedrooms and availability. Never the contact link (signed-in viewers only) and never
+ * the poster.
+ *
+ * Every store write names the listing it touched (store.js calls s.search.touch(id)); touched listings are synced a
+ * moment later, each in its own transaction. A sweep after every expiry tick that expired something, once a minute
+ * after boot and every ten minutes brings Search level with the table whatever happened in between: it re-sends only
+ * what changed (rent_index_revisions is the sequencer) and tombstones listings whose row is gone (an account deletion).
+ *
+ * const search = createSearchIndex({ config, s, log }); // outbox relay off until the events URL and the OAuth
+ * search.touch(id); search.sweep(); search.start(); search.stop(); // client secret are set; rows wait meanwhile
+ */
+const { createServiceOutbox } = require('openvibe-sdk/events');
+const { createSearchFeed } = require('openvibe-publishing/search-feed');
+const listings = require('./listings/listings');
+
+const EVENT_TYPES = ['rent.index_document.upserted', 'rent.index_document.deleted'];
+const FLUSH_MS = 250;
+const START_DELAY_MS = 60_000;
+const INTERVAL_MS = 10 * 60_000;
+
+/** One listing row → what its public page shows, as search-feed's document description. */
+function describe(row, nowMs) {
+ const kind = listings.KIND_TEXT[row.kind] || row.kind;
+ const place = [row.neighbourhood, row.city, row.region, row.country].filter(Boolean).join(', ');
+ const price = listings.priceText(row);
+ const facts = [kind, price, place, row.bedrooms == null ? '' : `${row.bedrooms} bedroom${Number(row.bedrooms) === 1 ? '' : 's'}`,
+ row.available_from ? `available from ${row.available_from}` : ''].filter(Boolean);
+ return {
+ listed: row.state === 'active' && Date.parse(row.expires_at) > nowMs,
+ title: row.title,
+ summary: facts.join(' · '),
+ body: [row.title, facts.join(' · '), row.description].filter(Boolean).join('\n'),
+ facets: {
+ kind: row.kind, country: row.country, city: row.city, currency: row.currency, period: row.period,
+ bedrooms: row.bedrooms == null ? null : Number(row.bedrooms),
+ },
+ authorship: { mode: 'human' },
+ publishedAt: row.created_at,
+ updatedAt: row.updated_at,
+ };
+}
+
+function createSearchIndex({ config, s, log = console, outbox = null }) {
+ const out = outbox || createServiceOutbox({
+ db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
+ clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,
+ });
+ const feed = createSearchFeed({
+ owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,
+ types: {
+ listing: {
+ page: (row) => `/listings/${row.id}`,
+ document: (row) => describe(row, s.now()),
+ rows: (after, limit) => s.db.many('SELECT * FROM rent_listings WHERE id > $1 ORDER BY id LIMIT $2', [after, limit]),
+ exists: async (ids) => (await s.db.many('SELECT id FROM rent_listings WHERE id = ANY($1)', [ids])).map((r) => r.id),
+ },
+ },
+ });
+ const pending = new Set();
+ const timers = [];
+ let flushTimer = null;
+ let flushing = null;
+
+ /** Sync one listing now: its document, or a tombstone when the row is gone. */
+ async function syncId(id) {
+ return await s.db.tx(async (t) => {
+ const row = await t.maybe('SELECT * FROM rent_listings WHERE id = $1', [id]);
+ return row ? await feed.sync(t, 'listing', row) : await feed.remove(t, 'listing', id);
+ });
+ }
+
+ async function flush() {
+ flushTimer = null;
+ if (flushing) { await flushing; }
+ const ids = [...pending];
+ pending.clear();
+ flushing = (async () => {
+ for (const id of ids) {
+ try { await syncId(id); } catch (err) { log.warn(`[Search] listing ${id}: ${(err && err.message) || err}`); }
+ }
+ if (ids.length && out.kick) out.kick().catch(() => {});
+ })();
+ try { await flushing; } finally { flushing = null; }
+ }
+
+ /** A store write changed this listing: sync it shortly (never awaited by the request, never fatal). */
+ function touch(id) {
+ if (!id) return;
+ pending.add(String(id));
+ if (!flushTimer) {
+ flushTimer = setTimeout(() => { flush().catch(() => {}); }, FLUSH_MS);
+ if (flushTimer.unref) flushTimer.unref();
+ }
+ }
+
+ async function sweep() {
+ const res = await feed.sweep();
+ const l = res.listing;
+ if (l.sent || l.removed || l.failed) log.log(`[Search] listings: ${l.sent} sent, ${l.removed} removed, ${l.failed} failed of ${l.seen}`);
+ if (out.kick) out.kick().catch(() => {});
+ return res;
+ }
+ const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
+
+ function start() {
+ if (timers.length) return false;
+ out.start();
+ const kick = setTimeout(quietly, START_DELAY_MS);
+ if (kick.unref) kick.unref();
+ const tick = setInterval(quietly, INTERVAL_MS);
+ if (tick.unref) tick.unref();
+ timers.push(kick, tick);
+ return true;
+ }
+
+ function stop() {
+ for (const t of timers.splice(0)) { clearTimeout(t); clearInterval(t); }
+ if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
+ out.stop();
+ }
+
+ /** Wait for touched listings to be synced (tests, and a graceful stop). */
+ async function settle() {
+ if (flushTimer) { clearTimeout(flushTimer); await flush(); }
+ if (flushing) await flushing;
+ }
+
+ return { feed, outbox: out, describe, touch, syncId, sweep, afterExpiry: quietly, settle, start, stop, status: () => out.status() };
+}
+
+module.exports = { createSearchIndex, describe, EVENT_TYPES };
@@ -69,6 +69,7 @@ async function boot(opts = {}) {
@@ -69,6 +69,7 @@ async function boot(opts = {}) {
base, network, config, ctx: built.ctx, get, signIn, dbDump,
base, network, config, ctx: built.ctx, get, signIn, dbDump,
logs: () => captured.join('\n'),
logs: () => captured.join('\n'),
async close() {
async close() {
+ built.ctx.search.stop();
await new Promise((r) => server.close(r));
await new Promise((r) => server.close(r));
built.ctx.keys.client.stop();
built.ctx.keys.client.stop();
await testdb.close();
await testdb.close();
new file mode 100644
new file mode 100644
@@ -0,0 +1,110 @@
@@ -0,0 +1,110 @@
+'use strict';
+/**
+ * Listing pages in OpenVibe.Search (server/search-index.js): a posted listing is one rent.index_document.upserted in
+ * the outbox, exactly the contract, with what the public page shows and never the contact link or the poster; an edit
+ * is the next revision; hiding, removing, expiring or deleting it is a tombstone (once); renewing an expired listing
+ * brings it back; a sweep re-sends nothing that did not change.
+ */
+const assert = require('assert');
+const contracts = require('openvibe-contracts');
+const { boot, check, done } = require('./helpers/boot');
+const { post } = require('./helpers/rent');
+const store = require('../server/listings/store');
+
+const DAY = 86_400_000;
+
+(async () => {
+ const clock = { t: Date.now() }; // real time: the test's sign-in tokens are issued now
+ const t = await boot({ now: () => clock.t });
+ const { s, search } = t.ctx;
+ const kim = t.network.addUser('kim');
+ const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);
+ const valid = (env) => {
+ const e = contracts.validate('events.event-envelope@1', env);
+ assert.ok(e.valid, `envelope: ${JSON.stringify(e.errors)}`);
+ const p = contracts.validate(env.event_type, env.payload);
+ assert.ok(p.valid, `${env.event_type}: ${JSON.stringify(p.errors)}`);
+ if (env.event_type.endsWith('.upserted')) assert.ok(contracts.validate('search.index-document@1', env.payload).valid);
+ assert.strictEqual(env.source, 'rent');
+ assert.strictEqual(env.visibility, 'internal');
+ return env;
+ };
+ const last = async () => valid((await outbox()).pop());
+ let id;
+
+ try {
+ await check('a posted listing is one document with what its public page shows, and never the contact link', async () => {
+ const made = await post(t, kim, { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', city: 'Porto', country: 'PT', neighbourhood: 'Ribeira', bedrooms: 1 });
+ assert.strictEqual(made.status, 201, made.text);
+ id = made.json().id;
+ await search.settle();
+ const envs = await outbox();
+ assert.strictEqual(envs.length, 1);
+ const doc = valid(envs[0]).payload;
+ assert.strictEqual(envs[0].event_type, 'rent.index_document.upserted');
+ assert.strictEqual(doc.id, id);
+ assert.strictEqual(doc.canonical_url, `https://openvibe.rent/listings/${id}`);
+ assert.strictEqual(doc.title, 'Bright room near the river');
+ assert.match(doc.summary, /^Room · 650 EUR a month · Ribeira, Porto, PT · 1 bedroom/);
+ assert.deepStrictEqual(doc.facets, { kind: 'room', country: 'PT', city: 'Porto', currency: 'EUR', period: 'month', bedrooms: 1 });
+ assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });
+ const text = JSON.stringify(envs[0]);
+ assert.ok(!text.includes('example.com/rooms'), 'the contact link stays off Search');
+ assert.ok(!text.includes('usr_'), 'the poster is not in the document');
+ });
+
+ await check('an edit is the next revision; a sweep then sends nothing', async () => {
+ await store.update(s, id, { title: 'Bright room near the river, bills included' });
+ await search.settle();
+ const env = await last();
+ assert.strictEqual(env.payload.revision, 2);
+ assert.match(env.payload.title, /bills included/);
+ const before = (await outbox()).length;
+ const res = await search.sweep();
+ assert.strictEqual(res.listing.sent, 0);
+ assert.strictEqual((await outbox()).length, before);
+ });
+
+ await check('hidden by staff is a tombstone, restored is a document again', async () => {
+ await store.setState(s, id, 'hidden');
+ await search.settle();
+ let env = await last();
+ assert.strictEqual(env.event_type, 'rent.index_document.deleted');
+ assert.deepStrictEqual(env.payload, { type: 'listing', id, revision: 3 });
+ await store.setState(s, id, 'active');
+ await search.settle();
+ env = await last();
+ assert.strictEqual(env.event_type, 'rent.index_document.upserted');
+ assert.strictEqual(env.payload.revision, 4);
+ });
+
+ await check('an expired listing leaves Search through the expiry sweep, and renewing brings it back', async () => {
+ clock.t += 31 * DAY;
+ assert.strictEqual(await store.expireListings(s), 1);
+ for (let i = 0; i < 100 && (await last()).event_type !== 'rent.index_document.deleted'; i += 1) await new Promise((r) => setTimeout(r, 50));
+ assert.strictEqual((await last()).event_type, 'rent.index_document.deleted', 'the expiry tick swept');
+ await store.renew(s, id);
+ await search.settle();
+ assert.strictEqual((await last()).event_type, 'rent.index_document.upserted');
+ });
+
+ await check('a deleted listing is a tombstone, sent once', async () => {
+ await store.remove(s, id);
+ await search.settle();
+ const env = await last();
+ assert.strictEqual(env.event_type, 'rent.index_document.deleted');
+ assert.strictEqual(env.payload.id, id);
+ const before = (await outbox()).length;
+ assert.strictEqual((await search.sweep()).listing.removed, 0, 'already a tombstone');
+ assert.strictEqual((await outbox()).length, before);
+ });
+
+ await check('the relay stays off without the events URL and the client secret; rows wait', async () => {
+ const st = await search.status();
+ assert.strictEqual(st.enabled, false);
+ assert.ok(st.pending > 0);
+ });
+ } finally {
+ await done(t);
+ }
+})();