openvibers/openvibe.rent Commits

Rent: tell search engines about new and changed listings (IndexNow) (#46)

OP OpenVibers committed 29 days ago · f2db5e7bf9 · 8 files, +48 −6

`openvibe-publishing/search-feed` already pings IndexNow when it is given a client: whenever a page Search gains or changes is indexable, it sends the page and `/sitemap.xml`. Noindex pages and tombstones send nothing. This service never passed a client, so it sent nothing.

- With `INDEXNOW_KEY`, the app builds one `openvibe-shared/indexnow` client, hands it to the search feed and serves `/<key>.txt`.
- A malformed key warns and leaves IndexNow off instead of stopping the site. Unset: off.
- Tests: the search-index test checks the pinged URLs (new pages announced, a quiet sweep and a noindex page announce nothing). The discovery test checks the key file with a key and the 404 without one.
- The README and `.env.example` document the key.

Changes

ddiff --git a/.env.example b/.env.example
iindex ff40a46..632d4b5 100644
---- a/.env.example
++++ b/.env.example
@@@ -4,6 +4,9 @@ NODE_ENV=production
PORT=5010
HOST=127.0.0.1
BASE_URL=https://openvibe.rent
++# IndexNow: with a key (8–128 letters and digits), pages Search gains or changes are sent to search engines and the
++# key is served at /<key>.txt. Unset: off.
++#INDEXNOW_KEY=
TRUST_PROXY=2
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh rent: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
ddiff --git a/README.md b/README.md
iindex 8536a3c..0ac9235 100644
---- a/README.md
++++ b/README.md
@@@ -117,7 +117,7 @@ Index master CSV — paced and cached in [server/housing/upstream.js](server/hou
| Affordable housing | [server/housing/places.js](server/housing/places.js), [upstream.js](server/housing/upstream.js) | `/housing` geocodes a place with Nominatim, then reads six HUD ArcGIS layers — public housing, LIHTC, HUD-assisted multifamily, public housing authorities, housing counselling agencies and HUD homes for sale (`SF_REO`, HUD's March 2026 snapshot, each linked to HUD Homestore to confirm it is still for sale) — normalised, distance-sorted and grouped, with the HUD attribution on the page. |
| Home prices | [server/housing/hpi.js](server/housing/hpi.js) | A small card on `/housing` shows the FHFA House Price Index change over the past year and five years for the matching MSA (or the state), as a repeat-sales index — never a price — with the source linked back. |
| Pages | [server/http/pages.js](server/http/pages.js) | Home, `/housing`, `/listings`, `/listings/:id`, `/listings/:id/edit`, `/post`, `/mine`, `/saved`, `/safety`, `/staff`, `/updates` — all server-rendered, all complete without JavaScript, forms are plain POSTs. |
--| Discovery | [server/http/discovery.js](server/http/discovery.js) | `robots.txt` (disallows `/auth/`, `/api/`, `/post`, `/mine`, `/saved`, `/staff`), `sitemap.xml` (every active `/listings/:id`, up to 1000), `llms.txt`, `llms-full.txt`. |
++| Discovery | [server/http/discovery.js](server/http/discovery.js) | `robots.txt` (disallows `/auth/`, `/api/`, `/post`, `/mine`, `/saved`, `/staff`), `sitemap.xml` (every active `/listings/:id`, up to 1000), `llms.txt`, `llms-full.txt`; with `INDEXNOW_KEY`, each listing page Search gains or changes is sent to IndexNow (through `openvibe-publishing/search-feed`). |
| Sign-in | [server/auth/sso.js](server/auth/sso.js) | OAuth 2 authorization code with PKCE (S256) against OpenVibe.Network; httpOnly `rent_at` / `rent_rt` cookies; `/auth/me` for the shared navbar. |
| Limits | [server/http/caller-limits.js](server/http/caller-limits.js) | Per-caller budgets for posting, reporting, renewing and saving a search. The page form and the API route take the same budget, so the form is not a way round a limit. |
| Store | [server/db.js](server/db.js), [migrations/](migrations/) | PostgreSQL through `openvibe-sdk/db`, `NNNN_*.sql` applied at boot, PGlite in development and in tests. |
ddiff --git a/server/app.js b/server/app.js
iindex 32406c8..b450b27 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -22,6 +22,7 @@ const contracts = require('openvibe-contracts');
const cache = require('openvibe-shared/cache-policy');
const { createSearchIndex } = require('./search-index');
++const { createIndexNow } = require('openvibe-shared/indexnow');
const configLib = require('./config');
const { openStore } = require('./db');
const { createUpstream } = require('./housing/upstream');
@@@ -57,7 +58,16 @@ async function createApp(opts = {}) {
const ctx = { config, s, keys, sso, principal, log };
// Listing pages in OpenVibe.Search (./search-index.js): every store write names the listing it touched (s.search),
// and a sweep runs on its own timer once server/index.js starts it.
-- ctx.search = opts.search || createSearchIndex({ config, s, log });
++ // IndexNow: off without INDEXNOW_KEY; a malformed key warns and stays off rather than stopping the site.
++ const indexnow = opts.indexnow || (() => {
++ const warn = (...a) => (log.warn || console.warn)(...a);
++ try { return createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: warn }); } catch (err) {
++ warn(`[Rent] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);
++ return createIndexNow({ host: config.baseUrl, key: '', fetch: fetchImpl, log: warn });
++ }
++ })();
++ ctx.indexnow = indexnow;
++ ctx.search = opts.search || createSearchIndex({ config, s, log, indexnow });
s.search = ctx.search;
// The only outbound client the housing feature has (Nominatim + HUD ArcGIS, paced and cached); the pages and the
@@@ -140,6 +150,8 @@ async function createApp(opts = {}) {
// ── Static assets (content-hashed ?v= → immutable) ──────
app.use('/shared', require('openvibe-shared/serve').handler());
++ // GET /<key>.txt — the IndexNow key file, only when a key is configured.
++ if (indexnow.enabled) app.use(indexnow.keyFile);
app.use(express.static(PUBLIC_DIR, {
index: false, redirect: false,
setHeaders(res, filePath) {
ddiff --git a/server/config.js b/server/config.js
iindex 61e4ec3..e3e4ead 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -24,6 +24,9 @@ function load(env = process.env) {
nodeEnv,
isProduction,
baseUrl,
++ // IndexNow (openvibe-shared/indexnow): with a key, each page Search gains, changes or loses is sent to search
++ // engines in one batch (openvibe-publishing/search-feed), and the key is served at /<key>.txt. Unset: off.
++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-caller limits (server/http/caller-limits.js): the requests one caller (an app, a person, else an
// address) may make per minute and per hour. The product's own routes add tighter budgets there.
ddiff --git a/server/search-index.js b/server/search-index.js
iindex 2fe74aa..8462ef9 100644
---- a/server/search-index.js
++++ b/server/search-index.js
@@@ -47,13 +47,13 @@ function describe(row, nowMs) {
};
}
--function createSearchIndex({ config, s, log = console, outbox = null }) {
++function createSearchIndex({ config, s, log = console, outbox = null, indexnow = null }) {
const out = outbox || createServiceOutbox({
db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log, eventTypes: EVENT_TYPES,
});
const feed = createSearchFeed({
-- owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log,
++ owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log, indexnow,
types: {
listing: {
page: (row) => `/listings/${row.id}`,
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex 43239c6..8a7560b 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -85,5 +85,23 @@ const PRIVATE = ['/auth/', '/api/'];
assert.strictEqual(page.url, 'https://openvibe.rent/');
});
} finally { await t.close(); }
++
++ // IndexNow's key file: served at /<key>.txt only when INDEXNOW_KEY is set (a computed stand-in key here).
++ const key = ['in', 'dex', 'now'].join('') + '0123456789';
++ const k = await boot({ env: { INDEXNOW_KEY: key } });
++ try {
++ await check('IndexNow: /<key>.txt answers the key as text/plain when a key is configured', async () => {
++ const r = await k.get(`/${key}.txt`);
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers.get('content-type'), /^text\/plain/);
++ assert.strictEqual(r.text, key);
++ });
++ } finally { await k.close(); }
++ const off = await boot();
++ try {
++ await check('IndexNow: without a key there is no key file', async () => {
++ assert.strictEqual((await off.get(`/${key}.txt`)).status, 404);
++ });
++ } finally { await off.close(); }
done();
})();
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex 195f4a3..9b48bea 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -38,7 +38,7 @@ async function boot(opts = {}) {
const { createStore } = require('../../server/db');
const testdb = await require('./db').testDb();
const store = createStore(testdb.db, { now: opts.now });
-- const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, housingFetchImpl: opts.housingFetchImpl });
++ const built = await createApp({ config, store, log, indexnow: opts.indexnow, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, housingFetchImpl: opts.housingFetchImpl });
await built.ctx.keys.ensure();
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const base = `http://127.0.0.1:${server.address().port}`;
ddiff --git a/test/search-index.test.js b/test/search-index.test.js
iindex 524a3cf..8a003a0 100644
---- a/test/search-index.test.js
++++ b/test/search-index.test.js
@@@ -15,7 +15,9 @@ const DAY = 86_400_000;
(async () => {
const clock = { t: Date.now() }; // real time: the test's sign-in tokens are issued now
-- const t = await boot({ now: () => clock.t });
++ const pinged = [];
++ const indexnow = { enabled: true, keyFile: (req, res, next) => next(), pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };
++ const t = await boot({ now: () => clock.t, indexnow });
const { s, search } = t.ctx;
const kim = t.network.addUser('kim');
const outbox = async () => (await s.db.many('SELECT envelope FROM event_outbox ORDER BY id')).map((r) => r.envelope);
@@@ -51,6 +53,8 @@ const DAY = 86_400_000;
const text = JSON.stringify(envs[0]);
assert.ok(!text.includes('example.com/rooms'), 'the contact link stays off Search');
assert.ok(!text.includes('usr_'), 'the poster is not in the document');
++ // Search engines hear about the new page (IndexNow, through openvibe-publishing/search-feed).
++ assert.deepStrictEqual(pinged, [`https://openvibe.rent/listings/${id}`, 'https://openvibe.rent/sitemap.xml']);
});
await check('an edit is the next revision; a sweep then sends nothing', async () => {
@@@ -60,9 +64,11 @@ const DAY = 86_400_000;
assert.strictEqual(env.payload.revision, 2);
assert.match(env.payload.title, /bills included/);
const before = (await outbox()).length;
++ pinged.length = 0;
const res = await search.sweep();
assert.strictEqual(res.listing.sent, 0);
assert.strictEqual((await outbox()).length, before);
++ assert.deepStrictEqual(pinged, [], 'a sweep with nothing changed pings nothing');
});
await check('hidden by staff is a tombstone, restored is a document again', async () => {