openvibers/openvibe.inventory Commits

Inventory: tell search engines about new and changed item pages (IndexNow) (#48)

OP OpenVibers committed 29 days ago · 30a30a09fc · 8 files, +52 −6

`openvibe-publishing/search-feed` already pings IndexNow when it is given a client: whenever a page Search gains or changes is indexable, it sends the page and `/sitemap.xml`. Noindex pages and tombstones send nothing. This service never passed a client, so it sent nothing.

- With `INDEXNOW_KEY`, the app builds one `openvibe-shared/indexnow` client, hands it to the search feed and serves `/<key>.txt`.
- A malformed key warns and leaves IndexNow off instead of stopping the site. Unset: off.
- Tests: the search-index test checks the pinged URLs (new pages announced, a quiet sweep and a noindex page announce nothing). The discovery test checks the key file with a key and the 404 without one.
- The README and `.env.example` document the key.

Changes

ddiff --git a/.env.example b/.env.example
iindex 44878ac..7d50c51 100644
---- a/.env.example
++++ b/.env.example
@@@ -4,6 +4,9 @@ NODE_ENV=production
PORT=5030
HOST=127.0.0.1
BASE_URL=https://inventory.openvibe.network
++# IndexNow: with a key (8–128 letters and digits), pages Search gains or changes are sent to search engines and the
++# key is served at /<key>.txt. Unset: off.
++#INDEXNOW_KEY=
TRUST_PROXY=2
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh inventory: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
ddiff --git a/README.md b/README.md
iindex 5522dcb..c3758d8 100644
---- a/README.md
++++ b/README.md
@@@ -128,7 +128,8 @@ them.
`/` (what it is), `/items` (every item by kind), `/items/:id` (one item: where it shows, who gives it, how many exist),
`/me` (your items; Wear and Take off are plain forms), `/u/:subject` (a person's public inventory) and `/updates`.
--Every page works without JavaScript.
++Every page works without JavaScript. With `INDEXNOW_KEY`, each item page Search gains or changes is sent to search
++engines (IndexNow, through `openvibe-publishing/search-feed`) and the key is served at `/<key>.txt`.
## Moving Live's cosmetics (ADR-054 §8)
ddiff --git a/server/app.js b/server/app.js
iindex d60dac8..34eb39d 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -35,6 +35,7 @@ const { createServiceReadiness } = require('./observability');
const { createCallerLimits } = require('./http/caller-limits');
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createSearchIndex } = require('./search-index');
++const { createIndexNow } = require('openvibe-shared/indexnow');
const { createNetworkSender } = require('openvibe-sdk/account-data');
const { createInventory } = require('./inventory/store');
const { seed } = require('./inventory/seed');
@@@ -69,7 +70,15 @@ async function createApp(opts = {}) {
await seed(inv, { log });
// Item pages in OpenVibe.Search (./search-index.js) through the same outbox: the definition writes name what they
// changed (watch), and a sweep runs on its own timer once server/index.js starts it.
-- const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log });
++ // IndexNow: off without INDEXNOW_KEY; a malformed key warns and stays off rather than stopping the site.
++ const indexnow = opts.indexnow || (() => {
++ const warn = (...a) => (log.warn || console.warn)(...a);
++ try { return createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: warn }); } catch (err) {
++ warn(`[Inventory] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);
++ return createIndexNow({ host: config.baseUrl, key: '', fetch: fetchImpl, log: warn });
++ }
++ })();
++ const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log, indexnow });
searchIndex.watch(inv);
// Account export and deletion (ADR-033): the table map, and the sender to Network's internal routes with this
// service's own client-credentials token.
@@@ -140,6 +149,8 @@ async function createApp(opts = {}) {
// ── Static assets (content-hashed ?v= → immutable) ──────
app.use('/shared', require('openvibe-shared/serve').handler());
++ // GET /<key>.txt — the IndexNow key file, only when a key is configured.
++ if (indexnow.enabled) app.use(indexnow.keyFile);
app.use(express.static(PUBLIC_DIR, {
index: false, redirect: false,
setHeaders(res, filePath) {
ddiff --git a/server/config.js b/server/config.js
iindex 0f081b0..b516ace 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -24,6 +24,9 @@ function load(env = process.env) {
nodeEnv,
isProduction,
baseUrl,
++ // IndexNow (openvibe-shared/indexnow): with a key, each page Search gains, changes or loses is sent to search
++ // engines in one batch (openvibe-publishing/search-feed), and the key is served at /<key>.txt. Unset: off.
++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-caller limits (server/http/caller-limits.js): the requests one caller (an app, a person, else an
// address) may make per minute and per hour. The product's own routes add tighter budgets there.
ddiff --git a/server/search-index.js b/server/search-index.js
iindex de3c112..63c9303 100644
---- a/server/search-index.js
++++ b/server/search-index.js
@@@ -47,9 +47,9 @@ function describe(row) {
};
}
--function createSearchIndex({ config, s, outbox, log = console }) {
++function createSearchIndex({ config, s, outbox, log = console, indexnow = null }) {
const feed = createSearchFeed({
-- owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log,
++ owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log, indexnow,
types: {
definition: {
page: (row) => `/items/${row.id}`,
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex a034b57..46577fb 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -100,5 +100,23 @@ const PRIVATE = ['/auth/', '/api/'];
assert.strictEqual(page.url, 'https://inventory.openvibe.network/');
});
} finally { await t.close(); }
++
++ // IndexNow's key file: served at /<key>.txt only when INDEXNOW_KEY is set (a computed stand-in key here).
++ const key = ['in', 'dex', 'now'].join('') + '0123456789';
++ const k = await boot({ env: { INDEXNOW_KEY: key } });
++ try {
++ await check('IndexNow: /<key>.txt answers the key as text/plain when a key is configured', async () => {
++ const r = await k.get(`/${key}.txt`);
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers.get('content-type'), /^text\/plain/);
++ assert.strictEqual(r.text, key);
++ });
++ } finally { await k.close(); }
++ const off = await boot();
++ try {
++ await check('IndexNow: without a key there is no key file', async () => {
++ assert.strictEqual((await off.get(`/${key}.txt`)).status, 404);
++ });
++ } finally { await off.close(); }
done();
})();
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex e94c956..72ff87c 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -38,7 +38,7 @@ async function boot(opts = {}) {
const { createStore } = require('../../server/db');
const testdb = await require('./db').testDb();
const store = createStore(testdb.db, { now: opts.now });
-- const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });
++ const built = await createApp({ config, store, log, indexnow: opts.indexnow, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });
await built.ctx.keys.ensure();
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const base = `http://127.0.0.1:${server.address().port}`;
ddiff --git a/test/search-index.test.js b/test/search-index.test.js
iindex 11b976a..f2c414f 100644
---- a/test/search-index.test.js
++++ b/test/search-index.test.js
@@@ -10,7 +10,9 @@ const contracts = require('openvibe-contracts');
const { boot, check, done } = require('./helpers/boot');
(async () => {
-- const t = await boot();
++ const pinged = [];
++ const indexnow = { enabled: true, keyFile: (req, res, next) => next(), pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };
++ const t = await boot({ indexnow });
const { s, searchIndex } = t.ctx;
const docs = async () => (await s.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'inventory.index_document.%' ORDER BY id")).map((r) => r.envelope);
const valid = (env) => {
@@@ -37,7 +39,13 @@ const { boot, check, done } = require('./helpers/boot');
assert.strictEqual(doc.facets.rarity, published[0].rarity);
assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });
assert.strictEqual((await t.get(`/items/${id}`)).status, 200);
++ // Search engines hear about each item page and the sitemap (IndexNow, through openvibe-publishing/search-feed).
++ const sitemap = `${t.config.baseUrl}/sitemap.xml`;
++ assert.deepStrictEqual(pinged.filter((u) => u !== sitemap).sort(), published.map((d) => `${t.config.baseUrl}/items/${d.id}`).sort());
++ assert.ok(pinged.includes(sitemap));
++ pinged.length = 0;
assert.strictEqual((await searchIndex.sweep()).definition.sent, 0, 'a second sweep sends nothing');
++ assert.deepStrictEqual(pinged, [], 'nor pings anything');
});
await check('a retired item stays in Search marked noindex; a draft is a tombstone', async () => {
@@@ -46,10 +54,12 @@ const { boot, check, done } = require('./helpers/boot');
let env = valid((await docs()).pop());
assert.strictEqual(env.event_type, 'inventory.index_document.upserted');
assert.strictEqual(env.payload.indexability.decision, 'noindex');
++ assert.deepStrictEqual(pinged, [], 'a noindex page is not announced');
await s.db.exec("UPDATE inventory_definitions SET status = 'draft' WHERE id = $1", [id]);
await searchIndex.syncId(id);
env = valid((await docs()).pop());
assert.strictEqual(env.event_type, 'inventory.index_document.deleted');
++ assert.deepStrictEqual(pinged, [], 'a tombstone is not announced (the page leaves the sitemap)');
assert.strictEqual((await t.get(`/items/${id}`)).status, 404);
await s.db.exec("UPDATE inventory_definitions SET status = 'published' WHERE id = $1", [id]);
});