openvibers/openvibe.inventory Commits
Inventory: tell search engines about new and changed item pages (IndexNow) (#48)
Changes
@@@ -4,6 +4,9 @@ NODE_ENV=productionPORT=5030HOST=127.0.0.1BASE_URL=https://inventory.openvibe.network++# IndexNow: with a key (8–128 letters and digits), pages Search gains or changes are sent to search engines and the++# key is served at /<key>.txt. Unset: off.++#INDEXNOW_KEY=TRUST_PROXY=2# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh inventory: the runtime role through PgBouncer,# the owner on a direct connection (migrations), and Valkey (shared limit counters).@@@ -128,7 +128,8 @@ them.`/` (what it is), `/items` (every item by kind), `/items/:id` (one item: where it shows, who gives it, how many exist),`/me` (your items; Wear and Take off are plain forms), `/u/:subject` (a person's public inventory) and `/updates`.--Every page works without JavaScript.++Every page works without JavaScript. With `INDEXNOW_KEY`, each item page Search gains or changes is sent to search++engines (IndexNow, through `openvibe-publishing/search-feed`) and the key is served at `/<key>.txt`.## Moving Live's cosmetics (ADR-054 §8)@@@ -35,6 +35,7 @@ const { createServiceReadiness } = require('./observability');const { createCallerLimits } = require('./http/caller-limits');const { createServiceOutbox } = require('openvibe-sdk/events');const { createSearchIndex } = require('./search-index');++const { createIndexNow } = require('openvibe-shared/indexnow');const { createNetworkSender } = require('openvibe-sdk/account-data');const { createInventory } = require('./inventory/store');const { seed } = require('./inventory/seed');@@@ -69,7 +70,15 @@ async function createApp(opts = {}) {await seed(inv, { log });// Item pages in OpenVibe.Search (./search-index.js) through the same outbox: the definition writes name what they// changed (watch), and a sweep runs on its own timer once server/index.js starts it.-- const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log });++ // IndexNow: off without INDEXNOW_KEY; a malformed key warns and stays off rather than stopping the site.++ const indexnow = opts.indexnow || (() => {++ const warn = (...a) => (log.warn || console.warn)(...a);++ try { return createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: warn }); } catch (err) {++ warn(`[Inventory] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);++ return createIndexNow({ host: config.baseUrl, key: '', fetch: fetchImpl, log: warn });++ }++ })();++ const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log, indexnow });searchIndex.watch(inv);// Account export and deletion (ADR-033): the table map, and the sender to Network's internal routes with this// service's own client-credentials token.@@@ -140,6 +149,8 @@ async function createApp(opts = {}) {// ── Static assets (content-hashed ?v= → immutable) ──────app.use('/shared', require('openvibe-shared/serve').handler());++ // GET /<key>.txt — the IndexNow key file, only when a key is configured.++ if (indexnow.enabled) app.use(indexnow.keyFile);app.use(express.static(PUBLIC_DIR, {index: false, redirect: false,setHeaders(res, filePath) {@@@ -24,6 +24,9 @@ function load(env = process.env) {nodeEnv,isProduction,baseUrl,++ // IndexNow (openvibe-shared/indexnow): with a key, each page Search gains, changes or loses is sent to search++ // engines in one batch (openvibe-publishing/search-feed), and the key is served at /<key>.txt. Unset: off.++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,// Per-caller limits (server/http/caller-limits.js): the requests one caller (an app, a person, else an// address) may make per minute and per hour. The product's own routes add tighter budgets there.@@@ -47,9 +47,9 @@ function describe(row) {};}--function createSearchIndex({ config, s, outbox, log = console }) {++function createSearchIndex({ config, s, outbox, log = console, indexnow = null }) {const feed = createSearchFeed({-- owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log,++ owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log, indexnow,types: {definition: {page: (row) => `/items/${row.id}`,@@@ -100,5 +100,23 @@ const PRIVATE = ['/auth/', '/api/'];assert.strictEqual(page.url, 'https://inventory.openvibe.network/');});} finally { await t.close(); }++++ // IndexNow's key file: served at /<key>.txt only when INDEXNOW_KEY is set (a computed stand-in key here).++ const key = ['in', 'dex', 'now'].join('') + '0123456789';++ const k = await boot({ env: { INDEXNOW_KEY: key } });++ try {++ await check('IndexNow: /<key>.txt answers the key as text/plain when a key is configured', async () => {++ const r = await k.get(`/${key}.txt`);++ assert.strictEqual(r.status, 200);++ assert.match(r.headers.get('content-type'), /^text\/plain/);++ assert.strictEqual(r.text, key);++ });++ } finally { await k.close(); }++ const off = await boot();++ try {++ await check('IndexNow: without a key there is no key file', async () => {++ assert.strictEqual((await off.get(`/${key}.txt`)).status, 404);++ });++ } finally { await off.close(); }done();})();@@@ -38,7 +38,7 @@ async function boot(opts = {}) {const { createStore } = require('../../server/db');const testdb = await require('./db').testDb();const store = createStore(testdb.db, { now: opts.now });-- const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });++ const built = await createApp({ config, store, log, indexnow: opts.indexnow, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });await built.ctx.keys.ensure();const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });const base = `http://127.0.0.1:${server.address().port}`;@@@ -10,7 +10,9 @@ const contracts = require('openvibe-contracts');const { boot, check, done } = require('./helpers/boot');(async () => {-- const t = await boot();++ const pinged = [];++ const indexnow = { enabled: true, keyFile: (req, res, next) => next(), pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };++ const t = await boot({ indexnow });const { s, searchIndex } = t.ctx;const docs = async () => (await s.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'inventory.index_document.%' ORDER BY id")).map((r) => r.envelope);const valid = (env) => {@@@ -37,7 +39,13 @@ const { boot, check, done } = require('./helpers/boot');assert.strictEqual(doc.facets.rarity, published[0].rarity);assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });assert.strictEqual((await t.get(`/items/${id}`)).status, 200);++ // Search engines hear about each item page and the sitemap (IndexNow, through openvibe-publishing/search-feed).++ const sitemap = `${t.config.baseUrl}/sitemap.xml`;++ assert.deepStrictEqual(pinged.filter((u) => u !== sitemap).sort(), published.map((d) => `${t.config.baseUrl}/items/${d.id}`).sort());++ assert.ok(pinged.includes(sitemap));++ pinged.length = 0;assert.strictEqual((await searchIndex.sweep()).definition.sent, 0, 'a second sweep sends nothing');++ assert.deepStrictEqual(pinged, [], 'nor pings anything');});await check('a retired item stays in Search marked noindex; a draft is a tombstone', async () => {@@@ -46,10 +54,12 @@ const { boot, check, done } = require('./helpers/boot');let env = valid((await docs()).pop());assert.strictEqual(env.event_type, 'inventory.index_document.upserted');assert.strictEqual(env.payload.indexability.decision, 'noindex');++ assert.deepStrictEqual(pinged, [], 'a noindex page is not announced');await s.db.exec("UPDATE inventory_definitions SET status = 'draft' WHERE id = $1", [id]);await searchIndex.syncId(id);env = valid((await docs()).pop());assert.strictEqual(env.event_type, 'inventory.index_document.deleted');++ assert.deepStrictEqual(pinged, [], 'a tombstone is not announced (the page leaves the sitemap)');assert.strictEqual((await t.get(`/items/${id}`)).status, 404);await s.db.exec("UPDATE inventory_definitions SET status = 'published' WHERE id = $1", [id]);});
@@ -4,6 +4,9 @@ NODE_ENV=production
@@ -4,6 +4,9 @@ NODE_ENV=production
PORT=5030
PORT=5030
HOST=127.0.0.1
HOST=127.0.0.1
BASE_URL=https://inventory.openvibe.network
BASE_URL=https://inventory.openvibe.network
+# IndexNow: with a key (8–128 letters and digits), pages Search gains or changes are sent to search engines and the
+# key is served at /<key>.txt. Unset: off.
+#INDEXNOW_KEY=
TRUST_PROXY=2
TRUST_PROXY=2
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh inventory: the runtime role through PgBouncer,
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh inventory: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
@@ -128,7 +128,8 @@ them.
@@ -128,7 +128,8 @@ them.
`/` (what it is), `/items` (every item by kind), `/items/:id` (one item: where it shows, who gives it, how many exist),
`/` (what it is), `/items` (every item by kind), `/items/:id` (one item: where it shows, who gives it, how many exist),
`/me` (your items; Wear and Take off are plain forms), `/u/:subject` (a person's public inventory) and `/updates`.
`/me` (your items; Wear and Take off are plain forms), `/u/:subject` (a person's public inventory) and `/updates`.
-Every page works without JavaScript.
+Every page works without JavaScript. With `INDEXNOW_KEY`, each item page Search gains or changes is sent to search
+engines (IndexNow, through `openvibe-publishing/search-feed`) and the key is served at `/<key>.txt`.
## Moving Live's cosmetics (ADR-054 §8)
## Moving Live's cosmetics (ADR-054 §8)
@@ -35,6 +35,7 @@ const { createServiceReadiness } = require('./observability');
@@ -35,6 +35,7 @@ const { createServiceReadiness } = require('./observability');
const { createCallerLimits } = require('./http/caller-limits');
const { createCallerLimits } = require('./http/caller-limits');
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createSearchIndex } = require('./search-index');
const { createSearchIndex } = require('./search-index');
+const { createIndexNow } = require('openvibe-shared/indexnow');
const { createNetworkSender } = require('openvibe-sdk/account-data');
const { createNetworkSender } = require('openvibe-sdk/account-data');
const { createInventory } = require('./inventory/store');
const { createInventory } = require('./inventory/store');
const { seed } = require('./inventory/seed');
const { seed } = require('./inventory/seed');
@@ -69,7 +70,15 @@ async function createApp(opts = {}) {
@@ -69,7 +70,15 @@ async function createApp(opts = {}) {
await seed(inv, { log });
await seed(inv, { log });
// Item pages in OpenVibe.Search (./search-index.js) through the same outbox: the definition writes name what they
// Item pages in OpenVibe.Search (./search-index.js) through the same outbox: the definition writes name what they
// changed (watch), and a sweep runs on its own timer once server/index.js starts it.
// changed (watch), and a sweep runs on its own timer once server/index.js starts it.
- const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log });
+ // IndexNow: off without INDEXNOW_KEY; a malformed key warns and stays off rather than stopping the site.
+ const indexnow = opts.indexnow || (() => {
+ const warn = (...a) => (log.warn || console.warn)(...a);
+ try { return createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: warn }); } catch (err) {
+ warn(`[Inventory] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);
+ return createIndexNow({ host: config.baseUrl, key: '', fetch: fetchImpl, log: warn });
+ }
+ })();
+ const searchIndex = opts.searchIndex || createSearchIndex({ config, s, outbox, log, indexnow });
searchIndex.watch(inv);
searchIndex.watch(inv);
// Account export and deletion (ADR-033): the table map, and the sender to Network's internal routes with this
// Account export and deletion (ADR-033): the table map, and the sender to Network's internal routes with this
// service's own client-credentials token.
// service's own client-credentials token.
@@ -140,6 +149,8 @@ async function createApp(opts = {}) {
@@ -140,6 +149,8 @@ async function createApp(opts = {}) {
// ── Static assets (content-hashed ?v= → immutable) ──────
// ── Static assets (content-hashed ?v= → immutable) ──────
app.use('/shared', require('openvibe-shared/serve').handler());
app.use('/shared', require('openvibe-shared/serve').handler());
+ // GET /<key>.txt — the IndexNow key file, only when a key is configured.
+ if (indexnow.enabled) app.use(indexnow.keyFile);
app.use(express.static(PUBLIC_DIR, {
app.use(express.static(PUBLIC_DIR, {
index: false, redirect: false,
index: false, redirect: false,
setHeaders(res, filePath) {
setHeaders(res, filePath) {
@@ -24,6 +24,9 @@ function load(env = process.env) {
@@ -24,6 +24,9 @@ function load(env = process.env) {
nodeEnv,
nodeEnv,
isProduction,
isProduction,
baseUrl,
baseUrl,
+ // IndexNow (openvibe-shared/indexnow): with a key, each page Search gains, changes or loses is sent to search
+ // engines in one batch (openvibe-publishing/search-feed), and the key is served at /<key>.txt. Unset: off.
+ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-caller limits (server/http/caller-limits.js): the requests one caller (an app, a person, else an
// Per-caller limits (server/http/caller-limits.js): the requests one caller (an app, a person, else an
// address) may make per minute and per hour. The product's own routes add tighter budgets there.
// address) may make per minute and per hour. The product's own routes add tighter budgets there.
@@ -47,9 +47,9 @@ function describe(row) {
@@ -47,9 +47,9 @@ function describe(row) {
};
};
}
}
-function createSearchIndex({ config, s, outbox, log = console }) {
+function createSearchIndex({ config, s, outbox, log = console, indexnow = null }) {
const feed = createSearchFeed({
const feed = createSearchFeed({
- owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log,
+ owner: 'inventory', db: s.db, outbox, baseUrl: config.baseUrl, now: s.now, log, indexnow,
types: {
types: {
definition: {
definition: {
page: (row) => `/items/${row.id}`,
page: (row) => `/items/${row.id}`,
@@ -100,5 +100,23 @@ const PRIVATE = ['/auth/', '/api/'];
@@ -100,5 +100,23 @@ const PRIVATE = ['/auth/', '/api/'];
assert.strictEqual(page.url, 'https://inventory.openvibe.network/');
assert.strictEqual(page.url, 'https://inventory.openvibe.network/');
});
});
} finally { await t.close(); }
} finally { await t.close(); }
+
+ // IndexNow's key file: served at /<key>.txt only when INDEXNOW_KEY is set (a computed stand-in key here).
+ const key = ['in', 'dex', 'now'].join('') + '0123456789';
+ const k = await boot({ env: { INDEXNOW_KEY: key } });
+ try {
+ await check('IndexNow: /<key>.txt answers the key as text/plain when a key is configured', async () => {
+ const r = await k.get(`/${key}.txt`);
+ assert.strictEqual(r.status, 200);
+ assert.match(r.headers.get('content-type'), /^text\/plain/);
+ assert.strictEqual(r.text, key);
+ });
+ } finally { await k.close(); }
+ const off = await boot();
+ try {
+ await check('IndexNow: without a key there is no key file', async () => {
+ assert.strictEqual((await off.get(`/${key}.txt`)).status, 404);
+ });
+ } finally { await off.close(); }
done();
done();
})();
})();
@@ -38,7 +38,7 @@ async function boot(opts = {}) {
@@ -38,7 +38,7 @@ async function boot(opts = {}) {
const { createStore } = require('../../server/db');
const { createStore } = require('../../server/db');
const testdb = await require('./db').testDb();
const testdb = await require('./db').testDb();
const store = createStore(testdb.db, { now: opts.now });
const store = createStore(testdb.db, { now: opts.now });
- const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });
+ const built = await createApp({ config, store, log, indexnow: opts.indexnow, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, workshopMedia: opts.workshopMedia, people: opts.people });
await built.ctx.keys.ensure();
await built.ctx.keys.ensure();
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const base = `http://127.0.0.1:${server.address().port}`;
const base = `http://127.0.0.1:${server.address().port}`;
@@ -10,7 +10,9 @@ const contracts = require('openvibe-contracts');
@@ -10,7 +10,9 @@ const contracts = require('openvibe-contracts');
const { boot, check, done } = require('./helpers/boot');
const { boot, check, done } = require('./helpers/boot');
(async () => {
(async () => {
- const t = await boot();
+ const pinged = [];
+ const indexnow = { enabled: true, keyFile: (req, res, next) => next(), pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };
+ const t = await boot({ indexnow });
const { s, searchIndex } = t.ctx;
const { s, searchIndex } = t.ctx;
const docs = async () => (await s.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'inventory.index_document.%' ORDER BY id")).map((r) => r.envelope);
const docs = async () => (await s.db.many("SELECT envelope FROM event_outbox WHERE envelope->>'event_type' LIKE 'inventory.index_document.%' ORDER BY id")).map((r) => r.envelope);
const valid = (env) => {
const valid = (env) => {
@@ -37,7 +39,13 @@ const { boot, check, done } = require('./helpers/boot');
@@ -37,7 +39,13 @@ const { boot, check, done } = require('./helpers/boot');
assert.strictEqual(doc.facets.rarity, published[0].rarity);
assert.strictEqual(doc.facets.rarity, published[0].rarity);
assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });
assert.deepStrictEqual(doc.indexability, { decision: 'index', reasons: [] });
assert.strictEqual((await t.get(`/items/${id}`)).status, 200);
assert.strictEqual((await t.get(`/items/${id}`)).status, 200);
+ // Search engines hear about each item page and the sitemap (IndexNow, through openvibe-publishing/search-feed).
+ const sitemap = `${t.config.baseUrl}/sitemap.xml`;
+ assert.deepStrictEqual(pinged.filter((u) => u !== sitemap).sort(), published.map((d) => `${t.config.baseUrl}/items/${d.id}`).sort());
+ assert.ok(pinged.includes(sitemap));
+ pinged.length = 0;
assert.strictEqual((await searchIndex.sweep()).definition.sent, 0, 'a second sweep sends nothing');
assert.strictEqual((await searchIndex.sweep()).definition.sent, 0, 'a second sweep sends nothing');
+ assert.deepStrictEqual(pinged, [], 'nor pings anything');
});
});
await check('a retired item stays in Search marked noindex; a draft is a tombstone', async () => {
await check('a retired item stays in Search marked noindex; a draft is a tombstone', async () => {
@@ -46,10 +54,12 @@ const { boot, check, done } = require('./helpers/boot');
@@ -46,10 +54,12 @@ const { boot, check, done } = require('./helpers/boot');
let env = valid((await docs()).pop());
let env = valid((await docs()).pop());
assert.strictEqual(env.event_type, 'inventory.index_document.upserted');
assert.strictEqual(env.event_type, 'inventory.index_document.upserted');
assert.strictEqual(env.payload.indexability.decision, 'noindex');
assert.strictEqual(env.payload.indexability.decision, 'noindex');
+ assert.deepStrictEqual(pinged, [], 'a noindex page is not announced');
await s.db.exec("UPDATE inventory_definitions SET status = 'draft' WHERE id = $1", [id]);
await s.db.exec("UPDATE inventory_definitions SET status = 'draft' WHERE id = $1", [id]);
await searchIndex.syncId(id);
await searchIndex.syncId(id);
env = valid((await docs()).pop());
env = valid((await docs()).pop());
assert.strictEqual(env.event_type, 'inventory.index_document.deleted');
assert.strictEqual(env.event_type, 'inventory.index_document.deleted');
+ assert.deepStrictEqual(pinged, [], 'a tombstone is not announced (the page leaves the sitemap)');
assert.strictEqual((await t.get(`/items/${id}`)).status, 404);
assert.strictEqual((await t.get(`/items/${id}`)).status, 404);
await s.db.exec("UPDATE inventory_definitions SET status = 'published' WHERE id = $1", [id]);
await s.db.exec("UPDATE inventory_definitions SET status = 'published' WHERE id = $1", [id]);
});
});