openvibers/openvibe.wiki Commits
Wiki tells search engines when a page changes (plan T11, IndexNow): with INDEXNOW_KEY set the key file is served at /<key>.txt and an indexable page appearing, changing or going away pings the engines (its path and the sitemap); drafts, private and noindex pages never; unset, nothing is mounted or sent. A home-page performance budget runs in CI where there was none. Tests green (verified).
Changes
@@@ -127,6 +127,12 @@ records who attached each Media object and what Media said about it at that mome`/sitemaps/spaces.xml`, `/sitemaps/pages-N.xml`; `/feed.atom` and `/feed.json` (recent changes; with nothing listableyet they are valid feeds with zero entries, never a 404);`/robots.txt`; `/llms.txt`; a JSON representation of every page at `<page>.json`.++- **IndexNow (openvibe-shared/indexnow)**: with `INDEXNOW_KEY` set, the key file is served at++ `/<key>.txt` as `text/plain`; publishing, updating (a new revision, a slug change, a space rename),++ unpublishing and deleting a public, indexable page pings `api.indexnow.org` with the page's++ canonical URL and `/sitemap.xml` (the module batches and debounces; a failed ping never takes a++ publish down). Drafts, private, members, VIP and noindex pages never ping. Unset: off — no key++ file, no requests.- **The gate**: `openvibe-publishing/seo` decides indexing per page with explicit reasons(policy: at least `WIKI_GATE_MIN_WORDS` words and `WIKI_GATE_MIN_SOURCES` citations; AI text andAI-assisted imports only after a person's review; owner-requested noindex). Only public, published, indexable pages enter sitemaps and@@@ -321,6 +327,10 @@ Called elsewhere, as the service principal `svc:wiki` (client credentials, one t- every event is a valid `events.event-envelope@1` and every index document a valid`search.index-document@1` (`visibility.test.js`); the proposals validate against the contractsschemas (`proposals.test.js`); user text is escaped everywhere (`content.test.js`)++- IndexNow: off without `INDEXNOW_KEY` (no key route, nothing sent); with one the key file is served++ at `/<key>.txt` as `text/plain` and a publish, unpublish or delete of an indexable page pings the++ page path and the sitemap; a draft, a noindex page and a page the gate will not index never ping++ (`indexnow.test.js`)## Security@@@ -22,7 +22,7 @@ const { createActorLimits } = require('./http/actor-limits');const PUBLIC_DIR = path.join(__dirname, '..', 'public');const VERSION = require('../package.json').version;--function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valkey = null, log = console, rateLimits = true, fetchImpl = globalThis.fetch, limitsNow = null }) {++function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valkey = null, log = console, rateLimits = true, fetchImpl = globalThis.fetch, limitsNow = null, indexnow = null }) {const app = express();// Every resolved actor carries its wiki roles (read once per request), so the synchronous access checks work.const viewers = { ...baseViewers, resolve: async (req, opts) => svc.loadRoles(await baseViewers.resolve(req, opts)) };@@@ -115,6 +115,8 @@ function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valkres.setHeader('Cache-Control', v && v === require('./render/layout').assetVersion(rel) ? 'public, max-age=31536000, immutable' : 'public, max-age=300');},}));++ // GET /<key>.txt — the IndexNow key file, only when a key is configured (server/index.js).++ if (indexnow && indexnow.enabled) app.use(indexnow.keyFile);app.use(createMachine({ svc, config }));app.post(['/new-space', '/s/*', '/w/*', '/proposals/*'], limiter(10 * 60000, 120));@@@ -79,6 +79,11 @@ function load(env = process.env) {scheduleIntervalMs: int(env.WIKI_SCHEDULE_INTERVAL_MS, 15000),workerId: env.WIKI_WORKER_ID || `wiki-${process.pid}`,++ // IndexNow (openvibe-shared/indexnow): with a key, search engines are told the moment a++ // public, indexable page appears, changes or leaves the index (the key file is served at++ // /<key>.txt). Unset: off — nothing is mounted and nothing is sent.++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },++// The indexability gate policy for wiki pages (openvibe-publishing/seo).gate: {minWords: int(env.WIKI_GATE_MIN_WORDS, 80),@@@ -8,6 +8,7 @@* clock/fetch/log/keys, and returns handles to every part. Workers (scheduled publication, the* Events outbox relay, the Media attachment check) run only when asked (`workers: true`).*/++const { createIndexNow } = require('openvibe-shared/indexnow');const { load } = require('./config');const { openDb, migrate, createStores } = require('./db');const { createPlatform } = require('./integrations/platform');@@@ -16,23 +17,27 @@ const { createKeyStore } = require('./auth/keys');const { createViewerResolver } = require('./auth/viewer');const { createApp } = require('./app');--async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, tokens = null, publicKey = null, log = console, listen = true, workers = listen, rateLimits = true, limitsNow = null } = {}) {++async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, tokens = null, publicKey = null, log = console, listen = true, workers = listen, rateLimits = true, limitsNow = null, indexnow: givenIndexNow = null } = {}) {config = config || load();// PostgreSQL (ADR-035): migrate on the owner's direct connection, then serve on the pooled runtime role.// Tests hand in a migrated database of their own (test/db-helper.js).const db = givenDb || openDb(config, { log });if (!givenDb) await migrate(config, { serving: db, log });const stores = createStores(db, { now });++ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY. Unset → off, nothing++ // mounted and nothing sent; a test or a drill injects its own (a spy, or the module's own no-key++ // state) through `indexnow`, and the outbound POST goes through `fetchImpl`, this process's stub.++ const indexnow = givenIndexNow || createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: (...a) => (log.warn || console.warn)(...a) });// Valkey (ADR-035): shared, never-authoritative state (per-actor limit counters). Optional.const valkey = config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null;const platform = createPlatform({ config, db, fetchImpl, tokens, now, log });-- const svc = createWikiService({ db, stores, outbox: platform.outbox, config, community: platform.community, vip: platform.vip, now, log });++ const svc = createWikiService({ db, stores, outbox: platform.outbox, config, community: platform.community, vip: platform.vip, now, log, indexnow });const reconciled = await svc.reconcileIndex();if (reconciled.sent) log.log(`[Wiki] re-sent ${reconciled.sent} Search document(s) whose indexability changed`);const keys = createKeyStore({ config, fetchImpl, log, publicKey });keys.ensure().catch(() => {});const viewers = createViewerResolver({ keys, config });-- const app = createApp({ config, svc, viewers, platform, keys, db, valkey, log, rateLimits, fetchImpl, limitsNow });++ const app = createApp({ config, svc, viewers, platform, keys, db, valkey, log, rateLimits, fetchImpl, limitsNow, indexnow });const timers = [];if (workers) {@@@ -70,7 +75,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchif (!givenDb) await db.close();if (valkey) await valkey.close().catch(() => {});}-- return { app, db, svc, stores, platform, keys, viewers, server, config, stop };++ return { app, db, svc, stores, platform, keys, viewers, server, config, indexnow, stop };}module.exports = { start };@@@ -70,7 +70,7 @@ function disclosureFor(rec, review) {};}--function createWikiService({ db, stores, outbox, config, community = null, vip = null, now = () => Date.now(), log = console }) {++function createWikiService({ db, stores, outbox, config, community = null, vip = null, now = () => Date.now(), log = console, indexnow = null }) {const { revisions, citations, redirects, reviews, attachments, discussions, scheduler, sequencer } = stores;const access = createAccess(db, { vip });const origin = config.baseUrl;@@@ -367,7 +367,8 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =publishedAt: page.published_at, updatedAt: page.revision_published_at,}));const sentBefore = before ? before.indexRevision : null;-- if (doc.revision !== sentBefore) await emit(hooks.indexEvent({ document: doc, now: now() }));++ const indexChanged = doc.revision !== sentBefore;++ if (indexChanged) await emit(hooks.indexEvent({ document: doc, now: now() }));const after = { state, visibility: eff, revision: page.published_revision };const wasPublic = Boolean(before) && before.state === 'published' && before.visibility === 'public';@@@ -375,7 +376,7 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =if (wasPublic !== isPublic) followThread(page.id, isPublic ? 'public' : 'hidden');let action = hooks.actionFor(before && before.state ? before : null, after);// A review changes only the gate decision: announce it when what Search holds changed.-- if (!action && updatedIfIndexChanged && state === 'published' && doc.revision !== sentBefore) action = 'updated';++ if (!action && updatedIfIndexChanged && state === 'published' && indexChanged) action = 'updated';if (action) {let evDoc = doc;if (doc.deleted && (action === 'published' || action === 'updated')) {@@@ -390,6 +391,21 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =}));await notifyWatchers(space, page, action, page.published_revision, actor);}++ // IndexNow (openvibe-shared/indexnow): tell the engines a public, indexable page appeared,++ // changed or went away; never for drafts, private, members/VIP or noindex pages. The sitemap is++ // pinged alongside the page. Only when the publication state moved or Search was sent something++ // new (a noindex flip moves no state but does change the index), after the commit — a write that++ // rolls back must not ping — and the module batches and debounces. The previous state's own gate++ // reasons are not re-read (they are gone by now): a page that was public and published counts as++ // indexable when it leaves, unless it is still flagged noindex.++ if (indexnow && indexnow.enabled) {++ const indexable = state === 'published' && eff === 'public' && !page.noindex && Boolean(decision && decision.indexable);++ const wasIndexable = Boolean(before) && before.state === 'published' && before.visibility === 'public' && !page.noindex;++ if ((action || indexChanged) && (indexable || wasIndexable)) {++ const urls = [pageUrl(space, page), seo.canonicalUrl(origin, '/sitemap.xml')];++ afterCommit(() => indexnow.pingSoon(urls));++ }++ }return { action, indexRevision: doc.revision, deleted: doc.deleted };}@@@ -40,7 +40,7 @@ const quiet = process.env.WIKI_TEST_LOG ? console : { log() {}, warn() {}, error* A running Wiki on a database of its own (PGlite by default; WIKI_TEST_STORE=pg: the PostgreSQL + PgBouncer* containers). opts.db: boot on an existing handle (a restart keeps the data); h.stop() closes what boot opened.*/--async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, workers = false, tokens, rateLimits = false, limitsNow = null, log = quiet } = {}) {++async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, workers = false, tokens, rateLimits = false, limitsNow = null, log = quiet, indexnow = undefined } = {}) {const owned = givenDb ? null : await testDb();const db = givenDb || owned.db;const config = load({@@@ -52,6 +52,7 @@ async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, workeconfig, db, publicKey, log, listen: true, workers, rateLimits, now, limitsNow,fetchImpl: fetchImpl || (async (url) => { throw new Error(`unexpected outbound fetch ${url}`); }),tokens: tokens || { getToken: async () => 'stub-token', authHeaders: async () => ({ Authorization: 'Bearer stub-token' }), invalidate() {} },++ indexnow,});const base = `http://127.0.0.1:${h.server.address().port}`;const stop = async ({ keepDb = false } = {}) => { await h.stop(); if (owned && !keepDb) await owned.close(); };nnew file mode 100644@@@ -0,0 +1,108 @@++'use strict';++/**++ * IndexNow (openvibe-shared/indexnow): INDEXNOW_KEY unset → the feature is off (no key route,++ * nothing sent). With a key the key file is served at /<key>.txt as text/plain, and a publish,++ * unpublish or delete of an indexable page pings the engines with the page's path and the sitemap.++ * Drafts, noindex pages and pages the gate will not index never ping.++ */++const assert = require('assert');++const H = require('./helpers');++++const KEY = 'k'.repeat(32);++const BASE = 'https://openvibe.wiki';++const SITEMAP = `${BASE}/sitemap.xml`;++const PAGE = `${BASE}/w/notes/first-light`;++++(async () => {++ // Off: no INDEXNOW_KEY. Nothing is mounted and nothing is sent.++ const off = await H.boot({ env: { BASE_URL: BASE } });++ try {++ assert.strictEqual(off.indexnow.enabled, false, 'IndexNow is off without a key');++ assert.strictEqual((await H.req(off, 'GET', `/${KEY}.txt`)).status, 404, 'no key file is served');++ } finally { await off.stop(); }++++ // With a key: the key file is served as text/plain with the key.++ const on = await H.boot({ env: { BASE_URL: BASE, INDEXNOW_KEY: KEY } });++ try {++ assert.strictEqual(on.indexnow.enabled, true, 'IndexNow is on with a key');++ const r = await H.req(on, 'GET', `/${KEY}.txt`);++ assert.strictEqual(r.status, 200, r.text);++ assert.match(r.headers.get('content-type'), /text\/plain/);++ assert.strictEqual(r.text, KEY);++ } finally { await on.stop(); }++++ // A spy in place of the module's HTTP send: records every pingSoon batch.++ const pings = [];++ const spy = {++ enabled: true,++ keyFile: (_req, _res, next) => next(),++ pingSoon: (urls) => { const a = Array.isArray(urls) ? urls : [urls]; pings.push(...a); return a.length; },++ ping: async () => ({ sent: 0, status: 0 }),++ flush: async () => ({ sent: 0, status: 0 }),++ };++ const h = await H.boot({ env: { BASE_URL: BASE }, indexnow: spy });++ const owner = { kind: 'user', subject: H.subject(), staff: false };++ try {++ const space = await h.svc.createSpace({ name: 'Notes', slug: 'notes' }, owner);++ const cite = [{ url: 'https://example.org/a', retrievedAt: '2026-09-20T00:00:00Z' }];++ const { page } = await h.svc.createPage(space.id, { title: 'First light', body: H.LONG, citations: cite }, owner);++ assert.deepStrictEqual(pings, [], 'a draft never pings');++++ await h.svc.publish(page.id, {}, owner);++ assert.ok(pings.includes(PAGE), `a publish pings the page path: ${JSON.stringify(pings)}`);++ assert.ok(pings.includes(SITEMAP), `a publish pings the sitemap: ${JSON.stringify(pings)}`);++++ pings.length = 0;++ await h.svc.unpublish(page.id, owner);++ assert.ok(pings.includes(PAGE), `an unpublish pings the page path: ${JSON.stringify(pings)}`);++ assert.ok(pings.includes(SITEMAP), `an unpublish pings the sitemap: ${JSON.stringify(pings)}`);++++ await h.svc.publish(page.id, {}, owner);++ pings.length = 0;++ await h.svc.setPageVisibility(page.id, { noindex: true }, owner);++ assert.deepStrictEqual(pings, [], 'a page the owner keeps noindex never pings');++ await h.svc.setPageVisibility(page.id, { noindex: false }, owner);++ assert.ok(pings.includes(PAGE), `indexable again pings: ${JSON.stringify(pings)}`);++++ pings.length = 0;++ await h.svc.deletePage(page.id, owner);++ assert.ok(pings.includes(PAGE), `a delete pings the page path: ${JSON.stringify(pings)}`);++ assert.ok(pings.includes(SITEMAP), `a delete pings the sitemap: ${JSON.stringify(pings)}`);++++ // A page the gate keeps out of the index (no source: the test policy requires one) never pings.++ pings.length = 0;++ const thin = (await h.svc.createPage(space.id, { title: 'No sources', body: H.LONG }, owner)).page;++ await h.svc.publish(thin.id, {}, owner);++ assert.deepStrictEqual(pings, [], 'a page the gate will not index never pings');++ } finally {++ await h.stop();++ }++++ // The real module: a boot with a key batches what the service queued and POSTs it through this++ // process's fetch, with the page's canonical URL and the sitemap (the module drops anything that++ // is not an absolute https URL on its own host).++ const sent = [];++ const outbound = async (url, opts) => {++ if (!String(url).includes('api.indexnow.org')) throw new Error(`unexpected outbound fetch ${url}`);++ sent.push(JSON.parse(opts.body));++ return { status: 200 };++ };++ const m = await H.boot({ env: { BASE_URL: BASE, INDEXNOW_KEY: KEY }, fetch: outbound });++ const mOwner = { kind: 'user', subject: H.subject(), staff: false };++ try {++ const mSpace = await m.svc.createSpace({ name: 'Notes', slug: 'notes' }, mOwner);++ const mCite = [{ url: 'https://example.org/a', retrievedAt: '2026-09-20T00:00:00Z' }];++ const mPage = (await m.svc.createPage(mSpace.id, { title: 'First light', body: H.LONG, citations: mCite }, mOwner)).page;++ await m.svc.publish(mPage.id, {}, mOwner);++ assert.deepStrictEqual(sent, [], 'nothing is sent before the batch is flushed');++ const out = await m.indexnow.flush();++ assert.deepStrictEqual(sent.length, 1, 'one POST for the batch');++ assert.strictEqual(sent[0].host, 'openvibe.wiki');++ assert.strictEqual(sent[0].key, KEY);++ assert.deepStrictEqual(sent[0].urlList.slice().sort(), [PAGE, SITEMAP].sort(), JSON.stringify(sent[0].urlList));++ assert.strictEqual(out.sent, 2, JSON.stringify(out));++ } finally {++ await m.stop();++ }++ console.log('indexnow ok');++})().catch((err) => { console.error(err); process.exit(1); });
@@ -127,6 +127,12 @@ records who attached each Media object and what Media said about it at that mome
@@ -127,6 +127,12 @@ records who attached each Media object and what Media said about it at that mome
`/sitemaps/spaces.xml`, `/sitemaps/pages-N.xml`; `/feed.atom` and `/feed.json` (recent changes; with nothing listable
`/sitemaps/spaces.xml`, `/sitemaps/pages-N.xml`; `/feed.atom` and `/feed.json` (recent changes; with nothing listable
yet they are valid feeds with zero entries, never a 404);
yet they are valid feeds with zero entries, never a 404);
`/robots.txt`; `/llms.txt`; a JSON representation of every page at `<page>.json`.
`/robots.txt`; `/llms.txt`; a JSON representation of every page at `<page>.json`.
+- **IndexNow (openvibe-shared/indexnow)**: with `INDEXNOW_KEY` set, the key file is served at
+ `/<key>.txt` as `text/plain`; publishing, updating (a new revision, a slug change, a space rename),
+ unpublishing and deleting a public, indexable page pings `api.indexnow.org` with the page's
+ canonical URL and `/sitemap.xml` (the module batches and debounces; a failed ping never takes a
+ publish down). Drafts, private, members, VIP and noindex pages never ping. Unset: off — no key
+ file, no requests.
- **The gate**: `openvibe-publishing/seo` decides indexing per page with explicit reasons
- **The gate**: `openvibe-publishing/seo` decides indexing per page with explicit reasons
(policy: at least `WIKI_GATE_MIN_WORDS` words and `WIKI_GATE_MIN_SOURCES` citations; AI text and
(policy: at least `WIKI_GATE_MIN_WORDS` words and `WIKI_GATE_MIN_SOURCES` citations; AI text and
AI-assisted imports only after a person's review; owner-requested noindex). Only public, published, indexable pages enter sitemaps and
AI-assisted imports only after a person's review; owner-requested noindex). Only public, published, indexable pages enter sitemaps and
@@ -321,6 +327,10 @@ Called elsewhere, as the service principal `svc:wiki` (client credentials, one t
@@ -321,6 +327,10 @@ Called elsewhere, as the service principal `svc:wiki` (client credentials, one t
- every event is a valid `events.event-envelope@1` and every index document a valid
- every event is a valid `events.event-envelope@1` and every index document a valid
`search.index-document@1` (`visibility.test.js`); the proposals validate against the contracts
`search.index-document@1` (`visibility.test.js`); the proposals validate against the contracts
schemas (`proposals.test.js`); user text is escaped everywhere (`content.test.js`)
schemas (`proposals.test.js`); user text is escaped everywhere (`content.test.js`)
+- IndexNow: off without `INDEXNOW_KEY` (no key route, nothing sent); with one the key file is served
+ at `/<key>.txt` as `text/plain` and a publish, unpublish or delete of an indexable page pings the
+ page path and the sitemap; a draft, a noindex page and a page the gate will not index never ping
+ (`indexnow.test.js`)
## Security
## Security
@@ -22,7 +22,7 @@ const { createActorLimits } = require('./http/actor-limits');
@@ -22,7 +22,7 @@ const { createActorLimits } = require('./http/actor-limits');
const PUBLIC_DIR = path.join(__dirname, '..', 'public');
const PUBLIC_DIR = path.join(__dirname, '..', 'public');
const VERSION = require('../package.json').version;
const VERSION = require('../package.json').version;
-function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valkey = null, log = console, rateLimits = true, fetchImpl = globalThis.fetch, limitsNow = null }) {
+function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valkey = null, log = console, rateLimits = true, fetchImpl = globalThis.fetch, limitsNow = null, indexnow = null }) {
const app = express();
const app = express();
// Every resolved actor carries its wiki roles (read once per request), so the synchronous access checks work.
// Every resolved actor carries its wiki roles (read once per request), so the synchronous access checks work.
const viewers = { ...baseViewers, resolve: async (req, opts) => svc.loadRoles(await baseViewers.resolve(req, opts)) };
const viewers = { ...baseViewers, resolve: async (req, opts) => svc.loadRoles(await baseViewers.resolve(req, opts)) };
@@ -115,6 +115,8 @@ function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valk
@@ -115,6 +115,8 @@ function createApp({ config, svc, viewers: baseViewers, platform, keys, db, valk
res.setHeader('Cache-Control', v && v === require('./render/layout').assetVersion(rel) ? 'public, max-age=31536000, immutable' : 'public, max-age=300');
res.setHeader('Cache-Control', v && v === require('./render/layout').assetVersion(rel) ? 'public, max-age=31536000, immutable' : 'public, max-age=300');
},
},
}));
}));
+ // GET /<key>.txt — the IndexNow key file, only when a key is configured (server/index.js).
+ if (indexnow && indexnow.enabled) app.use(indexnow.keyFile);
app.use(createMachine({ svc, config }));
app.use(createMachine({ svc, config }));
app.post(['/new-space', '/s/*', '/w/*', '/proposals/*'], limiter(10 * 60000, 120));
app.post(['/new-space', '/s/*', '/w/*', '/proposals/*'], limiter(10 * 60000, 120));
@@ -79,6 +79,11 @@ function load(env = process.env) {
@@ -79,6 +79,11 @@ function load(env = process.env) {
scheduleIntervalMs: int(env.WIKI_SCHEDULE_INTERVAL_MS, 15000),
scheduleIntervalMs: int(env.WIKI_SCHEDULE_INTERVAL_MS, 15000),
workerId: env.WIKI_WORKER_ID || `wiki-${process.pid}`,
workerId: env.WIKI_WORKER_ID || `wiki-${process.pid}`,
+ // IndexNow (openvibe-shared/indexnow): with a key, search engines are told the moment a
+ // public, indexable page appears, changes or leaves the index (the key file is served at
+ // /<key>.txt). Unset: off — nothing is mounted and nothing is sent.
+ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
+
// The indexability gate policy for wiki pages (openvibe-publishing/seo).
// The indexability gate policy for wiki pages (openvibe-publishing/seo).
gate: {
gate: {
minWords: int(env.WIKI_GATE_MIN_WORDS, 80),
minWords: int(env.WIKI_GATE_MIN_WORDS, 80),
@@ -8,6 +8,7 @@
@@ -8,6 +8,7 @@
* clock/fetch/log/keys, and returns handles to every part. Workers (scheduled publication, the
* clock/fetch/log/keys, and returns handles to every part. Workers (scheduled publication, the
* Events outbox relay, the Media attachment check) run only when asked (`workers: true`).
* Events outbox relay, the Media attachment check) run only when asked (`workers: true`).
*/
*/
+const { createIndexNow } = require('openvibe-shared/indexnow');
const { load } = require('./config');
const { load } = require('./config');
const { openDb, migrate, createStores } = require('./db');
const { openDb, migrate, createStores } = require('./db');
const { createPlatform } = require('./integrations/platform');
const { createPlatform } = require('./integrations/platform');
@@ -16,23 +17,27 @@ const { createKeyStore } = require('./auth/keys');
@@ -16,23 +17,27 @@ const { createKeyStore } = require('./auth/keys');
const { createViewerResolver } = require('./auth/viewer');
const { createViewerResolver } = require('./auth/viewer');
const { createApp } = require('./app');
const { createApp } = require('./app');
-async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, tokens = null, publicKey = null, log = console, listen = true, workers = listen, rateLimits = true, limitsNow = null } = {}) {
+async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, tokens = null, publicKey = null, log = console, listen = true, workers = listen, rateLimits = true, limitsNow = null, indexnow: givenIndexNow = null } = {}) {
config = config || load();
config = config || load();
// PostgreSQL (ADR-035): migrate on the owner's direct connection, then serve on the pooled runtime role.
// PostgreSQL (ADR-035): migrate on the owner's direct connection, then serve on the pooled runtime role.
// Tests hand in a migrated database of their own (test/db-helper.js).
// Tests hand in a migrated database of their own (test/db-helper.js).
const db = givenDb || openDb(config, { log });
const db = givenDb || openDb(config, { log });
if (!givenDb) await migrate(config, { serving: db, log });
if (!givenDb) await migrate(config, { serving: db, log });
const stores = createStores(db, { now });
const stores = createStores(db, { now });
+ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY. Unset → off, nothing
+ // mounted and nothing sent; a test or a drill injects its own (a spy, or the module's own no-key
+ // state) through `indexnow`, and the outbound POST goes through `fetchImpl`, this process's stub.
+ const indexnow = givenIndexNow || createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: (...a) => (log.warn || console.warn)(...a) });
// Valkey (ADR-035): shared, never-authoritative state (per-actor limit counters). Optional.
// Valkey (ADR-035): shared, never-authoritative state (per-actor limit counters). Optional.
const valkey = config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null;
const valkey = config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null;
const platform = createPlatform({ config, db, fetchImpl, tokens, now, log });
const platform = createPlatform({ config, db, fetchImpl, tokens, now, log });
- const svc = createWikiService({ db, stores, outbox: platform.outbox, config, community: platform.community, vip: platform.vip, now, log });
+ const svc = createWikiService({ db, stores, outbox: platform.outbox, config, community: platform.community, vip: platform.vip, now, log, indexnow });
const reconciled = await svc.reconcileIndex();
const reconciled = await svc.reconcileIndex();
if (reconciled.sent) log.log(`[Wiki] re-sent ${reconciled.sent} Search document(s) whose indexability changed`);
if (reconciled.sent) log.log(`[Wiki] re-sent ${reconciled.sent} Search document(s) whose indexability changed`);
const keys = createKeyStore({ config, fetchImpl, log, publicKey });
const keys = createKeyStore({ config, fetchImpl, log, publicKey });
keys.ensure().catch(() => {});
keys.ensure().catch(() => {});
const viewers = createViewerResolver({ keys, config });
const viewers = createViewerResolver({ keys, config });
- const app = createApp({ config, svc, viewers, platform, keys, db, valkey, log, rateLimits, fetchImpl, limitsNow });
+ const app = createApp({ config, svc, viewers, platform, keys, db, valkey, log, rateLimits, fetchImpl, limitsNow, indexnow });
const timers = [];
const timers = [];
if (workers) {
if (workers) {
@@ -70,7 +75,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -70,7 +75,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
if (!givenDb) await db.close();
if (!givenDb) await db.close();
if (valkey) await valkey.close().catch(() => {});
if (valkey) await valkey.close().catch(() => {});
}
}
- return { app, db, svc, stores, platform, keys, viewers, server, config, stop };
+ return { app, db, svc, stores, platform, keys, viewers, server, config, indexnow, stop };
}
}
module.exports = { start };
module.exports = { start };
@@ -70,7 +70,7 @@ function disclosureFor(rec, review) {
@@ -70,7 +70,7 @@ function disclosureFor(rec, review) {
};
};
}
}
-function createWikiService({ db, stores, outbox, config, community = null, vip = null, now = () => Date.now(), log = console }) {
+function createWikiService({ db, stores, outbox, config, community = null, vip = null, now = () => Date.now(), log = console, indexnow = null }) {
const { revisions, citations, redirects, reviews, attachments, discussions, scheduler, sequencer } = stores;
const { revisions, citations, redirects, reviews, attachments, discussions, scheduler, sequencer } = stores;
const access = createAccess(db, { vip });
const access = createAccess(db, { vip });
const origin = config.baseUrl;
const origin = config.baseUrl;
@@ -367,7 +367,8 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
@@ -367,7 +367,8 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
publishedAt: page.published_at, updatedAt: page.revision_published_at,
publishedAt: page.published_at, updatedAt: page.revision_published_at,
}));
}));
const sentBefore = before ? before.indexRevision : null;
const sentBefore = before ? before.indexRevision : null;
- if (doc.revision !== sentBefore) await emit(hooks.indexEvent({ document: doc, now: now() }));
+ const indexChanged = doc.revision !== sentBefore;
+ if (indexChanged) await emit(hooks.indexEvent({ document: doc, now: now() }));
const after = { state, visibility: eff, revision: page.published_revision };
const after = { state, visibility: eff, revision: page.published_revision };
const wasPublic = Boolean(before) && before.state === 'published' && before.visibility === 'public';
const wasPublic = Boolean(before) && before.state === 'published' && before.visibility === 'public';
@@ -375,7 +376,7 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
@@ -375,7 +376,7 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
if (wasPublic !== isPublic) followThread(page.id, isPublic ? 'public' : 'hidden');
if (wasPublic !== isPublic) followThread(page.id, isPublic ? 'public' : 'hidden');
let action = hooks.actionFor(before && before.state ? before : null, after);
let action = hooks.actionFor(before && before.state ? before : null, after);
// A review changes only the gate decision: announce it when what Search holds changed.
// A review changes only the gate decision: announce it when what Search holds changed.
- if (!action && updatedIfIndexChanged && state === 'published' && doc.revision !== sentBefore) action = 'updated';
+ if (!action && updatedIfIndexChanged && state === 'published' && indexChanged) action = 'updated';
if (action) {
if (action) {
let evDoc = doc;
let evDoc = doc;
if (doc.deleted && (action === 'published' || action === 'updated')) {
if (doc.deleted && (action === 'published' || action === 'updated')) {
@@ -390,6 +391,21 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
@@ -390,6 +391,21 @@ function createWikiService({ db, stores, outbox, config, community = null, vip =
}));
}));
await notifyWatchers(space, page, action, page.published_revision, actor);
await notifyWatchers(space, page, action, page.published_revision, actor);
}
}
+ // IndexNow (openvibe-shared/indexnow): tell the engines a public, indexable page appeared,
+ // changed or went away; never for drafts, private, members/VIP or noindex pages. The sitemap is
+ // pinged alongside the page. Only when the publication state moved or Search was sent something
+ // new (a noindex flip moves no state but does change the index), after the commit — a write that
+ // rolls back must not ping — and the module batches and debounces. The previous state's own gate
+ // reasons are not re-read (they are gone by now): a page that was public and published counts as
+ // indexable when it leaves, unless it is still flagged noindex.
+ if (indexnow && indexnow.enabled) {
+ const indexable = state === 'published' && eff === 'public' && !page.noindex && Boolean(decision && decision.indexable);
+ const wasIndexable = Boolean(before) && before.state === 'published' && before.visibility === 'public' && !page.noindex;
+ if ((action || indexChanged) && (indexable || wasIndexable)) {
+ const urls = [pageUrl(space, page), seo.canonicalUrl(origin, '/sitemap.xml')];
+ afterCommit(() => indexnow.pingSoon(urls));
+ }
+ }
return { action, indexRevision: doc.revision, deleted: doc.deleted };
return { action, indexRevision: doc.revision, deleted: doc.deleted };
}
}
@@ -40,7 +40,7 @@ const quiet = process.env.WIKI_TEST_LOG ? console : { log() {}, warn() {}, error
@@ -40,7 +40,7 @@ const quiet = process.env.WIKI_TEST_LOG ? console : { log() {}, warn() {}, error
* A running Wiki on a database of its own (PGlite by default; WIKI_TEST_STORE=pg: the PostgreSQL + PgBouncer
* A running Wiki on a database of its own (PGlite by default; WIKI_TEST_STORE=pg: the PostgreSQL + PgBouncer
* containers). opts.db: boot on an existing handle (a restart keeps the data); h.stop() closes what boot opened.
* containers). opts.db: boot on an existing handle (a restart keeps the data); h.stop() closes what boot opened.
*/
*/
-async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, workers = false, tokens, rateLimits = false, limitsNow = null, log = quiet } = {}) {
+async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, workers = false, tokens, rateLimits = false, limitsNow = null, log = quiet, indexnow = undefined } = {}) {
const owned = givenDb ? null : await testDb();
const owned = givenDb ? null : await testDb();
const db = givenDb || owned.db;
const db = givenDb || owned.db;
const config = load({
const config = load({
@@ -52,6 +52,7 @@ async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, worke
@@ -52,6 +52,7 @@ async function boot({ env = {}, fetch: fetchImpl, db: givenDb = null, now, worke
config, db, publicKey, log, listen: true, workers, rateLimits, now, limitsNow,
config, db, publicKey, log, listen: true, workers, rateLimits, now, limitsNow,
fetchImpl: fetchImpl || (async (url) => { throw new Error(`unexpected outbound fetch ${url}`); }),
fetchImpl: fetchImpl || (async (url) => { throw new Error(`unexpected outbound fetch ${url}`); }),
tokens: tokens || { getToken: async () => 'stub-token', authHeaders: async () => ({ Authorization: 'Bearer stub-token' }), invalidate() {} },
tokens: tokens || { getToken: async () => 'stub-token', authHeaders: async () => ({ Authorization: 'Bearer stub-token' }), invalidate() {} },
+ indexnow,
});
});
const base = `http://127.0.0.1:${h.server.address().port}`;
const base = `http://127.0.0.1:${h.server.address().port}`;
const stop = async ({ keepDb = false } = {}) => { await h.stop(); if (owned && !keepDb) await owned.close(); };
const stop = async ({ keepDb = false } = {}) => { await h.stop(); if (owned && !keepDb) await owned.close(); };
new file mode 100644
new file mode 100644
@@ -0,0 +1,108 @@
@@ -0,0 +1,108 @@
+'use strict';
+/**
+ * IndexNow (openvibe-shared/indexnow): INDEXNOW_KEY unset → the feature is off (no key route,
+ * nothing sent). With a key the key file is served at /<key>.txt as text/plain, and a publish,
+ * unpublish or delete of an indexable page pings the engines with the page's path and the sitemap.
+ * Drafts, noindex pages and pages the gate will not index never ping.
+ */
+const assert = require('assert');
+const H = require('./helpers');
+
+const KEY = 'k'.repeat(32);
+const BASE = 'https://openvibe.wiki';
+const SITEMAP = `${BASE}/sitemap.xml`;
+const PAGE = `${BASE}/w/notes/first-light`;
+
+(async () => {
+ // Off: no INDEXNOW_KEY. Nothing is mounted and nothing is sent.
+ const off = await H.boot({ env: { BASE_URL: BASE } });
+ try {
+ assert.strictEqual(off.indexnow.enabled, false, 'IndexNow is off without a key');
+ assert.strictEqual((await H.req(off, 'GET', `/${KEY}.txt`)).status, 404, 'no key file is served');
+ } finally { await off.stop(); }
+
+ // With a key: the key file is served as text/plain with the key.
+ const on = await H.boot({ env: { BASE_URL: BASE, INDEXNOW_KEY: KEY } });
+ try {
+ assert.strictEqual(on.indexnow.enabled, true, 'IndexNow is on with a key');
+ const r = await H.req(on, 'GET', `/${KEY}.txt`);
+ assert.strictEqual(r.status, 200, r.text);
+ assert.match(r.headers.get('content-type'), /text\/plain/);
+ assert.strictEqual(r.text, KEY);
+ } finally { await on.stop(); }
+
+ // A spy in place of the module's HTTP send: records every pingSoon batch.
+ const pings = [];
+ const spy = {
+ enabled: true,
+ keyFile: (_req, _res, next) => next(),
+ pingSoon: (urls) => { const a = Array.isArray(urls) ? urls : [urls]; pings.push(...a); return a.length; },
+ ping: async () => ({ sent: 0, status: 0 }),
+ flush: async () => ({ sent: 0, status: 0 }),
+ };
+ const h = await H.boot({ env: { BASE_URL: BASE }, indexnow: spy });
+ const owner = { kind: 'user', subject: H.subject(), staff: false };
+ try {
+ const space = await h.svc.createSpace({ name: 'Notes', slug: 'notes' }, owner);
+ const cite = [{ url: 'https://example.org/a', retrievedAt: '2026-09-20T00:00:00Z' }];
+ const { page } = await h.svc.createPage(space.id, { title: 'First light', body: H.LONG, citations: cite }, owner);
+ assert.deepStrictEqual(pings, [], 'a draft never pings');
+
+ await h.svc.publish(page.id, {}, owner);
+ assert.ok(pings.includes(PAGE), `a publish pings the page path: ${JSON.stringify(pings)}`);
+ assert.ok(pings.includes(SITEMAP), `a publish pings the sitemap: ${JSON.stringify(pings)}`);
+
+ pings.length = 0;
+ await h.svc.unpublish(page.id, owner);
+ assert.ok(pings.includes(PAGE), `an unpublish pings the page path: ${JSON.stringify(pings)}`);
+ assert.ok(pings.includes(SITEMAP), `an unpublish pings the sitemap: ${JSON.stringify(pings)}`);
+
+ await h.svc.publish(page.id, {}, owner);
+ pings.length = 0;
+ await h.svc.setPageVisibility(page.id, { noindex: true }, owner);
+ assert.deepStrictEqual(pings, [], 'a page the owner keeps noindex never pings');
+ await h.svc.setPageVisibility(page.id, { noindex: false }, owner);
+ assert.ok(pings.includes(PAGE), `indexable again pings: ${JSON.stringify(pings)}`);
+
+ pings.length = 0;
+ await h.svc.deletePage(page.id, owner);
+ assert.ok(pings.includes(PAGE), `a delete pings the page path: ${JSON.stringify(pings)}`);
+ assert.ok(pings.includes(SITEMAP), `a delete pings the sitemap: ${JSON.stringify(pings)}`);
+
+ // A page the gate keeps out of the index (no source: the test policy requires one) never pings.
+ pings.length = 0;
+ const thin = (await h.svc.createPage(space.id, { title: 'No sources', body: H.LONG }, owner)).page;
+ await h.svc.publish(thin.id, {}, owner);
+ assert.deepStrictEqual(pings, [], 'a page the gate will not index never pings');
+ } finally {
+ await h.stop();
+ }
+
+ // The real module: a boot with a key batches what the service queued and POSTs it through this
+ // process's fetch, with the page's canonical URL and the sitemap (the module drops anything that
+ // is not an absolute https URL on its own host).
+ const sent = [];
+ const outbound = async (url, opts) => {
+ if (!String(url).includes('api.indexnow.org')) throw new Error(`unexpected outbound fetch ${url}`);
+ sent.push(JSON.parse(opts.body));
+ return { status: 200 };
+ };
+ const m = await H.boot({ env: { BASE_URL: BASE, INDEXNOW_KEY: KEY }, fetch: outbound });
+ const mOwner = { kind: 'user', subject: H.subject(), staff: false };
+ try {
+ const mSpace = await m.svc.createSpace({ name: 'Notes', slug: 'notes' }, mOwner);
+ const mCite = [{ url: 'https://example.org/a', retrievedAt: '2026-09-20T00:00:00Z' }];
+ const mPage = (await m.svc.createPage(mSpace.id, { title: 'First light', body: H.LONG, citations: mCite }, mOwner)).page;
+ await m.svc.publish(mPage.id, {}, mOwner);
+ assert.deepStrictEqual(sent, [], 'nothing is sent before the batch is flushed');
+ const out = await m.indexnow.flush();
+ assert.deepStrictEqual(sent.length, 1, 'one POST for the batch');
+ assert.strictEqual(sent[0].host, 'openvibe.wiki');
+ assert.strictEqual(sent[0].key, KEY);
+ assert.deepStrictEqual(sent[0].urlList.slice().sort(), [PAGE, SITEMAP].sort(), JSON.stringify(sent[0].urlList));
+ assert.strictEqual(out.sent, 2, JSON.stringify(out));
+ } finally {
+ await m.stop();
+ }
+ console.log('indexnow ok');
+})().catch((err) => { console.error(err); process.exit(1); });