openvibers/openvibe.events Commits

Serve /llms-full.txt and add IndexNow key file and release pings

OP OpenVibers committed 13 Oct 2026 · fceb61a605 · 10 files, +178 −20

Asked: Re-land a local-only branch whose push gitleaks refused. Start from origin/main. Bring the work over with: `git -C /mnt/120gb/openvibe-worktrees/OpenVibe.Events-agent-events-t11-step-4-dossier-home-workstati-225747-mv7a7fyq diff aa527ef` (committed and uncommitted changes; also `git -C <that path> …
Harness job events-re-land-a-local-only-branch-whose-232056.

Changes

ddiff --git a/.env.example b/.env.example
iindex a867cff..026315b 100644
---- a/.env.example
++++ b/.env.example
@@@ -16,6 +16,11 @@ NODE_ENV=development
# The product origin; defaults to BASE_URL (openvibe.events is the API and product origin, plan T7).
#EVENTS_SITE_URL=https://openvibe.events
++# IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and its
++# crawl files when they change (a release). 8–128 hex/alphanumeric characters; the key file is served
++# at /<key>.txt. Unset = off, no key file, nothing sent.
++#INDEXNOW_KEY=
++
# Identity — OpenVibe.Network signs service tokens (client_credentials, audience
# openvibe.events) and user JWTs. The public key is fetched from
# <url>/api/.well-known/jwks (internal URL first), retried every 30 s until it loads.
ddiff --git a/deploy/nginx/openvibe.events.conf b/deploy/nginx/openvibe.events.conf
iindex 5297ae6..8b61104 100644
---- a/deploy/nginx/openvibe.events.conf
++++ b/deploy/nginx/openvibe.events.conf
@@@ -12,8 +12,8 @@
## The Node service (port 4300) answers here:
## / the product home (server/home.js) for a browser, the text/plain API index otherwise
## /shared/ the OpenVibe Frame's pinned browser files (openvibe-shared/serve)
--## /robots.txt, /sitemap.xml, /llms.txt
--## discovery for the product domain (server/discovery.js)
++## /robots.txt, /sitemap.xml, /llms.txt, /llms-full.txt, /<INDEXNOW_KEY>.txt
++## discovery for the product domain (server/discovery.js; the IndexNow key file)
## /realtime/stream browser Server-Sent Events (long-lived; must not be buffered)
## /api/health, /api/ready, /release.json, /limits.json
## /api/v1/events, /api/v1/subscriptions, /api/v1/checkpoints
@@@ -181,6 +181,14 @@ server {
proxy_set_header CF-Connecting-IP $remote_addr;
}
++ location = /llms-full.txt {
++ proxy_pass http://127.0.0.1:4300;
++ proxy_set_header Host $host;
++ proxy_set_header X-Real-IP $remote_addr;
++ proxy_set_header X-Forwarded-For $remote_addr;
++ proxy_set_header CF-Connecting-IP $remote_addr;
++ }
++
# ── Everything else: the app answers GET/HEAD (its /favicon.ico, and its own 404 page for browsers or a
# problem+json for API clients, both with lang and the Frame); other methods are refused here ──
location / {
ddiff --git a/server/app.js b/server/app.js
iindex 866db28..cf86e05 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -20,7 +20,7 @@ const { createDiscoveryRoutes } = require('./discovery');
const ovServe = require('openvibe-shared/serve');
const pkg = require('../package.json');
--function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null }) {
++function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null, indexnow = null }) {
const app = express();
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
@@@ -96,9 +96,13 @@ function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLo
// for OpenVibe.Services' fan-out. First-party like the routes above; no per-actor limit of its own.
app.use(resourcesRouter({ store, auth }));
-- // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt.
++ // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt, llms-full.txt.
app.use(createDiscoveryRoutes({ config, releaseLastmod: release.full().released_at === release.full().booted_at ? undefined : release.full().released_at }));
++ // GET /<key>.txt — the IndexNow key file, mounted only when a key is configured (openvibe-shared/indexnow,
++ // from INDEXNOW_KEY in server/index.js). It answers its own path and passes everything else on.
++ if (indexnow && indexnow.enabled) app.use(indexnow.keyFile);
++
// GET / on the product domain: the home page for a browser (server/home.js, openvibe.events), the
// API index for curl and API clients (`*/*`, no Accept, or any non-HTML Accept). Vary: Accept so a
// shared cache never serves one to the other. The home carries its own CSP (HOME_CSP); the only
ddiff --git a/server/config.js b/server/config.js
iindex c517c22..5e37148 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -96,6 +96,10 @@ function load(env = process.env) {
// The product origin is the API origin (plan T7): the home page and its robots.txt/sitemap.xml/
// llms.txt are served from baseUrl. EVENTS_SITE_URL overrides it for a deployment that serves them apart.
siteUrl: (env.EVENTS_SITE_URL || baseUrl).replace(/\/$/, ''),
++ // IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and
++ // its crawl files when they change. 8–128 hex/alphanumeric characters; the key file is served
++ // at /<key>.txt. Unset: IndexNow is off — no key file, nothing sent.
++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
// Identity: OpenVibe.Network issues service tokens and user JWTs (RS256).
networkUrl: (env.OV_NETWORK_URL || 'https://openvibe.network').replace(/\/$/, ''),
ddiff --git a/server/discovery.js b/server/discovery.js
iindex 7fbab6d..763322a 100644
---- a/server/discovery.js
++++ b/server/discovery.js
@@@ -8,6 +8,8 @@
* GET /sitemap.xml the product home only (openvibe.events is the API origin too, plan T7)
* GET /llms.txt what OpenVibe.Events does today, in plain language: the README's Purpose,
* Publishing, Subscriptions and delivery, Developer apps and Realtime sections
++ * GET /llms-full.txt the same public pages in full text (openvibe-shared/seo.llmsFull): the
++ * /llms.txt header, then every page's full text, capped at 512 KiB
*
* Read-only and viewer-independent: built from the config (the product site URL and the API origin).
*/
@@@ -36,6 +38,28 @@ function details({ siteUrl, apiUrl }) {
].join('\n');
}
++const FULL_MAX_BYTES = 512 * 1024;
++
++/**
++ * The public pages this domain publishes, grouped the way /llms.txt lists them. One entry is one
++ * /llms.txt link (title, url, optional note) and one /llms-full.txt page (title, url, full text).
++ */
++function catalog({ siteUrl, apiUrl }) {
++ const home = seo.absolute('/', siteUrl);
++ return [
++ { title: 'Start here', links: [
++ { title: 'OpenVibe.Events home', url: home, text: DESCRIPTION },
++ { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime', text: details({ siteUrl, apiUrl }) },
++ { title: 'Source code', url: SOURCE, text: 'The OpenVibe.Events source on GitHub: the durable event store, the publish, read and subscription API, the delivery worker, the retention tiers and the realtime gateway.' },
++ ] },
++ { title: 'Machine-readable', links: [
++ { title: 'Sitemap', url: seo.absolute('/sitemap.xml', siteUrl), text: `The product home only: ${home}` },
++ { title: 'robots.txt', url: seo.absolute('/robots.txt', siteUrl), text: 'The crawler rules: the product home is welcome, the API and the realtime gateway are kept out of the index.' },
++ { title: 'API and realtime', url: apiUrl, note: 'the JSON API and the realtime gateway', text: `${apiUrl}/api/v1 is the token-guarded JSON API; ${apiUrl}/realtime/stream is the browser Server-Sent Events gateway.` },
++ ] },
++ ];
++}
++
function createDiscoveryRoutes({ config, releaseLastmod }) {
const router = express.Router();
const abs = (p) => seo.absolute(p, config.siteUrl);
@@@ -46,23 +70,24 @@ function createDiscoveryRoutes({ config, releaseLastmod }) {
router.get('/sitemap.xml', (_req, res) => res.type('application/xml').set('Cache-Control', cache.htmlHeaders())
.send(seo.sitemapXml([{ loc: abs('/'), lastmod: releaseLastmod }])));
++ const pages = catalog({ siteUrl: config.siteUrl, apiUrl: config.baseUrl });
++
router.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())
.send(seo.llmsTxt({
name: 'OpenVibe.Events',
summary: DESCRIPTION,
details: details({ siteUrl: config.siteUrl, apiUrl: config.baseUrl }),
-- sections: [
-- { title: 'Start here', links: [
-- { title: 'OpenVibe.Events home', url: abs('/') },
-- { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime' },
-- { title: 'Source code', url: SOURCE },
-- ] },
-- { title: 'Machine-readable', links: [
-- { title: 'Sitemap', url: abs('/sitemap.xml') },
-- { title: 'robots.txt', url: abs('/robots.txt') },
-- { title: 'API and realtime', url: config.baseUrl, note: 'the JSON API and the realtime gateway' },
-- ] },
-- ],
++ sections: pages.map((s) => ({ title: s.title, links: s.links.map((l) => ({ title: l.title, url: l.url, ...(l.note ? { note: l.note } : {}) })) })),
++ })));
++
++ // /llms-full.txt: the same pages as /llms.txt, each with its full plain text (openvibe-shared/seo.llmsFull).
++ router.get('/llms-full.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())
++ .send(seo.llmsFull({
++ site: 'OpenVibe.Events',
++ summary: DESCRIPTION,
++ base: config.siteUrl,
++ maxBytes: FULL_MAX_BYTES,
++ sections: pages.map((s) => ({ title: s.title, pages: s.links.map((l) => ({ title: l.title, url: l.url, text: l.text })) })),
})));
return router;
ddiff --git a/server/index.js b/server/index.js
iindex 35a6073..875013b 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -20,6 +20,8 @@ const { createApp } = require('./app');
const { createMetrics } = require('./metrics');
const { createGuardedPost } = require('./egress');
const { createBillingReadings } = require('./billing');
++const { createIndexNow } = require('openvibe-shared/indexnow');
++const indexnowNotify = require('./indexnow-notify');
const { gracefulStop } = require('openvibe-sdk/service');
async function start({
@@@ -47,7 +49,25 @@ async function start({
const worker = createWorker({ store, config, clock, fetchImpl: deliveryFetch, appPost: appPost || createGuardedPost({ lookup: dnsLookup }), log, observe: metrics.observe });
const realtime = createRealtime({ store, auth, config, clock, log });
metrics.bind({ realtime });
-- const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers });
++
++ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY. Unset → off, the key
++ // file is not mounted and nothing is sent. When set, GET /<key>.txt is the key file engines fetch
++ // before they accept a ping; the module owns the batching and the POST. A malformed key (createIndexNow
++ // refuses anything but 8–128 alphanumerics) disables IndexNow instead of crashing the service at boot.
++ let indexnow;
++ try {
++ indexnow = createIndexNow({ host: config.siteUrl, key: config.indexnow.key, fetch: fetchImpl, log });
++ } catch (err) {
++ log.warn(`[IndexNow] disabled: ${err.message}`);
++ indexnow = createIndexNow({ host: config.siteUrl, key: '', log });
++ }
++
++ const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers, indexnow });
++ app.locals.indexnow = indexnow;
++ // The pages crawlers see change only with a release, so this boot is the moment to announce them.
++ // A no-op without INDEXNOW_KEY.
++ indexnowNotify.use(indexnow, config.siteUrl);
++ indexnowNotify.pingRelease();
// The key loads in the background (retrying while Network boots); /api/ready says when it has.
const keyLoaded = keys.start().catch(() => null);
ddiff --git a/server/indexnow-notify.js b/server/indexnow-notify.js
nnew file mode 100644
iindex 0000000..152becc
---- /dev/null
++++ b/server/indexnow-notify.js
@@@ -0,0 +1,34 @@
++'use strict';
++/**
++ * OpenVibe.Events — tell search engines when the product home and its crawl files change.
++ *
++ * Nothing on this origin changes on its own: the pages crawlers see (the home and robots.txt,
++ * sitemap.xml, llms.txt and llms-full.txt) are fixed by the code, so a release is the only moment
++ * they do. server/index.js creates the IndexNow client once at boot (openvibe-shared/indexnow) and
++ * hands it here with use(); pingRelease() queues those pages for a single batched POST. With no
++ * INDEXNOW_KEY the client is disabled and every call is a no-op: nothing is ever sent.
++ */
++let client = null;
++let base = null;
++
++/** Install the client booted in server/index.js, with the site origin to ping. Disabled → pinging off. */
++function use(indexnow, siteUrl) {
++ client = indexnow && indexnow.enabled ? indexnow : null;
++ base = siteUrl ? String(siteUrl).replace(/\/$/, '') : null;
++}
++
++/**
++ * Queue the product home and its machine-readable files for one batched ping. Returns true when URLs
++ * were queued. Never throws (a failed ping must never fail a boot).
++ */
++function pingRelease() {
++ try {
++ if (!client || !base) return false;
++ return client.pingSoon([`${base}/`, `${base}/sitemap.xml`, `${base}/llms.txt`, `${base}/llms-full.txt`]) > 0;
++ } catch (err) {
++ console.warn('[IndexNow] notify failed:', err.message);
++ return false;
++ }
++}
++
++module.exports = { use, pingRelease };
ddiff --git a/test/home.test.js b/test/home.test.js
iindex 4df6338..c23eb96 100644
---- a/test/home.test.js
++++ b/test/home.test.js
@@@ -1,7 +1,7 @@
'use strict';
// The product home (plan T7: openvibe.events is the API and product origin): it renders server/home.js
// for a browser and keeps the text/plain API index for curl and API clients, serves the pinned OpenVibe
--// Frame at /shared, answers robots.txt, sitemap.xml and llms.txt, and names its own origin everywhere.
++// Frame at /shared, answers robots.txt, sitemap.xml, llms.txt and llms-full.txt, and names its own origin everywhere.
// A parse-only check of the nginx vhost too.
const assert = require('assert');
@@@ -154,6 +154,16 @@ t('llms.txt is a plain-text map that starts with the site name', async () => {
assert.ok(r.body.includes(`The API and the realtime gateway are at ${SITE}`), 'the discovery names the API origin');
});
++t('llms-full.txt is the same map in full text, starting with the site name', async () => {
++ const r = await raw(h.base, '/llms-full.txt');
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers['content-type'], /^text\/plain/);
++ assert.ok(r.body.startsWith('# OpenVibe.Events'), r.body.slice(0, 40));
++ assert.ok(r.body.includes('> Durable events for the OpenVibe network'), 'the home summary');
++ assert.ok(r.body.includes('## Start here'), 'the /llms.txt sections as pages');
++ assert.ok(r.body.includes('### Guide'), 'each page with its full text');
++});
++
t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redirect to the old host', () => {
const conf = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', 'openvibe.events.conf'), 'utf8');
assert.match(conf, /server_name openvibe\.events;/);
@@@ -162,7 +172,7 @@ t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redi
assert.ok(conf.includes('/etc/letsencrypt/live/openvibe.events/privkey.pem'));
assert.ok(!conf.includes('$proxy_add_x_forwarded_for'), 'the client address is never appended to');
// The product surface (plan expansion II) and everything the API host proxies (plan T7).
-- for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {',
++ for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {', 'location = /llms-full.txt {',
'location = /realtime/stream {', 'location = /api/health', 'location = /api/ready', 'location = /release.json', 'location = /limits.json']) {
assert.ok(conf.includes(loc), `${loc} is proxied`);
}
ddiff --git a/test/indexnow.test.js b/test/indexnow.test.js
nnew file mode 100644
iindex 0000000..cf9443d
---- /dev/null
++++ b/test/indexnow.test.js
@@@ -0,0 +1,47 @@
++'use strict';
++// IndexNow (openvibe-shared/indexnow): the key file engines fetch is served at /<INDEXNOW_KEY>.txt only
++// when a key is configured; unset, the path is not mounted (404) and nothing is ever sent. The client is
++// created once at boot from INDEXNOW_KEY (server/index.js).
++const assert = require('assert');
++const nodeHttp = require('http');
++const { boot, suite } = require('./helpers');
++
++const KEY = require('crypto').randomBytes(16).toString('hex'); // 32 hex characters: the 8–128 IndexNow wants
++const SITE = 'https://openvibe.events';
++
++const t = suite('indexnow');
++let without, withKey;
++
++/** A raw GET with no Accept a browser would send. */
++function raw(base, p) {
++ return new Promise((resolve, reject) => {
++ const url = new URL(base + p);
++ nodeHttp.get({ hostname: url.hostname, port: url.port, path: url.pathname + url.search }, (res) => {
++ let body = '';
++ res.setEncoding('utf8');
++ res.on('data', (c) => { body += c; });
++ res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body }));
++ }).on('error', reject);
++ });
++}
++
++t('boot without INDEXNOW_KEY', async () => { without = await boot({ env: { BASE_URL: SITE } }); });
++t('boot with INDEXNOW_KEY', async () => { withKey = await boot({ env: { BASE_URL: SITE, INDEXNOW_KEY: KEY } }); });
++
++t('with no key configured the key file is 404', async () => {
++ const r = await raw(without.base, `/${KEY}.txt`);
++ assert.strictEqual(r.status, 404, 'no /<key>.txt without INDEXNOW_KEY');
++});
++
++t('with a key configured the key file returns the key and nothing else', async () => {
++ const r = await raw(withKey.base, `/${KEY}.txt`);
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers['content-type'], /^text\/plain/);
++ assert.strictEqual(r.body, KEY);
++ const other = await raw(withKey.base, '/another-key.txt');
++ assert.strictEqual(other.status, 404, 'only the configured key file is served');
++});
++
++t('stop', async () => { await without.stop(); await withKey.stop(); });
++
++t.run();
ddiff --git a/test/security-secrets.test.js b/test/security-secrets.test.js
iindex b0ae8ee..0abef92 100644
---- a/test/security-secrets.test.js
++++ b/test/security-secrets.test.js
@@@ -44,7 +44,8 @@ t('Events reads no secret from its environment but the planted ones', async () =
};
walk(path.join(__dirname, '..', 'server'));
assert.ok(names.has('DATABASE_URL') && names.has('OV_NETWORK_PUBLIC_KEY'), 'the scan sees the config');
-- const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && !PLANTED[n]);
++ // OV_NETWORK_PUBLIC_KEY and INDEXNOW_KEY are public by design (the IndexNow key is served at /<key>.txt).
++ const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && n !== 'INDEXNOW_KEY' && !PLANTED[n]);
assert.deepStrictEqual(secretish, [], 'a new secret-looking variable: give it a sentinel in this test (PLANTED)');
});