openvibers/openvibe.events Commits
Serve /llms-full.txt and add IndexNow key file and release pings
Changes
@@@ -16,6 +16,11 @@ NODE_ENV=development# The product origin; defaults to BASE_URL (openvibe.events is the API and product origin, plan T7).#EVENTS_SITE_URL=https://openvibe.events++# IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and its++# crawl files when they change (a release). 8–128 hex/alphanumeric characters; the key file is served++# at /<key>.txt. Unset = off, no key file, nothing sent.++#INDEXNOW_KEY=++# Identity — OpenVibe.Network signs service tokens (client_credentials, audience# openvibe.events) and user JWTs. The public key is fetched from# <url>/api/.well-known/jwks (internal URL first), retried every 30 s until it loads.@@@ -12,8 +12,8 @@## The Node service (port 4300) answers here:## / the product home (server/home.js) for a browser, the text/plain API index otherwise## /shared/ the OpenVibe Frame's pinned browser files (openvibe-shared/serve)--## /robots.txt, /sitemap.xml, /llms.txt--## discovery for the product domain (server/discovery.js)++## /robots.txt, /sitemap.xml, /llms.txt, /llms-full.txt, /<INDEXNOW_KEY>.txt++## discovery for the product domain (server/discovery.js; the IndexNow key file)## /realtime/stream browser Server-Sent Events (long-lived; must not be buffered)## /api/health, /api/ready, /release.json, /limits.json## /api/v1/events, /api/v1/subscriptions, /api/v1/checkpoints@@@ -181,6 +181,14 @@ server {proxy_set_header CF-Connecting-IP $remote_addr;}++ location = /llms-full.txt {++ proxy_pass http://127.0.0.1:4300;++ proxy_set_header Host $host;++ proxy_set_header X-Real-IP $remote_addr;++ proxy_set_header X-Forwarded-For $remote_addr;++ proxy_set_header CF-Connecting-IP $remote_addr;++ }++# ── Everything else: the app answers GET/HEAD (its /favicon.ico, and its own 404 page for browsers or a# problem+json for API clients, both with lang and the Frame); other methods are refused here ──location / {@@@ -20,7 +20,7 @@ const { createDiscoveryRoutes } = require('./discovery');const ovServe = require('openvibe-shared/serve');const pkg = require('../package.json');--function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null }) {++function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null, indexnow = null }) {const app = express();app.disable('x-powered-by');app.set('trust proxy', 'loopback');@@@ -96,9 +96,13 @@ function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLo// for OpenVibe.Services' fan-out. First-party like the routes above; no per-actor limit of its own.app.use(resourcesRouter({ store, auth }));-- // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt.++ // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt, llms-full.txt.app.use(createDiscoveryRoutes({ config, releaseLastmod: release.full().released_at === release.full().booted_at ? undefined : release.full().released_at }));++ // GET /<key>.txt — the IndexNow key file, mounted only when a key is configured (openvibe-shared/indexnow,++ // from INDEXNOW_KEY in server/index.js). It answers its own path and passes everything else on.++ if (indexnow && indexnow.enabled) app.use(indexnow.keyFile);++// GET / on the product domain: the home page for a browser (server/home.js, openvibe.events), the// API index for curl and API clients (`*/*`, no Accept, or any non-HTML Accept). Vary: Accept so a// shared cache never serves one to the other. The home carries its own CSP (HOME_CSP); the only@@@ -96,6 +96,10 @@ function load(env = process.env) {// The product origin is the API origin (plan T7): the home page and its robots.txt/sitemap.xml/// llms.txt are served from baseUrl. EVENTS_SITE_URL overrides it for a deployment that serves them apart.siteUrl: (env.EVENTS_SITE_URL || baseUrl).replace(/\/$/, ''),++ // IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and++ // its crawl files when they change. 8–128 hex/alphanumeric characters; the key file is served++ // at /<key>.txt. Unset: IndexNow is off — no key file, nothing sent.++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },// Identity: OpenVibe.Network issues service tokens and user JWTs (RS256).networkUrl: (env.OV_NETWORK_URL || 'https://openvibe.network').replace(/\/$/, ''),@@@ -8,6 +8,8 @@* GET /sitemap.xml the product home only (openvibe.events is the API origin too, plan T7)* GET /llms.txt what OpenVibe.Events does today, in plain language: the README's Purpose,* Publishing, Subscriptions and delivery, Developer apps and Realtime sections++ * GET /llms-full.txt the same public pages in full text (openvibe-shared/seo.llmsFull): the++ * /llms.txt header, then every page's full text, capped at 512 KiB** Read-only and viewer-independent: built from the config (the product site URL and the API origin).*/@@@ -36,6 +38,28 @@ function details({ siteUrl, apiUrl }) {].join('\n');}++const FULL_MAX_BYTES = 512 * 1024;++++/**++ * The public pages this domain publishes, grouped the way /llms.txt lists them. One entry is one++ * /llms.txt link (title, url, optional note) and one /llms-full.txt page (title, url, full text).++ */++function catalog({ siteUrl, apiUrl }) {++ const home = seo.absolute('/', siteUrl);++ return [++ { title: 'Start here', links: [++ { title: 'OpenVibe.Events home', url: home, text: DESCRIPTION },++ { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime', text: details({ siteUrl, apiUrl }) },++ { title: 'Source code', url: SOURCE, text: 'The OpenVibe.Events source on GitHub: the durable event store, the publish, read and subscription API, the delivery worker, the retention tiers and the realtime gateway.' },++ ] },++ { title: 'Machine-readable', links: [++ { title: 'Sitemap', url: seo.absolute('/sitemap.xml', siteUrl), text: `The product home only: ${home}` },++ { title: 'robots.txt', url: seo.absolute('/robots.txt', siteUrl), text: 'The crawler rules: the product home is welcome, the API and the realtime gateway are kept out of the index.' },++ { title: 'API and realtime', url: apiUrl, note: 'the JSON API and the realtime gateway', text: `${apiUrl}/api/v1 is the token-guarded JSON API; ${apiUrl}/realtime/stream is the browser Server-Sent Events gateway.` },++ ] },++ ];++}++function createDiscoveryRoutes({ config, releaseLastmod }) {const router = express.Router();const abs = (p) => seo.absolute(p, config.siteUrl);@@@ -46,23 +70,24 @@ function createDiscoveryRoutes({ config, releaseLastmod }) {router.get('/sitemap.xml', (_req, res) => res.type('application/xml').set('Cache-Control', cache.htmlHeaders()).send(seo.sitemapXml([{ loc: abs('/'), lastmod: releaseLastmod }])));++ const pages = catalog({ siteUrl: config.siteUrl, apiUrl: config.baseUrl });++router.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders()).send(seo.llmsTxt({name: 'OpenVibe.Events',summary: DESCRIPTION,details: details({ siteUrl: config.siteUrl, apiUrl: config.baseUrl }),-- sections: [-- { title: 'Start here', links: [-- { title: 'OpenVibe.Events home', url: abs('/') },-- { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime' },-- { title: 'Source code', url: SOURCE },-- ] },-- { title: 'Machine-readable', links: [-- { title: 'Sitemap', url: abs('/sitemap.xml') },-- { title: 'robots.txt', url: abs('/robots.txt') },-- { title: 'API and realtime', url: config.baseUrl, note: 'the JSON API and the realtime gateway' },-- ] },-- ],++ sections: pages.map((s) => ({ title: s.title, links: s.links.map((l) => ({ title: l.title, url: l.url, ...(l.note ? { note: l.note } : {}) })) })),++ })));++++ // /llms-full.txt: the same pages as /llms.txt, each with its full plain text (openvibe-shared/seo.llmsFull).++ router.get('/llms-full.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())++ .send(seo.llmsFull({++ site: 'OpenVibe.Events',++ summary: DESCRIPTION,++ base: config.siteUrl,++ maxBytes: FULL_MAX_BYTES,++ sections: pages.map((s) => ({ title: s.title, pages: s.links.map((l) => ({ title: l.title, url: l.url, text: l.text })) })),})));return router;@@@ -20,6 +20,8 @@ const { createApp } = require('./app');const { createMetrics } = require('./metrics');const { createGuardedPost } = require('./egress');const { createBillingReadings } = require('./billing');++const { createIndexNow } = require('openvibe-shared/indexnow');++const indexnowNotify = require('./indexnow-notify');const { gracefulStop } = require('openvibe-sdk/service');async function start({@@@ -47,7 +49,25 @@ async function start({const worker = createWorker({ store, config, clock, fetchImpl: deliveryFetch, appPost: appPost || createGuardedPost({ lookup: dnsLookup }), log, observe: metrics.observe });const realtime = createRealtime({ store, auth, config, clock, log });metrics.bind({ realtime });-- const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers });++++ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY. Unset → off, the key++ // file is not mounted and nothing is sent. When set, GET /<key>.txt is the key file engines fetch++ // before they accept a ping; the module owns the batching and the POST. A malformed key (createIndexNow++ // refuses anything but 8–128 alphanumerics) disables IndexNow instead of crashing the service at boot.++ let indexnow;++ try {++ indexnow = createIndexNow({ host: config.siteUrl, key: config.indexnow.key, fetch: fetchImpl, log });++ } catch (err) {++ log.warn(`[IndexNow] disabled: ${err.message}`);++ indexnow = createIndexNow({ host: config.siteUrl, key: '', log });++ }++++ const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers, indexnow });++ app.locals.indexnow = indexnow;++ // The pages crawlers see change only with a release, so this boot is the moment to announce them.++ // A no-op without INDEXNOW_KEY.++ indexnowNotify.use(indexnow, config.siteUrl);++ indexnowNotify.pingRelease();// The key loads in the background (retrying while Network boots); /api/ready says when it has.const keyLoaded = keys.start().catch(() => null);nnew file mode 100644@@@ -0,0 +1,34 @@++'use strict';++/**++ * OpenVibe.Events — tell search engines when the product home and its crawl files change.++ *++ * Nothing on this origin changes on its own: the pages crawlers see (the home and robots.txt,++ * sitemap.xml, llms.txt and llms-full.txt) are fixed by the code, so a release is the only moment++ * they do. server/index.js creates the IndexNow client once at boot (openvibe-shared/indexnow) and++ * hands it here with use(); pingRelease() queues those pages for a single batched POST. With no++ * INDEXNOW_KEY the client is disabled and every call is a no-op: nothing is ever sent.++ */++let client = null;++let base = null;++++/** Install the client booted in server/index.js, with the site origin to ping. Disabled → pinging off. */++function use(indexnow, siteUrl) {++ client = indexnow && indexnow.enabled ? indexnow : null;++ base = siteUrl ? String(siteUrl).replace(/\/$/, '') : null;++}++++/**++ * Queue the product home and its machine-readable files for one batched ping. Returns true when URLs++ * were queued. Never throws (a failed ping must never fail a boot).++ */++function pingRelease() {++ try {++ if (!client || !base) return false;++ return client.pingSoon([`${base}/`, `${base}/sitemap.xml`, `${base}/llms.txt`, `${base}/llms-full.txt`]) > 0;++ } catch (err) {++ console.warn('[IndexNow] notify failed:', err.message);++ return false;++ }++}++++module.exports = { use, pingRelease };@@@ -1,7 +1,7 @@'use strict';// The product home (plan T7: openvibe.events is the API and product origin): it renders server/home.js// for a browser and keeps the text/plain API index for curl and API clients, serves the pinned OpenVibe--// Frame at /shared, answers robots.txt, sitemap.xml and llms.txt, and names its own origin everywhere.++// Frame at /shared, answers robots.txt, sitemap.xml, llms.txt and llms-full.txt, and names its own origin everywhere.// A parse-only check of the nginx vhost too.const assert = require('assert');@@@ -154,6 +154,16 @@ t('llms.txt is a plain-text map that starts with the site name', async () => {assert.ok(r.body.includes(`The API and the realtime gateway are at ${SITE}`), 'the discovery names the API origin');});++t('llms-full.txt is the same map in full text, starting with the site name', async () => {++ const r = await raw(h.base, '/llms-full.txt');++ assert.strictEqual(r.status, 200);++ assert.match(r.headers['content-type'], /^text\/plain/);++ assert.ok(r.body.startsWith('# OpenVibe.Events'), r.body.slice(0, 40));++ assert.ok(r.body.includes('> Durable events for the OpenVibe network'), 'the home summary');++ assert.ok(r.body.includes('## Start here'), 'the /llms.txt sections as pages');++ assert.ok(r.body.includes('### Guide'), 'each page with its full text');++});++t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redirect to the old host', () => {const conf = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', 'openvibe.events.conf'), 'utf8');assert.match(conf, /server_name openvibe\.events;/);@@@ -162,7 +172,7 @@ t('deploy/nginx/openvibe.events.conf is the API and product origin, with no rediassert.ok(conf.includes('/etc/letsencrypt/live/openvibe.events/privkey.pem'));assert.ok(!conf.includes('$proxy_add_x_forwarded_for'), 'the client address is never appended to');// The product surface (plan expansion II) and everything the API host proxies (plan T7).-- for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {',++ for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {', 'location = /llms-full.txt {','location = /realtime/stream {', 'location = /api/health', 'location = /api/ready', 'location = /release.json', 'location = /limits.json']) {assert.ok(conf.includes(loc), `${loc} is proxied`);}nnew file mode 100644@@@ -0,0 +1,47 @@++'use strict';++// IndexNow (openvibe-shared/indexnow): the key file engines fetch is served at /<INDEXNOW_KEY>.txt only++// when a key is configured; unset, the path is not mounted (404) and nothing is ever sent. The client is++// created once at boot from INDEXNOW_KEY (server/index.js).++const assert = require('assert');++const nodeHttp = require('http');++const { boot, suite } = require('./helpers');++++const KEY = require('crypto').randomBytes(16).toString('hex'); // 32 hex characters: the 8–128 IndexNow wants++const SITE = 'https://openvibe.events';++++const t = suite('indexnow');++let without, withKey;++++/** A raw GET with no Accept a browser would send. */++function raw(base, p) {++ return new Promise((resolve, reject) => {++ const url = new URL(base + p);++ nodeHttp.get({ hostname: url.hostname, port: url.port, path: url.pathname + url.search }, (res) => {++ let body = '';++ res.setEncoding('utf8');++ res.on('data', (c) => { body += c; });++ res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body }));++ }).on('error', reject);++ });++}++++t('boot without INDEXNOW_KEY', async () => { without = await boot({ env: { BASE_URL: SITE } }); });++t('boot with INDEXNOW_KEY', async () => { withKey = await boot({ env: { BASE_URL: SITE, INDEXNOW_KEY: KEY } }); });++++t('with no key configured the key file is 404', async () => {++ const r = await raw(without.base, `/${KEY}.txt`);++ assert.strictEqual(r.status, 404, 'no /<key>.txt without INDEXNOW_KEY');++});++++t('with a key configured the key file returns the key and nothing else', async () => {++ const r = await raw(withKey.base, `/${KEY}.txt`);++ assert.strictEqual(r.status, 200);++ assert.match(r.headers['content-type'], /^text\/plain/);++ assert.strictEqual(r.body, KEY);++ const other = await raw(withKey.base, '/another-key.txt');++ assert.strictEqual(other.status, 404, 'only the configured key file is served');++});++++t('stop', async () => { await without.stop(); await withKey.stop(); });++++t.run();@@@ -44,7 +44,8 @@ t('Events reads no secret from its environment but the planted ones', async () =};walk(path.join(__dirname, '..', 'server'));assert.ok(names.has('DATABASE_URL') && names.has('OV_NETWORK_PUBLIC_KEY'), 'the scan sees the config');-- const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && !PLANTED[n]);++ // OV_NETWORK_PUBLIC_KEY and INDEXNOW_KEY are public by design (the IndexNow key is served at /<key>.txt).++ const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && n !== 'INDEXNOW_KEY' && !PLANTED[n]);assert.deepStrictEqual(secretish, [], 'a new secret-looking variable: give it a sentinel in this test (PLANTED)');});
@@ -16,6 +16,11 @@ NODE_ENV=development
@@ -16,6 +16,11 @@ NODE_ENV=development
# The product origin; defaults to BASE_URL (openvibe.events is the API and product origin, plan T7).
# The product origin; defaults to BASE_URL (openvibe.events is the API and product origin, plan T7).
#EVENTS_SITE_URL=https://openvibe.events
#EVENTS_SITE_URL=https://openvibe.events
+# IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and its
+# crawl files when they change (a release). 8–128 hex/alphanumeric characters; the key file is served
+# at /<key>.txt. Unset = off, no key file, nothing sent.
+#INDEXNOW_KEY=
+
# Identity — OpenVibe.Network signs service tokens (client_credentials, audience
# Identity — OpenVibe.Network signs service tokens (client_credentials, audience
# openvibe.events) and user JWTs. The public key is fetched from
# openvibe.events) and user JWTs. The public key is fetched from
# <url>/api/.well-known/jwks (internal URL first), retried every 30 s until it loads.
# <url>/api/.well-known/jwks (internal URL first), retried every 30 s until it loads.
@@ -12,8 +12,8 @@
@@ -12,8 +12,8 @@
## The Node service (port 4300) answers here:
## The Node service (port 4300) answers here:
## / the product home (server/home.js) for a browser, the text/plain API index otherwise
## / the product home (server/home.js) for a browser, the text/plain API index otherwise
## /shared/ the OpenVibe Frame's pinned browser files (openvibe-shared/serve)
## /shared/ the OpenVibe Frame's pinned browser files (openvibe-shared/serve)
-## /robots.txt, /sitemap.xml, /llms.txt
-## discovery for the product domain (server/discovery.js)
+## /robots.txt, /sitemap.xml, /llms.txt, /llms-full.txt, /<INDEXNOW_KEY>.txt
+## discovery for the product domain (server/discovery.js; the IndexNow key file)
## /realtime/stream browser Server-Sent Events (long-lived; must not be buffered)
## /realtime/stream browser Server-Sent Events (long-lived; must not be buffered)
## /api/health, /api/ready, /release.json, /limits.json
## /api/health, /api/ready, /release.json, /limits.json
## /api/v1/events, /api/v1/subscriptions, /api/v1/checkpoints
## /api/v1/events, /api/v1/subscriptions, /api/v1/checkpoints
@@ -181,6 +181,14 @@ server {
@@ -181,6 +181,14 @@ server {
proxy_set_header CF-Connecting-IP $remote_addr;
proxy_set_header CF-Connecting-IP $remote_addr;
}
}
+ location = /llms-full.txt {
+ proxy_pass http://127.0.0.1:4300;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header CF-Connecting-IP $remote_addr;
+ }
+
# ── Everything else: the app answers GET/HEAD (its /favicon.ico, and its own 404 page for browsers or a
# ── Everything else: the app answers GET/HEAD (its /favicon.ico, and its own 404 page for browsers or a
# problem+json for API clients, both with lang and the Frame); other methods are refused here ──
# problem+json for API clients, both with lang and the Frame); other methods are refused here ──
location / {
location / {
@@ -20,7 +20,7 @@ const { createDiscoveryRoutes } = require('./discovery');
@@ -20,7 +20,7 @@ const { createDiscoveryRoutes } = require('./discovery');
const ovServe = require('openvibe-shared/serve');
const ovServe = require('openvibe-shared/serve');
const pkg = require('../package.json');
const pkg = require('../package.json');
-function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null }) {
+function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log = console, valkey = null, policies = null, planner = null, carriers = null, indexnow = null }) {
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
app.set('trust proxy', 'loopback');
@@ -96,9 +96,13 @@ function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLo
@@ -96,9 +96,13 @@ function createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLo
// for OpenVibe.Services' fan-out. First-party like the routes above; no per-actor limit of its own.
// for OpenVibe.Services' fan-out. First-party like the routes above; no per-actor limit of its own.
app.use(resourcesRouter({ store, auth }));
app.use(resourcesRouter({ store, auth }));
- // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt.
+ // Discovery for the product domain (openvibe.events): robots.txt, sitemap.xml, llms.txt, llms-full.txt.
app.use(createDiscoveryRoutes({ config, releaseLastmod: release.full().released_at === release.full().booted_at ? undefined : release.full().released_at }));
app.use(createDiscoveryRoutes({ config, releaseLastmod: release.full().released_at === release.full().booted_at ? undefined : release.full().released_at }));
+ // GET /<key>.txt — the IndexNow key file, mounted only when a key is configured (openvibe-shared/indexnow,
+ // from INDEXNOW_KEY in server/index.js). It answers its own path and passes everything else on.
+ if (indexnow && indexnow.enabled) app.use(indexnow.keyFile);
+
// GET / on the product domain: the home page for a browser (server/home.js, openvibe.events), the
// GET / on the product domain: the home page for a browser (server/home.js, openvibe.events), the
// API index for curl and API clients (`*/*`, no Accept, or any non-HTML Accept). Vary: Accept so a
// API index for curl and API clients (`*/*`, no Accept, or any non-HTML Accept). Vary: Accept so a
// shared cache never serves one to the other. The home carries its own CSP (HOME_CSP); the only
// shared cache never serves one to the other. The home carries its own CSP (HOME_CSP); the only
@@ -96,6 +96,10 @@ function load(env = process.env) {
@@ -96,6 +96,10 @@ function load(env = process.env) {
// The product origin is the API origin (plan T7): the home page and its robots.txt/sitemap.xml/
// The product origin is the API origin (plan T7): the home page and its robots.txt/sitemap.xml/
// llms.txt are served from baseUrl. EVENTS_SITE_URL overrides it for a deployment that serves them apart.
// llms.txt are served from baseUrl. EVENTS_SITE_URL overrides it for a deployment that serves them apart.
siteUrl: (env.EVENTS_SITE_URL || baseUrl).replace(/\/$/, ''),
siteUrl: (env.EVENTS_SITE_URL || baseUrl).replace(/\/$/, ''),
+ // IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl the product home and
+ // its crawl files when they change. 8–128 hex/alphanumeric characters; the key file is served
+ // at /<key>.txt. Unset: IndexNow is off — no key file, nothing sent.
+ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
// Identity: OpenVibe.Network issues service tokens and user JWTs (RS256).
// Identity: OpenVibe.Network issues service tokens and user JWTs (RS256).
networkUrl: (env.OV_NETWORK_URL || 'https://openvibe.network').replace(/\/$/, ''),
networkUrl: (env.OV_NETWORK_URL || 'https://openvibe.network').replace(/\/$/, ''),
@@ -8,6 +8,8 @@
@@ -8,6 +8,8 @@
* GET /sitemap.xml the product home only (openvibe.events is the API origin too, plan T7)
* GET /sitemap.xml the product home only (openvibe.events is the API origin too, plan T7)
* GET /llms.txt what OpenVibe.Events does today, in plain language: the README's Purpose,
* GET /llms.txt what OpenVibe.Events does today, in plain language: the README's Purpose,
* Publishing, Subscriptions and delivery, Developer apps and Realtime sections
* Publishing, Subscriptions and delivery, Developer apps and Realtime sections
+ * GET /llms-full.txt the same public pages in full text (openvibe-shared/seo.llmsFull): the
+ * /llms.txt header, then every page's full text, capped at 512 KiB
*
*
* Read-only and viewer-independent: built from the config (the product site URL and the API origin).
* Read-only and viewer-independent: built from the config (the product site URL and the API origin).
*/
*/
@@ -36,6 +38,28 @@ function details({ siteUrl, apiUrl }) {
@@ -36,6 +38,28 @@ function details({ siteUrl, apiUrl }) {
].join('\n');
].join('\n');
}
}
+const FULL_MAX_BYTES = 512 * 1024;
+
+/**
+ * The public pages this domain publishes, grouped the way /llms.txt lists them. One entry is one
+ * /llms.txt link (title, url, optional note) and one /llms-full.txt page (title, url, full text).
+ */
+function catalog({ siteUrl, apiUrl }) {
+ const home = seo.absolute('/', siteUrl);
+ return [
+ { title: 'Start here', links: [
+ { title: 'OpenVibe.Events home', url: home, text: DESCRIPTION },
+ { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime', text: details({ siteUrl, apiUrl }) },
+ { title: 'Source code', url: SOURCE, text: 'The OpenVibe.Events source on GitHub: the durable event store, the publish, read and subscription API, the delivery worker, the retention tiers and the realtime gateway.' },
+ ] },
+ { title: 'Machine-readable', links: [
+ { title: 'Sitemap', url: seo.absolute('/sitemap.xml', siteUrl), text: `The product home only: ${home}` },
+ { title: 'robots.txt', url: seo.absolute('/robots.txt', siteUrl), text: 'The crawler rules: the product home is welcome, the API and the realtime gateway are kept out of the index.' },
+ { title: 'API and realtime', url: apiUrl, note: 'the JSON API and the realtime gateway', text: `${apiUrl}/api/v1 is the token-guarded JSON API; ${apiUrl}/realtime/stream is the browser Server-Sent Events gateway.` },
+ ] },
+ ];
+}
+
function createDiscoveryRoutes({ config, releaseLastmod }) {
function createDiscoveryRoutes({ config, releaseLastmod }) {
const router = express.Router();
const router = express.Router();
const abs = (p) => seo.absolute(p, config.siteUrl);
const abs = (p) => seo.absolute(p, config.siteUrl);
@@ -46,23 +70,24 @@ function createDiscoveryRoutes({ config, releaseLastmod }) {
@@ -46,23 +70,24 @@ function createDiscoveryRoutes({ config, releaseLastmod }) {
router.get('/sitemap.xml', (_req, res) => res.type('application/xml').set('Cache-Control', cache.htmlHeaders())
router.get('/sitemap.xml', (_req, res) => res.type('application/xml').set('Cache-Control', cache.htmlHeaders())
.send(seo.sitemapXml([{ loc: abs('/'), lastmod: releaseLastmod }])));
.send(seo.sitemapXml([{ loc: abs('/'), lastmod: releaseLastmod }])));
+ const pages = catalog({ siteUrl: config.siteUrl, apiUrl: config.baseUrl });
+
router.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())
router.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())
.send(seo.llmsTxt({
.send(seo.llmsTxt({
name: 'OpenVibe.Events',
name: 'OpenVibe.Events',
summary: DESCRIPTION,
summary: DESCRIPTION,
details: details({ siteUrl: config.siteUrl, apiUrl: config.baseUrl }),
details: details({ siteUrl: config.siteUrl, apiUrl: config.baseUrl }),
- sections: [
- { title: 'Start here', links: [
- { title: 'OpenVibe.Events home', url: abs('/') },
- { title: 'Guide', url: GUIDE, note: 'the README: publishing, subscriptions, developer apps and realtime' },
- { title: 'Source code', url: SOURCE },
- ] },
- { title: 'Machine-readable', links: [
- { title: 'Sitemap', url: abs('/sitemap.xml') },
- { title: 'robots.txt', url: abs('/robots.txt') },
- { title: 'API and realtime', url: config.baseUrl, note: 'the JSON API and the realtime gateway' },
- ] },
- ],
+ sections: pages.map((s) => ({ title: s.title, links: s.links.map((l) => ({ title: l.title, url: l.url, ...(l.note ? { note: l.note } : {}) })) })),
+ })));
+
+ // /llms-full.txt: the same pages as /llms.txt, each with its full plain text (openvibe-shared/seo.llmsFull).
+ router.get('/llms-full.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders())
+ .send(seo.llmsFull({
+ site: 'OpenVibe.Events',
+ summary: DESCRIPTION,
+ base: config.siteUrl,
+ maxBytes: FULL_MAX_BYTES,
+ sections: pages.map((s) => ({ title: s.title, pages: s.links.map((l) => ({ title: l.title, url: l.url, text: l.text })) })),
})));
})));
return router;
return router;
@@ -20,6 +20,8 @@ const { createApp } = require('./app');
@@ -20,6 +20,8 @@ const { createApp } = require('./app');
const { createMetrics } = require('./metrics');
const { createMetrics } = require('./metrics');
const { createGuardedPost } = require('./egress');
const { createGuardedPost } = require('./egress');
const { createBillingReadings } = require('./billing');
const { createBillingReadings } = require('./billing');
+const { createIndexNow } = require('openvibe-shared/indexnow');
+const indexnowNotify = require('./indexnow-notify');
const { gracefulStop } = require('openvibe-sdk/service');
const { gracefulStop } = require('openvibe-sdk/service');
async function start({
async function start({
@@ -47,7 +49,25 @@ async function start({
@@ -47,7 +49,25 @@ async function start({
const worker = createWorker({ store, config, clock, fetchImpl: deliveryFetch, appPost: appPost || createGuardedPost({ lookup: dnsLookup }), log, observe: metrics.observe });
const worker = createWorker({ store, config, clock, fetchImpl: deliveryFetch, appPost: appPost || createGuardedPost({ lookup: dnsLookup }), log, observe: metrics.observe });
const realtime = createRealtime({ store, auth, config, clock, log });
const realtime = createRealtime({ store, auth, config, clock, log });
metrics.bind({ realtime });
metrics.bind({ realtime });
- const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers });
+
+ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY. Unset → off, the key
+ // file is not mounted and nothing is sent. When set, GET /<key>.txt is the key file engines fetch
+ // before they accept a ping; the module owns the batching and the POST. A malformed key (createIndexNow
+ // refuses anything but 8–128 alphanumerics) disables IndexNow instead of crashing the service at boot.
+ let indexnow;
+ try {
+ indexnow = createIndexNow({ host: config.siteUrl, key: config.indexnow.key, fetch: fetchImpl, log });
+ } catch (err) {
+ log.warn(`[IndexNow] disabled: ${err.message}`);
+ indexnow = createIndexNow({ host: config.siteUrl, key: '', log });
+ }
+
+ const app = createApp({ config, store, auth, keys, worker, realtime, metrics, dnsLookup, clock, log, valkey, policies, planner, carriers, indexnow });
+ app.locals.indexnow = indexnow;
+ // The pages crawlers see change only with a release, so this boot is the moment to announce them.
+ // A no-op without INDEXNOW_KEY.
+ indexnowNotify.use(indexnow, config.siteUrl);
+ indexnowNotify.pingRelease();
// The key loads in the background (retrying while Network boots); /api/ready says when it has.
// The key loads in the background (retrying while Network boots); /api/ready says when it has.
const keyLoaded = keys.start().catch(() => null);
const keyLoaded = keys.start().catch(() => null);
new file mode 100644
new file mode 100644
@@ -0,0 +1,34 @@
@@ -0,0 +1,34 @@
+'use strict';
+/**
+ * OpenVibe.Events — tell search engines when the product home and its crawl files change.
+ *
+ * Nothing on this origin changes on its own: the pages crawlers see (the home and robots.txt,
+ * sitemap.xml, llms.txt and llms-full.txt) are fixed by the code, so a release is the only moment
+ * they do. server/index.js creates the IndexNow client once at boot (openvibe-shared/indexnow) and
+ * hands it here with use(); pingRelease() queues those pages for a single batched POST. With no
+ * INDEXNOW_KEY the client is disabled and every call is a no-op: nothing is ever sent.
+ */
+let client = null;
+let base = null;
+
+/** Install the client booted in server/index.js, with the site origin to ping. Disabled → pinging off. */
+function use(indexnow, siteUrl) {
+ client = indexnow && indexnow.enabled ? indexnow : null;
+ base = siteUrl ? String(siteUrl).replace(/\/$/, '') : null;
+}
+
+/**
+ * Queue the product home and its machine-readable files for one batched ping. Returns true when URLs
+ * were queued. Never throws (a failed ping must never fail a boot).
+ */
+function pingRelease() {
+ try {
+ if (!client || !base) return false;
+ return client.pingSoon([`${base}/`, `${base}/sitemap.xml`, `${base}/llms.txt`, `${base}/llms-full.txt`]) > 0;
+ } catch (err) {
+ console.warn('[IndexNow] notify failed:', err.message);
+ return false;
+ }
+}
+
+module.exports = { use, pingRelease };
@@ -1,7 +1,7 @@
@@ -1,7 +1,7 @@
'use strict';
'use strict';
// The product home (plan T7: openvibe.events is the API and product origin): it renders server/home.js
// The product home (plan T7: openvibe.events is the API and product origin): it renders server/home.js
// for a browser and keeps the text/plain API index for curl and API clients, serves the pinned OpenVibe
// for a browser and keeps the text/plain API index for curl and API clients, serves the pinned OpenVibe
-// Frame at /shared, answers robots.txt, sitemap.xml and llms.txt, and names its own origin everywhere.
+// Frame at /shared, answers robots.txt, sitemap.xml, llms.txt and llms-full.txt, and names its own origin everywhere.
// A parse-only check of the nginx vhost too.
// A parse-only check of the nginx vhost too.
const assert = require('assert');
const assert = require('assert');
@@ -154,6 +154,16 @@ t('llms.txt is a plain-text map that starts with the site name', async () => {
@@ -154,6 +154,16 @@ t('llms.txt is a plain-text map that starts with the site name', async () => {
assert.ok(r.body.includes(`The API and the realtime gateway are at ${SITE}`), 'the discovery names the API origin');
assert.ok(r.body.includes(`The API and the realtime gateway are at ${SITE}`), 'the discovery names the API origin');
});
});
+t('llms-full.txt is the same map in full text, starting with the site name', async () => {
+ const r = await raw(h.base, '/llms-full.txt');
+ assert.strictEqual(r.status, 200);
+ assert.match(r.headers['content-type'], /^text\/plain/);
+ assert.ok(r.body.startsWith('# OpenVibe.Events'), r.body.slice(0, 40));
+ assert.ok(r.body.includes('> Durable events for the OpenVibe network'), 'the home summary');
+ assert.ok(r.body.includes('## Start here'), 'the /llms.txt sections as pages');
+ assert.ok(r.body.includes('### Guide'), 'each page with its full text');
+});
+
t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redirect to the old host', () => {
t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redirect to the old host', () => {
const conf = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', 'openvibe.events.conf'), 'utf8');
const conf = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', 'openvibe.events.conf'), 'utf8');
assert.match(conf, /server_name openvibe\.events;/);
assert.match(conf, /server_name openvibe\.events;/);
@@ -162,7 +172,7 @@ t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redi
@@ -162,7 +172,7 @@ t('deploy/nginx/openvibe.events.conf is the API and product origin, with no redi
assert.ok(conf.includes('/etc/letsencrypt/live/openvibe.events/privkey.pem'));
assert.ok(conf.includes('/etc/letsencrypt/live/openvibe.events/privkey.pem'));
assert.ok(!conf.includes('$proxy_add_x_forwarded_for'), 'the client address is never appended to');
assert.ok(!conf.includes('$proxy_add_x_forwarded_for'), 'the client address is never appended to');
// The product surface (plan expansion II) and everything the API host proxies (plan T7).
// The product surface (plan expansion II) and everything the API host proxies (plan T7).
- for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {',
+ for (const loc of ['location = / {', 'location ^~ /shared/ {', 'location = /robots.txt {', 'location = /sitemap.xml {', 'location = /llms.txt {', 'location = /llms-full.txt {',
'location = /realtime/stream {', 'location = /api/health', 'location = /api/ready', 'location = /release.json', 'location = /limits.json']) {
'location = /realtime/stream {', 'location = /api/health', 'location = /api/ready', 'location = /release.json', 'location = /limits.json']) {
assert.ok(conf.includes(loc), `${loc} is proxied`);
assert.ok(conf.includes(loc), `${loc} is proxied`);
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+'use strict';
+// IndexNow (openvibe-shared/indexnow): the key file engines fetch is served at /<INDEXNOW_KEY>.txt only
+// when a key is configured; unset, the path is not mounted (404) and nothing is ever sent. The client is
+// created once at boot from INDEXNOW_KEY (server/index.js).
+const assert = require('assert');
+const nodeHttp = require('http');
+const { boot, suite } = require('./helpers');
+
+const KEY = require('crypto').randomBytes(16).toString('hex'); // 32 hex characters: the 8–128 IndexNow wants
+const SITE = 'https://openvibe.events';
+
+const t = suite('indexnow');
+let without, withKey;
+
+/** A raw GET with no Accept a browser would send. */
+function raw(base, p) {
+ return new Promise((resolve, reject) => {
+ const url = new URL(base + p);
+ nodeHttp.get({ hostname: url.hostname, port: url.port, path: url.pathname + url.search }, (res) => {
+ let body = '';
+ res.setEncoding('utf8');
+ res.on('data', (c) => { body += c; });
+ res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body }));
+ }).on('error', reject);
+ });
+}
+
+t('boot without INDEXNOW_KEY', async () => { without = await boot({ env: { BASE_URL: SITE } }); });
+t('boot with INDEXNOW_KEY', async () => { withKey = await boot({ env: { BASE_URL: SITE, INDEXNOW_KEY: KEY } }); });
+
+t('with no key configured the key file is 404', async () => {
+ const r = await raw(without.base, `/${KEY}.txt`);
+ assert.strictEqual(r.status, 404, 'no /<key>.txt without INDEXNOW_KEY');
+});
+
+t('with a key configured the key file returns the key and nothing else', async () => {
+ const r = await raw(withKey.base, `/${KEY}.txt`);
+ assert.strictEqual(r.status, 200);
+ assert.match(r.headers['content-type'], /^text\/plain/);
+ assert.strictEqual(r.body, KEY);
+ const other = await raw(withKey.base, '/another-key.txt');
+ assert.strictEqual(other.status, 404, 'only the configured key file is served');
+});
+
+t('stop', async () => { await without.stop(); await withKey.stop(); });
+
+t.run();
@@ -44,7 +44,8 @@ t('Events reads no secret from its environment but the planted ones', async () =
@@ -44,7 +44,8 @@ t('Events reads no secret from its environment but the planted ones', async () =
};
};
walk(path.join(__dirname, '..', 'server'));
walk(path.join(__dirname, '..', 'server'));
assert.ok(names.has('DATABASE_URL') && names.has('OV_NETWORK_PUBLIC_KEY'), 'the scan sees the config');
assert.ok(names.has('DATABASE_URL') && names.has('OV_NETWORK_PUBLIC_KEY'), 'the scan sees the config');
- const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && !PLANTED[n]);
+ // OV_NETWORK_PUBLIC_KEY and INDEXNOW_KEY are public by design (the IndexNow key is served at /<key>.txt).
+ const secretish = [...names].filter((n) => /SECRET|PASSWORD|TOKEN|PRIVATE|CREDENTIAL|_KEY$/.test(n) && n !== 'OV_NETWORK_PUBLIC_KEY' && n !== 'INDEXNOW_KEY' && !PLANTED[n]);
assert.deepStrictEqual(secretish, [], 'a new secret-looking variable: give it a sentinel in this test (PLANTED)');
assert.deepStrictEqual(secretish, [], 'a new secret-looking variable: give it a sentinel in this test (PLANTED)');
});
});