openvibers/openvibe.search Commits

Serve /llms-full.txt route index and the IndexNow key file

OP OpenVibers committed 4 Oct 2026 · fe1f83cf91 · 6 files, +104 −1

Asked: Re-land T11 step 4 c+d (llms-full.txt route index + IndexNow key file) for OpenVibe.Search. The finished work is a local-only commit bd001d3 on branch agent/search-t11-step-4-c-d-221219-muudma01 in the worktree /mnt/120gb/openvibe-worktrees/OpenVibe.Search-agent-search-t11-step-4-c-d-221219-muudma0…
Harness job search-re-land-t11-step-4-c-d-221617.

Changes

ddiff --git a/.env.example b/.env.example
iindex 6b70120..64b4a69 100644
---- a/.env.example
++++ b/.env.example
@@@ -69,6 +69,10 @@ VALKEY_PREFIX=
#SEARCH_LIMITS_MINUTE=120
#SEARCH_LIMITS_HOUR=3000
++# IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so only the
++# key file is served, at /<key>.txt, when INDEXNOW_KEY is set. Unset: off (nothing mounted, nothing sent).
++#INDEXNOW_KEY=
++
# Removal purge queue (README "Removals, caches and the CDN"). Every removal is recorded for
# its owner (GET /api/v1/owners/:owner/removals). The Cloudflare purge of a formerly public URL
# runs only when CLOUDFLARE_PURGE_TOKEN is set (inert otherwise). The token needs only
ddiff --git a/server/app.js b/server/app.js
iindex 7923d40..52fb3e4 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -12,6 +12,7 @@ const { savedRouter } = require('./api/saved');
const { pageRouter } = require('./web/page');
const { webhookRouter } = require('./api/webhook');
const { createLimits } = require('./actor-limits');
++const { createIndexNow } = require('openvibe-shared/indexnow');
const pkg = require('../package.json');
function createApp({ config, db, store, engine, auth, outbox, relay, purges, purger, saved, searcher: givenSearcher = null, notifier = null, log = console, now }) {
@@@ -34,6 +35,12 @@ function createApp({ config, db, store, engine, auth, outbox, relay, purges, pur
});
// The webhook reads its raw body itself (the signature covers the exact bytes).
++ // IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so this
++ // service has nothing to ping — only the key file is served at /<key>.txt, when INDEXNOW_KEY is
++ // set. Mounted before the static handlers so nothing shadows it; an unset key means off.
++ const indexnow = createIndexNow({ host: config.baseUrl, key: config.indexnow.key, log });
++ if (indexnow.enabled) app.use(indexnow.keyFile);
++
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
app.use('/shared', require('openvibe-shared/serve').handler());
app.use(webhookRouter({ config, db, store, relay, log, now }));
ddiff --git a/server/config.js b/server/config.js
iindex 5f9d38d..74a7f6b 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -113,6 +113,11 @@ function load(env = process.env) {
hour: Math.max(1, int(env.SEARCH_LIMITS_HOUR, 3000)),
},
++ // IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so
++ // there is nothing to ping — only the key file is served, at /<key>.txt, when a key is set.
++ // Unset: off, nothing is mounted and nothing is ever sent.
++ indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
++
// Removal purge queue (server/purge.js). Every removal is recorded for owners; the
// Cloudflare purge of a formerly public URL runs only when the token is set.
purge: {
ddiff --git a/server/web/page.js b/server/web/page.js
iindex 9b88a5f..602fa02 100644
---- a/server/web/page.js
++++ b/server/web/page.js
@@@ -7,6 +7,7 @@
* GET / search form; ?q= results, ?owner= / ?type= filters, ?cursor= next page
* GET /robots.txt the front page and /updates may be crawled, result pages and the API not
* GET /llms.txt what this site is, its public pages and its public endpoints (openvibe-shared/seo)
++ * GET /llms-full.txt the public routes and public JSON endpoints, one line of text each
* GET /sitemap.xml the public pages, with lastmod from the newest public document (never the clock)
*
* Search answers are no-store (a document that leaves the index leaves this page at once) and result
@@@ -135,6 +136,38 @@ function llmsTxt(origin) {
});
}
++/**
++ * /llms-full.txt: the public route index and the public JSON endpoints, one line of text each, so an
++ * agent sees what this origin serves without fetching anything. It carries routes only — never a
++ * search result, a document or a person's data — and maxBytes caps it like the shared module's other
++ * full-text files (the list is fixed, so it sits far under the cap).
++ */
++function llmsFullTxt(origin) {
++ return seo.llmsFull({
++ site: SITE_NAME,
++ summary: `The public routes and public JSON endpoints of ${SITE_NAME}, one line each. ${SITE_DESCRIPTION}`,
++ base: origin,
++ maxBytes: 512 * 1024,
++ sections: [
++ { title: 'Public routes', pages: [
++ { title: 'Search page', url: `${origin}/`, text: 'The HTML search page; ?q=, ?owner= and ?type= search. Result pages are noindex.' },
++ { title: 'What shipped', url: `${origin}/updates`, text: "This site's update log." },
++ { title: 'robots.txt', url: `${origin}/robots.txt`, text: 'Crawl rules: the front page and /updates may be crawled, result pages and the API may not.' },
++ { title: 'Sitemap', url: `${origin}/sitemap.xml`, text: 'The public pages, with lastmod from the newest public document.' },
++ { title: 'llms.txt', url: `${origin}/llms.txt`, text: "This site's map for language-model crawlers." },
++ { title: 'llms-full.txt', url: `${origin}/llms-full.txt`, text: 'This file: the routes above and the JSON endpoints below, one line each.' },
++ ] },
++ { title: 'Public JSON endpoints', pages: [
++ { title: 'Search API', url: `${origin}/api/v1/search?q=`, text: 'JSON results; anonymous callers see public, published, indexable documents only.' },
++ { title: 'Suggest API', url: `${origin}/api/v1/suggest?q=`, text: 'JSON title suggestions for the given prefix.' },
++ { title: 'Health', url: `${origin}/api/health`, text: 'Liveness: status, service and version (JSON).' },
++ { title: 'Readiness', url: `${origin}/api/ready`, text: 'Readiness of the database and the optional dependencies (JSON).' },
++ { title: 'Release', url: `${origin}/release.json`, text: 'The deployed release manifest (JSON).' },
++ ] },
++ ],
++ });
++}
++
/**
* /sitemap.xml over the public pages. lastmod comes from the data, never from the clock: the front
* page carries the newest content time among the public, indexable documents (so an empty index
@@@ -243,6 +276,10 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(llmsTxt(origin));
});
++ router.get('/llms-full.txt', (_req, res) => {
++ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(llmsFullTxt(origin));
++ });
++
router.get('/sitemap.xml', async (_req, res, next) => {
let indexLastmod = null;
try {
@@@ -306,4 +343,4 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
return router;
}
--module.exports = { pageRouter, setRelease, TEXT_INDEX, robotsTxt, llmsTxt, sitemapXml, homeJsonLd };
++module.exports = { pageRouter, setRelease, TEXT_INDEX, robotsTxt, llmsTxt, llmsFullTxt, sitemapXml, homeJsonLd };
ddiff --git a/test/asset-cache.test.js b/test/asset-cache.test.js
iindex a0ce0f1..ec17a31 100644
---- a/test/asset-cache.test.js
++++ b/test/asset-cache.test.js
@@@ -36,6 +36,7 @@ t('the crawl files and the update log use the estate HTML policy', async () => {
for (const [p, expected] of [
['/robots.txt', 'public, max-age=3600, stale-while-revalidate=3600'],
['/llms.txt', 'public, max-age=3600, stale-while-revalidate=3600'],
++ ['/llms-full.txt', 'public, max-age=3600, stale-while-revalidate=3600'],
['/sitemap.xml', 'public, max-age=3600, stale-while-revalidate=3600'],
['/updates', 'public, max-age=60, stale-while-revalidate=3600'],
]) {
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex 46f7e59..9346edb 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -6,8 +6,11 @@
* rules the site already had, and never name a private path or a private document.
*/
const assert = require('assert');
++const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { boot, request, doc, suite } = require('./helpers');
++const { llmsFullTxt } = require('../server/web/page');
++const { createApp } = require('../server/app');
const t = suite('discovery');
let svc;
@@@ -64,6 +67,52 @@ t('GET /sitemap.xml: the public pages, lastmod from the newest public document,
for (const p of ['/api/', '/internal/', '/?', 'openvibe.wiki']) assert.ok(!r.text.includes(p), `no ${p}`);
});
++t('GET /llms-full.txt: text/plain, the public routes and JSON endpoints, one line each, nothing private', async () => {
++ const r = await request(svc.base, 'GET', '/llms-full.txt');
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers.get('content-type'), /text\/plain/);
++ assert.strictEqual(r.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');
++ // Every public route and public JSON endpoint is named.
++ for (const p of ['/', '/updates', '/robots.txt', '/sitemap.xml', '/llms.txt', '/llms-full.txt',
++ '/api/v1/search?q=', '/api/v1/suggest?q=', '/api/health', '/api/ready', '/release.json']) {
++ assert.ok(r.text.includes(`${origin()}${p}`), `names ${p}`);
++ }
++ // One line of text each: every route line carries a description after the URL.
++ assert.match(r.text, /URL: \S+\n\n?\S/, 'a one-line description follows each URL');
++ // Nothing private or per-user, and never search results or document data.
++ assert.ok(!/saved-searches|\/api\/v1\/owners|\/internal\/events|\/api\/v1\/documents/.test(r.text));
++ assert.ok(!r.text.includes('crawl-pub') && !r.text.includes('Published page'), 'no document or result data');
++});
++
++t('GET /<key>.txt: 404 when INDEXNOW_KEY is unset', async () => {
++ const r = await request(svc.base, 'GET', '/no-indexnow-key.txt');
++ assert.strictEqual(r.status, 404);
++});
++
++t('GET /<key>.txt: the IndexNow key file answers when INDEXNOW_KEY is set', async () => {
++ const key = crypto.randomBytes(16).toString('hex');
++ const on = await boot({ env: { INDEXNOW_KEY: key } });
++ try {
++ const r = await request(on.base, 'GET', `/${key}.txt`);
++ assert.strictEqual(r.status, 200);
++ assert.match(r.headers.get('content-type'), /text\/plain/);
++ assert.strictEqual(r.text, key);
++ // Another path is not the key file.
++ const other = await request(on.base, 'GET', '/not-the-key.txt');
++ assert.strictEqual(other.status, 404);
++ } finally { await on.stop(); }
++});
++
++t('the route index is built from routes only: every entry gets a line of text, none is private', async () => {
++ const text = llmsFullTxt(origin());
++ assert.ok(typeof createApp === 'function', 'the app wires the key file');
++ const urls = text.match(/^URL: (.+)$/gm);
++ assert.ok(urls && urls.length >= 8, 'the public routes and JSON endpoints are listed');
++ // Each route is followed by a non-empty line of text, and nothing private or per-user appears.
++ assert.match(text, /URL: \S+\n\n?\S/, 'a one-line description follows each URL');
++ assert.ok(!/saved-searches|\/api\/v1\/owners|\/internal\/events|\/api\/v1\/documents/.test(text));
++});
++
t('shutdown', async () => { await svc.stop(); });
t.run();