openvibers/openvibe.search Commits
Proxy /llms-full.txt and the IndexNow key file; tolerate a bad key
Changes
@@@ -7,7 +7,9 @@## The Node service (port 4710, systemd unit openvibe-search) serves:## / the search page (HTML for browsers, a text route index## otherwise) and the crawl files: /robots.txt, /llms.txt,--## /sitemap.xml. Public, no-store, result pages noindex.++## /llms-full.txt, /sitemap.xml and, when INDEXNOW_KEY is set,++## the IndexNow key file /<key>.txt. Public, no-store, result++## pages noindex.## /api/v1/search, /api/v1/suggest, GET /api/v1/documents/... public query API (anonymous =## public documents only; the ov_token cookie or a Bearer adds## the caller's ACL matches). Responses are Cache-Control: no-store.@@@ -81,7 +83,10 @@ server {}location = /robots.txt { proxy_pass http://127.0.0.1:4710; }location = /llms.txt { proxy_pass http://127.0.0.1:4710; }++ location = /llms-full.txt { proxy_pass http://127.0.0.1:4710; }location = /sitemap.xml { proxy_pass http://127.0.0.1:4710; }++ # The IndexNow key file (8-128 letters and digits); Node answers 404 unless it is INDEXNOW_KEY.++ location ~ "^/[A-Za-z0-9]{8,128}\.txt$" { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }# This site's own pinned copy of the Frame's browser files (openvibe-shared/serve).location ^~ /shared/ { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }# What shipped on this site (the shared log).@@@ -37,8 +37,14 @@ function createApp({ config, db, store, engine, auth, outbox, relay, purges, pur// The webhook reads its raw body itself (the signature covers the exact bytes).// IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so this// service has nothing to ping — only the key file is served at /<key>.txt, when INDEXNOW_KEY is-- // set. Mounted before the static handlers so nothing shadows it; an unset key means off.-- const indexnow = createIndexNow({ host: config.baseUrl, key: config.indexnow.key, log });++ // set. Mounted before the static handlers so nothing shadows it; an unset key means off, and a++ // malformed one (not 8-128 letters and digits) is logged and left off rather than stopping boot.++ let indexnow = { enabled: false };++ try {++ indexnow = createIndexNow({ host: config.baseUrl, key: config.indexnow.key, log });++ } catch (err) {++ log.warn(`[indexnow] INDEXNOW_KEY ignored, key file not served: ${err.message}`);++ }if (indexnow.enabled) app.use(indexnow.keyFile);// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).@@@ -10,7 +10,6 @@ const crypto = require('crypto');const { ids } = require('openvibe-contracts');const { boot, request, doc, suite } = require('./helpers');const { llmsFullTxt } = require('../server/web/page');--const { createApp } = require('../server/app');const t = suite('discovery');let svc;@@@ -103,9 +102,18 @@ t('GET /<key>.txt: the IndexNow key file answers when INDEXNOW_KEY is set', asyn} finally { await on.stop(); }});++t('GET /<key>.txt: a malformed INDEXNOW_KEY leaves the key file off without stopping boot', async () => {++ const on = await boot({ env: { INDEXNOW_KEY: 'bad key!' } });++ try {++ const health = await request(on.base, 'GET', '/api/health');++ assert.strictEqual(health.status, 200);++ const r = await request(on.base, 'GET', '/bad key!.txt');++ assert.strictEqual(r.status, 404);++ } finally { await on.stop(); }++});++t('the route index is built from routes only: every entry gets a line of text, none is private', async () => {const text = llmsFullTxt(origin());-- assert.ok(typeof createApp === 'function', 'the app wires the key file');const urls = text.match(/^URL: (.+)$/gm);assert.ok(urls && urls.length >= 8, 'the public routes and JSON endpoints are listed');// Each route is followed by a non-empty line of text, and nothing private or per-user appears.
@@ -7,7 +7,9 @@
@@ -7,7 +7,9 @@
## The Node service (port 4710, systemd unit openvibe-search) serves:
## The Node service (port 4710, systemd unit openvibe-search) serves:
## / the search page (HTML for browsers, a text route index
## / the search page (HTML for browsers, a text route index
## otherwise) and the crawl files: /robots.txt, /llms.txt,
## otherwise) and the crawl files: /robots.txt, /llms.txt,
-## /sitemap.xml. Public, no-store, result pages noindex.
+## /llms-full.txt, /sitemap.xml and, when INDEXNOW_KEY is set,
+## the IndexNow key file /<key>.txt. Public, no-store, result
+## pages noindex.
## /api/v1/search, /api/v1/suggest, GET /api/v1/documents/... public query API (anonymous =
## /api/v1/search, /api/v1/suggest, GET /api/v1/documents/... public query API (anonymous =
## public documents only; the ov_token cookie or a Bearer adds
## public documents only; the ov_token cookie or a Bearer adds
## the caller's ACL matches). Responses are Cache-Control: no-store.
## the caller's ACL matches). Responses are Cache-Control: no-store.
@@ -81,7 +83,10 @@ server {
@@ -81,7 +83,10 @@ server {
}
}
location = /robots.txt { proxy_pass http://127.0.0.1:4710; }
location = /robots.txt { proxy_pass http://127.0.0.1:4710; }
location = /llms.txt { proxy_pass http://127.0.0.1:4710; }
location = /llms.txt { proxy_pass http://127.0.0.1:4710; }
+ location = /llms-full.txt { proxy_pass http://127.0.0.1:4710; }
location = /sitemap.xml { proxy_pass http://127.0.0.1:4710; }
location = /sitemap.xml { proxy_pass http://127.0.0.1:4710; }
+ # The IndexNow key file (8-128 letters and digits); Node answers 404 unless it is INDEXNOW_KEY.
+ location ~ "^/[A-Za-z0-9]{8,128}\.txt$" { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
# This site's own pinned copy of the Frame's browser files (openvibe-shared/serve).
# This site's own pinned copy of the Frame's browser files (openvibe-shared/serve).
location ^~ /shared/ { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
location ^~ /shared/ { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
# What shipped on this site (the shared log).
# What shipped on this site (the shared log).
@@ -37,8 +37,14 @@ function createApp({ config, db, store, engine, auth, outbox, relay, purges, pur
@@ -37,8 +37,14 @@ function createApp({ config, db, store, engine, auth, outbox, relay, purges, pur
// The webhook reads its raw body itself (the signature covers the exact bytes).
// The webhook reads its raw body itself (the signature covers the exact bytes).
// IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so this
// IndexNow (openvibe-shared/indexnow): Search publishes no pages of its own to submit, so this
// service has nothing to ping — only the key file is served at /<key>.txt, when INDEXNOW_KEY is
// service has nothing to ping — only the key file is served at /<key>.txt, when INDEXNOW_KEY is
- // set. Mounted before the static handlers so nothing shadows it; an unset key means off.
- const indexnow = createIndexNow({ host: config.baseUrl, key: config.indexnow.key, log });
+ // set. Mounted before the static handlers so nothing shadows it; an unset key means off, and a
+ // malformed one (not 8-128 letters and digits) is logged and left off rather than stopping boot.
+ let indexnow = { enabled: false };
+ try {
+ indexnow = createIndexNow({ host: config.baseUrl, key: config.indexnow.key, log });
+ } catch (err) {
+ log.warn(`[indexnow] INDEXNOW_KEY ignored, key file not served: ${err.message}`);
+ }
if (indexnow.enabled) app.use(indexnow.keyFile);
if (indexnow.enabled) app.use(indexnow.keyFile);
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
@@ -10,7 +10,6 @@ const crypto = require('crypto');
@@ -10,7 +10,6 @@ const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { ids } = require('openvibe-contracts');
const { boot, request, doc, suite } = require('./helpers');
const { boot, request, doc, suite } = require('./helpers');
const { llmsFullTxt } = require('../server/web/page');
const { llmsFullTxt } = require('../server/web/page');
-const { createApp } = require('../server/app');
const t = suite('discovery');
const t = suite('discovery');
let svc;
let svc;
@@ -103,9 +102,18 @@ t('GET /<key>.txt: the IndexNow key file answers when INDEXNOW_KEY is set', asyn
@@ -103,9 +102,18 @@ t('GET /<key>.txt: the IndexNow key file answers when INDEXNOW_KEY is set', asyn
} finally { await on.stop(); }
} finally { await on.stop(); }
});
});
+t('GET /<key>.txt: a malformed INDEXNOW_KEY leaves the key file off without stopping boot', async () => {
+ const on = await boot({ env: { INDEXNOW_KEY: 'bad key!' } });
+ try {
+ const health = await request(on.base, 'GET', '/api/health');
+ assert.strictEqual(health.status, 200);
+ const r = await request(on.base, 'GET', '/bad key!.txt');
+ assert.strictEqual(r.status, 404);
+ } finally { await on.stop(); }
+});
+
t('the route index is built from routes only: every entry gets a line of text, none is private', async () => {
t('the route index is built from routes only: every entry gets a line of text, none is private', async () => {
const text = llmsFullTxt(origin());
const text = llmsFullTxt(origin());
- assert.ok(typeof createApp === 'function', 'the app wires the key file');
const urls = text.match(/^URL: (.+)$/gm);
const urls = text.match(/^URL: (.+)$/gm);
assert.ok(urls && urls.length >= 8, 'the public routes and JSON endpoints are listed');
assert.ok(urls && urls.length >= 8, 'the public routes and JSON endpoints are listed');
// Each route is followed by a non-empty line of text, and nothing private or per-user appears.
// Each route is followed by a non-empty line of text, and nothing private or per-user appears.