openvibers/openvibe.network Commits
Added GET /robots.txt and GET /sitemap.xml in server/seo/routes.js,… (#4)
Changes
@@@ -765,6 +765,9 @@ app.get(['/', '/index.html'], (req, res) => {return require('./home/render').sendHome(req, res);});app.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(require('./home/render').llmsTxt()));++// The crawl files the front door owes crawlers, built from openvibe-shared/seo (plan T11 lane D), ahead++// of express.static below so they answer the generated copy rather than public/.++app.use(require('./seo/routes').createSeoRoutes({ release }));// Account hub (my.html) — the apex hosts the account hub under /my// plus its client-routed sections.nnew file mode 100644@@@ -0,0 +1,45 @@++'use strict';++// Crawl files for the public front door, mounted before express.static.++const express = require('express');++const fs = require('fs');++const path = require('path');++const seo = require('openvibe-shared/seo');++++const HOST = 'https://openvibe.network';++// This is read once at boot if there is no release manifest.++const FALLBACK_LASTMOD = fs.statSync(path.join(__dirname, '../home/render.js')).mtime.toISOString();++++// Public HTML routes; sign-in, account and admin pages are omitted.++const PAGES = [++ { path: '/', changefreq: 'daily', priority: 1.0 },++ { path: '/status', changefreq: 'daily', priority: 0.8 },++ { path: '/updates', changefreq: 'daily', priority: 0.8 },++ { path: '/terms', changefreq: 'yearly', priority: 0.3 },++ { path: '/privacy', changefreq: 'yearly', priority: 0.3 },++ { path: '/dmca', changefreq: 'yearly', priority: 0.2 },++];++++function robotsTxt() {++ return seo.robotsTxt({ sitemaps: [`${HOST}/sitemap.xml`] });++}++++function sitemapXml({ release = null } = {}) {++ const lastmod = release?.full()?.released_at || FALLBACK_LASTMOD;++ const day = String(lastmod).slice(0, 10);++ return seo.sitemapXml(PAGES.map((page) => ({ loc: `${HOST}${page.path}`, lastmod: day, changefreq: page.changefreq, priority: page.priority })));++}++++function createSeoRoutes({ release = null } = {}) {++ const r = express.Router();++ const robots = robotsTxt();++ const sitemap = sitemapXml({ release });++ r.get('/robots.txt', (_req, res) => {++ res.set('Content-Type', 'text/plain; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(robots);++ });++ r.get('/sitemap.xml', (_req, res) => {++ res.set('Content-Type', 'application/xml; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(sitemap);++ });++ return r;++}++++module.exports = { createSeoRoutes, robotsTxt, sitemapXml, PAGES, HOST };@@@ -41,7 +41,19 @@ const freePort = () => new Promise((resolve) => { const s = net.createServer();if (!up) await new Promise((r) => setTimeout(r, 100));}assert.ok(up, `the server did not start:\n${stderr}`);-- const m = await measure({ base });++ // Node's fetch pools keep-alive sockets; the server closes an idle one after its default++ // 5 s keepAliveTimeout, and a slow asset walk under load can hit that window and fail with++ // UND_ERR_SOCKET ("other side closed"). This test measures sizes, not connection reuse.++ const retryFetch = async (url, init) => {++ for (let attempt = 0; ; attempt++) {++ try { return await fetch(url, init); }++ catch (e) {++ if (attempt >= 2 || e?.cause?.code !== 'UND_ERR_SOCKET') throw e;++ await new Promise((r) => setTimeout(r, 250));++ }++ }++ };++ const m = await measure({ base, fetch: retryFetch });const over = check(m, BUDGETS);assert.deepStrictEqual(over, [], format(m, over));console.log(format(m));nnew file mode 100644@@@ -0,0 +1,64 @@++'use strict';++// The front door's crawl files (plan T11 lane D, audit §5 item 8): /robots.txt and /sitemap.xml are++// generated from openvibe-shared/seo rather than served as static files, so a crawler is told about the++// sitemap and gets lastmods that come from the release, not from the clock at request time.++// node test/seo-routes.test.js++const assert = require('assert');++const { bootServer } = require('./helpers/boot-server');++const seo = require('openvibe-shared/seo');++const seoRoutes = require('../server/seo/routes');++++(async () => {++ const srv = await bootServer();++ try {++ const get = (p) => fetch(srv.base + p).then(async r => ({ status: r.status, type: r.headers.get('content-type') || '', body: await r.text() }));++++ // ── /robots.txt ──++ const robots = await get('/robots.txt');++ assert.strictEqual(robots.status, 200, '/robots.txt is 200');++ assert.ok(robots.type.startsWith('text/plain'), `/robots.txt is text/plain (got ${robots.type})`);++ assert.ok(/^User-agent: \*/m.test(robots.body), 'robots.txt has the default group');++ assert.ok(/^Allow: \/$/m.test(robots.body), 'robots.txt allows the root');++ assert.ok(!robots.body.includes('Disallow:'), 'robots.txt allows all paths and crawlers');++ for (const bot of seo.AI_AND_SEARCH_BOTS) assert.ok(robots.body.includes(`User-agent: ${bot}`), `robots.txt names ${bot}`);++ assert.ok(robots.body.includes('Sitemap: https://openvibe.network/sitemap.xml'), 'robots.txt points at the sitemap');++ for (const line of robots.body.split('\n')) if (line) assert.ok(/^[A-Za-z-]+: /.test(line), `every robots.txt line is a directive (got "${line}")`);++++ // ── /sitemap.xml ──++ const sitemap = await get('/sitemap.xml');++ assert.strictEqual(sitemap.status, 200, '/sitemap.xml is 200');++ assert.ok(sitemap.type.startsWith('application/xml'), `/sitemap.xml is application/xml (got ${sitemap.type})`);++ assert.ok(sitemap.body.startsWith('<?xml version="1.0" encoding="UTF-8"?>'), 'the sitemap has the XML declaration');++ const opened = (body) => {++ const tags = [...body.matchAll(/<(\/?)([a-z0-9]+:?[a-z0-9]*)\b[^>]*>/g)];++ const stack = [];++ for (const [, closing, name] of tags) {++ if (closing) { assert.strictEqual(stack.pop(), name, `the sitemap closes <${name}>`); } else if (!/^<\?/.test(name)) stack.push(name);++ }++ assert.deepStrictEqual(stack, [], 'the sitemap has no unclosed tags');++ };++ opened(sitemap.body);++ assert.ok(sitemap.body.includes('<loc>https://openvibe.network/</loc>'), 'the sitemap lists the home page');++ assert.ok(sitemap.body.includes('<lastmod>'), 'every entry carries a lastmod');++ const locs = [...sitemap.body.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m => m[1]);++ assert.deepStrictEqual(locs, seoRoutes.PAGES.map(p => `${seoRoutes.HOST}${p.path}`), 'the sitemap lists exactly the public HTML pages');++ assert.ok(locs.includes('https://openvibe.network/status'), 'the sitemap lists the status page');++ assert.ok(!locs.some(l => /\/admin|\/login/.test(l)), 'private pages are not in the sitemap');++ const lastmods = new Set([...sitemap.body.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m => m[1]));++ assert.strictEqual(lastmods.size, 1, 'one lastmod for the whole sitemap');++ assert.ok(/^\d{4}-\d{2}-\d{2}$/.test([...lastmods][0]), 'lastmod is a date, not a timestamp');++++ // ── lastmod is the release, not the request clock ──++ const again = await get('/sitemap.xml');++ assert.strictEqual(again.body, sitemap.body, 'two requests a moment apart produce the same sitemap');++ const shipped = seoRoutes.sitemapXml({ release: { full: () => ({ released_at: '2026-01-02T03:04:05Z' }) } });++ assert.ok(shipped.includes('<lastmod>2026-01-02</lastmod>'), 'lastmod follows the release manifest');++++ console.log('seo routes: all checks passed');++ } catch (err) {++ console.error(srv.logs());++ throw err;++ } finally {++ await srv.stop();++ }++})().then(() => process.exit(0)).catch((err) => { console.error(err); process.exit(1); });@@@ -26,6 +26,11 @@ const { ACCOUNT_HUB_PATHS } = require('../server/not-found');assert.strictEqual(r.status, 200, `${p} is a real page: 200 (got ${r.status})`);}assert.strictEqual((await get('/sso/check')).status, 400, '/sso/check is a route (400 without its parameters), not a 404');++ // The crawl files are generated routes (server/seo/routes.js), not the static files in public/.++ const robots = await get('/robots.txt');++ assert.ok(robots.type.startsWith('text/plain') && robots.text.includes('Sitemap: https://openvibe.network/sitemap.xml'), '/robots.txt is the generated text/plain copy');++ const sitemap = await get('/sitemap.xml');++ assert.ok(sitemap.type.startsWith('application/xml') && sitemap.text.includes('<loc>https://openvibe.network/</loc>'), '/sitemap.xml is the generated XML copy');const hub = await get('/themes');assert.ok(hub.text.includes('<html') && !hub.text.includes('Page not found'), 'an account hub section is the account hub');// Redirects stay redirects.
@@ -765,6 +765,9 @@ app.get(['/', '/index.html'], (req, res) => {
@@ -765,6 +765,9 @@ app.get(['/', '/index.html'], (req, res) => {
return require('./home/render').sendHome(req, res);
return require('./home/render').sendHome(req, res);
});
});
app.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(require('./home/render').llmsTxt()));
app.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(require('./home/render').llmsTxt()));
+// The crawl files the front door owes crawlers, built from openvibe-shared/seo (plan T11 lane D), ahead
+// of express.static below so they answer the generated copy rather than public/.
+app.use(require('./seo/routes').createSeoRoutes({ release }));
// Account hub (my.html) — the apex hosts the account hub under /my
// Account hub (my.html) — the apex hosts the account hub under /my
// plus its client-routed sections.
// plus its client-routed sections.
new file mode 100644
new file mode 100644
@@ -0,0 +1,45 @@
@@ -0,0 +1,45 @@
+'use strict';
+// Crawl files for the public front door, mounted before express.static.
+const express = require('express');
+const fs = require('fs');
+const path = require('path');
+const seo = require('openvibe-shared/seo');
+
+const HOST = 'https://openvibe.network';
+// This is read once at boot if there is no release manifest.
+const FALLBACK_LASTMOD = fs.statSync(path.join(__dirname, '../home/render.js')).mtime.toISOString();
+
+// Public HTML routes; sign-in, account and admin pages are omitted.
+const PAGES = [
+ { path: '/', changefreq: 'daily', priority: 1.0 },
+ { path: '/status', changefreq: 'daily', priority: 0.8 },
+ { path: '/updates', changefreq: 'daily', priority: 0.8 },
+ { path: '/terms', changefreq: 'yearly', priority: 0.3 },
+ { path: '/privacy', changefreq: 'yearly', priority: 0.3 },
+ { path: '/dmca', changefreq: 'yearly', priority: 0.2 },
+];
+
+function robotsTxt() {
+ return seo.robotsTxt({ sitemaps: [`${HOST}/sitemap.xml`] });
+}
+
+function sitemapXml({ release = null } = {}) {
+ const lastmod = release?.full()?.released_at || FALLBACK_LASTMOD;
+ const day = String(lastmod).slice(0, 10);
+ return seo.sitemapXml(PAGES.map((page) => ({ loc: `${HOST}${page.path}`, lastmod: day, changefreq: page.changefreq, priority: page.priority })));
+}
+
+function createSeoRoutes({ release = null } = {}) {
+ const r = express.Router();
+ const robots = robotsTxt();
+ const sitemap = sitemapXml({ release });
+ r.get('/robots.txt', (_req, res) => {
+ res.set('Content-Type', 'text/plain; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(robots);
+ });
+ r.get('/sitemap.xml', (_req, res) => {
+ res.set('Content-Type', 'application/xml; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(sitemap);
+ });
+ return r;
+}
+
+module.exports = { createSeoRoutes, robotsTxt, sitemapXml, PAGES, HOST };
@@ -41,7 +41,19 @@ const freePort = () => new Promise((resolve) => { const s = net.createServer();
@@ -41,7 +41,19 @@ const freePort = () => new Promise((resolve) => { const s = net.createServer();
if (!up) await new Promise((r) => setTimeout(r, 100));
if (!up) await new Promise((r) => setTimeout(r, 100));
}
}
assert.ok(up, `the server did not start:\n${stderr}`);
assert.ok(up, `the server did not start:\n${stderr}`);
- const m = await measure({ base });
+ // Node's fetch pools keep-alive sockets; the server closes an idle one after its default
+ // 5 s keepAliveTimeout, and a slow asset walk under load can hit that window and fail with
+ // UND_ERR_SOCKET ("other side closed"). This test measures sizes, not connection reuse.
+ const retryFetch = async (url, init) => {
+ for (let attempt = 0; ; attempt++) {
+ try { return await fetch(url, init); }
+ catch (e) {
+ if (attempt >= 2 || e?.cause?.code !== 'UND_ERR_SOCKET') throw e;
+ await new Promise((r) => setTimeout(r, 250));
+ }
+ }
+ };
+ const m = await measure({ base, fetch: retryFetch });
const over = check(m, BUDGETS);
const over = check(m, BUDGETS);
assert.deepStrictEqual(over, [], format(m, over));
assert.deepStrictEqual(over, [], format(m, over));
console.log(format(m));
console.log(format(m));
new file mode 100644
new file mode 100644
@@ -0,0 +1,64 @@
@@ -0,0 +1,64 @@
+'use strict';
+// The front door's crawl files (plan T11 lane D, audit §5 item 8): /robots.txt and /sitemap.xml are
+// generated from openvibe-shared/seo rather than served as static files, so a crawler is told about the
+// sitemap and gets lastmods that come from the release, not from the clock at request time.
+// node test/seo-routes.test.js
+const assert = require('assert');
+const { bootServer } = require('./helpers/boot-server');
+const seo = require('openvibe-shared/seo');
+const seoRoutes = require('../server/seo/routes');
+
+(async () => {
+ const srv = await bootServer();
+ try {
+ const get = (p) => fetch(srv.base + p).then(async r => ({ status: r.status, type: r.headers.get('content-type') || '', body: await r.text() }));
+
+ // ── /robots.txt ──
+ const robots = await get('/robots.txt');
+ assert.strictEqual(robots.status, 200, '/robots.txt is 200');
+ assert.ok(robots.type.startsWith('text/plain'), `/robots.txt is text/plain (got ${robots.type})`);
+ assert.ok(/^User-agent: \*/m.test(robots.body), 'robots.txt has the default group');
+ assert.ok(/^Allow: \/$/m.test(robots.body), 'robots.txt allows the root');
+ assert.ok(!robots.body.includes('Disallow:'), 'robots.txt allows all paths and crawlers');
+ for (const bot of seo.AI_AND_SEARCH_BOTS) assert.ok(robots.body.includes(`User-agent: ${bot}`), `robots.txt names ${bot}`);
+ assert.ok(robots.body.includes('Sitemap: https://openvibe.network/sitemap.xml'), 'robots.txt points at the sitemap');
+ for (const line of robots.body.split('\n')) if (line) assert.ok(/^[A-Za-z-]+: /.test(line), `every robots.txt line is a directive (got "${line}")`);
+
+ // ── /sitemap.xml ──
+ const sitemap = await get('/sitemap.xml');
+ assert.strictEqual(sitemap.status, 200, '/sitemap.xml is 200');
+ assert.ok(sitemap.type.startsWith('application/xml'), `/sitemap.xml is application/xml (got ${sitemap.type})`);
+ assert.ok(sitemap.body.startsWith('<?xml version="1.0" encoding="UTF-8"?>'), 'the sitemap has the XML declaration');
+ const opened = (body) => {
+ const tags = [...body.matchAll(/<(\/?)([a-z0-9]+:?[a-z0-9]*)\b[^>]*>/g)];
+ const stack = [];
+ for (const [, closing, name] of tags) {
+ if (closing) { assert.strictEqual(stack.pop(), name, `the sitemap closes <${name}>`); } else if (!/^<\?/.test(name)) stack.push(name);
+ }
+ assert.deepStrictEqual(stack, [], 'the sitemap has no unclosed tags');
+ };
+ opened(sitemap.body);
+ assert.ok(sitemap.body.includes('<loc>https://openvibe.network/</loc>'), 'the sitemap lists the home page');
+ assert.ok(sitemap.body.includes('<lastmod>'), 'every entry carries a lastmod');
+ const locs = [...sitemap.body.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m => m[1]);
+ assert.deepStrictEqual(locs, seoRoutes.PAGES.map(p => `${seoRoutes.HOST}${p.path}`), 'the sitemap lists exactly the public HTML pages');
+ assert.ok(locs.includes('https://openvibe.network/status'), 'the sitemap lists the status page');
+ assert.ok(!locs.some(l => /\/admin|\/login/.test(l)), 'private pages are not in the sitemap');
+ const lastmods = new Set([...sitemap.body.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m => m[1]));
+ assert.strictEqual(lastmods.size, 1, 'one lastmod for the whole sitemap');
+ assert.ok(/^\d{4}-\d{2}-\d{2}$/.test([...lastmods][0]), 'lastmod is a date, not a timestamp');
+
+ // ── lastmod is the release, not the request clock ──
+ const again = await get('/sitemap.xml');
+ assert.strictEqual(again.body, sitemap.body, 'two requests a moment apart produce the same sitemap');
+ const shipped = seoRoutes.sitemapXml({ release: { full: () => ({ released_at: '2026-01-02T03:04:05Z' }) } });
+ assert.ok(shipped.includes('<lastmod>2026-01-02</lastmod>'), 'lastmod follows the release manifest');
+
+ console.log('seo routes: all checks passed');
+ } catch (err) {
+ console.error(srv.logs());
+ throw err;
+ } finally {
+ await srv.stop();
+ }
+})().then(() => process.exit(0)).catch((err) => { console.error(err); process.exit(1); });
@@ -26,6 +26,11 @@ const { ACCOUNT_HUB_PATHS } = require('../server/not-found');
@@ -26,6 +26,11 @@ const { ACCOUNT_HUB_PATHS } = require('../server/not-found');
assert.strictEqual(r.status, 200, `${p} is a real page: 200 (got ${r.status})`);
assert.strictEqual(r.status, 200, `${p} is a real page: 200 (got ${r.status})`);
}
}
assert.strictEqual((await get('/sso/check')).status, 400, '/sso/check is a route (400 without its parameters), not a 404');
assert.strictEqual((await get('/sso/check')).status, 400, '/sso/check is a route (400 without its parameters), not a 404');
+ // The crawl files are generated routes (server/seo/routes.js), not the static files in public/.
+ const robots = await get('/robots.txt');
+ assert.ok(robots.type.startsWith('text/plain') && robots.text.includes('Sitemap: https://openvibe.network/sitemap.xml'), '/robots.txt is the generated text/plain copy');
+ const sitemap = await get('/sitemap.xml');
+ assert.ok(sitemap.type.startsWith('application/xml') && sitemap.text.includes('<loc>https://openvibe.network/</loc>'), '/sitemap.xml is the generated XML copy');
const hub = await get('/themes');
const hub = await get('/themes');
assert.ok(hub.text.includes('<html') && !hub.text.includes('Page not found'), 'an account hub section is the account hub');
assert.ok(hub.text.includes('<html') && !hub.text.includes('Page not found'), 'an account hub section is the account hub');
// Redirects stay redirects.
// Redirects stay redirects.