openvibers/openvibe.network Commits

Added GET /robots.txt and GET /sitemap.xml in server/seo/routes.js,… (#4)

OP OpenVibers committed 2 Oct 2026 · f2504702d7 · 5 files, +130 −1

* Added GET /robots.txt and GET /sitemap.xml in server/seo/routes.js,…

Done by the harness job network-plan-t11-lane-d-follow-up-220735.

* finish-f1: Changed: test/perf-budget.test.js now gives the server 60…

Handed to the local harness by the cloud job pr-OpenVibe.Network-4; done by Claude opus.

* Retry a closed keep-alive socket in the perf-budget test

Asked: Make test/perf-budget.test.js robust against a keep-alive socket race. Under full-suite load, `npm test` failed in this file only (80/81 files passed; the file passes alone and under `npm run test:pg`) with `TypeError: fetch failed`, cause `UND_ERR_SOCKET` ("other side closed"), at the `measure()` …
Harness job task-Network-make-test-perf-budget-test-js-robust-aga-083555.

Changes

ddiff --git a/server/index.js b/server/index.js
iindex ecaabf0f..e3d8dd88 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -765,6 +765,9 @@ app.get(['/', '/index.html'], (req, res) => {
return require('./home/render').sendHome(req, res);
});
app.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(require('./home/render').llmsTxt()));
++// The crawl files the front door owes crawlers, built from openvibe-shared/seo (plan T11 lane D), ahead
++// of express.static below so they answer the generated copy rather than public/.
++app.use(require('./seo/routes').createSeoRoutes({ release }));
// Account hub (my.html) — the apex hosts the account hub under /my
// plus its client-routed sections.
ddiff --git a/server/seo/routes.js b/server/seo/routes.js
nnew file mode 100644
iindex 00000000..5e4333d2
---- /dev/null
++++ b/server/seo/routes.js
@@@ -0,0 +1,45 @@
++'use strict';
++// Crawl files for the public front door, mounted before express.static.
++const express = require('express');
++const fs = require('fs');
++const path = require('path');
++const seo = require('openvibe-shared/seo');
++
++const HOST = 'https://openvibe.network';
++// This is read once at boot if there is no release manifest.
++const FALLBACK_LASTMOD = fs.statSync(path.join(__dirname, '../home/render.js')).mtime.toISOString();
++
++// Public HTML routes; sign-in, account and admin pages are omitted.
++const PAGES = [
++ { path: '/', changefreq: 'daily', priority: 1.0 },
++ { path: '/status', changefreq: 'daily', priority: 0.8 },
++ { path: '/updates', changefreq: 'daily', priority: 0.8 },
++ { path: '/terms', changefreq: 'yearly', priority: 0.3 },
++ { path: '/privacy', changefreq: 'yearly', priority: 0.3 },
++ { path: '/dmca', changefreq: 'yearly', priority: 0.2 },
++];
++
++function robotsTxt() {
++ return seo.robotsTxt({ sitemaps: [`${HOST}/sitemap.xml`] });
++}
++
++function sitemapXml({ release = null } = {}) {
++ const lastmod = release?.full()?.released_at || FALLBACK_LASTMOD;
++ const day = String(lastmod).slice(0, 10);
++ return seo.sitemapXml(PAGES.map((page) => ({ loc: `${HOST}${page.path}`, lastmod: day, changefreq: page.changefreq, priority: page.priority })));
++}
++
++function createSeoRoutes({ release = null } = {}) {
++ const r = express.Router();
++ const robots = robotsTxt();
++ const sitemap = sitemapXml({ release });
++ r.get('/robots.txt', (_req, res) => {
++ res.set('Content-Type', 'text/plain; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(robots);
++ });
++ r.get('/sitemap.xml', (_req, res) => {
++ res.set('Content-Type', 'application/xml; charset=utf-8').set('Cache-Control', 'public, max-age=3600').send(sitemap);
++ });
++ return r;
++}
++
++module.exports = { createSeoRoutes, robotsTxt, sitemapXml, PAGES, HOST };
ddiff --git a/test/perf-budget.test.js b/test/perf-budget.test.js
iindex ace47b64..38a2f4ee 100644
---- a/test/perf-budget.test.js
++++ b/test/perf-budget.test.js
@@@ -41,7 +41,19 @@ const freePort = () => new Promise((resolve) => { const s = net.createServer();
if (!up) await new Promise((r) => setTimeout(r, 100));
}
assert.ok(up, `the server did not start:\n${stderr}`);
-- const m = await measure({ base });
++ // Node's fetch pools keep-alive sockets; the server closes an idle one after its default
++ // 5 s keepAliveTimeout, and a slow asset walk under load can hit that window and fail with
++ // UND_ERR_SOCKET ("other side closed"). This test measures sizes, not connection reuse.
++ const retryFetch = async (url, init) => {
++ for (let attempt = 0; ; attempt++) {
++ try { return await fetch(url, init); }
++ catch (e) {
++ if (attempt >= 2 || e?.cause?.code !== 'UND_ERR_SOCKET') throw e;
++ await new Promise((r) => setTimeout(r, 250));
++ }
++ }
++ };
++ const m = await measure({ base, fetch: retryFetch });
const over = check(m, BUDGETS);
assert.deepStrictEqual(over, [], format(m, over));
console.log(format(m));
ddiff --git a/test/seo-routes.test.js b/test/seo-routes.test.js
nnew file mode 100644
iindex 00000000..0a42032b
---- /dev/null
++++ b/test/seo-routes.test.js
@@@ -0,0 +1,64 @@
++'use strict';
++// The front door's crawl files (plan T11 lane D, audit §5 item 8): /robots.txt and /sitemap.xml are
++// generated from openvibe-shared/seo rather than served as static files, so a crawler is told about the
++// sitemap and gets lastmods that come from the release, not from the clock at request time.
++// node test/seo-routes.test.js
++const assert = require('assert');
++const { bootServer } = require('./helpers/boot-server');
++const seo = require('openvibe-shared/seo');
++const seoRoutes = require('../server/seo/routes');
++
++(async () => {
++ const srv = await bootServer();
++ try {
++ const get = (p) => fetch(srv.base + p).then(async r => ({ status: r.status, type: r.headers.get('content-type') || '', body: await r.text() }));
++
++ // ── /robots.txt ──
++ const robots = await get('/robots.txt');
++ assert.strictEqual(robots.status, 200, '/robots.txt is 200');
++ assert.ok(robots.type.startsWith('text/plain'), `/robots.txt is text/plain (got ${robots.type})`);
++ assert.ok(/^User-agent: \*/m.test(robots.body), 'robots.txt has the default group');
++ assert.ok(/^Allow: \/$/m.test(robots.body), 'robots.txt allows the root');
++ assert.ok(!robots.body.includes('Disallow:'), 'robots.txt allows all paths and crawlers');
++ for (const bot of seo.AI_AND_SEARCH_BOTS) assert.ok(robots.body.includes(`User-agent: ${bot}`), `robots.txt names ${bot}`);
++ assert.ok(robots.body.includes('Sitemap: https://openvibe.network/sitemap.xml'), 'robots.txt points at the sitemap');
++ for (const line of robots.body.split('\n')) if (line) assert.ok(/^[A-Za-z-]+: /.test(line), `every robots.txt line is a directive (got "${line}")`);
++
++ // ── /sitemap.xml ──
++ const sitemap = await get('/sitemap.xml');
++ assert.strictEqual(sitemap.status, 200, '/sitemap.xml is 200');
++ assert.ok(sitemap.type.startsWith('application/xml'), `/sitemap.xml is application/xml (got ${sitemap.type})`);
++ assert.ok(sitemap.body.startsWith('<?xml version="1.0" encoding="UTF-8"?>'), 'the sitemap has the XML declaration');
++ const opened = (body) => {
++ const tags = [...body.matchAll(/<(\/?)([a-z0-9]+:?[a-z0-9]*)\b[^>]*>/g)];
++ const stack = [];
++ for (const [, closing, name] of tags) {
++ if (closing) { assert.strictEqual(stack.pop(), name, `the sitemap closes <${name}>`); } else if (!/^<\?/.test(name)) stack.push(name);
++ }
++ assert.deepStrictEqual(stack, [], 'the sitemap has no unclosed tags');
++ };
++ opened(sitemap.body);
++ assert.ok(sitemap.body.includes('<loc>https://openvibe.network/</loc>'), 'the sitemap lists the home page');
++ assert.ok(sitemap.body.includes('<lastmod>'), 'every entry carries a lastmod');
++ const locs = [...sitemap.body.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m => m[1]);
++ assert.deepStrictEqual(locs, seoRoutes.PAGES.map(p => `${seoRoutes.HOST}${p.path}`), 'the sitemap lists exactly the public HTML pages');
++ assert.ok(locs.includes('https://openvibe.network/status'), 'the sitemap lists the status page');
++ assert.ok(!locs.some(l => /\/admin|\/login/.test(l)), 'private pages are not in the sitemap');
++ const lastmods = new Set([...sitemap.body.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m => m[1]));
++ assert.strictEqual(lastmods.size, 1, 'one lastmod for the whole sitemap');
++ assert.ok(/^\d{4}-\d{2}-\d{2}$/.test([...lastmods][0]), 'lastmod is a date, not a timestamp');
++
++ // ── lastmod is the release, not the request clock ──
++ const again = await get('/sitemap.xml');
++ assert.strictEqual(again.body, sitemap.body, 'two requests a moment apart produce the same sitemap');
++ const shipped = seoRoutes.sitemapXml({ release: { full: () => ({ released_at: '2026-01-02T03:04:05Z' }) } });
++ assert.ok(shipped.includes('<lastmod>2026-01-02</lastmod>'), 'lastmod follows the release manifest');
++
++ console.log('seo routes: all checks passed');
++ } catch (err) {
++ console.error(srv.logs());
++ throw err;
++ } finally {
++ await srv.stop();
++ }
++})().then(() => process.exit(0)).catch((err) => { console.error(err); process.exit(1); });
ddiff --git a/test/unknown-paths.test.js b/test/unknown-paths.test.js
iindex 1ceef1d8..cbaa8452 100644
---- a/test/unknown-paths.test.js
++++ b/test/unknown-paths.test.js
@@@ -26,6 +26,11 @@ const { ACCOUNT_HUB_PATHS } = require('../server/not-found');
assert.strictEqual(r.status, 200, `${p} is a real page: 200 (got ${r.status})`);
}
assert.strictEqual((await get('/sso/check')).status, 400, '/sso/check is a route (400 without its parameters), not a 404');
++ // The crawl files are generated routes (server/seo/routes.js), not the static files in public/.
++ const robots = await get('/robots.txt');
++ assert.ok(robots.type.startsWith('text/plain') && robots.text.includes('Sitemap: https://openvibe.network/sitemap.xml'), '/robots.txt is the generated text/plain copy');
++ const sitemap = await get('/sitemap.xml');
++ assert.ok(sitemap.type.startsWith('application/xml') && sitemap.text.includes('<loc>https://openvibe.network/</loc>'), '/sitemap.xml is the generated XML copy');
const hub = await get('/themes');
assert.ok(hub.text.includes('<html') && !hub.text.includes('Page not found'), 'an account hub section is the account hub');
// Redirects stay redirects.