Markdown twins of the public pages, named in the head and the Link header (SEO-1) (#116)

ddiff --git a/server/agent-twins.js b/server/agent-twins.js
nnew file mode 100644
iindex 0000000..04cb54a
---- /dev/null
++++ b/server/agent-twins.js
@@@ -0,0 +1,104 @@
++'use strict';
++// The Markdown twins of this site's public pages (SEO-1, openvibe-shared/agent-layer).
++//
++// - A twin is the page's own render, signed out: the site's page router runs for a request that carries no session,
++//   so the twin says exactly what a visitor who is not signed in reads. Only the public pages the sitemap lists have
++//   one (TWIN_PATH_RE).
++// - `/x.md`, `/index.md`, `?format=markdown` or `Accept: text/markdown` asks for a twin; every other request goes on
++//   to the pages.
++// - Such a page names its twin in the head and in a Link: header when it answered 200 to a visitor who is not signed
++//   in. A signed-in page is that person's own and names none.
++const express = require('express');
++const agentLayer = require('openvibe-shared/agent-layer');
++
++const TWIN_PATH_RE = /^\/(?:docs|benchmarks(?:\/bmk_[A-Za-z0-9]+)?)?$/;
++
++/** A response the page code can write to; the first body written is the page's HTML. */
++function mockRes() {
++    return {
++        statusCode: 200, headers: {}, headersSent: false, locals: {}, finished: false,
++        setHeader(k, v) { this.headers[String(k).toLowerCase()] = v; return this; },
++        getHeader(k) { return this.headers[String(k).toLowerCase()]; },
++        removeHeader(k) { delete this.headers[String(k).toLowerCase()]; },
++        set(k, v) { if (k && typeof k === 'object') { for (const [a, b] of Object.entries(k)) this.setHeader(a, b); return this; } return this.setHeader(k, v); },
++        get(k) { return this.getHeader(k); },
++        header(k, v) { return this.set(k, v); },
++        append(k, v) { return this.setHeader(k, v); },
++        status(code) { this.statusCode = code; return this; },
++        type(t) { return this.setHeader('content-type', t); },
++        vary() { return this; },
++        location(url) { return this.setHeader('location', url); },
++        redirect(a, b) { this.statusCode = typeof a === 'number' ? a : 302; this.headers.location = typeof a === 'number' ? b : a; this.finished = true; return this; },
++        cookie() { return this; }, clearCookie() { return this; }, attachment() { return this; }, links() { return this; },
++        format() { return this; }, render() { return this; },
++        write(chunk) { this.body = (this.body || '') + chunk; return true; },
++        on() { return this; }, once() { return this; }, emit() { return false; },
++    };
++}
++
++/** A request without a session, so the page renders signed out. */
++function mockReq(path, host) {
++    return {
++        method: 'GET', url: path, originalUrl: path, path, baseUrl: '', params: {}, query: {}, body: {},
++        headers: { host, accept: 'text/html' }, hostname: host, protocol: 'https', secure: true,
++        ip: '127.0.0.1', ips: [], socket: {}, connection: {}, cookies: {}, signedCookies: {},
++        viewer: { kind: 'anonymous' }, ov: {},
++        get(name) { return this.headers[String(name).toLowerCase()]; },
++        header(name) { return this.get(name); },
++        accepts(types) { return Array.isArray(types) ? types[0] : (types || 'html'); },
++        is() { return false; },
++    };
++}
++
++/** Run the site's page router for one path; the HTML of a 200 page, or null. */
++function renderWithRouter(router, path, host) {
++    return new Promise((resolve) => {
++        const res = mockRes();
++        const keep = (body) => { resolve(res.statusCode === 200 && typeof body === 'string' ? body : null); return res; };
++        res.send = keep; res.end = keep; res.json = () => keep(null); res.sendStatus = () => keep(null);
++        try {
++            router.handle(mockReq(path, host), res, () => resolve(null));
++        } catch {
++            resolve(null);
++        }
++    });
++}
++
++const wantsTwin = (req) => {
++    const p = String(req.path || '');
++    return p.endsWith('.md') || (req.query && req.query.format === 'markdown') || req.accepts(['html', 'text/markdown']) === 'text/markdown';
++};
++
++/** Mount before the page routes: router is the site's page router, baseUrl its own origin. */
++function mountTwins(app, { router, descriptor, baseUrl, llms = '/llms.txt' }) {
++    const origin = String(baseUrl).replace(/\/+$/, '');
++    const host = new URL(origin).host;
++    const twins = express.Router();
++    agentLayer.mount(twins, {
++        descriptor,
++        render: async (path) => {
++            const p = String(path || '/').replace(/(.)\/+$/, '$1');
++            if (!TWIN_PATH_RE.test(p)) return null;
++            const html = await renderWithRouter(router, p, host);
++            return html ? { html, title: null, updated: null } : null;
++        },
++    });
++    app.use((req, res, next) => (wantsTwin(req) ? twins(req, res, next) : next()));
++    app.use((req, res, next) => {
++        // A page with a query (a search, a filter) is not the page its twin renders: it names none.
++        if ((req.method !== 'GET' && req.method !== 'HEAD') || !TWIN_PATH_RE.test(req.path) || String(req.originalUrl || '').includes('?')) return next();
++        const send = res.send;
++        res.send = function sendNamingTheTwin(body) {
++            if (typeof body === 'string' && res.statusCode === 200 && !(req.viewer && req.viewer.kind === 'user')
++                && /^text\/html\b/i.test(String(res.get('Content-Type') || 'text/html'))) {
++                const links = agentLayer.linkHeaderTags(`${origin}${req.path}`, llms);
++                if (!/<link\b[^>]*text\/markdown/i.test(body)) body = body.replace(/<\/head>/i, `${links.head}\n</head>`);
++                if (!res.get('Link')) res.set('Link', links.header);
++            }
++            return send.call(this, body);
++        };
++        next();
++    });
++}
++
++module.exports = { mountTwins, TWIN_PATH_RE, renderWithRouter };
ddiff --git a/server/app.js b/server/app.js
iindex ba03cbc..7e509c7 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -256,7 +256,11 @@ async function createApp(opts = {}) {
      app.use('/battle', createBattlePages(ctx));
      app.use('/ratings', createRatingsPages(ctx));
      app.use(require('./http/imports').createImportRoutes(ctx));
--    app.use(createPageRoutes(ctx));
++    const pageRoutes = createPageRoutes(ctx);
++    // The Markdown twins of the public pages, and the head link and Link: header that name them (SEO-1,
++    // server/agent-twins.js): a twin is the page rendered signed out.
++    require('./agent-twins').mountTwins(app, { router: pageRoutes, descriptor: require('./agent-descriptor'), baseUrl: ctx.config.baseUrl });
++    app.use(pageRoutes);
      app.use((req, res) => send(res, 404, {
          viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
          body: raw(require('./views').notFound()),
ddiff --git a/test/agent-twins.test.js b/test/agent-twins.test.js
nnew file mode 100644
iindex 0000000..b5cba6c
---- /dev/null
++++ b/test/agent-twins.test.js
@@@ -0,0 +1,63 @@
++'use strict';
++// SEO-1: the public pages' Markdown twins (server/agent-twins.js). A twin is the page rendered signed out, and the
++// page names it in its head and in the Link: header.
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { boot } = require('./helpers/boot');
++const { TWIN_PATH_RE } = require('../server/agent-twins');
++
++// Public pages from the sitemap that render without data.
++const PAGES = ['/', '/docs', '/benchmarks'];
++
++(async () => {
++    const t = await boot();
++    const origin = String(t.config.baseUrl).replace(/\/+$/, '');
++    const reEsc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
++    let failed = 0;
++    const check = async (name, fn) => {
++        try { await fn(); console.log(`  ✓ ${name}`); } catch (e) { failed++; console.log(`  ✗ ${name}\n      ${String(e.stack || e).split('\n').slice(0, 6).join('\n      ')}`); }
++    };
++    try {
++        await check('every listed page has a twin, and the page names it', async () => {
++            for (const p of PAGES) {
++                assert.ok(TWIN_PATH_RE.test(p), `${p} is a twin path`);
++                const page = await t.get(p);
++                assert.strictEqual(page.status, 200, p);
++                const twinPath = p === '/' ? '/index.md' : `${p}.md`;
++                const href = `${origin}${twinPath}`;
++                assert.ok(page.text.includes(`<link rel="alternate" type="text/markdown" href="${href}">`), `${p} names its twin in the head`);
++                assert.match(String(page.headers.get('link') || ''), new RegExp(`<${reEsc(href)}>; rel="alternate"; type="text/markdown"`), `${p} names it in the Link header`);
++                const twin = await t.get(twinPath);
++                assert.strictEqual(twin.status, 200, twinPath);
++                assert.match(String(twin.headers.get('content-type') || ''), /^text\/markdown/, `${twinPath} is Markdown`);
++                assert.ok(twin.text.startsWith('---\ntitle:') && /\ncanonical: "/.test(twin.text), `${twinPath} carries the page's title and address`);
++                // Markup only inside fenced code (a page's own code samples), never the page's scripts or styles.
++                const prose = twin.text.replace(/~~~~[\s\S]*?~~~~/g, '');
++                assert.ok(!/<(?:script|style)\b/i.test(prose) && twin.text.length > 150, `${twinPath} is the page's words`);
++            }
++        });
++        await check('a path with no twin answers a Markdown 404, and a missing page names no twin', async () => {
++            const twin = await t.get('/no-such-page-here.md');
++            assert.strictEqual(twin.status, 404);
++            const page = await t.get('/no-such-page-here');
++            assert.ok(!/type="text\/markdown"/.test(page.text) && !/text\/markdown/.test(String(page.headers.get('link') || '')));
++        });
++        if (t.network && typeof t.network.addUser === 'function') {
++            await check('a page drawn for a signed-in person names no twin', async () => {
++                const page = await t.get(PAGES[0], { as: t.network.addUser('twin-reader') });
++                assert.ok(!/type="text\/markdown"/.test(page.text), 'no twin in the head');
++                assert.ok(!/text\/markdown/.test(String(page.headers.get('link') || '')), 'no twin in the Link header');
++            });
++        }
++        await check('the twins are mounted before the page routes', async () => {
++            const src = fs.readFileSync(path.join(__dirname, '..', 'server', 'app.js'), 'utf8');
++            const at = src.indexOf("require('./agent-twins').mountTwins(app,");
++            assert.ok(at > 0 && at < src.indexOf('app.use(pageRoutes);'));
++        });
++    } finally {
++        await t.close();
++    }
++    console.log(failed ? `\n${failed} failed` : '\nall passed');
++    process.exit(failed ? 1 : 0);
++})();