openvibe.pics: Markdown twins of its public pages (SEO-1) (#278)

ddiff --git a/server/app.js b/server/app.js
iindex f885e68..9f29398 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -194,6 +194,21 @@ async function createApp(opts = {}) {
              return next();
          });
      }
++    // The Markdown twins of the pics host's public pages (SEO-1): a twin request only; the manifest stays the pics
++    // routes' own (/.well-known/agent.json in server/pics/pages.js).
++    if (picsHost) {
++        const picsAgent = express.Router();
++        require('openvibe-shared/agent-layer').mount(picsAgent, {
++            descriptor: require('./pics/agent-descriptor'),
++            render: require('./pics/agent-pages').createAgentPages({ router: picsRoutes, host: picsHost }).renderPublicPage,
++        });
++        app.use((req, res, next) => {
++            if (req.hostname !== picsHost || (req.method !== 'GET' && req.method !== 'HEAD')) return next();
++            const p = String(req.path || '');
++            const twin = p.endsWith('.md') || req.query.format === 'markdown' || req.accepts(['html', 'text/markdown']) === 'text/markdown';
++            return twin ? picsAgent(req, res, next) : next();
++        });
++    }
      app.use((req, res, next) => {
          if (videoHost && req.hostname === videoHost) return videoRoutes(req, res, next);
          if (picsHost && req.hostname === picsHost) return picsRoutes(req, res, next);
ddiff --git a/server/pics/agent-pages.js b/server/pics/agent-pages.js
nnew file mode 100644
iindex 0000000..510b190
---- /dev/null
++++ b/server/pics/agent-pages.js
@@@ -0,0 +1,18 @@
++'use strict';
++// The Markdown twins of openvibe.pics' public pages (openvibe-shared/agent-layer, SEO-1): the front door and the
++// pages the sitemap lists. A twin is the page's own render: the pics router answers in-process for an anonymous
++// browser (server/video/agent-pages.js renderIn), so it cannot drift and never carries a signed-in person's view.
++// A picture's page has no twin: it is the uploader's, and may be unlisted.
++const { renderIn } = require('../video/agent-pages');
++
++const TWIN_PATH_RE = /^\/(?:exif|content-policy|ai-training|terms|privacy|dmca|updates)?$/;
++
++function createAgentPages({ router, host }) {
++    async function renderPublicPage(path) {
++        const p = String(path || '/').replace(/\/+$/, '') || '/';
++        return TWIN_PATH_RE.test(p) ? renderIn(router, host, p, '') : null;
++    }
++    return { renderPublicPage };
++}
++
++module.exports = { createAgentPages, TWIN_PATH_RE };
ddiff --git a/server/pics/layout.js b/server/pics/layout.js
iindex ffbd339..c924f85 100644
---- a/server/pics/layout.js
++++ b/server/pics/layout.js
@@@ -35,6 +35,15 @@ const asset = (rel) => `/${rel}?v=${assetVersion(rel)}`;
  /** o: title, description, body (html), baseUrl, path, index, noindex, viewer, private, crumbs [{label, href}], scripts [rel],
   *     styles [shared file], sharedScripts [shared file], jsonLd (structured data for the page), image (its share image, an
   *     https URL), ogType */
++// A page names its Markdown twin (server/pics/agent-pages.js) when it has one: a public page, answered 200, drawn for
++// someone who is not signed in, without a query.
++function hasTwin(o) {
++    const viewer = o.viewer || { kind: 'anonymous' };
++    const pathNow = String(o.path || '/');
++    return viewer.kind !== 'user' && (o.status == null || o.status === 200) && !pathNow.includes('?')
++        && require('./agent-pages').TWIN_PATH_RE.test(pathNow);
++}
++
  function renderPage(o) {
      // The front page's title says what the site is (search results show it; at most 60 characters).
      const title = o.title || (o.path === '/' ? `${SITE_NAME}: share pictures, fast and open` : SITE_NAME);
@@@ -70,6 +79,7 @@ ${render(o.body)}
          siteName: SITE_NAME,
          description: o.description || `${SITE_NAME} — ${TAGLINE}`,
          canonical,
++        ...(hasTwin(o) ? { agentLayer: true, llms: '/llms.txt' } : {}),
          robots: o.noindex ? 'noindex' : (o.index ? 'index, follow' : 'noindex, nofollow'),
          ...(o.jsonLd ? { jsonLd: o.jsonLd } : {}),
          ...(o.image ? { image: o.image, imageAlt: title } : {}),
@@@ -103,7 +113,11 @@ const GUEST_HTML = 'public, max-age=0, must-revalidate';
  function send(res, status, o) {
      if (!res.get('Cache-Control')) res.set('Cache-Control', o.cache || (o.private ? cache.htmlHeaders({ private: true }) : GUEST_HTML));
      res.set('Vary', 'Host');
--    res.status(status).type('html').send(renderPage(o));
++    if (hasTwin({ ...o, status })) {
++        const canonical = `${String(o.baseUrl).replace(/\/+$/, '')}${String(o.path || '/')}`;
++        res.set('Link', require('openvibe-shared/agent-layer').linkHeaderTags(canonical, '/llms.txt').header);
++    }
++    res.status(status).type('html').send(renderPage({ ...o, status }));
  }
  
  module.exports = { renderPage, send, asset, assetVersion, SITE_NAME, TAGLINE, raw, html };
ddiff --git a/server/video/agent-pages.js b/server/video/agent-pages.js
iindex 155fc1b..7734d69 100644
---- a/server/video/agent-pages.js
++++ b/server/video/agent-pages.js
@@@ -46,32 +46,36 @@ function captureResponse() {
      };
  }
  
++/** One page of `router`, answered in-process for an anonymous browser on `host`: { html } for a 200 HTML page, else null. */
++async function renderIn(router, host, p, search = '') {
++    const req = anonymousRequest(p, search, host);
++    const res = captureResponse();
++    await new Promise((resolve) => {
++        let settled = false;
++        const finish = () => { if (!settled) { settled = true; resolve(); } };
++        const send = res.send.bind(res);
++        res.send = (body) => { send(body); finish(); return res; };
++        const redirect = res.redirect.bind(res);
++        res.redirect = (code, url) => { redirect(code, url); finish(); return res; };
++        try { router(req, res, () => finish()); } catch { finish(); }
++        const timer = setTimeout(finish, 10000);
++        if (timer.unref) timer.unref();
++    });
++    // A page that is not public (a private recording, a missing item, a redirect) has no twin.
++    if (res.statusCode !== 200 || res.body == null) return null;
++    const html = Buffer.isBuffer(res.body) ? res.body.toString('utf8') : String(res.body);
++    if (!/^text\/html/i.test(String(res.headers['Content-Type'] || ''))) return null;
++    return { html };
++}
++
  function createAgentPages({ router, videoHost }) {
      async function renderPublicPage(path, { query } = {}) {
          const p = String(path || '/').replace(/\/+$/, '') || '/';
          if (p !== '/' && !TWIN_PATH_RE.test(p)) return null;
--        const search = query && query.toString() ? `?${query.toString()}` : '';
--        const req = anonymousRequest(p, search, videoHost);
--        const res = captureResponse();
--        await new Promise((resolve) => {
--            let settled = false;
--            const finish = () => { if (!settled) { settled = true; resolve(); } };
--            const send = res.send.bind(res);
--            res.send = (body) => { send(body); finish(); return res; };
--            const redirect = res.redirect.bind(res);
--            res.redirect = (code, url) => { redirect(code, url); finish(); return res; };
--            try { router(req, res, () => finish()); } catch { finish(); }
--            const timer = setTimeout(finish, 10000);
--            if (timer.unref) timer.unref();
--        });
--        // A page that is not public (a private recording, a missing item, a redirect) has no twin.
--        if (res.statusCode !== 200 || res.body == null) return null;
--        const html = Buffer.isBuffer(res.body) ? res.body.toString('utf8') : String(res.body);
--        if (!/^text\/html/i.test(String(res.headers['Content-Type'] || ''))) return null;
--        return { html };
++        return renderIn(router, videoHost, p, query && query.toString() ? `?${query.toString()}` : '');
      }
  
      return { renderPublicPage };
  }
  
--module.exports = { createAgentPages, TWIN_PATH_RE };
++module.exports = { createAgentPages, TWIN_PATH_RE, renderIn };
ddiff --git a/test/pics-twins.test.js b/test/pics-twins.test.js
nnew file mode 100644
iindex 0000000..797a54d
---- /dev/null
++++ b/test/pics-twins.test.js
@@@ -0,0 +1,73 @@
++'use strict';
++/**
++ * SEO-1: the Markdown twins of openvibe.pics' public pages (server/pics/agent-pages.js). A twin is the page's own
++ * render for an anonymous browser; the page names it in its head and in the Link: header when it answered 200 to
++ * someone who is not signed in. A picture's page, a signed-in page and a missing page name none.
++ */
++const assert = require('assert');
++const http = require('http');
++const { boot, check, done } = require('./helpers/boot');
++
++function get(t, p, { cookie, accept } = {}) {
++    return new Promise((resolve, reject) => {
++        const headers = { Host: 'openvibe.pics' };
++        if (cookie) headers.cookie = cookie;
++        if (accept) headers.accept = accept;
++        const u = new URL(t.base + p);
++        const req = http.request({ hostname: u.hostname, port: u.port, path: u.pathname + u.search, method: 'GET', headers }, (res) => {
++            const chunks = [];
++            res.on('data', (c) => chunks.push(c));
++            res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, text: Buffer.concat(chunks).toString('utf8') }));
++        });
++        req.on('error', reject);
++        req.end();
++    });
++}
++
++const PAGES = ['/', '/exif', '/content-policy', '/ai-training', '/updates'];
++
++(async () => {
++    const t = await boot();
++    try {
++        await check('each public page names its twin in the head and the Link header, and the twin is the page', async () => {
++            for (const p of PAGES) {
++                const page = await get(t, p);
++                assert.strictEqual(page.status, 200, p);
++                const twinPath = p === '/' ? '/index.md' : `${p}.md`;
++                const href = `https://openvibe.pics${twinPath}`;
++                assert.ok(page.text.includes(`<link rel="alternate" type="text/markdown" href="${href}">`), `${p} names its twin in the head`);
++                assert.ok(String(page.headers.link || '').includes(`<${href}>; rel="alternate"; type="text/markdown"`), `${p} names it in the Link header`);
++                const twin = await get(t, twinPath);
++                assert.strictEqual(twin.status, 200, twinPath);
++                assert.ok(/^text\/markdown/.test(twin.headers['content-type']), `${twinPath} is Markdown`);
++                assert.ok(twin.text.startsWith('---\ntitle:') && twin.text.includes(`canonical: "https://openvibe.pics${p}"`), `${twinPath} is the pics page`);
++                assert.ok(!/<(?:script|style)\b/i.test(twin.text.replace(/~~~~[\s\S]*?~~~~/g, '')) && twin.text.length > 150, `${twinPath} is the page's words`);
++            }
++        });
++
++        await check('the drive and the video host keep their own pages', async () => {
++            const drive = await new Promise((resolve, reject) => {
++                const u = new URL(t.base + '/index.md');
++                http.get({ hostname: u.hostname, port: u.port, path: '/index.md', headers: { Host: 'openvibe.download' } }, (res) => {
++                    const chunks = [];
++                    res.on('data', (c) => chunks.push(c));
++                    res.on('end', () => resolve({ status: res.statusCode, text: Buffer.concat(chunks).toString('utf8') }));
++                }).on('error', reject);
++            });
++            assert.ok(!drive.text.includes('canonical: "https://openvibe.pics'), "the drive's twin is not the pics home");
++        });
++
++        await check('a picture, a missing page and a signed-in page name no twin', async () => {
++            const missing = await get(t, '/no-such-page');
++            assert.ok(!/text\/markdown/.test(missing.text) && !/text\/markdown/.test(String(missing.headers.link || '')), 'a missing page');
++            assert.strictEqual((await get(t, '/i/nosuchpicture.md')).status, 404, 'a picture has no twin');
++            const kim = t.network.addUser('kim-pics');
++            const mine = await get(t, '/exif', { cookie: t.signIn(kim) });
++            assert.strictEqual(mine.status, 200);
++            assert.ok(!/type="text\/markdown"/.test(mine.text) && !/text\/markdown/.test(String(mine.headers.link || '')), 'a signed-in page');
++        });
++    } finally {
++        await t.close();
++    }
++    done();
++})();