openvibers/openvibe.food Commits

Pages send the Link: header for their Markdown twin and llms.txt (SEO-24) (#206)

OP OpenVibers committed 10 days ago · 4e7bedf05e · 3 files, +99 −0

The head already named both; the header said nothing, so a reader that fetches only the headers
found no twin. server/link-header.js copies the head's two links into the header for a successful
HTML page, so they never disagree; a page whose head names no twin sends none.

Changes

ddiff --git a/server/app.js b/server/app.js
iindex b823abf..ac7a46b 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -232,6 +232,7 @@ async function createApp(opts = {}) {
              return next();
          });
      }
++    app.use(require('./link-header').linkHeader());   // SEO-24: the head's twin and llms.txt links, as a Link: header
      app.use(pageRoutes);
      app.use((req, res) => send(res, 404, { viewer: req.viewer, config, path: req.originalUrl, title: 'Not found', body: html`<h1>Not found</h1><p>No page here. Try <a href="/">the home page</a> or <a href="/updates">the update log</a>.</p>` }));
  
ddiff --git a/server/link-header.js b/server/link-header.js
nnew file mode 100644
iindex 0000000..a73af46
---- /dev/null
++++ b/server/link-header.js
@@@ -0,0 +1,50 @@
++'use strict';
++// The Link: header for a page's Markdown twin and llms.txt (SEO-24). A page names both in its head
++// (<link rel="alternate" type="text/markdown"> and <link rel="describedby">, openvibe-shared/agent-layer); this says
++// the same in the header, for a reader that fetches only the headers or reads them first. It copies what the head
++// says, so the two never disagree, and a page whose head names no twin (signed in, not found) sends no header.
++
++const LINK_TAG = /<link\b[^>]*>/gi;
++const attr = (tag, name) => {
++    const m = tag.match(new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, 'i'));
++    return m ? (m[1] != null ? m[1] : m[2]) : null;
++};
++const unescape = (s) => s.replace(/&quot;/g, '"').replace(/&#0*39;/g, "'").replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&');
++// URL encoding keeps a page's address out of HTTP header syntax (the same as linkHeaderTags).
++const headerUrl = (s) => String(s).replace(/[<>"\r\n]/g, (c) => encodeURIComponent(c));
++
++/** The Link header value an HTML page's head implies, or null when its head names no twin and no llms.txt. */
++function linksOf(html) {
++    const end = String(html).search(/<\/head>/i);
++    if (end < 0) return null;
++    let twin = null;
++    let llms = null;
++    for (const tag of String(html).slice(0, end).match(LINK_TAG) || []) {
++        const rel = String(attr(tag, 'rel') || '').toLowerCase().split(/\s+/);
++        const href = attr(tag, 'href');
++        if (!href) continue;
++        if (!twin && rel.includes('alternate') && String(attr(tag, 'type') || '').toLowerCase() === 'text/markdown') twin = unescape(href);
++        else if (!llms && rel.includes('describedby')) llms = unescape(href);
++    }
++    if (!twin || !llms) return null;
++    return `<${headerUrl(twin)}>; rel="alternate"; type="text/markdown", <${headerUrl(llms)}>; rel="describedby"`;
++}
++
++/** Express middleware, mounted before the page routes: a successful HTML page sends the header its head implies. */
++function linkHeader() {
++    return function pageLinkHeader(req, res, next) {
++        if (req.method !== 'GET' && req.method !== 'HEAD') return next();
++        const send = res.send;
++        res.send = function sendWithLinkHeader(body) {
++            if (typeof body === 'string' && res.statusCode >= 200 && res.statusCode < 300 && !res.get('Link')
++                && /^text\/html\b/i.test(String(res.get('Content-Type') || 'text/html'))) {
++                const value = linksOf(body);
++                if (value) res.set('Link', value);
++            }
++            return send.call(this, body);
++        };
++        next();
++    };
++}
++
++module.exports = { linkHeader, linksOf };
ddiff --git a/test/link-header.test.js b/test/link-header.test.js
nnew file mode 100644
iindex 0000000..50d343b
---- /dev/null
++++ b/test/link-header.test.js
@@@ -0,0 +1,48 @@
++'use strict';
++// SEO-24: a page that names its Markdown twin and llms.txt in its head says the same in a Link: header.
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { linkHeader, linksOf } = require('../server/link-header');
++
++const page = (head) => `<!doctype html><html><head><title>A page</title>${head}</head><body><link rel="describedby" href="/not-the-head"></body></html>`;
++const TWIN = '<link rel="alternate" type="application/rss+xml" href="/feed.xml">'
++    + '<link rel="alternate" type="text/markdown" href="https://site.test/q?a=1&amp;b=2.md">\n<link rel="describedby" href="/llms.txt">';
++
++assert.strictEqual(linksOf(page(TWIN)), '<https://site.test/q?a=1&b=2.md>; rel="alternate"; type="text/markdown", </llms.txt>; rel="describedby"',
++    'the header names what the head names, unescaped once');
++assert.strictEqual(linksOf(page('<link rel="alternate" type="application/rss+xml" href="/feed.xml">')), null, 'a head with no twin: no header');
++assert.strictEqual(linksOf('<p>no head</p>'), null);
++assert.strictEqual(linksOf(page("<link type='text/markdown' rel='alternate' href='/x.md'><link href='/llms.txt' rel='describedby'>")),
++    '</x.md>; rel="alternate"; type="text/markdown", </llms.txt>; rel="describedby"', 'attribute order and quotes do not matter');
++assert.strictEqual(linksOf(page('<link rel="alternate" type="text/markdown" href="/a&quot;b&lt;c.md"><link rel="describedby" href="/llms.txt">')).split(', ')[0],
++    '</a%22b%3Cc.md>; rel="alternate"; type="text/markdown"', 'a quote or bracket in an address is encoded, never raw in the header');
++
++function run({ method = 'GET', status = 200, type, link, body }) {
++    const headers = {};
++    if (type) headers['content-type'] = type;
++    if (link) headers.link = link;
++    const res = {
++        statusCode: status,
++        get: (k) => headers[k.toLowerCase()],
++        set(k, v) { headers[k.toLowerCase()] = v; return this; },
++        send(b) { this.sent = b; return this; },
++    };
++    let called = false;
++    linkHeader()({ method }, res, () => { called = true; });
++    assert.ok(called, 'the middleware always passes on');
++    res.send(body);
++    return headers.link;
++}
++
++assert.match(run({ body: page(TWIN) }), /rel="alternate"; type="text\/markdown"/, 'a 200 HTML page gets the header');
++assert.match(run({ method: 'HEAD', type: 'text/html; charset=utf-8', body: page(TWIN) }), /rel="describedby"/, 'HEAD too');
++assert.strictEqual(run({ status: 404, body: page(TWIN) }), undefined, 'a page that was not found: none');
++assert.strictEqual(run({ type: 'application/json', body: page(TWIN) }), undefined, 'not HTML: none');
++assert.strictEqual(run({ method: 'POST', body: page(TWIN) }), undefined, 'a form post: none');
++assert.strictEqual(run({ link: '</other>; rel="next"', body: page(TWIN) }), '</other>; rel="next"', 'a header the page set itself stays');
++assert.strictEqual(run({ body: page('') }), undefined, 'a page whose head names no twin: none');
++
++const app = fs.readFileSync(path.join(__dirname, '..', 'server/app.js'), 'utf8');
++assert.match(app, /app\.use\(require\('\.\/link-header'\)\.linkHeader\(\)\);/, 'the app mounts it before its page routes');
++console.log('link-header: ok');