'use strict';
// Server-only discovery and representations of the site's own public render output.
const { parseHTML } = require('linkedom');
const esc = s => String(s).replace(/[&"<>]/g, c => ({ '&': '&', '"': '"', '<': '<', '>': '>' }[c]));
const md = s => String(s || '').replace(/\\/g, '\\\\').replace(/([\[\]*_`])/g, '\\$1');
function twinUrl(value) {
const url = new URL(value, 'https://agent.invalid');
url.searchParams.delete('format');
if (!url.pathname.endsWith('.md')) url.pathname = url.pathname === '/' ? '/index.md' : url.pathname.replace(/\/$/, '') + '.md';
return /^[a-z]+:/i.test(value) ? url.href : url.pathname + url.search + url.hash;
}
function agentManifest(descriptor = {}, overrides = {}) {
const d = { ...descriptor, ...overrides };
const home = d.home || d.canonical;
const abs = value => value && home ? new URL(value, home).href : value;
return {
network: 'OpenVibe', site: d.site || (home ? new URL(home).hostname : undefined),
id: d.id, name: d.name, purpose: d.purpose, home,
api: abs(d.api), openapi: abs(d.openapi),
mcp: d.mcp ? { transport: 'streamable-http', ...d.mcp, url: abs(d.mcp.url) } : undefined,
docs: abs(d.docs), llms: abs(d.llms || (home && '/llms.txt')),
llmsFull: abs(d.llmsFull || (home && '/llms-full.txt')), sitemap: abs(d.sitemap || (home && '/sitemap.xml')),
policy: { robots: 'https://openvibe.network/bot.json', ...d.policy },
};
}
function linkHeaderTags(canonical, llms = '/llms.txt') {
const twin = twinUrl(canonical);
// URL encoding keeps arbitrary caller input out of HTTP header syntax.
const headerUrl = s => String(s).replace(/[<>"\r\n]/g, c => encodeURIComponent(c));
return {
head: `\n`,
header: `<${headerUrl(twin)}>; rel="alternate"; type="text/markdown", <${headerUrl(llms)}>; rel="describedby"`,
};
}
function markdownTwin({ html, title, canonical, updated = null }) {
const { document } = parseHTML(html);
title = title || document.querySelector('title')?.textContent || '';
const walk = node => {
if (node.nodeType === 3) return md(node.textContent.replace(/\s+/g, ' '));
if (node.nodeType !== 1) return '';
const tag = node.localName;
// Controls are not content: buttons, inputs, icons, players, and a card's ⋯ menu (Save, Share) say nothing to a reader.
if (['script', 'style', 'template', 'nav', 'footer', 'button', 'svg', 'select', 'input', 'textarea', 'video', 'audio', 'dialog'].includes(tag)
|| node.hasAttribute('hidden') || node.getAttribute('aria-hidden') === 'true' || (tag === 'details' && /\bovk-more\b/.test(node.getAttribute('class') || ''))) return '';
// Pieces the page sets side by side with no space between them (a card's who · count · when, a tile's name and
// line) keep a space between them, so "Maticus12 viewsTue" reads "Maticus 12 views Tue".
const inside = () => Array.from(node.childNodes).reduce((out, child) => {
const piece = walk(child);
return out && piece && child.nodeType === 1 && !['sup', 'sub', 'mark', 'wbr', 'abbr'].includes(child.localName) && !/\s$/.test(out) && !/^\s/.test(piece) && !/[([]$/.test(out) ? `${out} ${piece}` : out + piece;
}, '');
if (/^h[1-6]$/.test(tag)) return `\n\n${'#'.repeat(Number(tag[1]))} ${inside().trim()}\n\n`;
if (tag === 'a') return node.hasAttribute('href') ? `[${inside()}](${destination(node.getAttribute('href'))})` : inside();
if (tag === 'img') return ` || '')})`;
if (tag === 'br') return '\n';
if (tag === 'strong' || tag === 'b') return `**${inside()}**`;
if (tag === 'em' || tag === 'i') return `*${inside()}*`;
if (tag === 'pre') return `\n\n~~~~\n${node.textContent}\n~~~~\n\n`;
if (tag === 'code') return '`' + node.textContent.replace(/`/g, '\\`') + '`';
if (tag === 'ul' || tag === 'ol') {
return '\n\n' + Array.from(node.children).filter(n => n.localName === 'li').map((n, i) => {
const text = Array.from(n.childNodes).map(walk).join('').trim();
const prefix = tag === 'ol' ? `${Number(node.getAttribute('start') || 1) + i}. ` : '- ';
return prefix + text.replace(/\n/g, '\n' + ' '.repeat(prefix.length));
}).join('\n') + '\n\n';
}
if (tag === 'table') {
const rows = Array.from(node.querySelectorAll('tr')).map(row => Array.from(row.children).filter(n => /^(th|td)$/.test(n.localName)).map(n => walk(n).trim().replace(/\|/g, '\\|').replace(/\n+/g, '
')));
if (!rows.length) return '';
const width = Math.max(...rows.map(r => r.length));
const line = row => '| ' + Array.from({ length: width }, (_, i) => row[i] || '').join(' | ') + ' |';
return '\n\n' + [line(rows[0]), line(Array(width).fill('---')), ...rows.slice(1).map(line)].join('\n') + '\n\n';
}
if (tag === 'dl') {
// A strip of numbers (kit.stats) or a definition list: one "- label: value" line each.
const pairs = [];
for (const n of node.querySelectorAll('dt')) {
let dd = n.nextElementSibling;
while (dd && dd.localName !== 'dd') dd = dd.nextElementSibling;
if (!dd) dd = n.previousElementSibling && n.previousElementSibling.localName === 'dd' ? n.previousElementSibling : null;
const k = walk(n).trim(), v = dd ? walk(dd).trim() : '';
if (k || v) pairs.push(`- ${k}${v ? `: ${v}` : ''}`);
}
return pairs.length ? `\n\n${pairs.join('\n')}\n\n` : '';
}
const content = inside();
return /^(p|div|section|article|main|blockquote|dt|dd)$/.test(tag) ? `\n\n${content}\n\n` : content;
};
const destination = value => String(value).replace(/[\s()<>]/g, c => encodeURIComponent(c).replace(/\(/g, '%28').replace(/\)/g, '%29'));
const content = walk(document.querySelector('main') || document.body).replace(/\n[ \t]+\n/g, '\n\n').replace(/\n{3,}/g, '\n\n').trim();
return `---\ntitle: ${JSON.stringify(title)}\ncanonical: ${JSON.stringify(canonical)}\nupdated: ${JSON.stringify(updated)}\n---\n\n# ${md(title)}\n\nHTML: [${md(title || 'Original page')}](${destination(canonical)})\n\n${content}\n`;
}
/** Mount before public HTML routes. render(path, { query }) returns { html, title?, updated? } or null.
* It must use the same public snapshot renderer as HTML and must never return personalised content.
*/
function mount(app, { descriptor = {}, render }) {
const manifest = agentManifest(descriptor);
app.use((req, res, next) => {
if (!['GET', 'HEAD'].includes(req.method)) return next();
const url = new URL(req.url, 'https://agent.invalid');
if (url.pathname === '/.well-known/agent.json') {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'public, max-age=3600');
return res.end(req.method === 'HEAD' ? undefined : JSON.stringify(manifest));
}
const isTwin = url.pathname.endsWith('.md') || url.searchParams.get('format') === 'markdown' || (req.headers.accept && req.accepts(['html', 'text/markdown']) === 'text/markdown');
let path = url.pathname;
if (path === '/index.md') path = '/';
else if (path.endsWith('.md')) path = path.slice(0, -3);
url.searchParams.delete('format');
if (!render) return next();
Promise.resolve().then(() => render(path, { query: url.searchParams })).then(page => {
if (!page) {
if (!isTwin) return next();
res.statusCode = 404;
res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
return res.end(req.method === 'HEAD' ? undefined : '# Not found\n');
}
const canonical = new URL(path + url.search, manifest.home).href;
const links = linkHeaderTags(path + url.search, manifest.llms || '/llms.txt');
res.setHeader('Link', [res.getHeader('Link'), links.header].filter(Boolean).join(', '));
res.setHeader('Vary', [res.getHeader('Vary'), 'Accept'].filter(Boolean).join(', '));
res.setHeader('Cache-Control', 'public, max-age=300');
res.setHeader('Content-Type', isTwin ? 'text/markdown; charset=utf-8' : 'text/html; charset=utf-8');
const body = isTwin ? markdownTwin({ ...page, canonical }) : page.html.replace(/]*\brel=["'](?:alternate|describedby)["'])(?=[^>]*(?:text\/markdown|rel=["']describedby["']))[^>]*>/gi, '').replace(/<\/head>/i, links.head + '\n');
res.end(req.method === 'HEAD' ? undefined : body);
}).catch(next);
});
}
module.exports = { mount, agentManifest, markdownTwin, linkHeaderTags, twinUrl };