openvibers/openvibe.shared agent-layer.js

agent-layer.js

main 8.9 KB · 136 lines · javascript

Raw HistoryBlame Repository
1'use strict';
2// Server-only discovery and representations of the site's own public render output.
3const { parseHTML } = require('linkedom');
4const esc = s => String(s).replace(/[&"<>]/g, c => ({ '&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;' }[c]));
5const md = s => String(s || '').replace(/\\/g, '\\\\').replace(/([\[\]*_`])/g, '\\$1');
6
7function twinUrl(value) {
8 const url = new URL(value, 'https://agent.invalid');
9 url.searchParams.delete('format');
10 if (!url.pathname.endsWith('.md')) url.pathname = url.pathname === '/' ? '/index.md' : url.pathname.replace(/\/$/, '') + '.md';
11 return /^[a-z]+:/i.test(value) ? url.href : url.pathname + url.search + url.hash;
12}
13
14function agentManifest(descriptor = {}, overrides = {}) {
15 const d = { ...descriptor, ...overrides };
16 const home = d.home || d.canonical;
17 const abs = value => value && home ? new URL(value, home).href : value;
18 return {
19 network: 'OpenVibe', site: d.site || (home ? new URL(home).hostname : undefined),
20 id: d.id, name: d.name, purpose: d.purpose, home,
21 api: abs(d.api), openapi: abs(d.openapi),
22 mcp: d.mcp ? { transport: 'streamable-http', ...d.mcp, url: abs(d.mcp.url) } : undefined,
23 docs: abs(d.docs), llms: abs(d.llms || (home && '/llms.txt')),
24 llmsFull: abs(d.llmsFull || (home && '/llms-full.txt')), sitemap: abs(d.sitemap || (home && '/sitemap.xml')),
25 policy: { robots: 'https://openvibe.network/bot.json', ...d.policy },
26 };
27}
28
29function linkHeaderTags(canonical, llms = '/llms.txt') {
30 const twin = twinUrl(canonical);
31 // URL encoding keeps arbitrary caller input out of HTTP header syntax.
32 const headerUrl = s => String(s).replace(/[<>"\r\n]/g, c => encodeURIComponent(c));
33 return {
34 head: `<link rel="alternate" type="text/markdown" href="${esc(twin)}">\n<link rel="describedby" href="${esc(llms)}">`,
35 header: `<${headerUrl(twin)}>; rel="alternate"; type="text/markdown", <${headerUrl(llms)}>; rel="describedby"`,
36 };
37}
38
39function markdownTwin({ html, title, canonical, updated = null }) {
40 const { document } = parseHTML(html);
41 title = title || document.querySelector('title')?.textContent || '';
42 const walk = node => {
43 if (node.nodeType === 3) return md(node.textContent.replace(/\s+/g, ' '));
44 if (node.nodeType !== 1) return '';
45 const tag = node.localName;
46 // Controls are not content: buttons, inputs, icons, players, and a card's ⋯ menu (Save, Share) say nothing to a reader.
47 if (['script', 'style', 'template', 'nav', 'footer', 'button', 'svg', 'select', 'input', 'textarea', 'video', 'audio', 'dialog'].includes(tag)
48 || node.hasAttribute('hidden') || node.getAttribute('aria-hidden') === 'true' || (tag === 'details' && /\bovk-more\b/.test(node.getAttribute('class') || ''))) return '';
49 // Pieces the page sets side by side with no space between them (a card's who · count · when, a tile's name and
50 // line) keep a space between them, so "Maticus12 viewsTue" reads "Maticus 12 views Tue".
51 const inside = () => Array.from(node.childNodes).reduce((out, child) => {
52 const piece = walk(child);
53 return out && piece && child.nodeType === 1 && !['sup', 'sub', 'mark', 'wbr', 'abbr'].includes(child.localName) && !/\s$/.test(out) && !/^\s/.test(piece) && !/[([]$/.test(out) ? `${out} ${piece}` : out + piece;
54 }, '');
55 if (/^h[1-6]$/.test(tag)) return `\n\n${'#'.repeat(Number(tag[1]))} ${inside().trim()}\n\n`;
56 if (tag === 'a') return node.hasAttribute('href') ? `[${inside()}](${destination(node.getAttribute('href'))})` : inside();
57 if (tag === 'img') return `![${md(node.getAttribute('alt') || '')}](${destination(node.getAttribute('src') || '')})`;
58 if (tag === 'br') return '\n';
59 if (tag === 'strong' || tag === 'b') return `**${inside()}**`;
60 if (tag === 'em' || tag === 'i') return `*${inside()}*`;
61 if (tag === 'pre') return `\n\n~~~~\n${node.textContent}\n~~~~\n\n`;
62 if (tag === 'code') return '`' + node.textContent.replace(/`/g, '\\`') + '`';
63 if (tag === 'ul' || tag === 'ol') {
64 return '\n\n' + Array.from(node.children).filter(n => n.localName === 'li').map((n, i) => {
65 const text = Array.from(n.childNodes).map(walk).join('').trim();
66 const prefix = tag === 'ol' ? `${Number(node.getAttribute('start') || 1) + i}. ` : '- ';
67 return prefix + text.replace(/\n/g, '\n' + ' '.repeat(prefix.length));
68 }).join('\n') + '\n\n';
69 }
70 if (tag === 'table') {
71 const rows = Array.from(node.querySelectorAll('tr')).map(row => Array.from(row.children).filter(n => /^(th|td)$/.test(n.localName)).map(n => walk(n).trim().replace(/\|/g, '\\|').replace(/\n+/g, '<br>')));
72 if (!rows.length) return '';
73 const width = Math.max(...rows.map(r => r.length));
74 const line = row => '| ' + Array.from({ length: width }, (_, i) => row[i] || '').join(' | ') + ' |';
75 return '\n\n' + [line(rows[0]), line(Array(width).fill('---')), ...rows.slice(1).map(line)].join('\n') + '\n\n';
76 }
77 if (tag === 'dl') {
78 // A strip of numbers (kit.stats) or a definition list: one "- label: value" line each.
79 const pairs = [];
80 for (const n of node.querySelectorAll('dt')) {
81 let dd = n.nextElementSibling;
82 while (dd && dd.localName !== 'dd') dd = dd.nextElementSibling;
83 if (!dd) dd = n.previousElementSibling && n.previousElementSibling.localName === 'dd' ? n.previousElementSibling : null;
84 const k = walk(n).trim(), v = dd ? walk(dd).trim() : '';
85 if (k || v) pairs.push(`- ${k}${v ? `: ${v}` : ''}`);
86 }
87 return pairs.length ? `\n\n${pairs.join('\n')}\n\n` : '';
88 }
89 const content = inside();
90 return /^(p|div|section|article|main|blockquote|dt|dd)$/.test(tag) ? `\n\n${content}\n\n` : content;
91 };
92 const destination = value => String(value).replace(/[\s()<>]/g, c => encodeURIComponent(c).replace(/\(/g, '%28').replace(/\)/g, '%29'));
93 const content = walk(document.querySelector('main') || document.body).replace(/\n[ \t]+\n/g, '\n\n').replace(/\n{3,}/g, '\n\n').trim();
94 return `---\ntitle: ${JSON.stringify(title)}\ncanonical: ${JSON.stringify(canonical)}\nupdated: ${JSON.stringify(updated)}\n---\n\n# ${md(title)}\n\nHTML: [${md(title || 'Original page')}](${destination(canonical)})\n\n${content}\n`;
95}
96
97/** Mount before public HTML routes. render(path, { query }) returns { html, title?, updated? } or null.
98 * It must use the same public snapshot renderer as HTML and must never return personalised content.
99 */
100function mount(app, { descriptor = {}, render }) {
101 const manifest = agentManifest(descriptor);
102 app.use((req, res, next) => {
103 if (!['GET', 'HEAD'].includes(req.method)) return next();
104 const url = new URL(req.url, 'https://agent.invalid');
105 if (url.pathname === '/.well-known/agent.json') {
106 res.setHeader('Content-Type', 'application/json; charset=utf-8');
107 res.setHeader('Cache-Control', 'public, max-age=3600');
108 return res.end(req.method === 'HEAD' ? undefined : JSON.stringify(manifest));
109 }
110 const isTwin = url.pathname.endsWith('.md') || url.searchParams.get('format') === 'markdown' || (req.headers.accept && req.accepts(['html', 'text/markdown']) === 'text/markdown');
111 let path = url.pathname;
112 if (path === '/index.md') path = '/';
113 else if (path.endsWith('.md')) path = path.slice(0, -3);
114 url.searchParams.delete('format');
115 if (!render) return next();
116 Promise.resolve().then(() => render(path, { query: url.searchParams })).then(page => {
117 if (!page) {
118 if (!isTwin) return next();
119 res.statusCode = 404;
120 res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
121 return res.end(req.method === 'HEAD' ? undefined : '# Not found\n');
122 }
123 const canonical = new URL(path + url.search, manifest.home).href;
124 const links = linkHeaderTags(path + url.search, manifest.llms || '/llms.txt');
125 res.setHeader('Link', [res.getHeader('Link'), links.header].filter(Boolean).join(', '));
126 res.setHeader('Vary', [res.getHeader('Vary'), 'Accept'].filter(Boolean).join(', '));
127 res.setHeader('Cache-Control', 'public, max-age=300');
128 res.setHeader('Content-Type', isTwin ? 'text/markdown; charset=utf-8' : 'text/html; charset=utf-8');
129 const body = isTwin ? markdownTwin({ ...page, canonical }) : page.html.replace(/<link\b(?=[^>]*\brel=["'](?:alternate|describedby)["'])(?=[^>]*(?:text\/markdown|rel=["']describedby["']))[^>]*>/gi, '').replace(/<\/head>/i, links.head + '\n</head>');
130 res.end(req.method === 'HEAD' ? undefined : body);
131 }).catch(next);
132 });
133}
134
135module.exports = { mount, agentManifest, markdownTwin, linkHeaderTags, twinUrl };
136