openvibers/openvibe.shared seo.js

seo.js

main 34.2 KB · 463 lines · javascript

Raw HistoryBlame Repository
1'use strict';
2/**
3 * openvibe-shared/seo.js — one SEO toolkit for every OpenVibe site (Node, no dependencies).
4 *
5 * const seo = require('openvibe-shared/seo');
6 * seo.headTags({ title, description, canonical, image, jsonLd: [seo.jsonLd.website({...})] })
7 * seo.sitemapXml([{ loc, lastmod, videos? }]) seo.robotsTxt({ sitemaps }) seo.llmsTxt({...})
8 *
9 * Rules it encodes so pages do not have to remember them: one canonical per page, titles cut at
10 * 60 characters on a word, descriptions at 160, og/twitter mirrored from the same values, JSON-LD
11 * serialised safely (no `</script>` break-outs), sitemaps with absolute URLs only, robots.txt that
12 * welcomes search and AI crawlers by default and always names the sitemap, and an /llms.txt so
13 * language-model crawlers get a clean map of the site.
14 */
15
16const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
17const xml = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;' }[c]));
18
19/** Cut on a word boundary with an ellipsis; never mid-word, never longer than `max`. */
20function clip(text, max) {
21 const t = String(text == null ? '' : text).replace(/\s+/g, ' ').trim();
22 if (t.length <= max) return t;
23 const cut = t.slice(0, max - 1);
24 const at = cut.lastIndexOf(' ');
25 return (at > max * 0.6 ? cut.slice(0, at) : cut).replace(/[\s,;:.\-–—]+$/, '') + '…';
26}
27
28function absolute(url, base) {
29 if (!url) return '';
30 try { return new URL(url, base || undefined).toString(); } catch { return String(url); }
31}
32
33/** JSON-LD script tag; `<` is escaped so content can never close the script element. */
34function jsonLdTag(obj) {
35 return `<script type="application/ld+json">${JSON.stringify(obj).replace(/</g, '\\u003c')}</script>`;
36}
37
38/**
39 * The whole <head> SEO block as an HTML string.
40 * @param {object} o title, description, canonical (absolute), image, imageAlt, type ('website'),
41 * siteName, robots ('index, follow'), locale ('en_US'), keywords (array|string), author,
42 * themeColor, twitterSite, alternates [{ hreflang, href }], prev, next, jsonLd (array|object),
43 * titleSuffix (appended when it fits and the title does not already name the site), noTitle (skip <title> when the page has its own)
44 */
45function headTags(o = {}) {
46 let title = String(o.title || o.siteName || 'OpenVibe');
47 // The suffix (" · OpenVibe.Work") is left off a title that already names the site: "What shipped on OpenVibe.Work
48 // · OpenVibe.Work" was every update log's title (11-02).
49 const named = String(o.titleSuffix || '').replace(/^[\s·|—–-]+/, '').trim().toLowerCase();
50 if (o.titleSuffix && !(named && title.toLowerCase().includes(named)) && (title + o.titleSuffix).length <= 60) title += o.titleSuffix;
51 title = clip(title, 60);
52 const description = clip(o.description || '', 160);
53 const canonical = absolute(o.canonical);
54 const image = o.image ? absolute(o.image, canonical) : '';
55 const keywords = Array.isArray(o.keywords) ? o.keywords.join(', ') : (o.keywords || '');
56 const robots = o.robots || 'index, follow, max-image-preview:large, max-snippet:-1';
57 const out = [];
58 if (!o.noTitle) out.push(`<title>${esc(title)}</title>`);
59 if (description) out.push(`<meta name="description" content="${esc(description)}">`);
60 if (keywords) out.push(`<meta name="keywords" content="${esc(keywords)}">`);
61 out.push(`<meta name="robots" content="${esc(robots)}">`);
62 if (o.author) out.push(`<meta name="author" content="${esc(o.author)}">`);
63 if (o.themeColor) out.push(`<meta name="theme-color" content="${esc(o.themeColor)}">`);
64 if (canonical) out.push(`<link rel="canonical" href="${esc(canonical)}">`);
65 // Every page's bar asks openvibe.network (the Frame, sign-in, the menu): open that connection early (4.135.0).
66 if (o.preconnect !== false && !/^https:\/\/openvibe\.network(\/|$)/.test(canonical || '')) out.push('<link rel="preconnect" href="https://openvibe.network" crossorigin>');
67 for (const a of o.alternates || []) if (a && a.href) out.push(`<link rel="alternate" hreflang="${esc(a.hreflang || 'x-default')}" href="${esc(absolute(a.href))}">`);
68 if (o.prev) out.push(`<link rel="prev" href="${esc(absolute(o.prev))}">`);
69 if (o.next) out.push(`<link rel="next" href="${esc(absolute(o.next))}">`);
70 out.push(`<meta property="og:type" content="${esc(o.type || 'website')}">`);
71 if (o.siteName) out.push(`<meta property="og:site_name" content="${esc(o.siteName)}">`);
72 out.push(`<meta property="og:title" content="${esc(title)}">`);
73 if (description) out.push(`<meta property="og:description" content="${esc(description)}">`);
74 if (canonical) out.push(`<meta property="og:url" content="${esc(canonical)}">`);
75 out.push(`<meta property="og:locale" content="${esc(o.locale || 'en_US')}">`);
76 if (image) {
77 out.push(`<meta property="og:image" content="${esc(image)}">`);
78 if (o.imageAlt) out.push(`<meta property="og:image:alt" content="${esc(o.imageAlt)}">`);
79 }
80 out.push(`<meta name="twitter:card" content="${image && o.largeImage !== false ? 'summary_large_image' : 'summary'}">`);
81 if (o.twitterSite) out.push(`<meta name="twitter:site" content="${esc(o.twitterSite)}">`);
82 out.push(`<meta name="twitter:title" content="${esc(title)}">`);
83 if (description) out.push(`<meta name="twitter:description" content="${esc(description)}">`);
84 if (image) out.push(`<meta name="twitter:image" content="${esc(image)}">`);
85 const ld = o.jsonLd ? (Array.isArray(o.jsonLd) ? o.jsonLd : [o.jsonLd]).filter(Boolean) : [];
86 // A WebSite names its publisher by @id (OpenVibe's Organization). A page with a WebSite but no Organization gets the
87 // node too, so the reference resolves on every site and search engines see one OpenVibe across the network.
88 const types = (x) => [].concat(x && x['@type'] || []);
89 if (ld.some((x) => types(x).includes('WebSite')) && !ld.some((x) => types(x).includes('Organization') || x['@id'] === ORG['@id'])) ld.push(jsonLd.organization());
90 for (const item of ld) out.push(jsonLdTag(item));
91 return out.join('\n');
92}
93
94// OpenVibe, said so search engines and AI cannot take it for something else (the owner, 2026-10-28: Google's AI Overview
95// for "whois openvibe" described an unrelated "Openvibe" mobile app): what it is, its other names, what it is not, and its
96// sites. Network's home page carries the same Organization (#org); every site's pages name it as their publisher.
97const ORG = {
98 '@type': 'Organization', '@id': 'https://openvibe.network/#org', name: 'OpenVibe', alternateName: ['OpenVibe Network', 'OpenVibers'],
99 url: 'https://openvibe.network/', logo: 'https://openvibe.network/assets/logo-512.png',
100 description: 'OpenVibe is an open-source, community-run network of websites that share one free account: live streaming from the browser (OpenVibe.Live), more than 180 online tools (OpenVibe.Tools), chat and the Vibe assistant (OpenVibe.Chat), games, a wiki, news, reviews, deals and an open developer platform.',
101 disambiguatingDescription: 'The open-source network of websites at openvibe.network, made by OpenVibers; not the Openvibe mobile social-media aggregator app.',
102 sameAs: ['https://github.com/OpenVibers', 'https://discord.gg/M6MuRUaeJj', 'https://openvibe.live', 'https://openvibe.tools', 'https://openvibe.chat', 'https://openvibe.games',
103 'https://openvibe.media', 'https://openvibe.video', 'https://openvibe.community', 'https://openvibe.space', 'https://openvibe.wiki', 'https://openvibe.blog',
104 'https://openvibe.news', 'https://openvibe.services', 'https://openre.stream'],
105};
106
107const jsonLd = {
108 organization(extra = {}) { return { '@context': 'https://schema.org', ...ORG, ...extra }; },
109 /** WebSite; `searchUrl` with {q} adds the sitelinks SearchAction. */
110 website({ name, url, description, searchUrl, inLanguage = 'en' } = {}) {
111 const o = { '@context': 'https://schema.org', '@type': 'WebSite', '@id': `${url}#site`, name, url, description, inLanguage, publisher: { '@id': ORG['@id'] } };
112 if (searchUrl) o.potentialAction = { '@type': 'SearchAction', target: { '@type': 'EntryPoint', urlTemplate: searchUrl.replace('{q}', '{search_term_string}') }, 'query-input': 'required name=search_term_string' };
113 return o;
114 },
115 webPage({ name, url, description, type = 'WebPage', siteUrl, dateModified } = {}) {
116 return { '@context': 'https://schema.org', '@type': type, '@id': `${url}#page`, name, url, description, ...(siteUrl ? { isPartOf: { '@id': `${siteUrl}#site` } } : {}), ...(dateModified ? { dateModified } : {}) };
117 },
118 /** A web tool. No price/offer fields: the platform makes no cost claims. */
119 softwareApp({ name, url, description, category = 'UtilitiesApplication', keywords, image, os = 'Any (web browser)' } = {}) {
120 return { '@context': 'https://schema.org', '@type': 'WebApplication', name, url, description, applicationCategory: category, operatingSystem: os, browserRequirements: 'Requires a modern web browser', ...(keywords ? { keywords: Array.isArray(keywords) ? keywords.join(', ') : keywords } : {}), ...(image ? { image } : {}), publisher: { '@id': ORG['@id'] } };
121 },
122 breadcrumbs(items = []) {
123 return { '@context': 'https://schema.org', '@type': 'BreadcrumbList', itemListElement: items.map((it, i) => ({ '@type': 'ListItem', position: i + 1, name: it.name, item: it.url })) };
124 },
125 itemList(name, items = [], { url } = {}) {
126 return { '@context': 'https://schema.org', '@type': 'ItemList', name, ...(url ? { url } : {}), numberOfItems: items.length, itemListElement: items.map((it, i) => ({ '@type': 'ListItem', position: i + 1, name: it.name, url: it.url, ...(it.description ? { description: it.description } : {}) })) };
127 },
128 faq(pairs = []) {
129 return { '@context': 'https://schema.org', '@type': 'FAQPage', mainEntity: pairs.map((p) => ({ '@type': 'Question', name: p.q, acceptedAnswer: { '@type': 'Answer', text: p.a } })) };
130 },
131 howTo({ name, description, steps = [] } = {}) {
132 return { '@context': 'https://schema.org', '@type': 'HowTo', name, description, step: steps.map((s, i) => ({ '@type': 'HowToStep', position: i + 1, name: s.name || `Step ${i + 1}`, text: s.text })) };
133 },
134 article({ headline, url, description, author, datePublished, dateModified, image } = {}) {
135 return { '@context': 'https://schema.org', '@type': 'Article', headline: clip(headline, 110), url, description, ...(author ? { author: { '@type': 'Person', name: author } } : {}), ...(datePublished ? { datePublished } : {}), ...(dateModified ? { dateModified } : {}), ...(image ? { image } : {}), publisher: { '@id': ORG['@id'] } };
136 },
137 video({ name, url, description, thumbnailUrl, uploadDate, duration, contentUrl, embedUrl } = {}) {
138 return { '@context': 'https://schema.org', '@type': 'VideoObject', name, url, description: description || name, thumbnailUrl, uploadDate, ...(duration ? { duration } : {}), ...(contentUrl ? { contentUrl } : {}), ...(embedUrl ? { embedUrl } : {}) };
139 },
140 videoObject(options = {}) { return jsonLd.video(options); },
141 product({ name, url, description, image, sku, brand, offers, aggregateRating, review } = {}) {
142 return { '@context': 'https://schema.org', '@type': 'Product', name, url, description,
143 ...(image ? { image } : {}), ...(sku ? { sku } : {}),
144 ...(brand ? { brand: typeof brand === 'string' ? { '@type': 'Brand', name: brand } : brand } : {}),
145 ...(offers ? { offers } : {}), ...(aggregateRating ? { aggregateRating } : {}), ...(review ? { review } : {}) };
146 },
147 review({ name, url, reviewBody, author, datePublished, itemReviewed, rating, bestRating = 5, worstRating = 1 } = {}) {
148 return { '@context': 'https://schema.org', '@type': 'Review', name, url, reviewBody,
149 ...(author ? { author: typeof author === 'string' ? { '@type': 'Person', name: author } : author } : {}),
150 ...(datePublished ? { datePublished } : {}), ...(itemReviewed ? { itemReviewed } : {}),
151 ...(rating != null ? { reviewRating: { '@type': 'Rating', ratingValue: rating, bestRating, worstRating } } : {}) };
152 },
153 aggregateRating({ ratingValue, ratingCount, reviewCount, bestRating = 5, worstRating = 1 } = {}) {
154 return { '@context': 'https://schema.org', '@type': 'AggregateRating', ratingValue,
155 ...(ratingCount != null ? { ratingCount } : {}), ...(reviewCount != null ? { reviewCount } : {}), bestRating, worstRating };
156 },
157 imageGallery(images = []) {
158 return images.map((image) => {
159 const item = typeof image === 'string' ? { url: image } : image;
160 return { '@context': 'https://schema.org', '@type': 'ImageObject', url: item.url,
161 ...(item.name ? { name: item.name } : {}), ...(item.caption ? { caption: item.caption } : {}),
162 ...(item.width != null ? { width: item.width } : {}), ...(item.height != null ? { height: item.height } : {}) };
163 });
164 },
165 /** An event that really is happening at a time the page shows. Only the parts the page has are written. */
166 event({ name, startDate, endDate, url, description, organizer, location, eventStatus, eventAttendanceMode, image } = {}) {
167 return { '@context': 'https://schema.org', '@type': 'Event', name, startDate,
168 ...(endDate ? { endDate } : {}), ...(description ? { description } : {}),
169 ...(eventStatus ? { eventStatus } : {}), ...(eventAttendanceMode ? { eventAttendanceMode } : {}),
170 ...(location ? { location } : {}),
171 ...(organizer ? { organizer: typeof organizer === 'string' ? { '@type': 'Organization', name: organizer } : organizer } : {}),
172 ...(image ? { image } : {}), ...(url ? { url } : {}) };
173 },
174 /** A public job listing. `description` is the page's own excerpt; nothing is invented for the fields it omits. */
175 jobPosting({ title, url, description, datePosted, validThrough, hiringOrganization, jobLocationType, employmentType, jobLocation, baseSalary, identifier, directApply } = {}) {
176 return { '@context': 'https://schema.org', '@type': 'JobPosting', title,
177 ...(description ? { description } : {}), ...(datePosted ? { datePosted } : {}), ...(validThrough ? { validThrough } : {}),
178 ...(hiringOrganization ? { hiringOrganization: typeof hiringOrganization === 'string' ? { '@type': 'Organization', name: hiringOrganization } : hiringOrganization } : {}),
179 ...(jobLocationType ? { jobLocationType } : {}), ...(employmentType ? { employmentType } : {}),
180 ...(jobLocation ? { jobLocation } : {}), ...(baseSalary ? { baseSalary } : {}),
181 ...(identifier ? { identifier } : {}), ...(directApply != null ? { directApply } : {}), ...(url ? { url } : {}) };
182 },
183 /** A recipe page's own ingredients and steps. Steps may be strings or ready HowToStep objects. */
184 recipe({ name, url, description, image, recipeIngredient, recipeInstructions, recipeYield, totalTime, author, datePublished } = {}) {
185 return { '@context': 'https://schema.org', '@type': 'Recipe', name,
186 ...(description ? { description } : {}), ...(image ? { image } : {}),
187 ...(recipeIngredient ? { recipeIngredient } : {}),
188 ...(recipeInstructions ? { recipeInstructions: recipeInstructions.map((s) => (typeof s === 'string' ? { '@type': 'HowToStep', text: s } : s)) } : {}),
189 ...(recipeYield ? { recipeYield } : {}), ...(totalTime ? { totalTime } : {}),
190 ...(author ? { author: typeof author === 'string' ? { '@type': 'Organization', name: author } : author } : {}),
191 ...(datePublished ? { datePublished } : {}), ...(url ? { url } : {}) };
192 },
193 /** An open dataset a page publishes. `license` only when the site names one. */
194 dataset({ name, url, description, license, creator, keywords, distribution } = {}) {
195 return { '@context': 'https://schema.org', '@type': 'Dataset', name,
196 ...(description ? { description } : {}), ...(license ? { license } : {}),
197 ...(creator ? { creator: typeof creator === 'string' ? { '@type': 'Organization', name: creator } : creator } : {}),
198 ...(keywords ? { keywords: Array.isArray(keywords) ? keywords.join(', ') : keywords } : {}),
199 ...(distribution ? { distribution } : {}), ...(url ? { url } : {}) };
200 },
201};
202
203const isAbs = (u) => typeof u === 'string' && /^https?:\/\//i.test(u);
204
205/**
206 * One <video:video> of a video sitemap (Google's sitemap-video/1.1), or '' when a required part is missing: a thumbnail,
207 * a title, a description and the video itself (content_loc, the file, or player_loc, a page or embed that plays it).
208 * Duration is whole seconds (1–28,800, longer is left out); publication_date an ISO date or date-time.
209 */
210function videoXml(v) {
211 if (!v || !isAbs(v.thumbnail_loc) || !v.title || !v.description || !(isAbs(v.content_loc) || isAbs(v.player_loc))) return '';
212 const parts = [`<video:thumbnail_loc>${xml(v.thumbnail_loc)}</video:thumbnail_loc>`,
213 `<video:title>${xml(String(v.title).slice(0, 100))}</video:title>`,
214 `<video:description>${xml(String(v.description).slice(0, 2048))}</video:description>`];
215 if (isAbs(v.content_loc)) parts.push(`<video:content_loc>${xml(v.content_loc)}</video:content_loc>`);
216 if (isAbs(v.player_loc)) parts.push(`<video:player_loc>${xml(v.player_loc)}</video:player_loc>`);
217 const d = Math.round(Number(v.duration));
218 if (d >= 1 && d <= 28800) parts.push(`<video:duration>${d}</video:duration>`);
219 if (v.view_count != null && Number.isFinite(Number(v.view_count))) parts.push(`<video:view_count>${Math.max(0, Math.floor(Number(v.view_count)))}</video:view_count>`);
220 if (v.publication_date) parts.push(`<video:publication_date>${xml(String(v.publication_date))}</video:publication_date>`);
221 if (v.family_friendly === false) parts.push('<video:family_friendly>no</video:family_friendly>');
222 if (v.uploader) {
223 const info = isAbs(v.uploader_info) ? ` info="${xml(v.uploader_info)}"` : '';
224 parts.push(`<video:uploader${info}>${xml(String(v.uploader).slice(0, 255))}</video:uploader>`);
225 }
226 if (v.live != null) parts.push(`<video:live>${v.live ? 'yes' : 'no'}</video:live>`);
227 return `<video:video>${parts.join('')}</video:video>`;
228}
229
230/**
231 * urls: [{ loc, lastmod?, changefreq?, priority?, alternates?: [{hreflang, href}], images?: [url], videos?: [video] }],
232 * absolute http(s) only. A video is { thumbnail_loc, title, description, content_loc and/or player_loc, duration?,
233 * view_count?, publication_date?, family_friendly?, uploader?, uploader_info?, live? } (see videoXml); the video
234 * namespace is declared only when a video is listed.
235 */
236function sitemapXml(urls = []) {
237 let videos = false;
238 const rows = urls.filter((u) => u && /^https?:\/\//i.test(u.loc)).map((u) => {
239 const parts = [`<loc>${xml(u.loc)}</loc>`];
240 if (u.lastmod) parts.push(`<lastmod>${xml(String(u.lastmod).slice(0, 10))}</lastmod>`);
241 if (u.changefreq) parts.push(`<changefreq>${xml(u.changefreq)}</changefreq>`);
242 if (u.priority != null) parts.push(`<priority>${Number(u.priority).toFixed(1)}</priority>`);
243 for (const a of u.alternates || []) parts.push(`<xhtml:link rel="alternate" hreflang="${xml(a.hreflang)}" href="${xml(a.href)}"/>`);
244 for (const img of u.images || []) parts.push(`<image:image><image:loc>${xml(img)}</image:loc></image:image>`);
245 for (const v of u.videos || []) { const x = videoXml(v); if (x) { parts.push(x); videos = true; } }
246 return ` <url>${parts.join('')}</url>`;
247 });
248 const videoNs = videos ? ' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"' : '';
249 return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"${videoNs}>\n${rows.join('\n')}\n</urlset>\n`;
250}
251
252function sitemapIndexXml(maps = []) {
253 const rows = maps.filter((m) => m && m.loc).map((m) => ` <sitemap><loc>${xml(m.loc)}</loc>${m.lastmod ? `<lastmod>${xml(String(m.lastmod).slice(0, 10))}</lastmod>` : ''}</sitemap>`);
254 return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${rows.join('\n')}\n</sitemapindex>\n`;
255}
256
257// Crawlers that build search and AI indexes — welcomed by name so a generic Disallow never hits them.
258const AI_AND_SEARCH_BOTS = ['Googlebot', 'Bingbot', 'DuckDuckBot', 'Applebot', 'GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'Claude-User', 'Claude-SearchBot', 'Claude-Web', 'anthropic-ai', 'PerplexityBot', 'Perplexity-User', 'CCBot', 'Google-Extended', 'Amazonbot', 'meta-externalagent', 'YouBot', 'Bytespider'];
259
260function robotsTxt({ sitemaps = [], disallow = [], allow = ['/'], allowAI = true, block = [], crawlDelay } = {}) {
261 const lines = [];
262 // Content signals (contentsignals.org): with allowAI the network says yes to search, AI answers and training in so many
263 // words (owner, 2026-10-25: "fully AI allowed and indexable"), so no crawler has to guess from silence.
264 if (allowAI) lines.push('# Content signals: you may use this site\'s content for search, AI answers (ai-input) and AI training.', 'Content-Signal: search=yes, ai-input=yes, ai-train=yes', '');
265 lines.push('User-agent: *');
266 for (const a of allow) lines.push(`Allow: ${a}`);
267 for (const d of disallow) lines.push(`Disallow: ${d}`);
268 if (crawlDelay) lines.push(`Crawl-delay: ${crawlDelay}`);
269 if (allowAI) {
270 lines.push('');
271 for (const bot of AI_AND_SEARCH_BOTS) lines.push(`User-agent: ${bot}`);
272 lines.push('Allow: /');
273 for (const d of disallow) lines.push(`Disallow: ${d}`);
274 }
275 for (const b of block) { lines.push('', `User-agent: ${b}`, 'Disallow: /'); }
276 lines.push('');
277 for (const s of sitemaps) lines.push(`Sitemap: ${s}`);
278 return lines.join('\n') + '\n';
279}
280
281// The endings that make a path a file rather than a page (4.108.1). Any dot used to count, so a page named after a
282// domain (Help's /sites/openvibe.blog, Host's /catalog/domains/co.uk) lost its twin, unless its ending was longer
283// than eight letters (/sites/openvibe.community kept it).
284const FILE_EXT = new Set(('txt md json jsonl xml rss atom csv tsv yaml yml toml ics vcf webmanifest html htm js mjs cjs css map wasm '
285 + 'png jpg jpeg gif webp avif svg ico bmp tif tiff heic pdf zip gz tgz tar bz2 xz 7z rar dmg exe msi deb rpm apk '
286 + 'mp3 mp4 m4a m4v webm ogg oga ogv wav flac mov avi mkv m3u8 ts srt vtt woff woff2 ttf otf eot sh ps1 py').split(' '));
287
288/**
289 * Whether llms.txt's markdown mode links `url` at its Markdown twin: one of the site's own pages. A file (a last path
290 * segment with a file ending: /openapi.json, /feed.xml, /llms-full.txt), an /api/ or /.well-known/ path and, when
291 * `base` is given, another origin's link keep their URL (4.108.0; before, /openapi.json became /openapi.json.md).
292 */
293function twinnable(url, base) {
294 let u;
295 try { u = new URL(url, base || 'https://agent.invalid'); } catch { return false; }
296 if (!/^https?:$/.test(u.protocol)) return false;
297 if (base && /^([a-z][a-z0-9+.-]*:|\/\/)/i.test(String(url)) && u.origin !== new URL(base).origin) return false;
298 if (/^\/(api|\.well-known)(\/|$)/.test(u.pathname)) return false;
299 const last = u.pathname.split('/').pop();
300 const dot = last.lastIndexOf('.');
301 return dot < 0 || !FILE_EXT.has(last.slice(dot + 1).toLowerCase());
302}
303
304/**
305 * /llms.txt (llmstxt.org): a markdown map of the site for language-model crawlers. `markdown: true` links each of the
306 * site's own pages at its Markdown twin (/hot → /hot.md, / → /index.md); `base` (the site's origin) says which absolute
307 * links are its own. A page with no twin takes `html: true` and keeps its URL, marked "(HTML)". `optional` is the
308 * llmstxt.org "## Optional" section: links an agent may skip when it needs a shorter context.
309 */
310function llmsTxt({ name, summary, details, sections = [], markdown = false, optional = [], base } = {}) {
311 const out = [`# ${name}`, '', `> ${String(summary || '').replace(/\s+/g, ' ').trim()}`, ''];
312 if (details) out.push(String(details).trim(), '');
313 const href = (l) => (markdown && !l.html && twinnable(l.url, base) ? require('./agent-layer').twinUrl(l.url) : l.url);
314 for (const sec of [...sections, ...(optional.length ? [{ title: 'Optional', links: optional }] : [])]) {
315 if (!sec || !sec.links || !sec.links.length) continue;
316 out.push(`## ${sec.title}`, '');
317 for (const l of sec.links) out.push(`- [${l.title}](${href(l)})${l.html ? ' (HTML)' : ''}${l.note ? `: ${String(l.note).replace(/\s+/g, ' ').trim()}` : ''}`);
318 out.push('');
319 }
320 return out.join('\n');
321}
322
323/** Preserve full-text paragraphs while enforcing a character limit, including the ellipsis. */
324function limitText(value, max) {
325 const text = String(value == null ? '' : value);
326 if (max == null || !Number.isFinite(max)) return text;
327 const size = Math.max(0, Math.floor(max));
328 if (text.length <= size) return text;
329 return size ? text.slice(0, size - 1).trimEnd() + '…' : '';
330}
331
332const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
333
334/** Plain text from HTML: scripts, styles and comments dropped, block ends kept as line breaks. */
335function htmlText(html) {
336 return String(html == null ? '' : html)
337 .replace(/<!--[\s\S]*?-->/g, '')
338 .replace(/<(script|style|template)\b[\s\S]*?<\/\1\s*>/gi, '')
339 .replace(/<br\s*\/?>|<\/(li|dt|dd|tr)\s*>/gi, '\n')
340 .replace(/<\/(p|div|section|article|header|footer|main|aside|nav|ul|ol|dl|table|h[1-6]|blockquote|pre|figure)\s*>/gi, '\n\n')
341 .replace(/<[^>]*>/g, '')
342 .replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
343 if (e[0] !== '#') return ENTITIES[e.toLowerCase()] || m;
344 const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
345 return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : m;
346 })
347 .split('\n').map((line) => line.replace(/\s+/g, ' ').trim()).join('\n')
348 .replace(/\n{3,}/g, '\n\n')
349 .trim();
350}
351
352/** One /llms-full.txt page: `### title`, `URL: …` and its full text (`text`, or `html` stripped to text). */
353function llmsPage(page, base) {
354 const url = absolute(page.url, base);
355 const text = page.text != null ? String(page.text).replace(/\r\n?/g, '\n').trim() : htmlText(page.html);
356 return `\n### ${page.title || ''}\n${url ? `\nURL: ${url}\n` : ''}${text ? `\n${text}\n` : ''}`;
357}
358
359/**
360 * /llms-full.txt: the llms.txt header followed by full, plain-text sections. A section with
361 * `pages: [{ title, url, text | html }]` renders `## title` then every page in input order, never
362 * clipped; `maxBytes` stops before the first page that would pass it and ends with a
363 * `(truncated: N more pages at <base>/llms.txt)` line. A section with `url`/`body` renders as in 2.4.0.
364 */
365function llmsFull({ site, summary, base, sections = [], maxChars, maxTotal, maxBytes } = {}) {
366 const name = site && typeof site === 'object' ? site.name || site.title : site;
367 base = base || (site && typeof site === 'object' ? site.url : site);
368 let out = llmsTxt({ name, summary });
369 let left = 0;
370 for (const section of sections) {
371 if (!section) continue;
372 if (Array.isArray(section.pages)) {
373 let heading = `\n## ${section.title || ''}\n`;
374 for (const page of section.pages) {
375 if (!page) continue;
376 const chunk = heading + llmsPage(page, base);
377 if (left || (maxBytes != null && Buffer.byteLength(out + chunk) > maxBytes)) { left++; continue; }
378 out += chunk;
379 heading = '';
380 }
381 continue;
382 }
383 if (left) continue;
384 const url = absolute(section.url, base);
385 const body = limitText(String(section.body == null ? '' : section.body).trim(), section.maxChars == null ? maxChars : section.maxChars);
386 out += `\n## ${section.title || ''}\n\n${url}\n\n${body}\n`;
387 }
388 if (left) out += `\n(truncated: ${left} more page${left === 1 ? '' : 's'} at ${absolute('/llms.txt', base)})\n`;
389 return limitText(out, maxTotal);
390}
391
392/**
393 * AI-readable page summary. As a string (`${seo.pageSummary(…)}`, `String(…)`) it is the HTML:
394 * `<meta name="ai-summary">` (clipped to 160 like the description), a WebPage JSON-LD tag and, when
395 * `facts` (strings) are given, a `<noscript><section data-ai-summary>` block. `.head` and `.body` hold
396 * the head part and the noscript block on their own; `.meta`, `.jsonLd` and `.html` are the 2.4.0 values.
397 */
398function pageSummary({ title, summary, facts = [], url, updated } = {}) {
399 const pageUrl = absolute(url);
400 const text = String(summary == null ? '' : summary);
401 const jsonLd = { '@context': 'https://schema.org', '@type': 'WebPage', name: title, url: pageUrl,
402 abstract: text, ...(updated ? { dateModified: updated } : {}) };
403 const rows = facts.filter(Array.isArray).map(([key, value]) => `<dt>${esc(key)}</dt><dd>${esc(value)}</dd>`).join('');
404 const html = `<section data-ov-summary hidden><h2>${esc(title)}</h2><p>${esc(text)}</p>${rows ? `<dl>${rows}</dl>` : ''}${pageUrl ? `<a href="${esc(pageUrl)}">${esc(pageUrl)}</a>` : ''}${updated ? `<time datetime="${esc(updated)}">${esc(updated)}</time>` : ''}</section>`;
405 const modified = feedDate(updated, 'iso');
406 const ld = { '@context': 'https://schema.org', '@type': 'WebPage', ...(title != null ? { name: String(title) } : {}),
407 description: text, ...(pageUrl ? { url: pageUrl } : {}), ...(modified ? { dateModified: modified } : {}), abstract: text };
408 const head = [text ? `<meta name="ai-summary" content="${esc(clip(text, 160))}">` : '', jsonLdTag(ld)].filter(Boolean).join('\n');
409 const items = facts.map((f) => (Array.isArray(f) ? f.filter((x) => x != null && x !== '').join(': ') : String(f == null ? '' : f)).trim()).filter(Boolean);
410 const body = items.length ? `<noscript><section data-ai-summary><h2>${esc(title)}</h2><p>${esc(text)}</p><ul>${items.map((f) => `<li>${esc(f)}</li>`).join('')}</ul></section></noscript>` : '';
411 const result = { meta: esc(clip(text, 160)), jsonLd, html };
412 const tags = body ? `${head}\n${body}` : head;
413 Object.defineProperties(result, { head: { value: head }, body: { value: body }, toString: { value: () => tags } });
414 return result;
415}
416
417function feedDate(value, format) {
418 if (!value) return '';
419 const date = new Date(value);
420 return Number.isNaN(date.getTime()) ? '' : (format === 'rss' ? date.toUTCString() : date.toISOString());
421}
422
423/**
424 * One feed shape for RSS 2.0 and Atom 1.0; `format` is 'rss' or 'atom', anything else throws.
425 * feed: { title, link, description, language?, updated?, selfUrl?, items }
426 * item: { title, link, guid?, description?, content?, author?, published, updated? } (the 2.4.0
427 * `url`, `id` and `summary` names still work). Dates are Dates or ISO strings: RFC 822 in RSS,
428 * RFC 3339 in Atom. Relative item links resolve against feed.link; items keep their order.
429 */
430function feedXml({ title, link, description, language, updated, selfUrl, items = [] } = {}, { format = 'rss' } = {}) {
431 if (format !== 'rss' && format !== 'atom') throw new TypeError('format must be rss or atom');
432 const home = absolute(link);
433 const self = absolute(selfUrl, home);
434 const times = items.map((item) => new Date(item.updated || item.published || 0).getTime()).filter((t) => t > 0);
435 const built = feedDate(updated || (times.length ? Math.max(...times) : Date.now()), format);
436 const rows = items.map((item) => {
437 const url = absolute(item.link || item.url, home);
438 const guid = item.guid || item.id;
439 const summary = item.description || item.summary;
440 const published = feedDate(item.published || item.updated, format);
441 const modified = feedDate(item.updated || item.published, format);
442 if (format === 'rss') return ` <item><title>${esc(item.title)}</title><link>${esc(url)}</link><guid isPermaLink="${guid ? 'false' : 'true'}">${esc(guid || url)}</guid>${summary ? `<description>${esc(summary)}</description>` : ''}${published ? `<pubDate>${esc(published)}</pubDate>` : ''}${item.author ? `<author>${esc(item.author)}</author>` : ''}</item>`;
443 return ` <entry><id>${esc(guid ? absolute(guid, home) : url)}</id><title>${esc(item.title)}</title><link href="${esc(url)}"/>${published ? `<published>${esc(published)}</published>` : ''}${modified ? `<updated>${esc(modified)}</updated>` : ''}${summary ? `<summary>${esc(summary)}</summary>` : ''}${item.content ? `<content type="html">${esc(item.content)}</content>` : ''}${item.author ? `<author><name>${esc(item.author)}</name></author>` : ''}</entry>`;
444 });
445 const body = rows.length ? `\n${rows.join('\n')}` : '';
446 if (format === 'rss') return `<?xml version="1.0" encoding="UTF-8"?>\n<rss version="2.0"${self ? ' xmlns:atom="http://www.w3.org/2005/Atom"' : ''}><channel><title>${esc(title)}</title><link>${esc(home)}</link><description>${esc(description)}</description>${language ? `<language>${esc(language)}</language>` : ''}<lastBuildDate>${esc(built)}</lastBuildDate>${self ? `<atom:link href="${esc(self)}" rel="self" type="application/rss+xml"/>` : ''}${body}\n</channel></rss>\n`;
447 return `<?xml version="1.0" encoding="UTF-8"?>\n<feed xmlns="http://www.w3.org/2005/Atom"${language ? ` xml:lang="${esc(language)}"` : ''}><id>${esc(home)}</id><title>${esc(title)}</title>${description ? `<subtitle>${esc(description)}</subtitle>` : ''}<updated>${esc(built)}</updated><link rel="alternate" href="${esc(home)}"/>${self ? `<link rel="self" href="${esc(self)}"/>` : ''}${body}\n</feed>\n`;
448}
449
450function feedLinkTags({ rss, atom, title = 'Feed' } = {}) {
451 const out = [];
452 if (rss) out.push(`<link rel="alternate" type="application/rss+xml" title="${esc(title)}" href="${esc(absolute(rss))}">`);
453 if (atom) out.push(`<link rel="alternate" type="application/atom+xml" title="${esc(title)}" href="${esc(absolute(atom))}">`);
454 return out.join('\n');
455}
456
457module.exports = { headTags, jsonLd, jsonLdTag, sitemapXml, sitemapIndexXml, robotsTxt, llmsTxt, llmsFull, pageSummary, feedXml, feedLinkTags, clip, esc, absolute, AI_AND_SEARCH_BOTS };
458
459// Lazy server-only agent helpers; ordinary SEO rendering need not load the HTML parser.
460for (const name of ['agentManifest', 'markdownTwin', 'linkHeaderTags']) {
461 module.exports[name] = (...args) => require('./agent-layer')[name](...args);
462}
463