openvibers/openvibe.chat Commits
Chat: serve llms-full.txt for every public page (SEO-33) (#248)
Changes
@@@ -1,11 +1,13 @@'use strict';/**-- * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt and-- * sitemap.xml, built from openvibe-shared/seo — the same toolkit the other OpenVibe sites use.++ * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt,++ * llms-full.txt and sitemap.xml, built from openvibe-shared/seo — the same toolkit the other++ * OpenVibe sites use.** Every answer comes from public data only and never from the viewer: the global chat's last public* message, the rooms that are public and unarchived, and the site's own fixed pages. Direct-- * messages, your conversations, room settings, sign-in and the API are in no artifact.++ * messages, your conversations, room settings, sign-in and the API are in no artifact. llms-full.txt++ * and the sitemap read the same public page catalogue.** lastmod is the data's own timestamp (the newest public message, the room's last_message_at or its* created_at) — never "today", which would tell crawlers the whole site changed on every fetch.@@@ -26,27 +28,49 @@ function dayOf(ts) {return m ? m[1] : null;}--/** Public, indexable pages: the fixed ones plus every public room, each with a real lastmod. */++/**++ * Public, indexable pages: the fixed ones plus every public room, each with a real lastmod.++ * This is the one page catalogue: /sitemap.xml takes its paths and /llms-full.txt takes its titles++ * and descriptions, so the two can never describe different sites.++ */async function publicPages() {const pages = (await rooms.list(null, { limit: 100 })).public;const out = [-- { path: '/', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },-- { path: '/vibe', changefreq: 'weekly', priority: 0.9 },-- { path: '/rooms', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },-- { path: '/servers', changefreq: 'daily', priority: 0.8 },-- { path: '/updates', changefreq: 'daily', priority: 0.5 },-- { path: '/developers', changefreq: 'weekly', priority: 0.6 },++ { path: '/', title: 'Global chat', text: 'The whole-network public chat: read the newest messages without an account and without JavaScript; signed-in people post here.', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },++ { path: '/vibe', title: 'Vibe, OpenVibe\'s assistant', text: 'Chat with Vibe: it searches the web and OpenVibe, reads pages and pictures, remembers what it is asked to and sets up recurring work; each conversation is private to its person.', changefreq: 'weekly', priority: 0.9 },++ { path: '/rooms', title: 'Public rooms', text: 'Every public room without a server, with its topic and activity; anyone can read a public room.', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },++ { path: '/servers', title: 'Public servers', text: 'Public servers: groups with text and voice channels, roles and invites; anyone can read, signed-in people join.', changefreq: 'daily', priority: 0.8 },++ { path: '/updates', title: 'What shipped on OpenVibe.Chat', text: 'The shared changelog for OpenVibe.Chat, newest first.', changefreq: 'daily', priority: 0.5 },++ { path: '/developers', title: 'Developers', text: 'Add a support inbox or a room to any website with one tag, use the visitor API, openvibe-sdk/chat, feeds from Claude Code and signed webhooks; the limits are stated here.', changefreq: 'weekly', priority: 0.6 },];for (const r of pages) {-- out.push({ path: `/r/${r.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at) });++ out.push({++ path: `/r/${r.slug}`,++ title: `${r.name} — public room`,++ text: `Public room${r.topic ? ` about ${r.topic}` : ''}; ${r.kind === 'call' ? 'a voice and video call in the page' : 'a text room'} with ${r.message_count || 0} messages. Anyone can read it.`,++ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at),++ });}// Public servers (server/rooms/servers.js): their front page, the first text channel.for (const sv of (await require('../rooms/servers').list(null, { limit: 100 })).public) {-- out.push({ path: `/s/${sv.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at) });++ out.push({++ path: `/s/${sv.slug}`,++ title: `${sv.name} — public server`,++ text: `Public server${sv.description ? `: ${sv.description}` : ''}, with ${sv.members || 0} members and its text and voice channels. Anyone can read it.`,++ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at),++ });}return out;}++/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */++function legalPages() {++ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };++ return require('openvibe-shared/legal').PATHS.map((path) => ({++ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Chat, served by the site itself.`,++ }));++}++async function newestGlobalMessage() {try {const row = (await historyStore.page('global', { limit: 1 })).messages[0];@@@ -118,6 +142,20 @@ function createDiscoveryRoutes({ config }) {}));});++ // The full AI map: the llms.txt header, then every public page with what it is. The page list comes++ // from publicPages() — the same catalogue the sitemap uses — plus the shared legal routes, so a page++ // cannot be added to one artifact and forgotten in the other.++ router.get('/llms-full.txt', async (req, res) => {++ const pages = [...(await publicPages()), ...legalPages()].map((p) => ({ url: abs(p.path), title: p.title, text: p.text }));++ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(seo.llmsFull({++ site: 'OpenVibe.Chat',++ summary: 'OpenVibe.Chat: the global chat for the whole OpenVibe network, public rooms and servers with text and voice channels, Vibe the assistant, direct messages and calls.',++ base: site,++ maxBytes: 512 * 1024,++ sections: [{ title: 'Pages', pages }],++ }));++ });++router.get('/sitemap.xml', async (req, res) => {const urls = [];for (const p of await publicPages()) {@@@ -45,6 +45,19 @@ t('/llms.txt describes the site, its public pages and its machine-readable endpofor (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms.txt (${re})`);});++t('/llms-full.txt carries every public page and what it is, at least 500 characters', async () => {++ const r = await req('/llms-full.txt');++ assert.strictEqual(r.status, 200);++ assert.match(r.type, /^text\/plain/);++ assert.match(r.cache, /public/, 'cached like llms.txt');++ assert.match(r.text, /^# OpenVibe\.Chat/, 'llmstxt.org: the name first');++ assert.ok(r.text.length >= 500, `real content, not a stub (${r.text.length} characters)`);++ const sitemap = await req('/sitemap.xml');++ for (const m of sitemap.text.matchAll(/<loc>([^<]+)<\/loc>/g)) assert.ok(r.text.includes(m[1]), `llms-full.txt names ${m[1]}`);++ for (const p of ['/terms', '/privacy', '/dmca']) assert.ok(r.text.includes(`${SITE}${p}`), `llms-full.txt names ${p}`);++ for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms-full.txt (${re})`);++});++t('/robots.txt keeps every Disallow and welcomes the search and AI crawlers', async () => {const r = await req('/robots.txt');assert.strictEqual(r.status, 200);
@@ -1,11 +1,13 @@
@@ -1,11 +1,13 @@
'use strict';
'use strict';
/**
/**
- * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt and
- * sitemap.xml, built from openvibe-shared/seo — the same toolkit the other OpenVibe sites use.
+ * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt,
+ * llms-full.txt and sitemap.xml, built from openvibe-shared/seo — the same toolkit the other
+ * OpenVibe sites use.
*
*
* Every answer comes from public data only and never from the viewer: the global chat's last public
* Every answer comes from public data only and never from the viewer: the global chat's last public
* message, the rooms that are public and unarchived, and the site's own fixed pages. Direct
* message, the rooms that are public and unarchived, and the site's own fixed pages. Direct
- * messages, your conversations, room settings, sign-in and the API are in no artifact.
+ * messages, your conversations, room settings, sign-in and the API are in no artifact. llms-full.txt
+ * and the sitemap read the same public page catalogue.
*
*
* lastmod is the data's own timestamp (the newest public message, the room's last_message_at or its
* lastmod is the data's own timestamp (the newest public message, the room's last_message_at or its
* created_at) — never "today", which would tell crawlers the whole site changed on every fetch.
* created_at) — never "today", which would tell crawlers the whole site changed on every fetch.
@@ -26,27 +28,49 @@ function dayOf(ts) {
@@ -26,27 +28,49 @@ function dayOf(ts) {
return m ? m[1] : null;
return m ? m[1] : null;
}
}
-/** Public, indexable pages: the fixed ones plus every public room, each with a real lastmod. */
+/**
+ * Public, indexable pages: the fixed ones plus every public room, each with a real lastmod.
+ * This is the one page catalogue: /sitemap.xml takes its paths and /llms-full.txt takes its titles
+ * and descriptions, so the two can never describe different sites.
+ */
async function publicPages() {
async function publicPages() {
const pages = (await rooms.list(null, { limit: 100 })).public;
const pages = (await rooms.list(null, { limit: 100 })).public;
const out = [
const out = [
- { path: '/', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },
- { path: '/vibe', changefreq: 'weekly', priority: 0.9 },
- { path: '/rooms', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },
- { path: '/servers', changefreq: 'daily', priority: 0.8 },
- { path: '/updates', changefreq: 'daily', priority: 0.5 },
- { path: '/developers', changefreq: 'weekly', priority: 0.6 },
+ { path: '/', title: 'Global chat', text: 'The whole-network public chat: read the newest messages without an account and without JavaScript; signed-in people post here.', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },
+ { path: '/vibe', title: 'Vibe, OpenVibe\'s assistant', text: 'Chat with Vibe: it searches the web and OpenVibe, reads pages and pictures, remembers what it is asked to and sets up recurring work; each conversation is private to its person.', changefreq: 'weekly', priority: 0.9 },
+ { path: '/rooms', title: 'Public rooms', text: 'Every public room without a server, with its topic and activity; anyone can read a public room.', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },
+ { path: '/servers', title: 'Public servers', text: 'Public servers: groups with text and voice channels, roles and invites; anyone can read, signed-in people join.', changefreq: 'daily', priority: 0.8 },
+ { path: '/updates', title: 'What shipped on OpenVibe.Chat', text: 'The shared changelog for OpenVibe.Chat, newest first.', changefreq: 'daily', priority: 0.5 },
+ { path: '/developers', title: 'Developers', text: 'Add a support inbox or a room to any website with one tag, use the visitor API, openvibe-sdk/chat, feeds from Claude Code and signed webhooks; the limits are stated here.', changefreq: 'weekly', priority: 0.6 },
];
];
for (const r of pages) {
for (const r of pages) {
- out.push({ path: `/r/${r.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at) });
+ out.push({
+ path: `/r/${r.slug}`,
+ title: `${r.name} — public room`,
+ text: `Public room${r.topic ? ` about ${r.topic}` : ''}; ${r.kind === 'call' ? 'a voice and video call in the page' : 'a text room'} with ${r.message_count || 0} messages. Anyone can read it.`,
+ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at),
+ });
}
}
// Public servers (server/rooms/servers.js): their front page, the first text channel.
// Public servers (server/rooms/servers.js): their front page, the first text channel.
for (const sv of (await require('../rooms/servers').list(null, { limit: 100 })).public) {
for (const sv of (await require('../rooms/servers').list(null, { limit: 100 })).public) {
- out.push({ path: `/s/${sv.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at) });
+ out.push({
+ path: `/s/${sv.slug}`,
+ title: `${sv.name} — public server`,
+ text: `Public server${sv.description ? `: ${sv.description}` : ''}, with ${sv.members || 0} members and its text and voice channels. Anyone can read it.`,
+ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at),
+ });
}
}
return out;
return out;
}
}
+/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */
+function legalPages() {
+ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };
+ return require('openvibe-shared/legal').PATHS.map((path) => ({
+ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Chat, served by the site itself.`,
+ }));
+}
+
async function newestGlobalMessage() {
async function newestGlobalMessage() {
try {
try {
const row = (await historyStore.page('global', { limit: 1 })).messages[0];
const row = (await historyStore.page('global', { limit: 1 })).messages[0];
@@ -118,6 +142,20 @@ function createDiscoveryRoutes({ config }) {
@@ -118,6 +142,20 @@ function createDiscoveryRoutes({ config }) {
}));
}));
});
});
+ // The full AI map: the llms.txt header, then every public page with what it is. The page list comes
+ // from publicPages() — the same catalogue the sitemap uses — plus the shared legal routes, so a page
+ // cannot be added to one artifact and forgotten in the other.
+ router.get('/llms-full.txt', async (req, res) => {
+ const pages = [...(await publicPages()), ...legalPages()].map((p) => ({ url: abs(p.path), title: p.title, text: p.text }));
+ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(seo.llmsFull({
+ site: 'OpenVibe.Chat',
+ summary: 'OpenVibe.Chat: the global chat for the whole OpenVibe network, public rooms and servers with text and voice channels, Vibe the assistant, direct messages and calls.',
+ base: site,
+ maxBytes: 512 * 1024,
+ sections: [{ title: 'Pages', pages }],
+ }));
+ });
+
router.get('/sitemap.xml', async (req, res) => {
router.get('/sitemap.xml', async (req, res) => {
const urls = [];
const urls = [];
for (const p of await publicPages()) {
for (const p of await publicPages()) {
@@ -45,6 +45,19 @@ t('/llms.txt describes the site, its public pages and its machine-readable endpo
@@ -45,6 +45,19 @@ t('/llms.txt describes the site, its public pages and its machine-readable endpo
for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms.txt (${re})`);
for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms.txt (${re})`);
});
});
+t('/llms-full.txt carries every public page and what it is, at least 500 characters', async () => {
+ const r = await req('/llms-full.txt');
+ assert.strictEqual(r.status, 200);
+ assert.match(r.type, /^text\/plain/);
+ assert.match(r.cache, /public/, 'cached like llms.txt');
+ assert.match(r.text, /^# OpenVibe\.Chat/, 'llmstxt.org: the name first');
+ assert.ok(r.text.length >= 500, `real content, not a stub (${r.text.length} characters)`);
+ const sitemap = await req('/sitemap.xml');
+ for (const m of sitemap.text.matchAll(/<loc>([^<]+)<\/loc>/g)) assert.ok(r.text.includes(m[1]), `llms-full.txt names ${m[1]}`);
+ for (const p of ['/terms', '/privacy', '/dmca']) assert.ok(r.text.includes(`${SITE}${p}`), `llms-full.txt names ${p}`);
+ for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms-full.txt (${re})`);
+});
+
t('/robots.txt keeps every Disallow and welcomes the search and AI crawlers', async () => {
t('/robots.txt keeps every Disallow and welcomes the search and AI crawlers', async () => {
const r = await req('/robots.txt');
const r = await req('/robots.txt');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);