openvibers/openvibe.chat Commits

Chat: serve llms-full.txt for every public page (SEO-33) (#248)

OP OpenVibers committed 16 days ago · 6ef569c7ef · 2 files, +63 −12

Changes

ddiff --git a/server/web/discovery.js b/server/web/discovery.js
iindex 88883fb..4dc3d49 100644
---- a/server/web/discovery.js
++++ b/server/web/discovery.js
@@@ -1,11 +1,13 @@
'use strict';
/**
-- * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt and
-- * sitemap.xml, built from openvibe-shared/seo — the same toolkit the other OpenVibe sites use.
++ * Crawl and machine-readability artifacts for openvibe.chat (plan T11): robots.txt, llms.txt,
++ * llms-full.txt and sitemap.xml, built from openvibe-shared/seo — the same toolkit the other
++ * OpenVibe sites use.
*
* Every answer comes from public data only and never from the viewer: the global chat's last public
* message, the rooms that are public and unarchived, and the site's own fixed pages. Direct
-- * messages, your conversations, room settings, sign-in and the API are in no artifact.
++ * messages, your conversations, room settings, sign-in and the API are in no artifact. llms-full.txt
++ * and the sitemap read the same public page catalogue.
*
* lastmod is the data's own timestamp (the newest public message, the room's last_message_at or its
* created_at) — never "today", which would tell crawlers the whole site changed on every fetch.
@@@ -26,27 +28,49 @@ function dayOf(ts) {
return m ? m[1] : null;
}
--/** Public, indexable pages: the fixed ones plus every public room, each with a real lastmod. */
++/**
++ * Public, indexable pages: the fixed ones plus every public room, each with a real lastmod.
++ * This is the one page catalogue: /sitemap.xml takes its paths and /llms-full.txt takes its titles
++ * and descriptions, so the two can never describe different sites.
++ */
async function publicPages() {
const pages = (await rooms.list(null, { limit: 100 })).public;
const out = [
-- { path: '/', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },
-- { path: '/vibe', changefreq: 'weekly', priority: 0.9 },
-- { path: '/rooms', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },
-- { path: '/servers', changefreq: 'daily', priority: 0.8 },
-- { path: '/updates', changefreq: 'daily', priority: 0.5 },
-- { path: '/developers', changefreq: 'weekly', priority: 0.6 },
++ { path: '/', title: 'Global chat', text: 'The whole-network public chat: read the newest messages without an account and without JavaScript; signed-in people post here.', changefreq: 'hourly', priority: 1.0, lastmod: async () => await newestGlobalMessage() },
++ { path: '/vibe', title: 'Vibe, OpenVibe\'s assistant', text: 'Chat with Vibe: it searches the web and OpenVibe, reads pages and pictures, remembers what it is asked to and sets up recurring work; each conversation is private to its person.', changefreq: 'weekly', priority: 0.9 },
++ { path: '/rooms', title: 'Public rooms', text: 'Every public room without a server, with its topic and activity; anyone can read a public room.', changefreq: 'daily', priority: 0.8, lastmod: () => newestRoomActivity(pages) },
++ { path: '/servers', title: 'Public servers', text: 'Public servers: groups with text and voice channels, roles and invites; anyone can read, signed-in people join.', changefreq: 'daily', priority: 0.8 },
++ { path: '/updates', title: 'What shipped on OpenVibe.Chat', text: 'The shared changelog for OpenVibe.Chat, newest first.', changefreq: 'daily', priority: 0.5 },
++ { path: '/developers', title: 'Developers', text: 'Add a support inbox or a room to any website with one tag, use the visitor API, openvibe-sdk/chat, feeds from Claude Code and signed webhooks; the limits are stated here.', changefreq: 'weekly', priority: 0.6 },
];
for (const r of pages) {
-- out.push({ path: `/r/${r.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at) });
++ out.push({
++ path: `/r/${r.slug}`,
++ title: `${r.name} — public room`,
++ text: `Public room${r.topic ? ` about ${r.topic}` : ''}; ${r.kind === 'call' ? 'a voice and video call in the page' : 'a text room'} with ${r.message_count || 0} messages. Anyone can read it.`,
++ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(r.last_message_at || r.created_at),
++ });
}
// Public servers (server/rooms/servers.js): their front page, the first text channel.
for (const sv of (await require('../rooms/servers').list(null, { limit: 100 })).public) {
-- out.push({ path: `/s/${sv.slug}`, changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at) });
++ out.push({
++ path: `/s/${sv.slug}`,
++ title: `${sv.name} — public server`,
++ text: `Public server${sv.description ? `: ${sv.description}` : ''}, with ${sv.members || 0} members and its text and voice channels. Anyone can read it.`,
++ changefreq: 'daily', priority: 0.6, lastmod: () => dayOf(sv.created_at),
++ });
}
return out;
}
++/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */
++function legalPages() {
++ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };
++ return require('openvibe-shared/legal').PATHS.map((path) => ({
++ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Chat, served by the site itself.`,
++ }));
++}
++
async function newestGlobalMessage() {
try {
const row = (await historyStore.page('global', { limit: 1 })).messages[0];
@@@ -118,6 +142,20 @@ function createDiscoveryRoutes({ config }) {
}));
});
++ // The full AI map: the llms.txt header, then every public page with what it is. The page list comes
++ // from publicPages() — the same catalogue the sitemap uses — plus the shared legal routes, so a page
++ // cannot be added to one artifact and forgotten in the other.
++ router.get('/llms-full.txt', async (req, res) => {
++ const pages = [...(await publicPages()), ...legalPages()].map((p) => ({ url: abs(p.path), title: p.title, text: p.text }));
++ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(seo.llmsFull({
++ site: 'OpenVibe.Chat',
++ summary: 'OpenVibe.Chat: the global chat for the whole OpenVibe network, public rooms and servers with text and voice channels, Vibe the assistant, direct messages and calls.',
++ base: site,
++ maxBytes: 512 * 1024,
++ sections: [{ title: 'Pages', pages }],
++ }));
++ });
++
router.get('/sitemap.xml', async (req, res) => {
const urls = [];
for (const p of await publicPages()) {
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex e3bf561..21d4a6c 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -45,6 +45,19 @@ t('/llms.txt describes the site, its public pages and its machine-readable endpo
for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms.txt (${re})`);
});
++t('/llms-full.txt carries every public page and what it is, at least 500 characters', async () => {
++ const r = await req('/llms-full.txt');
++ assert.strictEqual(r.status, 200);
++ assert.match(r.type, /^text\/plain/);
++ assert.match(r.cache, /public/, 'cached like llms.txt');
++ assert.match(r.text, /^# OpenVibe\.Chat/, 'llmstxt.org: the name first');
++ assert.ok(r.text.length >= 500, `real content, not a stub (${r.text.length} characters)`);
++ const sitemap = await req('/sitemap.xml');
++ for (const m of sitemap.text.matchAll(/<loc>([^<]+)<\/loc>/g)) assert.ok(r.text.includes(m[1]), `llms-full.txt names ${m[1]}`);
++ for (const p of ['/terms', '/privacy', '/dmca']) assert.ok(r.text.includes(`${SITE}${p}`), `llms-full.txt names ${p}`);
++ for (const re of PRIVATE) assert.ok(!re.test(r.text), `no private path in llms-full.txt (${re})`);
++});
++
t('/robots.txt keeps every Disallow and welcomes the search and AI crawlers', async () => {
const r = await req('/robots.txt');
assert.strictEqual(r.status, 200);