openvibers/openvibe.host Commits

llms.txt is a map, not the catalog (AGT-5) (#277)

OP OpenVibers committed 10 days ago · 9302c8ef3d · 2 files, +30 −1

It listed every domain ending and plan: 1,361 links and 114 KB. Now the main pages and every provider, with notes saying
each ending and plan has a page (all in the sitemap and llms-full.txt); the legal pages, promotions and updates are
Optional.

Changes

ddiff --git a/server/app.js b/server/app.js
iindex 32365d9..68de01d 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -175,7 +175,31 @@ async function createApp(opts = {}) {
// links as they are). /terms, /privacy and /dmca are shared legal pages with no twin, so they are linked as HTML;
// what an agent may skip (the update log, /catalog/promos) is in `optional`, written last.
const LLMS_TWINLESS = new Set(legalPaths);
-- app.get('/llms.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsTxt({ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl, sections: [{ title: 'Public pages', links: (await allPages()).map((p) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) })) }], optional: [{ title: 'Hosting promotions', url: `${config.baseUrl}/catalog/promos`, note: 'current provider promotions' }, { title: 'What shipped on OpenVibe.Host', url: `${config.baseUrl}/updates`, note: 'the public update log' }, { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml` }, { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt` }] })); } catch (err) { next(err); } });
++ // llms.txt is a map, not the catalog (11-03: 1,361 links and 114 KB, one per domain ending and plan). It names the
++ // pages and every provider; the endings and plans are counted in a note, and each is in the sitemap and llms-full.txt.
++ const LLMS_SECONDARY = new Set(['/catalog/promos', '/updates', ...legalPaths]);
++ const llmsLink = (p, note) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(note ? { note } : {}), ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) });
++ app.get('/llms.txt', async (_req, res, next) => {
++ try {
++ const [providers, endings, plans] = await Promise.all([catalog.providers(), catalog.domains.sitemap(), catalog.sitemapPlans()]);
++ const notes = {
++ '/catalog': `each of the ${plans.length} plans has a page at /catalog/plans/<id> with its sourced price history`,
++ '/catalog/domains': `each of the ${endings.length} endings has a page at /catalog/domains/<ending> (for example /catalog/domains/dev)`,
++ };
++ res.type('text/plain').send(sharedSeo.llmsTxt({
++ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl,
++ sections: [
++ { title: 'Start here', links: publicPages.filter((p) => !LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p, notes[p.url])) },
++ { title: 'Providers', links: providers.map((p) => llmsLink({ title: p.name, url: `/catalog/providers/${p.slug}` })) },
++ ],
++ optional: [
++ ...publicPages.filter((p) => LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p)),
++ { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml`, note: 'every plan and domain ending page' },
++ { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt`, note: 'every page, with what it holds' },
++ ],
++ }));
++ } catch (err) { next(err); }
++ });
app.get('/llms-full.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsFull({ site: { name: 'OpenVibe.Host', url: config.baseUrl }, summary, base: config.baseUrl, maxBytes: 512 * 1024, sections: [{ title: 'Public pages', pages: await allPages() }] })); } catch (err) { next(err); } });
// /auth/me is the navbar's who-is-signed-in read on every page, so it is not counted with sign-ins: 60 page views in 15 minutes made a signed-in visitor look signed out.
app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, skip: (req) => req.path === '/me', standardHeaders: true, legacyHeaders: false }));
ddiff --git a/test/catalog-domains.test.js b/test/catalog-domains.test.js
iindex 80c0bf8..2bf40cc 100644
---- a/test/catalog-domains.test.js
++++ b/test/catalog-domains.test.js
@@@ -170,6 +170,11 @@ function fakeRegistry() {
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/catalog\/providers\/porkbun\.md\)/, 'a catalog page is its twin');
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/terms\) \(HTML\)/, 'the shared legal pages have no twin');
assert.match(r.text, /\n## Optional\n/);
++ // A map, not the catalog: the endings and plans are counted in notes, never listed one by one.
++ assert.ok(!/\/catalog\/domains\/[a-z]/.test(r.text.replace(/\/catalog\/domains\/<ending>|\/catalog\/domains\/dev\)/g, '')), 'no ending is listed by itself');
++ assert.match(r.text, /each of the \d+ endings has a page at \/catalog\/domains\/<ending>/);
++ assert.ok(Buffer.byteLength(r.text) < 16 * 1024, `${Buffer.byteLength(r.text)} bytes`);
++ assert.ok(r.text.indexOf('(https://openvibe.host/terms) (HTML)') > r.text.indexOf('\n## Optional\n'), 'the legal pages are optional');
});
await check('a name check: typed name read, registry asked, the same name priced at other endings; bad names are 422', async () => {