openvibers/openvibe.host Commits
llms.txt is a map, not the catalog (AGT-5) (#277)
Changes
@@@ -175,7 +175,31 @@ async function createApp(opts = {}) {// links as they are). /terms, /privacy and /dmca are shared legal pages with no twin, so they are linked as HTML;// what an agent may skip (the update log, /catalog/promos) is in `optional`, written last.const LLMS_TWINLESS = new Set(legalPaths);-- app.get('/llms.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsTxt({ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl, sections: [{ title: 'Public pages', links: (await allPages()).map((p) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) })) }], optional: [{ title: 'Hosting promotions', url: `${config.baseUrl}/catalog/promos`, note: 'current provider promotions' }, { title: 'What shipped on OpenVibe.Host', url: `${config.baseUrl}/updates`, note: 'the public update log' }, { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml` }, { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt` }] })); } catch (err) { next(err); } });++ // llms.txt is a map, not the catalog (11-03: 1,361 links and 114 KB, one per domain ending and plan). It names the++ // pages and every provider; the endings and plans are counted in a note, and each is in the sitemap and llms-full.txt.++ const LLMS_SECONDARY = new Set(['/catalog/promos', '/updates', ...legalPaths]);++ const llmsLink = (p, note) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(note ? { note } : {}), ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) });++ app.get('/llms.txt', async (_req, res, next) => {++ try {++ const [providers, endings, plans] = await Promise.all([catalog.providers(), catalog.domains.sitemap(), catalog.sitemapPlans()]);++ const notes = {++ '/catalog': `each of the ${plans.length} plans has a page at /catalog/plans/<id> with its sourced price history`,++ '/catalog/domains': `each of the ${endings.length} endings has a page at /catalog/domains/<ending> (for example /catalog/domains/dev)`,++ };++ res.type('text/plain').send(sharedSeo.llmsTxt({++ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl,++ sections: [++ { title: 'Start here', links: publicPages.filter((p) => !LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p, notes[p.url])) },++ { title: 'Providers', links: providers.map((p) => llmsLink({ title: p.name, url: `/catalog/providers/${p.slug}` })) },++ ],++ optional: [++ ...publicPages.filter((p) => LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p)),++ { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml`, note: 'every plan and domain ending page' },++ { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt`, note: 'every page, with what it holds' },++ ],++ }));++ } catch (err) { next(err); }++ });app.get('/llms-full.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsFull({ site: { name: 'OpenVibe.Host', url: config.baseUrl }, summary, base: config.baseUrl, maxBytes: 512 * 1024, sections: [{ title: 'Public pages', pages: await allPages() }] })); } catch (err) { next(err); } });// /auth/me is the navbar's who-is-signed-in read on every page, so it is not counted with sign-ins: 60 page views in 15 minutes made a signed-in visitor look signed out.app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, skip: (req) => req.path === '/me', standardHeaders: true, legacyHeaders: false }));@@@ -170,6 +170,11 @@ function fakeRegistry() {assert.match(r.text, /\]\(https:\/\/openvibe\.host\/catalog\/providers\/porkbun\.md\)/, 'a catalog page is its twin');assert.match(r.text, /\]\(https:\/\/openvibe\.host\/terms\) \(HTML\)/, 'the shared legal pages have no twin');assert.match(r.text, /\n## Optional\n/);++ // A map, not the catalog: the endings and plans are counted in notes, never listed one by one.++ assert.ok(!/\/catalog\/domains\/[a-z]/.test(r.text.replace(/\/catalog\/domains\/<ending>|\/catalog\/domains\/dev\)/g, '')), 'no ending is listed by itself');++ assert.match(r.text, /each of the \d+ endings has a page at \/catalog\/domains\/<ending>/);++ assert.ok(Buffer.byteLength(r.text) < 16 * 1024, `${Buffer.byteLength(r.text)} bytes`);++ assert.ok(r.text.indexOf('(https://openvibe.host/terms) (HTML)') > r.text.indexOf('\n## Optional\n'), 'the legal pages are optional');});await check('a name check: typed name read, registry asked, the same name priced at other endings; bad names are 422', async () => {
@@ -175,7 +175,31 @@ async function createApp(opts = {}) {
@@ -175,7 +175,31 @@ async function createApp(opts = {}) {
// links as they are). /terms, /privacy and /dmca are shared legal pages with no twin, so they are linked as HTML;
// links as they are). /terms, /privacy and /dmca are shared legal pages with no twin, so they are linked as HTML;
// what an agent may skip (the update log, /catalog/promos) is in `optional`, written last.
// what an agent may skip (the update log, /catalog/promos) is in `optional`, written last.
const LLMS_TWINLESS = new Set(legalPaths);
const LLMS_TWINLESS = new Set(legalPaths);
- app.get('/llms.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsTxt({ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl, sections: [{ title: 'Public pages', links: (await allPages()).map((p) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) })) }], optional: [{ title: 'Hosting promotions', url: `${config.baseUrl}/catalog/promos`, note: 'current provider promotions' }, { title: 'What shipped on OpenVibe.Host', url: `${config.baseUrl}/updates`, note: 'the public update log' }, { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml` }, { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt` }] })); } catch (err) { next(err); } });
+ // llms.txt is a map, not the catalog (11-03: 1,361 links and 114 KB, one per domain ending and plan). It names the
+ // pages and every provider; the endings and plans are counted in a note, and each is in the sitemap and llms-full.txt.
+ const LLMS_SECONDARY = new Set(['/catalog/promos', '/updates', ...legalPaths]);
+ const llmsLink = (p, note) => ({ title: p.title, url: `${config.baseUrl}${p.url}`, ...(note ? { note } : {}), ...(LLMS_TWINLESS.has(p.url) ? { html: true } : {}) });
+ app.get('/llms.txt', async (_req, res, next) => {
+ try {
+ const [providers, endings, plans] = await Promise.all([catalog.providers(), catalog.domains.sitemap(), catalog.sitemapPlans()]);
+ const notes = {
+ '/catalog': `each of the ${plans.length} plans has a page at /catalog/plans/<id> with its sourced price history`,
+ '/catalog/domains': `each of the ${endings.length} endings has a page at /catalog/domains/<ending> (for example /catalog/domains/dev)`,
+ };
+ res.type('text/plain').send(sharedSeo.llmsTxt({
+ name: 'OpenVibe.Host', summary, markdown: true, base: config.baseUrl,
+ sections: [
+ { title: 'Start here', links: publicPages.filter((p) => !LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p, notes[p.url])) },
+ { title: 'Providers', links: providers.map((p) => llmsLink({ title: p.name, url: `/catalog/providers/${p.slug}` })) },
+ ],
+ optional: [
+ ...publicPages.filter((p) => LLMS_SECONDARY.has(p.url)).map((p) => llmsLink(p)),
+ { title: 'Sitemap', url: `${config.baseUrl}/sitemap.xml`, note: 'every plan and domain ending page' },
+ { title: 'Full text for language models', url: `${config.baseUrl}/llms-full.txt`, note: 'every page, with what it holds' },
+ ],
+ }));
+ } catch (err) { next(err); }
+ });
app.get('/llms-full.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsFull({ site: { name: 'OpenVibe.Host', url: config.baseUrl }, summary, base: config.baseUrl, maxBytes: 512 * 1024, sections: [{ title: 'Public pages', pages: await allPages() }] })); } catch (err) { next(err); } });
app.get('/llms-full.txt', async (_req, res, next) => { try { res.type('text/plain').send(sharedSeo.llmsFull({ site: { name: 'OpenVibe.Host', url: config.baseUrl }, summary, base: config.baseUrl, maxBytes: 512 * 1024, sections: [{ title: 'Public pages', pages: await allPages() }] })); } catch (err) { next(err); } });
// /auth/me is the navbar's who-is-signed-in read on every page, so it is not counted with sign-ins: 60 page views in 15 minutes made a signed-in visitor look signed out.
// /auth/me is the navbar's who-is-signed-in read on every page, so it is not counted with sign-ins: 60 page views in 15 minutes made a signed-in visitor look signed out.
app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, skip: (req) => req.path === '/me', standardHeaders: true, legacyHeaders: false }));
app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, skip: (req) => req.path === '/me', standardHeaders: true, legacyHeaders: false }));
@@ -170,6 +170,11 @@ function fakeRegistry() {
@@ -170,6 +170,11 @@ function fakeRegistry() {
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/catalog\/providers\/porkbun\.md\)/, 'a catalog page is its twin');
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/catalog\/providers\/porkbun\.md\)/, 'a catalog page is its twin');
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/terms\) \(HTML\)/, 'the shared legal pages have no twin');
assert.match(r.text, /\]\(https:\/\/openvibe\.host\/terms\) \(HTML\)/, 'the shared legal pages have no twin');
assert.match(r.text, /\n## Optional\n/);
assert.match(r.text, /\n## Optional\n/);
+ // A map, not the catalog: the endings and plans are counted in notes, never listed one by one.
+ assert.ok(!/\/catalog\/domains\/[a-z]/.test(r.text.replace(/\/catalog\/domains\/<ending>|\/catalog\/domains\/dev\)/g, '')), 'no ending is listed by itself');
+ assert.match(r.text, /each of the \d+ endings has a page at \/catalog\/domains\/<ending>/);
+ assert.ok(Buffer.byteLength(r.text) < 16 * 1024, `${Buffer.byteLength(r.text)} bytes`);
+ assert.ok(r.text.indexOf('(https://openvibe.host/terms) (HTML)') > r.text.indexOf('\n## Optional\n'), 'the legal pages are optional');
});
});
await check('a name check: typed name read, registry asked, the same name priced at other endings; bad names are 422', async () => {
await check('a name check: typed name read, registry asked, the same name priced at other endings; bad names are 422', async () => {