openvibers/openvibe.wiki Commits
SEO-4: llms-full.txt names the wikis, not just indexable pages (#140)
Changes
@@@ -151,20 +151,34 @@ function createMachine({ svc, config }) {}));/**-- * /llms-full.txt: the llms.txt header, then one section of every indexable page. The same gate-- * as the sitemap decides who is here (a public, published page still unreviewed stays out), and-- * each entry is an excerpt — the page's summary, or its opening text — never the article. The-- * byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.++ * /llms-full.txt: the llms.txt header, then the wikis /llms.txt names and one entry per indexable++ * page. The same gate as the sitemap decides who is here (a public, published page still++ * unreviewed stays out), and each page entry is an excerpt — the page's summary, or its opening++ * text — never the article. The wikis are the site's own navigation pages, so a wiki whose pages++ * are all still unreviewed still answers with what it is and where it is, never with a bare++ * header. The byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.*/router.get('/llms-full.txt', wrap(async (_req, res) => {-- const pages = (await svc.publishedPublic()).filter(({ decision }) => decision.indexable).map(({ space, page, rev }) => ({++ const [published, allSpaces] = await Promise.all([svc.publishedPublic(), svc.listSpaces({ kind: 'anonymous' })]);++ const listed = published.filter(({ decision }) => decision.indexable);++ const perSpace = new Map();++ for (const { space } of published) perSpace.set(space.id, (perSpace.get(space.id) || 0) + 1);++ const spaces = allSpaces.filter((s) => s.visibility === 'public').map((s) => ({++ title: s.name,++ url: `${origin}${svc.spacePath(s)}`,++ text: [++ s.description ? s.description.replace(/\.?\s*$/, '.') : `The ${s.name} wiki on OpenVibe.Wiki.`,++ `${perSpace.get(s.id) || 0} published page${(perSpace.get(s.id) || 0) === 1 ? '' : 's'}${s.updated_at ? `, updated ${new Date(Number(s.updated_at)).toISOString().slice(0, 10)}` : ''}.`,++ ].join(' '),++ }));++ const pages = listed.map(({ space, page, rev }) => ({title: rev.fields.title,url: svc.pageUrl(space, page),text: rev.fields.summary || ssr.markdownToText(rev.content, LLMS_EXCERPT),}));setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({site: 'OpenVibe.Wiki', summary: AI_SUMMARY, base: origin,-- sections: [{ title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,++ sections: [{ title: 'Wikis', pages: spaces }, { title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,}));}));@@@ -64,6 +64,10 @@ async function lastIndexEvent(h, pageId) {assert.ok(r.text.includes('http://wiki.test/w/notes/open-page') && r.text.includes('http://wiki.test/w/notes/second-page'));assert.ok(!r.text.includes('unsourced-page'), 'only indexable pages, not merely listable ones');assert.ok(Buffer.byteLength(r.text) <= 512 * 1024, 'the page stays within maxBytes');++ // The wikis /llms.txt names are listed too, so a wiki whose pages are all still unreviewed++ // answers with what it is and where it is, never with a bare header.++ assert.match(r.text, /\n## Wikis\n/);++ assert.ok(r.text.includes('http://wiki.test/s/notes'), 'each public wiki has its own address');assert.ok((await H.req(h, 'GET', '/feed.json')).json.items.some((i) => i.url === 'http://wiki.test/w/notes/open-page'));let ev = (await lastIndexEvent(h, page.id));assert.strictEqual(ev.event_type, 'wiki.index_document.upserted');
@@ -151,20 +151,34 @@ function createMachine({ svc, config }) {
@@ -151,20 +151,34 @@ function createMachine({ svc, config }) {
}));
}));
/**
/**
- * /llms-full.txt: the llms.txt header, then one section of every indexable page. The same gate
- * as the sitemap decides who is here (a public, published page still unreviewed stays out), and
- * each entry is an excerpt — the page's summary, or its opening text — never the article. The
- * byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.
+ * /llms-full.txt: the llms.txt header, then the wikis /llms.txt names and one entry per indexable
+ * page. The same gate as the sitemap decides who is here (a public, published page still
+ * unreviewed stays out), and each page entry is an excerpt — the page's summary, or its opening
+ * text — never the article. The wikis are the site's own navigation pages, so a wiki whose pages
+ * are all still unreviewed still answers with what it is and where it is, never with a bare
+ * header. The byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.
*/
*/
router.get('/llms-full.txt', wrap(async (_req, res) => {
router.get('/llms-full.txt', wrap(async (_req, res) => {
- const pages = (await svc.publishedPublic()).filter(({ decision }) => decision.indexable).map(({ space, page, rev }) => ({
+ const [published, allSpaces] = await Promise.all([svc.publishedPublic(), svc.listSpaces({ kind: 'anonymous' })]);
+ const listed = published.filter(({ decision }) => decision.indexable);
+ const perSpace = new Map();
+ for (const { space } of published) perSpace.set(space.id, (perSpace.get(space.id) || 0) + 1);
+ const spaces = allSpaces.filter((s) => s.visibility === 'public').map((s) => ({
+ title: s.name,
+ url: `${origin}${svc.spacePath(s)}`,
+ text: [
+ s.description ? s.description.replace(/\.?\s*$/, '.') : `The ${s.name} wiki on OpenVibe.Wiki.`,
+ `${perSpace.get(s.id) || 0} published page${(perSpace.get(s.id) || 0) === 1 ? '' : 's'}${s.updated_at ? `, updated ${new Date(Number(s.updated_at)).toISOString().slice(0, 10)}` : ''}.`,
+ ].join(' '),
+ }));
+ const pages = listed.map(({ space, page, rev }) => ({
title: rev.fields.title,
title: rev.fields.title,
url: svc.pageUrl(space, page),
url: svc.pageUrl(space, page),
text: rev.fields.summary || ssr.markdownToText(rev.content, LLMS_EXCERPT),
text: rev.fields.summary || ssr.markdownToText(rev.content, LLMS_EXCERPT),
}));
}));
setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({
setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({
site: 'OpenVibe.Wiki', summary: AI_SUMMARY, base: origin,
site: 'OpenVibe.Wiki', summary: AI_SUMMARY, base: origin,
- sections: [{ title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,
+ sections: [{ title: 'Wikis', pages: spaces }, { title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,
}));
}));
}));
}));
@@ -64,6 +64,10 @@ async function lastIndexEvent(h, pageId) {
@@ -64,6 +64,10 @@ async function lastIndexEvent(h, pageId) {
assert.ok(r.text.includes('http://wiki.test/w/notes/open-page') && r.text.includes('http://wiki.test/w/notes/second-page'));
assert.ok(r.text.includes('http://wiki.test/w/notes/open-page') && r.text.includes('http://wiki.test/w/notes/second-page'));
assert.ok(!r.text.includes('unsourced-page'), 'only indexable pages, not merely listable ones');
assert.ok(!r.text.includes('unsourced-page'), 'only indexable pages, not merely listable ones');
assert.ok(Buffer.byteLength(r.text) <= 512 * 1024, 'the page stays within maxBytes');
assert.ok(Buffer.byteLength(r.text) <= 512 * 1024, 'the page stays within maxBytes');
+ // The wikis /llms.txt names are listed too, so a wiki whose pages are all still unreviewed
+ // answers with what it is and where it is, never with a bare header.
+ assert.match(r.text, /\n## Wikis\n/);
+ assert.ok(r.text.includes('http://wiki.test/s/notes'), 'each public wiki has its own address');
assert.ok((await H.req(h, 'GET', '/feed.json')).json.items.some((i) => i.url === 'http://wiki.test/w/notes/open-page'));
assert.ok((await H.req(h, 'GET', '/feed.json')).json.items.some((i) => i.url === 'http://wiki.test/w/notes/open-page'));
let ev = (await lastIndexEvent(h, page.id));
let ev = (await lastIndexEvent(h, page.id));
assert.strictEqual(ev.event_type, 'wiki.index_document.upserted');
assert.strictEqual(ev.event_type, 'wiki.index_document.upserted');