openvibers/openvibe.wiki Commits

SEO-4: llms-full.txt names the wikis, not just indexable pages (#140)

OP OpenVibers committed 15 days ago · 09a073293d · 2 files, +24 −6

/llms-full.txt was the llms.txt header and nothing else on the live site (164 bytes): it only
listed indexable pages, and every published page right now is AI-written and still unreviewed,
so the gate correctly keeps them all out — and an agent that fetched the file learned nothing.

It now opens with the wikis /llms.txt names: each public wiki's own address, its description
and how many pages it has, then the indexable pages as before. The sitemap is unchanged (it
was never empty: the home and every public wiki are listed); the nav pages are what the AI map
was missing.

Co-authored-by: CommandCodeBot <[email protected]>

Changes

ddiff --git a/server/http/machine.js b/server/http/machine.js
iindex b62c045..0d9d448 100644
---- a/server/http/machine.js
++++ b/server/http/machine.js
@@@ -151,20 +151,34 @@ function createMachine({ svc, config }) {
}));
/**
-- * /llms-full.txt: the llms.txt header, then one section of every indexable page. The same gate
-- * as the sitemap decides who is here (a public, published page still unreviewed stays out), and
-- * each entry is an excerpt — the page's summary, or its opening text — never the article. The
-- * byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.
++ * /llms-full.txt: the llms.txt header, then the wikis /llms.txt names and one entry per indexable
++ * page. The same gate as the sitemap decides who is here (a public, published page still
++ * unreviewed stays out), and each page entry is an excerpt — the page's summary, or its opening
++ * text — never the article. The wikis are the site's own navigation pages, so a wiki whose pages
++ * are all still unreviewed still answers with what it is and where it is, never with a bare
++ * header. The byte cap keeps a large wiki bounded: llmsFull truncates whole entries and says so.
*/
router.get('/llms-full.txt', wrap(async (_req, res) => {
-- const pages = (await svc.publishedPublic()).filter(({ decision }) => decision.indexable).map(({ space, page, rev }) => ({
++ const [published, allSpaces] = await Promise.all([svc.publishedPublic(), svc.listSpaces({ kind: 'anonymous' })]);
++ const listed = published.filter(({ decision }) => decision.indexable);
++ const perSpace = new Map();
++ for (const { space } of published) perSpace.set(space.id, (perSpace.get(space.id) || 0) + 1);
++ const spaces = allSpaces.filter((s) => s.visibility === 'public').map((s) => ({
++ title: s.name,
++ url: `${origin}${svc.spacePath(s)}`,
++ text: [
++ s.description ? s.description.replace(/\.?\s*$/, '.') : `The ${s.name} wiki on OpenVibe.Wiki.`,
++ `${perSpace.get(s.id) || 0} published page${(perSpace.get(s.id) || 0) === 1 ? '' : 's'}${s.updated_at ? `, updated ${new Date(Number(s.updated_at)).toISOString().slice(0, 10)}` : ''}.`,
++ ].join(' '),
++ }));
++ const pages = listed.map(({ space, page, rev }) => ({
title: rev.fields.title,
url: svc.pageUrl(space, page),
text: rev.fields.summary || ssr.markdownToText(rev.content, LLMS_EXCERPT),
}));
setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({
site: 'OpenVibe.Wiki', summary: AI_SUMMARY, base: origin,
-- sections: [{ title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,
++ sections: [{ title: 'Wikis', pages: spaces }, { title: 'Pages', pages }], maxBytes: LLMS_MAX_BYTES,
}));
}));
ddiff --git a/test/visibility.test.js b/test/visibility.test.js
iindex 6d6131c..f821257 100644
---- a/test/visibility.test.js
++++ b/test/visibility.test.js
@@@ -64,6 +64,10 @@ async function lastIndexEvent(h, pageId) {
assert.ok(r.text.includes('http://wiki.test/w/notes/open-page') && r.text.includes('http://wiki.test/w/notes/second-page'));
assert.ok(!r.text.includes('unsourced-page'), 'only indexable pages, not merely listable ones');
assert.ok(Buffer.byteLength(r.text) <= 512 * 1024, 'the page stays within maxBytes');
++ // The wikis /llms.txt names are listed too, so a wiki whose pages are all still unreviewed
++ // answers with what it is and where it is, never with a bare header.
++ assert.match(r.text, /\n## Wikis\n/);
++ assert.ok(r.text.includes('http://wiki.test/s/notes'), 'each public wiki has its own address');
assert.ok((await H.req(h, 'GET', '/feed.json')).json.items.some((i) => i.url === 'http://wiki.test/w/notes/open-page'));
let ev = (await lastIndexEvent(h, page.id));
assert.strictEqual(ev.event_type, 'wiki.index_document.upserted');