SEO-4: the sitemap lists the site's own pages, and llms-full.txt is never a bare header (#127)

ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex 4f3ab9d..c5ecb91 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -7,7 +7,8 @@
   *   GET /llms.txt               orientation for language models
   *   GET /llms-full.txt          the same orientation plus one section per indexable story, each with
   *                               the editor's opening summary (never the source's full article text)
-- *   GET /sitemap.xml            sitemap index over the two sections below
++ *   GET /sitemap.xml            sitemap index over the three sections below
++ *   GET /sitemaps/pages.xml     the site's own pages: front page, topics index, update log
   *   GET /sitemaps/stories.xml   published, INDEXABLE stories only (the gate decides: retracted,
   *                               unsupported, AI-unreviewed and noindex stories never appear);
   *                               lastmod = the story's latest publication time
@@@ -19,11 +20,17 @@ const express = require('express');
  const seo = require('openvibe-publishing/seo');
  const sharedSeo = require('openvibe-shared/seo');
  const cache = require('openvibe-shared/cache-policy');
++const { createHome } = require('../domain/home');
  
  // The one-line site summary /llms.txt, /llms-full.txt and the home page all carry.
  const SITE_SUMMARY = 'Source-backed news: AI briefs and editors\' stories where every paragraph cites the reports it rests on, a live wire of what public sources (collected by OpenVibe.Sources) report, and topic pages.';
  
--function createDiscoveryRoutes({ config, store, publication, topics }) {
++// What /llms.txt and /llms-full.txt both say about how the publication works.
++const SITE_DETAILS = 'OpenVibe.News is not an automatic headline generator. Each story page lists its sources (headline, outlet, author and date as the source published them, with links), a timeline, editor-assigned perspective groupings and its full correction history. Full article text from sources is never republished; at most a short summary where the source’s terms allow it. AI-assisted drafts are never published or indexed before a person reviews them. Retracted stories stay at their URL with a retraction notice and are noindex. Append .json to any story URL for the same content as data, including every claim’s source numbers and the Sources item ids.';
++
++function createDiscoveryRoutes(ctx) {
++    const { config, store, publication, topics, stories, clusters } = ctx;
++    const homeData = ctx.home || createHome({ store });
      const router = express.Router();
      const { db } = store;
      const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
@@@ -55,7 +62,7 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
          res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsTxt({
              name: 'OpenVibe.News',
              summary: SITE_SUMMARY,
--            details: 'OpenVibe.News is not an automatic headline generator. Each story page lists its sources (headline, outlet, author and date as the source published them, with links), a timeline, editor-assigned perspective groupings and its full correction history. Full article text from sources is never republished; at most a short summary where the source’s terms allow it. AI-assisted drafts are never published or indexed before a person reviews them. Retracted stories stay at their URL with a retraction notice and are noindex. Append .json to any story URL for the same content as data, including every claim’s source numbers and the Sources item ids.',
++            details: SITE_DETAILS,
              sections: [
                  { title: 'Start here', links: [
                      { title: 'Latest stories', url: abs('/') },
@@@ -71,23 +78,72 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
          }));
      });
  
++    /**
++     * The site's own pages, which exist whatever the news day looks like: the front page, the topics
++     * index and the update log. A quiet stretch with no published story must not leave a crawler with
++     * an empty sitemap: these are the pages that are always there, and the front page carries the wire,
++     * the topics and the newest briefs even then. Each entry carries the decision the public router
++     * makes for the same page (server/http/public.js), so a page that is noindex never appears here.
++     */
++    async function siteEntries() {
++        const [{ total }, wire, stats, topicList] = await Promise.all([
++            stories.listPublished({ limit: 1 }), clusters.wire({ limit: 8 }), homeData.stats(), topics.list(),
++        ]);
++        const newestTopic = topicList.length ? Math.max(...topicList.map((t) => t.updated_at)) : undefined;
++        const page = (path, { empty = false, lastmod } = {}) => ({
++            loc: abs(path),
++            ...(lastmod ? { lastmod: new Date(lastmod).toISOString() } : {}),
++            decision: seo.evaluate({
++                state: empty ? 'draft' : 'published', visibility: empty ? 'private' : 'public',
++                canonicalUrl: abs(path), wordCount: 0,
++            }, { policy: { minWords: 0 }, now: store.now() }),
++        });
++        return [
++            page('/', { empty: total === 0 && !wire.length && !stats.week, lastmod: newestTopic }),
++            page('/topics', { lastmod: newestTopic }),
++            page('/updates'),
++        ];
++    }
++
      router.get('/llms-full.txt', async (_req, res) => {
--        // The gate decides: only indexable stories, and only their editor's opening summary — the
--        // source's full article text is never stored, so it can never appear here.
--        const entries = (await storyEntries()).filter((e) => e.decision.indexable);
++        // The gate decides for stories: only indexable ones, and only their editor's opening summary —
++        // the source's full article text is never stored, so it can never appear here. The site's own
++        // pages and the topics are always listed: they are what /llms.txt points at and what exists
++        // when no story has been published yet.
++        const [entries, topicList] = await Promise.all([storyEntries(), topics.list()]);
++        const indexed = entries.filter((e) => e.decision.indexable);
          res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsFull({
              site: 'OpenVibe.News',
              summary: SITE_SUMMARY,
              base: config.baseUrl,
              maxBytes: 512 * 1024,
--            sections: [{
--                title: 'Stories',
--                pages: entries.map((e) => ({
--                    title: e.rev.fields.headline,
--                    url: publication.storyUrl(e.story),
--                    text: e.rev.fields.paragraphs && e.rev.fields.paragraphs[0] ? e.rev.fields.paragraphs[0].text : '',
--                })),
--            }],
++            sections: [
++                { title: 'How OpenVibe.News works', url: abs('/'), body: SITE_DETAILS },
++                {
++                    title: 'Pages',
++                    pages: [
++                        { title: 'OpenVibe.News', url: abs('/'), text: SITE_SUMMARY },
++                        { title: 'Topics', url: abs('/topics'), text: `The ${topicList.length} topics OpenVibe.News follows: ${topicList.map((t) => t.name).join(', ')}. Each topic page carries its stories, its AI briefs (labelled, noindex) and the wire of reports on it.` },
++                        { title: 'What shipped on OpenVibe.News', url: abs('/updates'), text: 'Every change deployed to OpenVibe.News, newest first.' },
++                    ],
++                },
++                {
++                    title: 'Topics',
++                    pages: topicList.map((t) => ({
++                        title: t.name,
++                        url: abs(publication.topicPath(t)),
++                        text: t.description || `Stories, briefs and the wire on ${t.name}.`,
++                    })),
++                },
++                {
++                    title: 'Stories',
++                    pages: indexed.map((e) => ({
++                        title: e.rev.fields.headline,
++                        url: publication.storyUrl(e.story),
++                        text: e.rev.fields.paragraphs && e.rev.fields.paragraphs[0] ? e.rev.fields.paragraphs[0].text : '',
++                    })),
++                },
++            ],
          }));
      });
  
@@@ -110,16 +166,25 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
      }
  
      router.get('/sitemap.xml', async (_req, res) => {
--        const stories = await storyEntries();
--        const indexed = stories.filter((e) => e.decision.indexable);
--        const topics = (await topicEntries(stories)).filter((e) => e.decision.indexable);
--        const newest = (times) => times.length ? new Date(Math.max(...times)).toISOString() : undefined;
++        const [entries, site] = await Promise.all([storyEntries(), siteEntries()]);
++        const indexed = entries.filter((e) => e.decision.indexable);
++        const topics = (await topicEntries(entries)).filter((e) => e.decision.indexable);
++        const newest = (times) => {
++            const known = times.filter((t) => Number.isFinite(t));
++            return known.length ? new Date(Math.max(...known)).toISOString() : undefined;
++        };
          xml(res, seo.sitemapIndex([
--            { loc: abs('/sitemaps/stories.xml'), lastmod: newest(indexed.map((e) => e.story.published_at)) },
--            { loc: abs('/sitemaps/topics.xml'), lastmod: newest(topics.map((e) => Date.parse(e.lastmod))) },
++            { loc: abs('/sitemaps/pages.xml'), lastmod: newest(site.filter((e) => e.decision.indexable).map((e) => new Date(e.lastmod).getTime())) },
++            { loc: abs('/sitemaps/stories.xml'), lastmod: newest(indexed.map((e) => new Date(e.story.published_at).getTime())) },
++            { loc: abs('/sitemaps/topics.xml'), lastmod: newest(topics.map((e) => new Date(e.lastmod).getTime())) },
          ]));
      });
  
++    // The site's own pages (never empty): what /llms.txt lists and a crawler reaches from the home page.
++    router.get('/sitemaps/pages.xml', async (_req, res) => {
++        xml(res, seo.sitemap((await siteEntries()).filter((e) => e.decision.indexable)).files[0]);
++    });
++
      router.get('/sitemaps/stories.xml', async (_req, res) => {
          xml(res, seo.sitemap((await storyEntries()).map((e) => ({ loc: publication.storyUrl(e.story), lastmod: e.story.published_at, decision: e.decision }))).files[0]);
      });
ddiff --git a/test/indexability.test.js b/test/indexability.test.js
iindex 95d645c..5df813f 100644
---- a/test/indexability.test.js
++++ b/test/indexability.test.js
@@@ -121,8 +121,29 @@ async function storyOn(t, headline, items) {
          assert.ok(r.text.includes('Officials confirmed on Tuesday'), 'the editor summary text is the page text');
          assert.ok(Buffer.byteLength(r.text, 'utf8') <= 512 * 1024, 'within maxBytes');
      });
++    await check('the sitemap index leads to the site\'s own pages, not only to stories', async () => {
++        assert.match((await t.get('/sitemap.xml')).text, /<loc>https:\/\/openvibe\.news\/sitemaps\/pages\.xml<\/loc>/);
++        assert.match((await t.get('/sitemaps/pages.xml')).text, /<loc>https:\/\/openvibe\.news\/<\/loc>/, 'a site with content indexes its front page');
++    });
      await t.close();
  
++    // The quiet site: the seeded topics and nothing published yet. Its sitemap and its AI map must say
++    // what the site is and where its pages are — never answer with an empty <urlset> or a bare header.
++    const quiet = await boot();
++    await check('a quiet site still lists its own pages in the sitemap and its topics in llms-full.txt', async () => {
++        const pages = (await quiet.get('/sitemaps/pages.xml')).text;
++        assert.match(pages, /<loc>https:\/\/openvibe\.news\/topics<\/loc>/);
++        assert.match(pages, /<loc>https:\/\/openvibe\.news\/updates<\/loc>/);
++        assert.doesNotMatch(pages, /<loc>https:\/\/openvibe\.news\/<\/loc>/, 'the front page with no story, wire or report is noindex, and the sitemap agrees');
++        const full = await quiet.get('/llms-full.txt');
++        assert.strictEqual(full.status, 200);
++        assert.ok(Buffer.byteLength(full.text, 'utf8') > 2048, `not a bare header (${Buffer.byteLength(full.text, 'utf8')} bytes)`);
++        assert.match(full.text, /## Topics/);
++        assert.match(full.text, /https:\/\/openvibe\.news\/topics\/space/, 'each seeded topic page is listed');
++        assert.doesNotMatch(full.text, /## Stories/, 'no story section when no story is published');
++    });
++    await quiet.close();
++
      const one = await boot({ env: { NEWS_MIN_INDEPENDENT_SOURCES: '1' } });
      await check('NEWS_MIN_INDEPENDENT_SOURCES=1 restores the one-source gate', async () => {
          assert.strictEqual(one.ctx.publication.POLICY.minSources, 1);