Deals: the sitemap cache keeps index entries, not page views (#238)

ddiff --git a/CHANGELOG.md b/CHANGELOG.md
iindex 1483706..3ae5656 100644
---- a/CHANGELOG.md
++++ b/CHANGELOG.md
@@@ -2,6 +2,8 @@
  
  ## Unreleased
  
++- The sitemap cache holds index entries, not page views (memory on the host): `/sitemap.xml`, `/sitemap-<n>.xml` and `/llms-full.txt` kept every offer's and product's full view for `DEALS_SITEMAP_TTL_MS` — observations, sources, vote tallies and merge members — at ~6.7 KB per offer. On 2026-11-07 that was 667 MB resident (923 MB at measurement, 2 GB peak) for 50,000 offers, the most of any Node service on a 15 GB host that ran out of memory at 02:34. The cache now stores `publication.indexEntry` / `productIndexEntry`: the URL, lastmod and indexability decision, plus an `/llms-full.txt` summary for the first 10,000 indexable offers (the file truncates at 512 KB), ~0.6 KB an offer. Each offer needs two reads (its merge group and the group's latest observation) instead of ten, and `listings.indexable()` joins the store so no per-offer store lookup is needed; a product's lastmod and freshest observation come from one query. Output is unchanged (same URLs, same order, same lastmods, same gate). `test/memory-budget.test.js` boots the app on a 2,500-offer catalog, builds the sitemaps and runs one Sources import: the heap grows at most 12 MB past boot (measured 3.5 MB; the old full-view cache measured 12.8 MB at 2,000 offers) and stays under 200 MB overall.
++
  - Imported deals become searchable (the automatic review of imports): an offer imported from a configured feed source is checked when it is imported and passes when its link is https and points at that source's retailer domain, its title is not empty and at most 300 characters, its price is a positive number with a currency, it has not expired and it states none of the blocked/adult terms. A pass sets `review_state = auto_approved` (published, indexable, `reviewed_by = system:import-checks`); a fail stays `pending` with the failed check's name in `review_failed_check`. Offers people submit by hand keep today's review flow and a moderator can still hide any offer. `server/domain/import-review.js`, migration `0009_import_auto_review.sql`, `scripts/backfill-import-review.js` (batches of 500, `--dry-run`, counts only), `test/import-review.test.js`.
  
  - `llms.txt` lists only URLs that answer (SEO-34): the *Product JSON* link pointed at `/sitemaps/products.xml`, which no longer exists once the sitemap is an index, and `/sitemap.xml` itself was a single 24,908-URL file a crawler waited out. The link now names the index, and `test/pages.test.js` sweeps every URL in the file for an answer below 400.
ddiff --git a/server/domain/listings.js b/server/domain/listings.js
iindex 406e903..f7e76e3 100644
---- a/server/domain/listings.js
++++ b/server/domain/listings.js
@@@ -38,7 +38,12 @@ function createListings({ store }) {
          sellers: db.prepare(`SELECT seller, COUNT(*) AS n, MAX(created_at) AS latest FROM deal_offers WHERE seller IS NOT NULL AND merged_into IS NULL AND status <> 'disabled' GROUP BY seller ORDER BY n DESC, seller LIMIT 2000`),
          bySeller: db.prepare(`SELECT * FROM deal_offers WHERE seller = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
          countSeller: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE seller = ? AND merged_into IS NULL AND status <> 'disabled'`),
--        indexable: db.prepare(`SELECT * FROM deal_offers WHERE merged_into IS NULL AND status <> 'disabled' ORDER BY created_at DESC LIMIT 50000`),
++        // The store's name and domain come along: an index entry needs its label, and one join is cheaper than a
++        // lookup per offer (a 50,000-offer sitemap build was 50,000 extra queries).
++        indexable: db.prepare(`SELECT o.*, s.name AS store_name, s.domain AS store_domain
++                                 FROM deal_offers o LEFT JOIN deal_stores s ON s.id = o.store_id
++                                WHERE o.merged_into IS NULL AND o.status <> 'disabled'
++                                ORDER BY o.created_at DESC LIMIT 50000`),
          // The canonical offers with an observation that came of age (freshnessMs) in a window: their freshness, and so
          // their index document, may have changed with time alone. A superset is fine: an unchanged document sends nothing.
          agedInto: db.prepare(`SELECT r.* FROM deal_offers r WHERE r.merged_into IS NULL AND r.status <> 'disabled' AND r.id IN (
ddiff --git a/server/domain/publication.js b/server/domain/publication.js
iindex dfc5463..30e96d2 100644
---- a/server/domain/publication.js
++++ b/server/domain/publication.js
@@@ -227,7 +227,61 @@ function createPublication({ config, store, reads, catalog }) {
          });
      }
  
--    return { abs, offerPath, productPath, storePath, sellerPath, categoryPath, freshness, decideOffer, decideProduct, offerView, cardView, offerDto, observationDto, offerLd, offerJsonLd, productView, productJsonLd };
++    /** A short, whitespace-collapsed summary; the full source text is never included. */
++    const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);
++
++    /**
++     * What an index artifact (a sitemap chunk, /llms-full.txt) keeps about one offer: its URL, when it last changed,
++     * whether a crawler may index it, and the short summary /llms-full.txt prints. Two reads (the merge group and its
++     * latest observation) instead of offerView's ten, and nothing that grows with the group's price history is
++     * retained; the entry measured ~0.6 KB. The sitemap cache holds one per offer for DEALS_SITEMAP_TTL_MS; keeping
++     * the full views instead cost ~6.7 KB an offer — ~340 MB of a 50,000-offer catalog, held twice over once products
++     * were built (2026-11-07: the Deals process at 667 MB-923 MB, peaking at 2 GB).
++     */
++    async function indexEntry(offer, now = store.now()) {
++        const root = offer.merged_into ? await reads.root(offer) : offer;
++        const ids = await reads.groupIds(root.id);
++        const latest = await reads.latestObservation(ids);
++        // listings.indexable() joins the store onto each row; any other caller gets it looked up here.
++        const s = root.store_domain !== undefined
++            ? (root.store_domain ? { name: root.store_name, domain: root.store_domain } : null)
++            : (root.store_id ? await catalog.store(root.store_id) : null);
++        const price = latest && latest.price != null && latest.currency ? `${latest.price} ${latest.currency}` : '';
++        return {
++            loc: abs(offerPath(root)),
++            lastmod: latest ? Math.max(latest.observed_at, root.updated_at) : root.updated_at,
++            decision: decideOffer(root, latest, now),
++            title: root.title,
++            text: [brief(root.description), price, s ? `at ${s.name || s.domain}` : ''].filter(Boolean).join(' '),
++        };
++    }
++
++    // A product's lastmod and freshest active observation in one read: each offer's group latest observation (a merge
++    // group shares one price history), without loading every offer's view. The same facts productView derives below.
++    const productIndex = store.db.prepare(`SELECT MAX(GREATEST(o.updated_at, COALESCE(l.observed_at, 0))) AS lastmod,
++                                                  MAX(CASE WHEN o.status = 'active' THEN l.observed_at END) AS active_latest
++                                             FROM deal_offers o
++                                             LEFT JOIN LATERAL (
++                                                 WITH RECURSIVE g(id, depth) AS (SELECT o.id, 0 UNION
++                                                     SELECT c.id, g.depth + 1 FROM deal_offers c JOIN g ON c.merged_into = g.id WHERE g.depth < 50)
++                                                 SELECT p.observed_at FROM deal_price_observations p JOIN deal_offer_sources s ON s.id = p.source_id
++                                                  WHERE p.offer_id IN (SELECT id FROM g) AND s.removed_at IS NULL
++                                                  ORDER BY p.observed_at DESC, p.seq DESC LIMIT 1) l ON true
++                                            WHERE o.product_id = ? AND o.merged_into IS NULL AND o.status <> 'disabled'`);
++
++    /** What the sitemaps keep about one product page: its URL, when it last changed, whether it is indexable. */
++    async function productIndexEntry(product, now = store.now()) {
++        const r = await productIndex.get(product.id);
++        const activeLatest = r.active_latest == null ? null : Number(r.active_latest);
++        const freshest = activeLatest != null && now - activeLatest <= config.freshnessMs ? activeLatest : null;
++        return {
++            loc: abs(productPath(product)),
++            lastmod: Math.max(product.updated_at, Number(r.lastmod) || 0),
++            decision: decideProduct(product, freshest, now),
++        };
++    }
++
++    return { abs, offerPath, productPath, storePath, sellerPath, categoryPath, freshness, decideOffer, decideProduct, offerView, cardView, offerDto, observationDto, offerLd, offerJsonLd, productView, productJsonLd, indexEntry, productIndexEntry };
  }
  
  module.exports = { createPublication };
ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex 3709add..9bc24a1 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -49,6 +49,11 @@ async function mapLimit(items, limit, fn) {
      return out;
  }
  
++// The sitemap cache keeps a summary for at most this many indexable offers. /llms-full.txt truncates at 512 KB of
++// text (about 3,500 entries), so the rest would never be printed; a 50,000-offer catalog kept 10 MB of summaries
++// it could not use.
++const LLMS_SUMMARIES_MAX = 10000;
++
  /**
   * One build at a time, shared by the readers that ask while it runs, reused for ttlMs (0: the next read rebuilds).
   * With a ttl, an expired build is still served at once while the next one is made behind it: a cold build of every
@@@ -74,21 +79,26 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {
      const router = express.Router();
      const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
      const xml = (res, body) => res.type('application/xml').set('Cache-Control', cache.htmlHeaders({ maxAge: 300 })).send(body);
--    /** A short, whitespace-collapsed summary; the full source text is never included. */
--    const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);
  
--    // Every offer's and product's page view, for the sitemaps and /llms-full.txt: built VIEWS_PARALLEL at a time, once
--    // for concurrent readers, reused for config.sitemapTtlMs. All at once, each /sitemap.xml queued thousands of
++    // Every offer's and product's index entry, for the sitemaps and /llms-full.txt: built VIEWS_PARALLEL at a time,
++    // once for concurrent readers, reused for config.sitemapTtlMs. All at once, each /sitemap.xml queued thousands of
      // queries on the pool for a minute (2026-10-23), and every page waited behind them.
++    //
++    // The cache holds compact entries (url, lastmod, decision, and a summary for the first LLMS_SUMMARIES_MAX
++    // indexable offers), never the page views: full views cost ~6.7 KB an offer, so a 50,000-offer catalog held
++    // ~340 MB and rebuilding it held another copy (667 MB resident, 2 GB peak, 2026-11-07). An entry measured ~0.6 KB,
++    // and the cache is bounded by the listings' own 50,000-offer and 50,000-product caps.
      const ttl = Number(config.sitemapTtlMs) || 0;
--    const offerEntries = shared(async () => mapLimit(await listings.indexable(), VIEWS_PARALLEL, async (o) => {
--        const v = await publication.offerView(o);
--        return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };
--    }), ttl, store.now);
--    const productEntries = shared(async () => mapLimit(await listings.products(), VIEWS_PARALLEL, async (p) => {
--        const pv = await publication.productView(p);
--        return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };
--    }), ttl, store.now);
++    const offerEntries = shared(async () => {
++        const entries = await mapLimit(await listings.indexable(), VIEWS_PARALLEL, (o) => publication.indexEntry(o));
++        let kept = 0;
++        for (const e of entries) {
++            if (e.decision.indexable && ++kept <= LLMS_SUMMARIES_MAX) continue;
++            delete e.text; delete e.title;
++        }
++        return entries;
++    }, ttl, store.now);
++    const productEntries = shared(async () => mapLimit(await listings.products(), VIEWS_PARALLEL, (p) => publication.productIndexEntry(p)), ttl, store.now);
      // The first build of the process, made before any reader asks (DEALS_SITEMAP_WARM_MS after start; production 30 s):
      // after it, every read is served at once (shared above).
      const warmMs = Number(config.sitemapWarmMs) || 0;
@@@ -180,10 +190,7 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {
          const pages = [];
          for (const e of entries) {
              if (!e.decision.indexable) continue;
--            const v = e.view;
--            const price = v.latest ? `${v.latest.price} ${v.latest.currency}` : '';
--            const text = [brief(v.root.description), price, v.store ? `at ${v.store.name || v.store.domain}` : ''].filter(Boolean).join(' ');
--            pages.push({ title: v.root.title, url: e.loc, text });
++            pages.push({ title: e.title, url: e.loc, text: e.text });
          }
          res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsFull({
              site: 'OpenVibe.Deals',
ddiff --git a/test/memory-budget.test.js b/test/memory-budget.test.js
nnew file mode 100644
iindex 0000000..6aebb6a
---- /dev/null
++++ b/test/memory-budget.test.js
@@@ -0,0 +1,97 @@
++'use strict';
++/**
++ * What the sitemap cache holds, measured (server/http/discovery.js, server/domain/publication.js).
++ *
++ * The offer and product lists behind /sitemap.xml, /sitemap-<n>.xml and /llms-full.txt are reused for
++ * DEALS_SITEMAP_TTL_MS. They used to be the full page views (every observation, source, vote tally and member of every
++ * offer): about 6.7 KB an offer, so production's 50,000-offer catalog kept ~340 MB for ten minutes and a rebuild held a
++ * second copy (667 MB resident, 2 GB peak on 2026-11-07). They are now compact index entries (~0.6 KB an offer).
++ *
++ * Budgets (a 2,500-offer, 1,200-product fixture): the heap after boot plus a built sitemap cache plus one Sources
++ * import must grow no more than HEAP_GROWTH_MAX over the heap right after boot, and stay under HEAP_MAX overall.
++ *   - the compact cache for this fixture is ~3 MB, and one import of 50 items adds bookkeeping, not a catalog;
++ *   - 12 MB leaves room for V8's arenas and an unrelated allocation or two, while the old full-view cache measured
++ *     12.8 MB at 2,000 offers (~16 MB at this size), so it fails this test;
++ *   - HEAP_MAX catches a runaway leak of hundreds of megabytes on a fresh process.
++ * Raising a budget is a decision to state in the commit.
++ */
++const assert = require('assert');
++const v8 = require('v8');
++const vm = require('vm');
++const { boot, check, done } = require('./helpers/boot');
++const { sourceItem } = require('./helpers/mocks');
++
++const OFFERS = 2500;
++const PRODUCTS = 1200;
++const IMPORT_ITEMS = 50;
++const HEAP_GROWTH_MAX = 12 * 1024 * 1024;
++const HEAP_MAX = 200 * 1024 * 1024;
++
++v8.setFlagsFromString('--expose-gc');
++const gc = typeof global.gc === 'function' ? global.gc : vm.runInNewContext('gc');
++const heap = () => { gc(); gc(); return process.memoryUsage().heapUsed; };
++const mb = (n) => `${(n / 1048576).toFixed(1)} MB`;
++
++/** The fixture: a catalog of the shape production has (offers with a store, a product for most, one fresh observation). */
++async function seed(db, now) {
++    await db.prepare(`INSERT INTO deal_stores (id, domain, name, created_at, updated_at)
++                      SELECT 'dst_' || g, 'shop' || g || '.test', 'Shop ' || g, ?, ? FROM generate_series(1, 10) g`).run(now, now);
++    await db.prepare(`INSERT INTO deal_products (id, slug, name, brand, category, description, created_by, created_at, updated_at)
++                      SELECT 'dpr_' || g, 'product-' || g, 'Product ' || g, 'Brand', 'cat', 'A product description', NULL, ?, ?
++                        FROM generate_series(1, ${PRODUCTS}) g`).run(now, now);
++    await db.prepare(`INSERT INTO deal_offers (id, slug, title, description, url, url_norm, store_id, product_id, category, origin,
++                                               text_origin, review_state, status, created_at, updated_at)
++                      SELECT 'dof_' || g, 'deal-' || g, 'A deal title ' || g,
++                             'A deal description that is long enough to summarise: item ' || g || ' at a shop.',
++                             'https://shop' || (g % 10 + 1) || '.test/p/' || g, 'https://shop' || (g % 10 + 1) || '.test/p/' || g,
++                             'dst_' || (g % 10 + 1), CASE WHEN g <= ${PRODUCTS} THEN 'dpr_' || g END, 'cat',
++                             'community', 'human', 'not_required', 'active', ?::bigint - g * 1000, ?::bigint - g * 1000
++                        FROM generate_series(1, ${OFFERS}) g`).run(now, now);
++    await db.prepare(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, created_at, updated_at)
++                      SELECT 'dos_' || g, 'dof_' || g, 'submission', 'deals', 'submission', 'sub-' || g, ?, ?
++                        FROM generate_series(1, ${OFFERS}) g`).run(now, now);
++    await db.prepare(`INSERT INTO deal_price_observations (id, offer_id, source_id, observed_at, recorded_at, price, price_num, currency, origin)
++                      SELECT 'dpo_' || g, 'dof_' || g, 'dos_' || g, ?, ?, '19.99', 19.99, 'USD', 'community'
++                        FROM generate_series(1, ${OFFERS}) g`).run(now - 3600_000, now - 3600_000);
++}
++
++(async () => {
++    // DEALS_SITEMAP_TTL_MS is what keeps the build: without it every read rebuilds and nothing is retained (and the
++    // leak this test guards against could not show). The worker is off in boot(); the import runs by hand.
++    const t = await boot({ env: { DEALS_SITEMAP_TTL_MS: '600000' } });
++    try {
++        await check('a 2,500-offer catalog: the sitemap cache and one import stay well inside the heap budget', async () => {
++            const now = t.clock.now();
++            await seed(t.ctx.store.db, now);
++            const atBoot = heap();
++            assert.ok(atBoot < HEAP_MAX, `heap after boot is ${mb(atBoot)}, over ${mb(HEAP_MAX)}`);
++
++            for (const path of ['/sitemap.xml', '/sitemap-0.xml', '/llms-full.txt']) {
++                const r = await t.get(path);
++                assert.strictEqual(r.status, 200, path);
++            }
++            const built = heap();
++
++            for (let i = 0; i < IMPORT_ITEMS; i++) {
++                t.sources.put(sourceItem({
++                    id: `itm_MEM${String(i).padStart(21, '0')}`,
++                    title: `Imported deal ${i}`,
++                    url: `https://imported.example/p/${i}`,
++                    retrievedAt: now,
++                    fields: { price: '9.99', currency: 'USD', availability: 'InStock' },
++                }));
++            }
++            const pulled = await t.ctx.importer.pull();
++            assert.strictEqual(pulled.items, IMPORT_ITEMS);
++            const after = heap();
++
++            const grew = after - atBoot;
++            assert.ok(grew <= HEAP_GROWTH_MAX, `heap grew ${mb(grew)} from boot to after the sitemaps and the import (${mb(atBoot)} -> ${mb(built)} -> ${mb(after)}); budget ${mb(HEAP_GROWTH_MAX)}`);
++            assert.ok(after < HEAP_MAX, `heap after the import is ${mb(after)}, over ${mb(HEAP_MAX)}`);
++            console.log(`  heap after boot ${mb(atBoot)}, sitemap cache ${mb(built - atBoot)}, after ${IMPORT_ITEMS} imported items ${mb(after)}`);
++        });
++    } finally {
++        await t.close();
++    }
++    done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/merge.test.js b/test/merge.test.js
iindex cadce57..fda5f94 100644
---- a/test/merge.test.js
++++ b/test/merge.test.js
@@@ -48,6 +48,26 @@ const { boot, check, done, HOUR } = require('./helpers/boot');
          assert.strictEqual(rows, 4, 'no vote row was deleted, moved or copied');
      });
  
++    await check('the sitemap index entries agree with the page views for a merge group', async () => {
++        // The sitemap cache keeps compact entries, not full views (memory on the host, 2026-11-07). This pins that the
++        // cheap read still sees the group: lastmod, the gate's decision and the /llms-full.txt summary.
++        const { product } = await t.ctx.catalog.resolve({ name: 'Merged TV' });
++        await t.ctx.store.db.prepare('UPDATE deal_offers SET product_id = ? WHERE id IN (?, ?)').run(product.id, A.id, B.id);
++        const av = await t.ctx.publication.offerView(await t.ctx.reads.get(A.id));
++        const ae = await t.ctx.publication.indexEntry(await t.ctx.reads.get(A.id));
++        assert.strictEqual(ae.loc, `https://openvibe.deals/d/${A.slug}`);
++        assert.strictEqual(ae.lastmod, Math.max(av.latest.observed_at, av.root.updated_at));
++        assert.strictEqual(ae.decision.indexable, av.decision.indexable);
++        assert.match(ae.text, /389 USD at shop\.example/, 'the group latest observation and the store label');
++
++        const prod = await t.ctx.catalog.product(product.id);
++        const pv = await t.ctx.publication.productView(prod);
++        const pe = await t.ctx.publication.productIndexEntry(prod);
++        assert.strictEqual(pe.loc, `https://openvibe.deals/p/${prod.slug}`);
++        assert.strictEqual(pe.lastmod, Math.max(prod.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))));
++        assert.strictEqual(pe.decision.indexable, pv.decision.indexable);
++    });
++
      await check('the merged listing redirects to the canonical one (page and JSON), and Search drops it', async () => {
          const page = await t.get(`/d/${B.slug}`);
          assert.strictEqual(page.status, 301);
ddiff --git a/test/sitemap-build.test.js b/test/sitemap-build.test.js
iindex b3b08f4..92f00b2 100644
---- a/test/sitemap-build.test.js
++++ b/test/sitemap-build.test.js
@@@ -25,15 +25,13 @@ function makeOffers(count) {
  function routes(ttlMs, clock, { offerCount = 20, products = [] } = {}) {
      const offers = makeOffers(offerCount);
      const publication = {
--        async offerView(o) {
++        async indexEntry(o) {
              seen.now++; seen.views++; seen.max = Math.max(seen.max, seen.now);
              await tick();
              seen.now--;
--            return { root: { title: o.slug, description: '' }, latest: { observed_at: o.updated_at, price: '1', currency: 'USD' }, decision: indexable };
++            return { loc: `https://deals.test/d/${o.slug}`, lastmod: o.updated_at, decision: indexable, title: o.slug, text: '1 USD' };
          },
--        offerPath: (o) => `/d/${o.slug}`,
--        async productView() { return { offers: [], decision: indexable }; },
--        productPath: (p) => `/p/${p.slug}`,
++        async productIndexEntry(p) { return { loc: `https://deals.test/p/${p.slug}`, lastmod: p.updated_at, decision: indexable }; },
      };
      const listings = {
          indexable: async () => offers, products: async () => products, newest: async () => [], countActive: async () => offers.length,