openvibers/openvibe.deals Commits

The sitemaps' offer and product lists are built once for concurrent readers, 8 views at a time, and reused for 10 minutes in production: each /sitemap.xml fetch read every view at once (#82)

OP OpenVibers committed 21 days ago · c27ead8d2e · 4 files, +114 −14

Changes

ddiff --git a/.env.example b/.env.example
iindex 47090e0..6a352a5 100644
---- a/.env.example
++++ b/.env.example
@@@ -16,6 +16,9 @@ TRUST_PROXY=2
# Retry-After. Writes have their own numbers, above the abuse controls below (README).
#DEALS_LIMITS_MINUTE=120
#DEALS_LIMITS_HOUR=3000
++# How long the sitemaps' and /llms-full.txt's offer and product lists are reused, in ms (production default 600000; 0
++# rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.
++#DEALS_SITEMAP_TTL_MS=600000
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
ddiff --git a/server/config.js b/server/config.js
iindex 80b922a..cf04370 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -29,6 +29,10 @@ function load(env = process.env) {
isProduction,
// Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it.
baseUrl,
++ // The sitemaps' and /llms-full.txt's offer and product lists (every offer's and product's page view) are built
++ // at most once in this window and shared (one build serves concurrent readers either way). Each crawler fetch
++ // of /sitemap.xml rebuilt both: 5,000-24,000 slow queries a time on 2026-10-23. 0 = no reuse.
++ sitemapTtlMs: int(env.DEALS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may
// make per minute and per hour. Writes set tighter numbers there.
ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex e1dd2af..0954af7 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -15,8 +15,9 @@
* GET /sitemaps/pages.xml the front page and the lists: hot, new, ending soon, picks, every category and shop,
* crowdfunding (an empty list is noindex and left out)
*
-- * Built from the database on every request, never from the viewer. Stale, expired, disabled,
-- * merged and unreviewed offers never appear.
++ * Built from the database, never from the viewer (the offer and product lists at most once per
++ * config.sitemapTtlMs, shared by concurrent readers). Stale, expired, disabled, merged and unreviewed
++ * offers never appear.
*/
const express = require('express');
const seo = require('openvibe-publishing/seo');
@@@ -24,6 +25,28 @@ const sharedSeo = require('openvibe-shared/seo');
const cache = require('openvibe-shared/cache-policy');
const { SITE_SUMMARY } = require('../render/layout');
++// Offers or products whose page view a list build reads at once (the pool has a handful of connections for everyone).
++const VIEWS_PARALLEL = 8;
++
++/** fn over items, `limit` at a time; results in the items' order. */
++async function mapLimit(items, limit, fn) {
++ const out = new Array(items.length);
++ let next = 0;
++ const worker = async () => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]); } };
++ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
++ return out;
++}
++
++/** One build at a time, shared by the readers that ask while it runs, reused for ttlMs (0: the next read rebuilds). */
++function shared(build, ttlMs, now) {
++ let value = null; let at = 0; let running = null;
++ return async () => {
++ if (value && now() - at < ttlMs) return value;
++ if (!running) running = build().then((v) => { value = v; at = now(); return v; }).finally(() => { running = null; });
++ return await running;
++ };
++}
++
function createDiscoveryRoutes({ config, store, publication, listings }) {
const router = express.Router();
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
@@@ -31,18 +54,18 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {
/** A short, whitespace-collapsed summary; the full source text is never included. */
const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);
-- async function offerEntries() {
-- return Promise.all((await listings.indexable()).map(async (o) => {
-- const v = await publication.offerView(o);
-- return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };
-- }));
-- }
-- async function productEntries() {
-- return Promise.all((await listings.products()).map(async (p) => {
-- const pv = await publication.productView(p);
-- return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };
-- }));
-- }
++ // Every offer's and product's page view, for the sitemaps and /llms-full.txt: built VIEWS_PARALLEL at a time, once
++ // for concurrent readers, reused for config.sitemapTtlMs. All at once, each /sitemap.xml queued thousands of
++ // queries on the pool for a minute (2026-10-23), and every page waited behind them.
++ const ttl = Number(config.sitemapTtlMs) || 0;
++ const offerEntries = shared(async () => mapLimit(await listings.indexable(), VIEWS_PARALLEL, async (o) => {
++ const v = await publication.offerView(o);
++ return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };
++ }), ttl, store.now);
++ const productEntries = shared(async () => mapLimit(await listings.products(), VIEWS_PARALLEL, async (p) => {
++ const pv = await publication.productView(p);
++ return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };
++ }), ttl, store.now);
/** The lists a person browses (all indexable while they have something to show); lastmod = their newest deal. */
async function pageEntries() {
ddiff --git a/test/sitemap-build.test.js b/test/sitemap-build.test.js
nnew file mode 100644
iindex 0000000..ed10b1e
---- /dev/null
++++ b/test/sitemap-build.test.js
@@@ -0,0 +1,70 @@
++'use strict';
++/**
++ * The sitemaps' offer and product lists (server/http/discovery.js): one build serves every request that arrives while
++ * it runs, at most 8 page views at once, reused for DEALS_SITEMAP_TTL_MS (production: 10 minutes; 0 rebuilds). On
++ * 2026-10-23 each crawler fetch of /sitemap.xml read every offer's and product's view at once: 5,000-24,000 queries
++ * queued on the pool for a minute while every page waited behind them.
++ */
++const assert = require('assert');
++const http = require('http');
++const express = require('express');
++const { createDiscoveryRoutes } = require('../server/http/discovery');
++
++const seen = { now: 0, max: 0, views: 0 };
++const tick = () => new Promise((r) => setImmediate(r));
++const offers = Array.from({ length: 20 }, (_x, i) => ({ id: `dof_${i}`, slug: `deal-${i}`, updated_at: 1792700000000 + i }));
++const indexable = { indexable: true, listable: true, robots: 'index, follow', reasons: [], codes: [] };
++
++function routes(ttlMs, clock) {
++ const publication = {
++ async offerView(o) {
++ seen.now++; seen.views++; seen.max = Math.max(seen.max, seen.now);
++ await tick();
++ seen.now--;
++ return { root: { title: o.slug, description: '' }, latest: { observed_at: o.updated_at, price: '1', currency: 'USD' }, decision: indexable };
++ },
++ offerPath: (o) => `/d/${o.slug}`,
++ async productView() { return { offers: [], decision: indexable }; },
++ productPath: (p) => `/p/${p.slug}`,
++ };
++ const listings = {
++ indexable: async () => offers, products: async () => [], newest: async () => [], countActive: async () => offers.length,
++ countEnding: async () => 0, countPicks: async () => 0, categories: async () => [], sellers: async () => [],
++ };
++ return createDiscoveryRoutes({ config: { baseUrl: 'https://deals.test', sitemapTtlMs: ttlMs }, store: { now: () => clock.t }, publication, listings });
++}
++
++async function serve(router) {
++ const app = express().use(router);
++ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });
++ const base = `http://127.0.0.1:${server.address().port}`;
++ return { get: async (p) => { const res = await fetch(base + p); return { status: res.status, text: await res.text() }; }, close: () => server.close() };
++}
++
++(async () => {
++ const clock = { t: Date.parse('2026-10-23T21:26:00Z') };
++
++ // No reuse (tests, development): concurrent requests share one build; the next request rebuilds.
++ let s = await serve(routes(0, clock));
++ const [a, b, c] = await Promise.all([s.get('/sitemap.xml'), s.get('/sitemaps/offers.xml'), s.get('/llms-full.txt')]);
++ assert.deepStrictEqual([a.status, b.status, c.status], [200, 200, 200]);
++ assert.ok(b.text.includes('https://deals.test/d/deal-19'), 'every offer listed');
++ assert.strictEqual(seen.views, 20, `three readers, one build (${seen.views} views)`);
++ assert.ok(seen.max <= 8 && seen.max >= 2, `at most 8 views at once (saw ${seen.max})`);
++ await s.get('/sitemaps/offers.xml');
++ assert.strictEqual(seen.views, 40, 'no TTL: rebuilt');
++ s.close();
++
++ // DEALS_SITEMAP_TTL_MS: reused inside the window, rebuilt after.
++ seen.views = 0;
++ s = await serve(routes(600000, clock));
++ await s.get('/sitemap.xml');
++ clock.t += 9 * 60 * 1000;
++ await s.get('/sitemaps/offers.xml');
++ assert.strictEqual(seen.views, 20, 'reused inside ten minutes');
++ clock.t += 2 * 60 * 1000;
++ await s.get('/sitemaps/offers.xml');
++ assert.strictEqual(seen.views, 40, 'rebuilt after');
++ s.close();
++ console.log('sitemap-build: all checks passed');
++})().catch((e) => { console.error(e); process.exit(1); });