openvibers/openvibe.deals Commits
The sitemaps' offer and product lists are built once for concurrent readers, 8 views at a time, and reused for 10 minutes in production: each /sitemap.xml fetch read every view at once (#82)
Changes
@@@ -16,6 +16,9 @@ TRUST_PROXY=2# Retry-After. Writes have their own numbers, above the abuse controls below (README).#DEALS_LIMITS_MINUTE=120#DEALS_LIMITS_HOUR=3000++# How long the sitemaps' and /llms-full.txt's offer and product lists are reused, in ms (production default 600000; 0++# rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.++#DEALS_SITEMAP_TTL_MS=600000# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,# the owner on a direct connection (migrations), and Valkey (shared limit counters).@@@ -29,6 +29,10 @@ function load(env = process.env) {isProduction,// Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it.baseUrl,++ // The sitemaps' and /llms-full.txt's offer and product lists (every offer's and product's page view) are built++ // at most once in this window and shared (one build serves concurrent readers either way). Each crawler fetch++ // of /sitemap.xml rebuilt both: 5,000-24,000 slow queries a time on 2026-10-23. 0 = no reuse.++ sitemapTtlMs: int(env.DEALS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may// make per minute and per hour. Writes set tighter numbers there.@@@ -15,8 +15,9 @@* GET /sitemaps/pages.xml the front page and the lists: hot, new, ending soon, picks, every category and shop,* crowdfunding (an empty list is noindex and left out)*-- * Built from the database on every request, never from the viewer. Stale, expired, disabled,-- * merged and unreviewed offers never appear.++ * Built from the database, never from the viewer (the offer and product lists at most once per++ * config.sitemapTtlMs, shared by concurrent readers). Stale, expired, disabled, merged and unreviewed++ * offers never appear.*/const express = require('express');const seo = require('openvibe-publishing/seo');@@@ -24,6 +25,28 @@ const sharedSeo = require('openvibe-shared/seo');const cache = require('openvibe-shared/cache-policy');const { SITE_SUMMARY } = require('../render/layout');++// Offers or products whose page view a list build reads at once (the pool has a handful of connections for everyone).++const VIEWS_PARALLEL = 8;++++/** fn over items, `limit` at a time; results in the items' order. */++async function mapLimit(items, limit, fn) {++ const out = new Array(items.length);++ let next = 0;++ const worker = async () => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]); } };++ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));++ return out;++}++++/** One build at a time, shared by the readers that ask while it runs, reused for ttlMs (0: the next read rebuilds). */++function shared(build, ttlMs, now) {++ let value = null; let at = 0; let running = null;++ return async () => {++ if (value && now() - at < ttlMs) return value;++ if (!running) running = build().then((v) => { value = v; at = now(); return v; }).finally(() => { running = null; });++ return await running;++ };++}++function createDiscoveryRoutes({ config, store, publication, listings }) {const router = express.Router();const abs = (p) => seo.canonicalUrl(config.baseUrl, p);@@@ -31,18 +54,18 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {/** A short, whitespace-collapsed summary; the full source text is never included. */const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);-- async function offerEntries() {-- return Promise.all((await listings.indexable()).map(async (o) => {-- const v = await publication.offerView(o);-- return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };-- }));-- }-- async function productEntries() {-- return Promise.all((await listings.products()).map(async (p) => {-- const pv = await publication.productView(p);-- return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };-- }));-- }++ // Every offer's and product's page view, for the sitemaps and /llms-full.txt: built VIEWS_PARALLEL at a time, once++ // for concurrent readers, reused for config.sitemapTtlMs. All at once, each /sitemap.xml queued thousands of++ // queries on the pool for a minute (2026-10-23), and every page waited behind them.++ const ttl = Number(config.sitemapTtlMs) || 0;++ const offerEntries = shared(async () => mapLimit(await listings.indexable(), VIEWS_PARALLEL, async (o) => {++ const v = await publication.offerView(o);++ return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };++ }), ttl, store.now);++ const productEntries = shared(async () => mapLimit(await listings.products(), VIEWS_PARALLEL, async (p) => {++ const pv = await publication.productView(p);++ return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };++ }), ttl, store.now);/** The lists a person browses (all indexable while they have something to show); lastmod = their newest deal. */async function pageEntries() {nnew file mode 100644@@@ -0,0 +1,70 @@++'use strict';++/**++ * The sitemaps' offer and product lists (server/http/discovery.js): one build serves every request that arrives while++ * it runs, at most 8 page views at once, reused for DEALS_SITEMAP_TTL_MS (production: 10 minutes; 0 rebuilds). On++ * 2026-10-23 each crawler fetch of /sitemap.xml read every offer's and product's view at once: 5,000-24,000 queries++ * queued on the pool for a minute while every page waited behind them.++ */++const assert = require('assert');++const http = require('http');++const express = require('express');++const { createDiscoveryRoutes } = require('../server/http/discovery');++++const seen = { now: 0, max: 0, views: 0 };++const tick = () => new Promise((r) => setImmediate(r));++const offers = Array.from({ length: 20 }, (_x, i) => ({ id: `dof_${i}`, slug: `deal-${i}`, updated_at: 1792700000000 + i }));++const indexable = { indexable: true, listable: true, robots: 'index, follow', reasons: [], codes: [] };++++function routes(ttlMs, clock) {++ const publication = {++ async offerView(o) {++ seen.now++; seen.views++; seen.max = Math.max(seen.max, seen.now);++ await tick();++ seen.now--;++ return { root: { title: o.slug, description: '' }, latest: { observed_at: o.updated_at, price: '1', currency: 'USD' }, decision: indexable };++ },++ offerPath: (o) => `/d/${o.slug}`,++ async productView() { return { offers: [], decision: indexable }; },++ productPath: (p) => `/p/${p.slug}`,++ };++ const listings = {++ indexable: async () => offers, products: async () => [], newest: async () => [], countActive: async () => offers.length,++ countEnding: async () => 0, countPicks: async () => 0, categories: async () => [], sellers: async () => [],++ };++ return createDiscoveryRoutes({ config: { baseUrl: 'https://deals.test', sitemapTtlMs: ttlMs }, store: { now: () => clock.t }, publication, listings });++}++++async function serve(router) {++ const app = express().use(router);++ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });++ const base = `http://127.0.0.1:${server.address().port}`;++ return { get: async (p) => { const res = await fetch(base + p); return { status: res.status, text: await res.text() }; }, close: () => server.close() };++}++++(async () => {++ const clock = { t: Date.parse('2026-10-23T21:26:00Z') };++++ // No reuse (tests, development): concurrent requests share one build; the next request rebuilds.++ let s = await serve(routes(0, clock));++ const [a, b, c] = await Promise.all([s.get('/sitemap.xml'), s.get('/sitemaps/offers.xml'), s.get('/llms-full.txt')]);++ assert.deepStrictEqual([a.status, b.status, c.status], [200, 200, 200]);++ assert.ok(b.text.includes('https://deals.test/d/deal-19'), 'every offer listed');++ assert.strictEqual(seen.views, 20, `three readers, one build (${seen.views} views)`);++ assert.ok(seen.max <= 8 && seen.max >= 2, `at most 8 views at once (saw ${seen.max})`);++ await s.get('/sitemaps/offers.xml');++ assert.strictEqual(seen.views, 40, 'no TTL: rebuilt');++ s.close();++++ // DEALS_SITEMAP_TTL_MS: reused inside the window, rebuilt after.++ seen.views = 0;++ s = await serve(routes(600000, clock));++ await s.get('/sitemap.xml');++ clock.t += 9 * 60 * 1000;++ await s.get('/sitemaps/offers.xml');++ assert.strictEqual(seen.views, 20, 'reused inside ten minutes');++ clock.t += 2 * 60 * 1000;++ await s.get('/sitemaps/offers.xml');++ assert.strictEqual(seen.views, 40, 'rebuilt after');++ s.close();++ console.log('sitemap-build: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });
@@ -16,6 +16,9 @@ TRUST_PROXY=2
@@ -16,6 +16,9 @@ TRUST_PROXY=2
# Retry-After. Writes have their own numbers, above the abuse controls below (README).
# Retry-After. Writes have their own numbers, above the abuse controls below (README).
#DEALS_LIMITS_MINUTE=120
#DEALS_LIMITS_MINUTE=120
#DEALS_LIMITS_HOUR=3000
#DEALS_LIMITS_HOUR=3000
+# How long the sitemaps' and /llms-full.txt's offer and product lists are reused, in ms (production default 600000; 0
+# rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.
+#DEALS_SITEMAP_TTL_MS=600000
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
@@ -29,6 +29,10 @@ function load(env = process.env) {
@@ -29,6 +29,10 @@ function load(env = process.env) {
isProduction,
isProduction,
// Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it.
// Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it.
baseUrl,
baseUrl,
+ // The sitemaps' and /llms-full.txt's offer and product lists (every offer's and product's page view) are built
+ // at most once in this window and shared (one build serves concurrent readers either way). Each crawler fetch
+ // of /sitemap.xml rebuilt both: 5,000-24,000 slow queries a time on 2026-10-23. 0 = no reuse.
+ sitemapTtlMs: int(env.DEALS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may
// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may
// make per minute and per hour. Writes set tighter numbers there.
// make per minute and per hour. Writes set tighter numbers there.
@@ -15,8 +15,9 @@
@@ -15,8 +15,9 @@
* GET /sitemaps/pages.xml the front page and the lists: hot, new, ending soon, picks, every category and shop,
* GET /sitemaps/pages.xml the front page and the lists: hot, new, ending soon, picks, every category and shop,
* crowdfunding (an empty list is noindex and left out)
* crowdfunding (an empty list is noindex and left out)
*
*
- * Built from the database on every request, never from the viewer. Stale, expired, disabled,
- * merged and unreviewed offers never appear.
+ * Built from the database, never from the viewer (the offer and product lists at most once per
+ * config.sitemapTtlMs, shared by concurrent readers). Stale, expired, disabled, merged and unreviewed
+ * offers never appear.
*/
*/
const express = require('express');
const express = require('express');
const seo = require('openvibe-publishing/seo');
const seo = require('openvibe-publishing/seo');
@@ -24,6 +25,28 @@ const sharedSeo = require('openvibe-shared/seo');
@@ -24,6 +25,28 @@ const sharedSeo = require('openvibe-shared/seo');
const cache = require('openvibe-shared/cache-policy');
const cache = require('openvibe-shared/cache-policy');
const { SITE_SUMMARY } = require('../render/layout');
const { SITE_SUMMARY } = require('../render/layout');
+// Offers or products whose page view a list build reads at once (the pool has a handful of connections for everyone).
+const VIEWS_PARALLEL = 8;
+
+/** fn over items, `limit` at a time; results in the items' order. */
+async function mapLimit(items, limit, fn) {
+ const out = new Array(items.length);
+ let next = 0;
+ const worker = async () => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]); } };
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+ return out;
+}
+
+/** One build at a time, shared by the readers that ask while it runs, reused for ttlMs (0: the next read rebuilds). */
+function shared(build, ttlMs, now) {
+ let value = null; let at = 0; let running = null;
+ return async () => {
+ if (value && now() - at < ttlMs) return value;
+ if (!running) running = build().then((v) => { value = v; at = now(); return v; }).finally(() => { running = null; });
+ return await running;
+ };
+}
+
function createDiscoveryRoutes({ config, store, publication, listings }) {
function createDiscoveryRoutes({ config, store, publication, listings }) {
const router = express.Router();
const router = express.Router();
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
@@ -31,18 +54,18 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {
@@ -31,18 +54,18 @@ function createDiscoveryRoutes({ config, store, publication, listings }) {
/** A short, whitespace-collapsed summary; the full source text is never included. */
/** A short, whitespace-collapsed summary; the full source text is never included. */
const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);
const brief = (value, max = 200) => String(value == null ? '' : value).replace(/\s+/g, ' ').trim().slice(0, max);
- async function offerEntries() {
- return Promise.all((await listings.indexable()).map(async (o) => {
- const v = await publication.offerView(o);
- return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };
- }));
- }
- async function productEntries() {
- return Promise.all((await listings.products()).map(async (p) => {
- const pv = await publication.productView(p);
- return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };
- }));
- }
+ // Every offer's and product's page view, for the sitemaps and /llms-full.txt: built VIEWS_PARALLEL at a time, once
+ // for concurrent readers, reused for config.sitemapTtlMs. All at once, each /sitemap.xml queued thousands of
+ // queries on the pool for a minute (2026-10-23), and every page waited behind them.
+ const ttl = Number(config.sitemapTtlMs) || 0;
+ const offerEntries = shared(async () => mapLimit(await listings.indexable(), VIEWS_PARALLEL, async (o) => {
+ const v = await publication.offerView(o);
+ return { offer: o, view: v, loc: abs(publication.offerPath(o)), lastmod: v.latest ? Math.max(v.latest.observed_at, o.updated_at) : o.updated_at, decision: v.decision };
+ }), ttl, store.now);
+ const productEntries = shared(async () => mapLimit(await listings.products(), VIEWS_PARALLEL, async (p) => {
+ const pv = await publication.productView(p);
+ return { loc: abs(publication.productPath(p)), lastmod: Math.max(p.updated_at, ...pv.offers.map((v) => Math.max(v.root.updated_at, v.latest ? v.latest.observed_at : 0))), decision: pv.decision };
+ }), ttl, store.now);
/** The lists a person browses (all indexable while they have something to show); lastmod = their newest deal. */
/** The lists a person browses (all indexable while they have something to show); lastmod = their newest deal. */
async function pageEntries() {
async function pageEntries() {
new file mode 100644
new file mode 100644
@@ -0,0 +1,70 @@
@@ -0,0 +1,70 @@
+'use strict';
+/**
+ * The sitemaps' offer and product lists (server/http/discovery.js): one build serves every request that arrives while
+ * it runs, at most 8 page views at once, reused for DEALS_SITEMAP_TTL_MS (production: 10 minutes; 0 rebuilds). On
+ * 2026-10-23 each crawler fetch of /sitemap.xml read every offer's and product's view at once: 5,000-24,000 queries
+ * queued on the pool for a minute while every page waited behind them.
+ */
+const assert = require('assert');
+const http = require('http');
+const express = require('express');
+const { createDiscoveryRoutes } = require('../server/http/discovery');
+
+const seen = { now: 0, max: 0, views: 0 };
+const tick = () => new Promise((r) => setImmediate(r));
+const offers = Array.from({ length: 20 }, (_x, i) => ({ id: `dof_${i}`, slug: `deal-${i}`, updated_at: 1792700000000 + i }));
+const indexable = { indexable: true, listable: true, robots: 'index, follow', reasons: [], codes: [] };
+
+function routes(ttlMs, clock) {
+ const publication = {
+ async offerView(o) {
+ seen.now++; seen.views++; seen.max = Math.max(seen.max, seen.now);
+ await tick();
+ seen.now--;
+ return { root: { title: o.slug, description: '' }, latest: { observed_at: o.updated_at, price: '1', currency: 'USD' }, decision: indexable };
+ },
+ offerPath: (o) => `/d/${o.slug}`,
+ async productView() { return { offers: [], decision: indexable }; },
+ productPath: (p) => `/p/${p.slug}`,
+ };
+ const listings = {
+ indexable: async () => offers, products: async () => [], newest: async () => [], countActive: async () => offers.length,
+ countEnding: async () => 0, countPicks: async () => 0, categories: async () => [], sellers: async () => [],
+ };
+ return createDiscoveryRoutes({ config: { baseUrl: 'https://deals.test', sitemapTtlMs: ttlMs }, store: { now: () => clock.t }, publication, listings });
+}
+
+async function serve(router) {
+ const app = express().use(router);
+ const server = await new Promise((r) => { const s = http.createServer(app).listen(0, '127.0.0.1', () => r(s)); });
+ const base = `http://127.0.0.1:${server.address().port}`;
+ return { get: async (p) => { const res = await fetch(base + p); return { status: res.status, text: await res.text() }; }, close: () => server.close() };
+}
+
+(async () => {
+ const clock = { t: Date.parse('2026-10-23T21:26:00Z') };
+
+ // No reuse (tests, development): concurrent requests share one build; the next request rebuilds.
+ let s = await serve(routes(0, clock));
+ const [a, b, c] = await Promise.all([s.get('/sitemap.xml'), s.get('/sitemaps/offers.xml'), s.get('/llms-full.txt')]);
+ assert.deepStrictEqual([a.status, b.status, c.status], [200, 200, 200]);
+ assert.ok(b.text.includes('https://deals.test/d/deal-19'), 'every offer listed');
+ assert.strictEqual(seen.views, 20, `three readers, one build (${seen.views} views)`);
+ assert.ok(seen.max <= 8 && seen.max >= 2, `at most 8 views at once (saw ${seen.max})`);
+ await s.get('/sitemaps/offers.xml');
+ assert.strictEqual(seen.views, 40, 'no TTL: rebuilt');
+ s.close();
+
+ // DEALS_SITEMAP_TTL_MS: reused inside the window, rebuilt after.
+ seen.views = 0;
+ s = await serve(routes(600000, clock));
+ await s.get('/sitemap.xml');
+ clock.t += 9 * 60 * 1000;
+ await s.get('/sitemaps/offers.xml');
+ assert.strictEqual(seen.views, 20, 'reused inside ten minutes');
+ clock.t += 2 * 60 * 1000;
+ await s.get('/sitemaps/offers.xml');
+ assert.strictEqual(seen.views, 40, 'rebuilt after');
+ s.close();
+ console.log('sitemap-build: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });