openvibers/openvibe.reviews Commits

The sitemaps' entity list is built once for concurrent readers, 8 entities at a time, and reused for 10 minutes in production: each /sitemap.xml fetch fired ~30,000 queries at once (#93)

OP OpenVibers committed 21 days ago · 58ec028865 · 4 files, +87 −2

Changes

ddiff --git a/.env.example b/.env.example
iindex 72508eb..3d26166 100644
---- a/.env.example
++++ b/.env.example
@@@ -68,6 +68,10 @@ OV_OAUTH_CLIENT_SECRET=
#REVIEWS_GATE_MIN_WORDS=60
#REVIEWS_GATE_MIN_SIGNALS=1
++# How long the sitemaps' entity list (every entity's gate decision) is reused, in ms (production default 600000;
++# 0 rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.
++#REVIEWS_SITEMAP_TTL_MS=600000
++
# Release reporting (openvibe-shared/release): the deployed commit, read from git when unset.
#RELEASE_COMMIT=
#MIN_CLIENT_RELEASE=
ddiff --git a/server/config.js b/server/config.js
iindex f09f4b7..c717386 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -88,6 +88,11 @@ function load(env = process.env) {
queueIntervalMs: int(env.REVIEWS_IMPORT_QUEUE_INTERVAL_MS, 15000),
},
++ // The sitemaps' list of every active entity with its gate decision is built at most once in this window and
++ // shared (one build serves concurrent readers either way). Building it reads every entity's page view, so
++ // each crawler fetch of /sitemap.xml rebuilt it: ~30,000 queries a time on 2026-10-23. 0 = no reuse.
++ sitemapTtlMs: int(env.REVIEWS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),
++
// The indexability gate policy for entity pages (openvibe-publishing/seo).
gate: {
minWords: int(env.REVIEWS_GATE_MIN_WORDS, 60),
ddiff --git a/server/reviews/service.js b/server/reviews/service.js
iindex cc2daf1..8b18195 100644
---- a/server/reviews/service.js
++++ b/server/reviews/service.js
@@@ -42,6 +42,8 @@ const MAX_CORRECTION_NOTE = 2000;
const CORRECTIONS_PER_PERSON_DAY = 20;
const DAY_MS = 24 * 3600 * 1000;
const MAX_AUDIT_TEXT = 2000;
++// Entities whose page view publicEntities() builds at once (the pool has a handful of connections for every reader).
++const PUBLIC_ENTITIES_PARALLEL = 8;
// A summary is text with citations. Nothing in it can carry a score.
const SUMMARY_KEYS = new Set(['overview', 'pros', 'cons', 'overview_signals', 'expected_revision', 'expectedRevision', 'message', 'publish', 'workflow', 'stub_provider', 'stubProvider', 'note', 'correction_note', 'correction_id']);
const BODY_KEYS = ['overview', 'overview_signals', 'pros', 'cons'];
@@@ -79,6 +81,7 @@ function recordName(raw) {
}
function createReviewsService({ db, stores, outbox, config, now = () => Date.now(), log = console, indexnow = null }) {
++ const publicList = { value: null, at: 0, building: null }; // publicEntities(): the last build and the one running
const { revisions, reviews, redirects, discussions, sequencer } = stores;
const access = createAccess(config);
const origin = config.baseUrl;
@@@ -2021,9 +2024,28 @@ function createReviewsService({ db, stores, outbox, config, now = () => Date.now
}))).filter(Boolean);
},
-- /** Every active entity with its decision (sitemaps). */
++ /**
++ * Every active entity with its decision (sitemaps). One build at a time, reused for config.sitemapTtlMs, and
++ * PUBLIC_ENTITIES_PARALLEL entities at once: a build reads each entity's page view (about five queries), and
++ * all at once (6,138 entities, 2026-10-23) it queued ~30,000 queries on the pool for a minute, each waiting
++ * ~800 ms, while every page waited behind them.
++ */
async publicEntities() {
-- return (await Promise.all((await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all()).map(async (e) => ({ entity: e, decision: await decide(e) }))));
++ const ttl = Number(config.sitemapTtlMs) || 0;
++ if (publicList.value && now() - publicList.at < ttl) return publicList.value;
++ if (!publicList.building) {
++ publicList.building = (async () => {
++ const rows = await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all();
++ const out = new Array(rows.length);
++ let next = 0;
++ const worker = async () => { while (next < rows.length) { const i = next++; out[i] = { entity: rows[i], decision: await decide(rows[i]) }; } };
++ await Promise.all(Array.from({ length: Math.min(PUBLIC_ENTITIES_PARALLEL, rows.length) }, worker));
++ publicList.value = out;
++ publicList.at = now();
++ return out;
++ })().finally(() => { publicList.building = null; });
++ }
++ return await publicList.building;
},
/** Entities with at least one live signal or a published summary (the home page). */
ddiff --git a/test/sitemap-build.test.js b/test/sitemap-build.test.js
nnew file mode 100644
iindex 0000000..b3a7723
---- /dev/null
++++ b/test/sitemap-build.test.js
@@@ -0,0 +1,54 @@
++'use strict';
++/**
++ * The sitemaps' entity list (svc.publicEntities): one build serves every reader that asks while it runs; with
++ * REVIEWS_SITEMAP_TTL_MS it is reused for that long (production: 10 minutes); and a build reads at most 8 entities'
++ * page views at once. On 2026-10-23 each crawler fetch of /sitemap.xml read all 6,138 entities at once: ~30,000
++ * queries queued on the pool for a minute, each waiting ~800 ms, while every page waited behind them.
++ */
++const assert = require('assert');
++const { boot, req, suite, createEntity } = require('./helpers');
++
++const t = suite('sitemap-build');
++
++/** Count the merged-into lookups (one per entity view) running at once; prepared statements call db.query. */
++function watchViews(h) {
++ const seen = { now: 0, max: 0, total: 0 };
++ const query = h.db.query.bind(h.db);
++ h.db.query = async (text, params) => {
++ if (!/merged_into = \$1 AND state = 'merged'/.test(String(text))) return await query(text, params);
++ seen.now++; seen.total++; seen.max = Math.max(seen.max, seen.now);
++ try { await new Promise((r) => setImmediate(r)); return await query(text, params); } finally { seen.now--; }
++ };
++ return seen;
++}
++
++t('concurrent readers share one build, at most 8 entities at a time; without a TTL the next read rebuilds', async () => {
++ const h = await boot();
++ try {
++ for (let i = 0; i < 20; i++) await createEntity(h, { name: `Game ${i}`, kind: 'game' });
++ const seen = watchViews(h);
++ const [a, b, c] = await Promise.all([h.svc.publicEntities(), h.svc.publicEntities(), h.svc.publicEntities()]);
++ assert.strictEqual(a.length, 20);
++ assert.ok(a === b && b === c, 'one build for three readers');
++ assert.ok(seen.max >= 1 && seen.max <= 8, `at most 8 at once (saw ${seen.max})`);
++ const built = seen.total;
++ assert.ok(built >= 20, `each entity read once (${built})`);
++ await createEntity(h, { name: 'Game 20', kind: 'game' });
++ assert.strictEqual((await h.svc.publicEntities()).length, 21, 'no TTL (tests, development): the next read rebuilds');
++ assert.ok((await req(h, 'GET', '/sitemap.xml')).status === 200);
++ } finally { await h.stop(); }
++});
++
++t('with REVIEWS_SITEMAP_TTL_MS the list is reused until it runs out', async () => {
++ let clock = Date.parse('2026-10-23T21:36:00.000Z');
++ const h = await boot({ env: { REVIEWS_SITEMAP_TTL_MS: '600000' }, now: () => clock });
++ try {
++ await createEntity(h, { name: 'Portal 2', kind: 'game' });
++ const first = await h.svc.publicEntities();
++ await createEntity(h, { name: 'Portal 3', kind: 'game' });
++ clock += 9 * 60 * 1000;
++ assert.strictEqual(await h.svc.publicEntities(), first, 'reused inside ten minutes');
++ clock += 2 * 60 * 1000;
++ assert.strictEqual((await h.svc.publicEntities()).length, 2, 'rebuilt after');
++ } finally { await h.stop(); }
++});