openvibers/openvibe.reviews Commits
The sitemaps' entity list is built once for concurrent readers, 8 entities at a time, and reused for 10 minutes in production: each /sitemap.xml fetch fired ~30,000 queries at once (#93)
Changes
@@@ -68,6 +68,10 @@ OV_OAUTH_CLIENT_SECRET=#REVIEWS_GATE_MIN_WORDS=60#REVIEWS_GATE_MIN_SIGNALS=1++# How long the sitemaps' entity list (every entity's gate decision) is reused, in ms (production default 600000;++# 0 rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.++#REVIEWS_SITEMAP_TTL_MS=600000++# Release reporting (openvibe-shared/release): the deployed commit, read from git when unset.#RELEASE_COMMIT=#MIN_CLIENT_RELEASE=@@@ -88,6 +88,11 @@ function load(env = process.env) {queueIntervalMs: int(env.REVIEWS_IMPORT_QUEUE_INTERVAL_MS, 15000),},++ // The sitemaps' list of every active entity with its gate decision is built at most once in this window and++ // shared (one build serves concurrent readers either way). Building it reads every entity's page view, so++ // each crawler fetch of /sitemap.xml rebuilt it: ~30,000 queries a time on 2026-10-23. 0 = no reuse.++ sitemapTtlMs: int(env.REVIEWS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),++// The indexability gate policy for entity pages (openvibe-publishing/seo).gate: {minWords: int(env.REVIEWS_GATE_MIN_WORDS, 60),@@@ -42,6 +42,8 @@ const MAX_CORRECTION_NOTE = 2000;const CORRECTIONS_PER_PERSON_DAY = 20;const DAY_MS = 24 * 3600 * 1000;const MAX_AUDIT_TEXT = 2000;++// Entities whose page view publicEntities() builds at once (the pool has a handful of connections for every reader).++const PUBLIC_ENTITIES_PARALLEL = 8;// A summary is text with citations. Nothing in it can carry a score.const SUMMARY_KEYS = new Set(['overview', 'pros', 'cons', 'overview_signals', 'expected_revision', 'expectedRevision', 'message', 'publish', 'workflow', 'stub_provider', 'stubProvider', 'note', 'correction_note', 'correction_id']);const BODY_KEYS = ['overview', 'overview_signals', 'pros', 'cons'];@@@ -79,6 +81,7 @@ function recordName(raw) {}function createReviewsService({ db, stores, outbox, config, now = () => Date.now(), log = console, indexnow = null }) {++ const publicList = { value: null, at: 0, building: null }; // publicEntities(): the last build and the one runningconst { revisions, reviews, redirects, discussions, sequencer } = stores;const access = createAccess(config);const origin = config.baseUrl;@@@ -2021,9 +2024,28 @@ function createReviewsService({ db, stores, outbox, config, now = () => Date.now}))).filter(Boolean);},-- /** Every active entity with its decision (sitemaps). */++ /**++ * Every active entity with its decision (sitemaps). One build at a time, reused for config.sitemapTtlMs, and++ * PUBLIC_ENTITIES_PARALLEL entities at once: a build reads each entity's page view (about five queries), and++ * all at once (6,138 entities, 2026-10-23) it queued ~30,000 queries on the pool for a minute, each waiting++ * ~800 ms, while every page waited behind them.++ */async publicEntities() {-- return (await Promise.all((await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all()).map(async (e) => ({ entity: e, decision: await decide(e) }))));++ const ttl = Number(config.sitemapTtlMs) || 0;++ if (publicList.value && now() - publicList.at < ttl) return publicList.value;++ if (!publicList.building) {++ publicList.building = (async () => {++ const rows = await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all();++ const out = new Array(rows.length);++ let next = 0;++ const worker = async () => { while (next < rows.length) { const i = next++; out[i] = { entity: rows[i], decision: await decide(rows[i]) }; } };++ await Promise.all(Array.from({ length: Math.min(PUBLIC_ENTITIES_PARALLEL, rows.length) }, worker));++ publicList.value = out;++ publicList.at = now();++ return out;++ })().finally(() => { publicList.building = null; });++ }++ return await publicList.building;},/** Entities with at least one live signal or a published summary (the home page). */nnew file mode 100644@@@ -0,0 +1,54 @@++'use strict';++/**++ * The sitemaps' entity list (svc.publicEntities): one build serves every reader that asks while it runs; with++ * REVIEWS_SITEMAP_TTL_MS it is reused for that long (production: 10 minutes); and a build reads at most 8 entities'++ * page views at once. On 2026-10-23 each crawler fetch of /sitemap.xml read all 6,138 entities at once: ~30,000++ * queries queued on the pool for a minute, each waiting ~800 ms, while every page waited behind them.++ */++const assert = require('assert');++const { boot, req, suite, createEntity } = require('./helpers');++++const t = suite('sitemap-build');++++/** Count the merged-into lookups (one per entity view) running at once; prepared statements call db.query. */++function watchViews(h) {++ const seen = { now: 0, max: 0, total: 0 };++ const query = h.db.query.bind(h.db);++ h.db.query = async (text, params) => {++ if (!/merged_into = \$1 AND state = 'merged'/.test(String(text))) return await query(text, params);++ seen.now++; seen.total++; seen.max = Math.max(seen.max, seen.now);++ try { await new Promise((r) => setImmediate(r)); return await query(text, params); } finally { seen.now--; }++ };++ return seen;++}++++t('concurrent readers share one build, at most 8 entities at a time; without a TTL the next read rebuilds', async () => {++ const h = await boot();++ try {++ for (let i = 0; i < 20; i++) await createEntity(h, { name: `Game ${i}`, kind: 'game' });++ const seen = watchViews(h);++ const [a, b, c] = await Promise.all([h.svc.publicEntities(), h.svc.publicEntities(), h.svc.publicEntities()]);++ assert.strictEqual(a.length, 20);++ assert.ok(a === b && b === c, 'one build for three readers');++ assert.ok(seen.max >= 1 && seen.max <= 8, `at most 8 at once (saw ${seen.max})`);++ const built = seen.total;++ assert.ok(built >= 20, `each entity read once (${built})`);++ await createEntity(h, { name: 'Game 20', kind: 'game' });++ assert.strictEqual((await h.svc.publicEntities()).length, 21, 'no TTL (tests, development): the next read rebuilds');++ assert.ok((await req(h, 'GET', '/sitemap.xml')).status === 200);++ } finally { await h.stop(); }++});++++t('with REVIEWS_SITEMAP_TTL_MS the list is reused until it runs out', async () => {++ let clock = Date.parse('2026-10-23T21:36:00.000Z');++ const h = await boot({ env: { REVIEWS_SITEMAP_TTL_MS: '600000' }, now: () => clock });++ try {++ await createEntity(h, { name: 'Portal 2', kind: 'game' });++ const first = await h.svc.publicEntities();++ await createEntity(h, { name: 'Portal 3', kind: 'game' });++ clock += 9 * 60 * 1000;++ assert.strictEqual(await h.svc.publicEntities(), first, 'reused inside ten minutes');++ clock += 2 * 60 * 1000;++ assert.strictEqual((await h.svc.publicEntities()).length, 2, 'rebuilt after');++ } finally { await h.stop(); }++});
@@ -68,6 +68,10 @@ OV_OAUTH_CLIENT_SECRET=
@@ -68,6 +68,10 @@ OV_OAUTH_CLIENT_SECRET=
#REVIEWS_GATE_MIN_WORDS=60
#REVIEWS_GATE_MIN_WORDS=60
#REVIEWS_GATE_MIN_SIGNALS=1
#REVIEWS_GATE_MIN_SIGNALS=1
+# How long the sitemaps' entity list (every entity's gate decision) is reused, in ms (production default 600000;
+# 0 rebuilds on every read, the default elsewhere). Concurrent reads share one build either way.
+#REVIEWS_SITEMAP_TTL_MS=600000
+
# Release reporting (openvibe-shared/release): the deployed commit, read from git when unset.
# Release reporting (openvibe-shared/release): the deployed commit, read from git when unset.
#RELEASE_COMMIT=
#RELEASE_COMMIT=
#MIN_CLIENT_RELEASE=
#MIN_CLIENT_RELEASE=
@@ -88,6 +88,11 @@ function load(env = process.env) {
@@ -88,6 +88,11 @@ function load(env = process.env) {
queueIntervalMs: int(env.REVIEWS_IMPORT_QUEUE_INTERVAL_MS, 15000),
queueIntervalMs: int(env.REVIEWS_IMPORT_QUEUE_INTERVAL_MS, 15000),
},
},
+ // The sitemaps' list of every active entity with its gate decision is built at most once in this window and
+ // shared (one build serves concurrent readers either way). Building it reads every entity's page view, so
+ // each crawler fetch of /sitemap.xml rebuilt it: ~30,000 queries a time on 2026-10-23. 0 = no reuse.
+ sitemapTtlMs: int(env.REVIEWS_SITEMAP_TTL_MS, isProduction ? 10 * 60 * 1000 : 0),
+
// The indexability gate policy for entity pages (openvibe-publishing/seo).
// The indexability gate policy for entity pages (openvibe-publishing/seo).
gate: {
gate: {
minWords: int(env.REVIEWS_GATE_MIN_WORDS, 60),
minWords: int(env.REVIEWS_GATE_MIN_WORDS, 60),
@@ -42,6 +42,8 @@ const MAX_CORRECTION_NOTE = 2000;
@@ -42,6 +42,8 @@ const MAX_CORRECTION_NOTE = 2000;
const CORRECTIONS_PER_PERSON_DAY = 20;
const CORRECTIONS_PER_PERSON_DAY = 20;
const DAY_MS = 24 * 3600 * 1000;
const DAY_MS = 24 * 3600 * 1000;
const MAX_AUDIT_TEXT = 2000;
const MAX_AUDIT_TEXT = 2000;
+// Entities whose page view publicEntities() builds at once (the pool has a handful of connections for every reader).
+const PUBLIC_ENTITIES_PARALLEL = 8;
// A summary is text with citations. Nothing in it can carry a score.
// A summary is text with citations. Nothing in it can carry a score.
const SUMMARY_KEYS = new Set(['overview', 'pros', 'cons', 'overview_signals', 'expected_revision', 'expectedRevision', 'message', 'publish', 'workflow', 'stub_provider', 'stubProvider', 'note', 'correction_note', 'correction_id']);
const SUMMARY_KEYS = new Set(['overview', 'pros', 'cons', 'overview_signals', 'expected_revision', 'expectedRevision', 'message', 'publish', 'workflow', 'stub_provider', 'stubProvider', 'note', 'correction_note', 'correction_id']);
const BODY_KEYS = ['overview', 'overview_signals', 'pros', 'cons'];
const BODY_KEYS = ['overview', 'overview_signals', 'pros', 'cons'];
@@ -79,6 +81,7 @@ function recordName(raw) {
@@ -79,6 +81,7 @@ function recordName(raw) {
}
}
function createReviewsService({ db, stores, outbox, config, now = () => Date.now(), log = console, indexnow = null }) {
function createReviewsService({ db, stores, outbox, config, now = () => Date.now(), log = console, indexnow = null }) {
+ const publicList = { value: null, at: 0, building: null }; // publicEntities(): the last build and the one running
const { revisions, reviews, redirects, discussions, sequencer } = stores;
const { revisions, reviews, redirects, discussions, sequencer } = stores;
const access = createAccess(config);
const access = createAccess(config);
const origin = config.baseUrl;
const origin = config.baseUrl;
@@ -2021,9 +2024,28 @@ function createReviewsService({ db, stores, outbox, config, now = () => Date.now
@@ -2021,9 +2024,28 @@ function createReviewsService({ db, stores, outbox, config, now = () => Date.now
}))).filter(Boolean);
}))).filter(Boolean);
},
},
- /** Every active entity with its decision (sitemaps). */
+ /**
+ * Every active entity with its decision (sitemaps). One build at a time, reused for config.sitemapTtlMs, and
+ * PUBLIC_ENTITIES_PARALLEL entities at once: a build reads each entity's page view (about five queries), and
+ * all at once (6,138 entities, 2026-10-23) it queued ~30,000 queries on the pool for a minute, each waiting
+ * ~800 ms, while every page waited behind them.
+ */
async publicEntities() {
async publicEntities() {
- return (await Promise.all((await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all()).map(async (e) => ({ entity: e, decision: await decide(e) }))));
+ const ttl = Number(config.sitemapTtlMs) || 0;
+ if (publicList.value && now() - publicList.at < ttl) return publicList.value;
+ if (!publicList.building) {
+ publicList.building = (async () => {
+ const rows = await db.prepare("SELECT * FROM review_entities WHERE state = 'active' ORDER BY id").all();
+ const out = new Array(rows.length);
+ let next = 0;
+ const worker = async () => { while (next < rows.length) { const i = next++; out[i] = { entity: rows[i], decision: await decide(rows[i]) }; } };
+ await Promise.all(Array.from({ length: Math.min(PUBLIC_ENTITIES_PARALLEL, rows.length) }, worker));
+ publicList.value = out;
+ publicList.at = now();
+ return out;
+ })().finally(() => { publicList.building = null; });
+ }
+ return await publicList.building;
},
},
/** Entities with at least one live signal or a published summary (the home page). */
/** Entities with at least one live signal or a published summary (the home page). */
new file mode 100644
new file mode 100644
@@ -0,0 +1,54 @@
@@ -0,0 +1,54 @@
+'use strict';
+/**
+ * The sitemaps' entity list (svc.publicEntities): one build serves every reader that asks while it runs; with
+ * REVIEWS_SITEMAP_TTL_MS it is reused for that long (production: 10 minutes); and a build reads at most 8 entities'
+ * page views at once. On 2026-10-23 each crawler fetch of /sitemap.xml read all 6,138 entities at once: ~30,000
+ * queries queued on the pool for a minute, each waiting ~800 ms, while every page waited behind them.
+ */
+const assert = require('assert');
+const { boot, req, suite, createEntity } = require('./helpers');
+
+const t = suite('sitemap-build');
+
+/** Count the merged-into lookups (one per entity view) running at once; prepared statements call db.query. */
+function watchViews(h) {
+ const seen = { now: 0, max: 0, total: 0 };
+ const query = h.db.query.bind(h.db);
+ h.db.query = async (text, params) => {
+ if (!/merged_into = \$1 AND state = 'merged'/.test(String(text))) return await query(text, params);
+ seen.now++; seen.total++; seen.max = Math.max(seen.max, seen.now);
+ try { await new Promise((r) => setImmediate(r)); return await query(text, params); } finally { seen.now--; }
+ };
+ return seen;
+}
+
+t('concurrent readers share one build, at most 8 entities at a time; without a TTL the next read rebuilds', async () => {
+ const h = await boot();
+ try {
+ for (let i = 0; i < 20; i++) await createEntity(h, { name: `Game ${i}`, kind: 'game' });
+ const seen = watchViews(h);
+ const [a, b, c] = await Promise.all([h.svc.publicEntities(), h.svc.publicEntities(), h.svc.publicEntities()]);
+ assert.strictEqual(a.length, 20);
+ assert.ok(a === b && b === c, 'one build for three readers');
+ assert.ok(seen.max >= 1 && seen.max <= 8, `at most 8 at once (saw ${seen.max})`);
+ const built = seen.total;
+ assert.ok(built >= 20, `each entity read once (${built})`);
+ await createEntity(h, { name: 'Game 20', kind: 'game' });
+ assert.strictEqual((await h.svc.publicEntities()).length, 21, 'no TTL (tests, development): the next read rebuilds');
+ assert.ok((await req(h, 'GET', '/sitemap.xml')).status === 200);
+ } finally { await h.stop(); }
+});
+
+t('with REVIEWS_SITEMAP_TTL_MS the list is reused until it runs out', async () => {
+ let clock = Date.parse('2026-10-23T21:36:00.000Z');
+ const h = await boot({ env: { REVIEWS_SITEMAP_TTL_MS: '600000' }, now: () => clock });
+ try {
+ await createEntity(h, { name: 'Portal 2', kind: 'game' });
+ const first = await h.svc.publicEntities();
+ await createEntity(h, { name: 'Portal 3', kind: 'game' });
+ clock += 9 * 60 * 1000;
+ assert.strictEqual(await h.svc.publicEntities(), first, 'reused inside ten minutes');
+ clock += 2 * 60 * 1000;
+ assert.strictEqual((await h.svc.publicEntities()).length, 2, 'rebuilt after');
+ } finally { await h.stop(); }
+});