openvibers/openvibe.deals Commits
Deals /hot: one shared page build per minute, kept warm with the front page (#185)
Changes
@@@ -22,6 +22,9 @@ TRUST_PROXY=2# How long the front page's build is reused, in ms (production default 60000; 0 rebuilds on every read, the default# elsewhere). Past the window one background rebuild runs and readers keep the last copy for up to three minutes.#DEALS_HOME_TTL_MS=60000++# How long /hot's first page build is reused, in ms (production default 60000; 0 rebuilds on every read, the default++# elsewhere). The worker rebuilds the front page and /hot before their windows close while it is enabled.++#DEALS_HOT_TTL_MS=60000# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,# the owner on a direct connection (migrations), and Valkey (shared limit counters).@@@ -197,6 +197,10 @@ async function createApp(opts = {}) {// ── Discovery, public pages ─────────────────────────────app.use(createDiscoveryRoutes(ctx));const pages = createPages(ctx);++ // The worker keeps the shared page builds (the front page and /hot's first page) warm before++ // their windows close, so no visitor fills them (server/worker.js); it stays off in tests and++ // restore drills with DEALS_WORKER=off.++ ctx.worker.warm(pages.warm);// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public// pages. The pages themselves keep their own routes — a signed-in visitor's vote buttons and account bar are// theirs — so only the manifest and twin requests reach this mount; the pages advertise the twin themselves.@@@ -58,6 +58,9 @@ function load(env = process.env) {// 41,000 latest-snapshot lookups and ~40 cards' store/observation/source reads among them). Reused for this// window, then one background rebuild runs while readers keep the last copy for up to three minutes.homeTtlMs: int(env.DEALS_HOME_TTL_MS, isProduction ? 60 * 1000 : 0),++ // /hot's first page: the same latest-snapshot read over every active offer as the front page's hot strip,++ // plus 25 cards. One shared build per window, refreshed in the background like the front page.++ hotTtlMs: int(env.DEALS_HOT_TTL_MS, isProduction ? 60 * 1000 : 0),trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may// make per minute and per hour. Writes set tighter numbers there.@@@ -231,19 +231,32 @@ function createPages(ctx) {// snapshot for every active offer (41,000 index probes over production's 5.7M snapshots, 0.4-1.0 s) and the// cards add ~40 store/observation/source reads. Build it once per config.homeTtlMs and share it; past the// window one background rebuild runs and readers keep the last copy for at most three minutes.++ // Both builds below carry no viewer: any reader may reuse the data, send() renders the layout per request,++ // and cacheHeaders() marks only an ANONYMOUS response public, max-age=60 (signed-in stays private).const homeData = ttlCache(homePage, {ttlMs: config.homeTtlMs,maxStaleMs: 3 * 60 * 1000,now: store.now,});++ // /hot's first page is the front page's "Hot right now" strip with its own pager: the same++ // latest-hotness-snapshot read over every active offer (the 0.4-1.0 s query on production's++ // 5.7M snapshots), then 25 cards. Build it once per config.hotTtlMs and share it, exactly as++ // the front page does; deeper pages keep building for their reader.++ const hotData = ttlCache(() => listPage('hot', 1), {++ ttlMs: config.hotTtlMs,++ maxStaleMs: 3 * 60 * 1000,++ now: store.now,++ });++router.get('/', wrap(async (req, res) => {// The hot list's later pages moved to /hot with the front page.if (req.query.page && pageNumber(req) > 1) return res.redirect(301, `/hot?page=${pageNumber(req)}`);send(req, res, 200, await homeData(), { cacheable: true });}));const sendList = (kind) => wrap(async (req, res) => {-- const page = await listPage(kind, pageNumber(req));++ const n = pageNumber(req);++ const page = kind === 'hot' && n === 1 ? await hotData() : await listPage(kind, n);if (!page) return notFound(req, res);send(req, res, 200, page, { cacheable: true });});@@@ -678,7 +691,8 @@ function createPages(ctx) {categories: categoriesIndexPage, category: categoryPage, updates: updatesPage,};-- return { router, notFound, messagePage, publicViews };++ // The shared builds the worker keeps warm (server/worker.js): each refreshes on its own TTL.++ return { router, notFound, messagePage, publicViews, warm: [homeData, hotData] };}module.exports = { createPages };@@@ -9,6 +9,9 @@* first reader after a deploy and nobody else, and a failed rebuild never outlives maxStaleMs.** ttlMs = 0 turns it off: every read builds, exactly as before.++ *++ * The returned read carries the same refresh() the background warmer uses (worker.js), and the++ * window it was built with, so a timer can rebuild it before the window closes on a reader.*/function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {const ttl = Math.max(0, Number(ttlMs) || 0);@@@ -28,7 +31,7 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {return running;}-- return async function read() {++ async function read() {if (!ttl) return await refresh();if (value) {const age = Math.max(0, now() - at);@@@ -41,7 +44,11 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {}}return await refresh();-- };++ }++++ read.refresh = refresh;++ read.ttlMs = ttl;++ return read;}module.exports = { ttlCache };@@@ -14,11 +14,29 @@** Every step is idempotent, so a tick re-run after a crash changes nothing that already happened.*/--function createWorker({ config, store, offers, hotness, indexing, listings, importer, campaigns = null, limits, log = console }) {++function createWorker({ config, store, offers, hotness, indexing, listings, importer, campaigns = null, limits, log = console, timers = { setInterval, clearInterval, setTimeout } }) {let timer = null;let running = false;let lastError = null;let lastTickAt = null;++ // Shared page builds to keep warm (http/pages.js): the front page and /hot's first page. Their++ // refresh() rebuilds once even when several triggers meet.++ let warmCaches = [];++ let warmTimer = null;++++ /** How often to rebuild: at three quarters of the shortest window, so a reader never finds it closed. */++ function warmEveryMs() {++ const ttls = warmCaches.map((c) => Number(c.ttlMs) || 0).filter((ms) => ms > 0);++ return ttls.length ? Math.max(1000, Math.floor(Math.min(...ttls) * 3 / 4)) : 0;++ }++++ /** Rebuild every shared page now. A failure keeps the last copy; the next warm tries again. */++ async function warmNow() {++ for (const cache of warmCaches) {++ if (!Number(cache.ttlMs)) continue;++ try { await cache.refresh(); } catch (err) { log.error('[Deals] page warm failed:', err.stack || err.message); }++ }++ }async function tick() {if (running) return null;@@@ -50,14 +68,32 @@ function createWorker({ config, store, offers, hotness, indexing, listings, imporeturn {tick,++ /**++ * The shared page builds this worker keeps warm. Registered by app.js once the pages exist,++ * before start(); no timer runs while the worker is off (tests, restore drills).++ */++ warm(caches) { warmCaches = caches; },++ warmNow,start() {if (!config.worker.enabled) return;-- timer = setInterval(() => { tick().catch(() => {}); }, config.worker.intervalMs);++ timer = timers.setInterval(() => { tick().catch(() => {}); }, config.worker.intervalMs);timer.unref();-- setTimeout(() => { tick().catch(() => {}); }, 2000).unref();++ timers.setTimeout(() => { tick().catch(() => {}); }, 2000).unref();++ const everyMs = warmEveryMs();++ if (everyMs) {++ // Fill before the first reader, then rebuild before each window closes: a visitor's++ // request never pays for the front page's or /hot's latest-snapshot query.++ warmNow().catch(() => {});++ warmTimer = timers.setInterval(() => { warmNow().catch(() => {}); }, everyMs);++ warmTimer.unref();++ }++ },++ stop() {++ timers.clearInterval(timer);++ timers.clearInterval(warmTimer);++ warmTimer = null;},-- stop() { clearInterval(timer); },-- status: () => ({ enabled: config.worker.enabled, last_error: lastError, last_tick_at: lastTickAt ? new Date(lastTickAt).toISOString() : null }),++ status: () => ({ enabled: config.worker.enabled, warming: warmTimer != null, last_error: lastError, last_tick_at: lastTickAt ? new Date(lastTickAt).toISOString() : null }),};}nnew file mode 100644@@@ -0,0 +1,113 @@++'use strict';++/**++ * /hot's first page is one build shared by its readers (DEALS_HOT_TTL_MS), the same trade the++ * front page makes with DEALS_HOME_TTL_MS: a read inside the window does not run the latest-++ * snapshot query, a read after it gets the last copy while one background rebuild runs, and the++ * worker can rebuild it before the window closes so a visitor never pays for it. The build carries++ * no viewer; the layout is still rendered per request and only anonymous responses are public.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const configLib = require('../server/config');++const { createWorker } = require('../server/worker');++const { createPages } = require('../server/http/pages');++++const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));++++(async () => {++ const t = await boot({ env: { DEALS_HOME_TTL_MS: '60000', DEALS_HOT_TTL_MS: '60000' } });++ const alice = t.network.addUser('alice');++ await t.submit(alice, { url: 'https://a.example/hot-1', title: 'Hot cached one', price: '10', currency: 'USD' });++ const warm = await t.get('/hot');++ assert.strictEqual(warm.status, 200);++ assert.ok(warm.text.includes('Hot cached one'));++ assert.match(warm.headers.get('cache-control'), /public, max-age=60/);++++ await check('a /hot read inside the window reuses the build, byte for byte', async () => {++ await t.submit(alice, { url: 'https://a.example/hot-2', title: 'Hot cached two', price: '12', currency: 'USD' });++ const r = await t.get('/hot');++ assert.strictEqual(r.status, 200);++ assert.strictEqual(r.text, warm.text, 'the same HTML, without the deal submitted after the build');++ assert.ok(!r.text.includes('Hot cached two'));++ });++++ await check('after the window a reader gets the last copy and one background rebuild picks up the new deal', async () => {++ t.clock.advance(61_000);++ const stale = await t.get('/hot');++ assert.strictEqual(stale.status, 200);++ assert.ok(!stale.text.includes('Hot cached two'), 'served from the last build, not a blocking rebuild');++ let fresh = null;++ for (let i = 0; i < 100 && !fresh; i++) {++ await sleep(20);++ const r = await t.get('/hot');++ if (r.text.includes('Hot cached two')) fresh = r;++ }++ assert.ok(fresh, 'the background rebuild published the new deal');++ assert.ok(fresh.text.includes('Hot cached one'));++ });++++ await check('the worker rebuilds both shared pages before their windows close, so no reader pays', async () => {++ t.clock.advance(61_000);++ await t.ctx.worker.warmNow();++ const hot = await t.get('/hot');++ assert.strictEqual(hot.status, 200);++ assert.ok(hot.text.includes('Hot cached two'), 'the warmed /hot build, not the stale copy');++ const home = await t.get('/');++ assert.strictEqual(home.status, 200);++ assert.ok(home.text.includes('Hot cached two'), 'the warmed front page build');++ });++++ await check('the shared build is viewer-neutral: a signed-in reader gets their own navbar and a private response', async () => {++ const anon = await t.get('/hot');++ const user = await t.get('/hot', { as: alice });++ assert.strictEqual(user.status, 200);++ assert.match(user.headers.get('cache-control'), /private/);++ assert.match(anon.headers.get('cache-control'), /public, max-age=60/);++ assert.ok(user.text.includes('Sign out'), 'the account controls are rendered for the viewer, not cached');++ assert.ok(!anon.text.includes('Sign out'));++ assert.ok(user.text.includes('Hot cached two'), 'the same deals');++ });++++ await check('the warm timer is off with the worker (tests and restore drills): start() arms nothing', async () => {++ assert.strictEqual(t.ctx.config.worker.enabled, false, 'the test harness boots with DEALS_WORKER=off');++ t.ctx.worker.start();++ assert.strictEqual(t.ctx.worker.status().warming, false);++ });++++ await check('DEALS_HOT_TTL_MS is /hot\'s window: 60 s in production, off by default elsewhere', () => {++ assert.strictEqual(configLib.load({ NODE_ENV: 'production' }).hotTtlMs, 60_000);++ assert.strictEqual(configLib.load({ NODE_ENV: 'production', DEALS_HOT_TTL_MS: '15000' }).hotTtlMs, 15_000);++ assert.strictEqual(configLib.load({ NODE_ENV: 'test' }).hotTtlMs, 0);++ assert.strictEqual(typeof createPages, 'function', 'the routes and the shared builds live in pages.js');++ });++++ await check('the worker warms at three quarters of the shortest window and stop() clears the timer', async () => {++ const intervals = [];++ const cleared = [];++ const timers = {++ setInterval: (fn, ms) => { const h = { fn, ms, unref() {} }; intervals.push(h); return h; },++ clearInterval: (h) => cleared.push(h),++ setTimeout: (fn, ms) => ({ fn, ms, unref() {} }),++ };++ let refreshes = 0;++ const worker = createWorker({++ config: { worker: { enabled: true, intervalMs: 10 * 60_000, hotWindowDays: 14 } },++ store: { now: () => 0 }, offers: {}, hotness: {}, indexing: {}, listings: {}, importer: {}, limits: {},++ log: { error() {} }, timers,++ });++ worker.warm([{ ttlMs: 60_000, refresh: async () => { refreshes++; } }]);++ worker.start();++ const warmTimer = intervals.find((h) => h.ms === 45_000);++ assert.ok(warmTimer, 'one warm interval, at three quarters of the 60 s window');++ await new Promise((resolve) => setImmediate(resolve));++ assert.strictEqual(refreshes, 1, 'the first build fills the cache before a reader asks');++ await warmTimer.fn();++ await new Promise((resolve) => setImmediate(resolve));++ assert.strictEqual(refreshes, 2, 'the interval rebuilds it before the window closes');++ worker.stop();++ assert.ok(cleared.includes(warmTimer), 'stop() clears the warm timer');++ });++++ await t.close();++ done();++})().catch((err) => { console.error(err); process.exitCode = 1; });@@@ -84,5 +84,16 @@ const gate = () => { const g = {}; g.promise = new Promise((resolve) => { g.openassert.deepStrictEqual([await read(), await read(), await read()], [1, 2, 3]);});++ await check('refresh() rebuilds before the window closes, and the next read sees it without waiting', async () => {++ let now = 0;++ let builds = 0;++ const read = ttlCache(async () => ({ build: ++builds }), { ttlMs: 100, maxStaleMs: 300, now: () => now });++ assert.deepStrictEqual(await read(), { build: 1 });++ now = 50;++ assert.strictEqual(read.ttlMs, 100, 'the window is readable so the worker can warm before it closes');++ assert.deepStrictEqual(await read.refresh(), { build: 2 });++ assert.deepStrictEqual(await read(), { build: 2 });++ });++done();})().catch((err) => { console.error(err); process.exitCode = 1; });
@@ -22,6 +22,9 @@ TRUST_PROXY=2
@@ -22,6 +22,9 @@ TRUST_PROXY=2
# How long the front page's build is reused, in ms (production default 60000; 0 rebuilds on every read, the default
# How long the front page's build is reused, in ms (production default 60000; 0 rebuilds on every read, the default
# elsewhere). Past the window one background rebuild runs and readers keep the last copy for up to three minutes.
# elsewhere). Past the window one background rebuild runs and readers keep the last copy for up to three minutes.
#DEALS_HOME_TTL_MS=60000
#DEALS_HOME_TTL_MS=60000
+# How long /hot's first page build is reused, in ms (production default 60000; 0 rebuilds on every read, the default
+# elsewhere). The worker rebuilds the front page and /hot before their windows close while it is enabled.
+#DEALS_HOT_TTL_MS=60000
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,
# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh deals: the runtime role through PgBouncer,
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
# the owner on a direct connection (migrations), and Valkey (shared limit counters).
@@ -197,6 +197,10 @@ async function createApp(opts = {}) {
@@ -197,6 +197,10 @@ async function createApp(opts = {}) {
// ── Discovery, public pages ─────────────────────────────
// ── Discovery, public pages ─────────────────────────────
app.use(createDiscoveryRoutes(ctx));
app.use(createDiscoveryRoutes(ctx));
const pages = createPages(ctx);
const pages = createPages(ctx);
+ // The worker keeps the shared page builds (the front page and /hot's first page) warm before
+ // their windows close, so no visitor fills them (server/worker.js); it stays off in tests and
+ // restore drills with DEALS_WORKER=off.
+ ctx.worker.warm(pages.warm);
// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public
// The agent layer (openvibe-shared/agent-layer): /.well-known/agent.json and the Markdown twins of the public
// pages. The pages themselves keep their own routes — a signed-in visitor's vote buttons and account bar are
// pages. The pages themselves keep their own routes — a signed-in visitor's vote buttons and account bar are
// theirs — so only the manifest and twin requests reach this mount; the pages advertise the twin themselves.
// theirs — so only the manifest and twin requests reach this mount; the pages advertise the twin themselves.
@@ -58,6 +58,9 @@ function load(env = process.env) {
@@ -58,6 +58,9 @@ function load(env = process.env) {
// 41,000 latest-snapshot lookups and ~40 cards' store/observation/source reads among them). Reused for this
// 41,000 latest-snapshot lookups and ~40 cards' store/observation/source reads among them). Reused for this
// window, then one background rebuild runs while readers keep the last copy for up to three minutes.
// window, then one background rebuild runs while readers keep the last copy for up to three minutes.
homeTtlMs: int(env.DEALS_HOME_TTL_MS, isProduction ? 60 * 1000 : 0),
homeTtlMs: int(env.DEALS_HOME_TTL_MS, isProduction ? 60 * 1000 : 0),
+ // /hot's first page: the same latest-snapshot read over every active offer as the front page's hot strip,
+ // plus 25 cards. One shared build per window, refreshed in the background like the front page.
+ hotTtlMs: int(env.DEALS_HOT_TTL_MS, isProduction ? 60 * 1000 : 0),
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
trustProxy: env.TRUST_PROXY != null ? Number(env.TRUST_PROXY) : 2,
// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may
// Per-actor limits (server/http/actor-limits.js, roadmap WS-R task 4): the API reads one caller may
// make per minute and per hour. Writes set tighter numbers there.
// make per minute and per hour. Writes set tighter numbers there.
@@ -231,19 +231,32 @@ function createPages(ctx) {
@@ -231,19 +231,32 @@ function createPages(ctx) {
// snapshot for every active offer (41,000 index probes over production's 5.7M snapshots, 0.4-1.0 s) and the
// snapshot for every active offer (41,000 index probes over production's 5.7M snapshots, 0.4-1.0 s) and the
// cards add ~40 store/observation/source reads. Build it once per config.homeTtlMs and share it; past the
// cards add ~40 store/observation/source reads. Build it once per config.homeTtlMs and share it; past the
// window one background rebuild runs and readers keep the last copy for at most three minutes.
// window one background rebuild runs and readers keep the last copy for at most three minutes.
+ // Both builds below carry no viewer: any reader may reuse the data, send() renders the layout per request,
+ // and cacheHeaders() marks only an ANONYMOUS response public, max-age=60 (signed-in stays private).
const homeData = ttlCache(homePage, {
const homeData = ttlCache(homePage, {
ttlMs: config.homeTtlMs,
ttlMs: config.homeTtlMs,
maxStaleMs: 3 * 60 * 1000,
maxStaleMs: 3 * 60 * 1000,
now: store.now,
now: store.now,
});
});
+ // /hot's first page is the front page's "Hot right now" strip with its own pager: the same
+ // latest-hotness-snapshot read over every active offer (the 0.4-1.0 s query on production's
+ // 5.7M snapshots), then 25 cards. Build it once per config.hotTtlMs and share it, exactly as
+ // the front page does; deeper pages keep building for their reader.
+ const hotData = ttlCache(() => listPage('hot', 1), {
+ ttlMs: config.hotTtlMs,
+ maxStaleMs: 3 * 60 * 1000,
+ now: store.now,
+ });
+
router.get('/', wrap(async (req, res) => {
router.get('/', wrap(async (req, res) => {
// The hot list's later pages moved to /hot with the front page.
// The hot list's later pages moved to /hot with the front page.
if (req.query.page && pageNumber(req) > 1) return res.redirect(301, `/hot?page=${pageNumber(req)}`);
if (req.query.page && pageNumber(req) > 1) return res.redirect(301, `/hot?page=${pageNumber(req)}`);
send(req, res, 200, await homeData(), { cacheable: true });
send(req, res, 200, await homeData(), { cacheable: true });
}));
}));
const sendList = (kind) => wrap(async (req, res) => {
const sendList = (kind) => wrap(async (req, res) => {
- const page = await listPage(kind, pageNumber(req));
+ const n = pageNumber(req);
+ const page = kind === 'hot' && n === 1 ? await hotData() : await listPage(kind, n);
if (!page) return notFound(req, res);
if (!page) return notFound(req, res);
send(req, res, 200, page, { cacheable: true });
send(req, res, 200, page, { cacheable: true });
});
});
@@ -678,7 +691,8 @@ function createPages(ctx) {
@@ -678,7 +691,8 @@ function createPages(ctx) {
categories: categoriesIndexPage, category: categoryPage, updates: updatesPage,
categories: categoriesIndexPage, category: categoryPage, updates: updatesPage,
};
};
- return { router, notFound, messagePage, publicViews };
+ // The shared builds the worker keeps warm (server/worker.js): each refreshes on its own TTL.
+ return { router, notFound, messagePage, publicViews, warm: [homeData, hotData] };
}
}
module.exports = { createPages };
module.exports = { createPages };
@@ -9,6 +9,9 @@
@@ -9,6 +9,9 @@
* first reader after a deploy and nobody else, and a failed rebuild never outlives maxStaleMs.
* first reader after a deploy and nobody else, and a failed rebuild never outlives maxStaleMs.
*
*
* ttlMs = 0 turns it off: every read builds, exactly as before.
* ttlMs = 0 turns it off: every read builds, exactly as before.
+ *
+ * The returned read carries the same refresh() the background warmer uses (worker.js), and the
+ * window it was built with, so a timer can rebuild it before the window closes on a reader.
*/
*/
function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
const ttl = Math.max(0, Number(ttlMs) || 0);
const ttl = Math.max(0, Number(ttlMs) || 0);
@@ -28,7 +31,7 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
@@ -28,7 +31,7 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
return running;
return running;
}
}
- return async function read() {
+ async function read() {
if (!ttl) return await refresh();
if (!ttl) return await refresh();
if (value) {
if (value) {
const age = Math.max(0, now() - at);
const age = Math.max(0, now() - at);
@@ -41,7 +44,11 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
@@ -41,7 +44,11 @@ function ttlCache(build, { ttlMs = 0, maxStaleMs = 0, now = Date.now } = {}) {
}
}
}
}
return await refresh();
return await refresh();
- };
+ }
+
+ read.refresh = refresh;
+ read.ttlMs = ttl;
+ return read;
}
}
module.exports = { ttlCache };
module.exports = { ttlCache };
@@ -14,11 +14,29 @@
@@ -14,11 +14,29 @@
*
*
* Every step is idempotent, so a tick re-run after a crash changes nothing that already happened.
* Every step is idempotent, so a tick re-run after a crash changes nothing that already happened.
*/
*/
-function createWorker({ config, store, offers, hotness, indexing, listings, importer, campaigns = null, limits, log = console }) {
+function createWorker({ config, store, offers, hotness, indexing, listings, importer, campaigns = null, limits, log = console, timers = { setInterval, clearInterval, setTimeout } }) {
let timer = null;
let timer = null;
let running = false;
let running = false;
let lastError = null;
let lastError = null;
let lastTickAt = null;
let lastTickAt = null;
+ // Shared page builds to keep warm (http/pages.js): the front page and /hot's first page. Their
+ // refresh() rebuilds once even when several triggers meet.
+ let warmCaches = [];
+ let warmTimer = null;
+
+ /** How often to rebuild: at three quarters of the shortest window, so a reader never finds it closed. */
+ function warmEveryMs() {
+ const ttls = warmCaches.map((c) => Number(c.ttlMs) || 0).filter((ms) => ms > 0);
+ return ttls.length ? Math.max(1000, Math.floor(Math.min(...ttls) * 3 / 4)) : 0;
+ }
+
+ /** Rebuild every shared page now. A failure keeps the last copy; the next warm tries again. */
+ async function warmNow() {
+ for (const cache of warmCaches) {
+ if (!Number(cache.ttlMs)) continue;
+ try { await cache.refresh(); } catch (err) { log.error('[Deals] page warm failed:', err.stack || err.message); }
+ }
+ }
async function tick() {
async function tick() {
if (running) return null;
if (running) return null;
@@ -50,14 +68,32 @@ function createWorker({ config, store, offers, hotness, indexing, listings, impo
@@ -50,14 +68,32 @@ function createWorker({ config, store, offers, hotness, indexing, listings, impo
return {
return {
tick,
tick,
+ /**
+ * The shared page builds this worker keeps warm. Registered by app.js once the pages exist,
+ * before start(); no timer runs while the worker is off (tests, restore drills).
+ */
+ warm(caches) { warmCaches = caches; },
+ warmNow,
start() {
start() {
if (!config.worker.enabled) return;
if (!config.worker.enabled) return;
- timer = setInterval(() => { tick().catch(() => {}); }, config.worker.intervalMs);
+ timer = timers.setInterval(() => { tick().catch(() => {}); }, config.worker.intervalMs);
timer.unref();
timer.unref();
- setTimeout(() => { tick().catch(() => {}); }, 2000).unref();
+ timers.setTimeout(() => { tick().catch(() => {}); }, 2000).unref();
+ const everyMs = warmEveryMs();
+ if (everyMs) {
+ // Fill before the first reader, then rebuild before each window closes: a visitor's
+ // request never pays for the front page's or /hot's latest-snapshot query.
+ warmNow().catch(() => {});
+ warmTimer = timers.setInterval(() => { warmNow().catch(() => {}); }, everyMs);
+ warmTimer.unref();
+ }
+ },
+ stop() {
+ timers.clearInterval(timer);
+ timers.clearInterval(warmTimer);
+ warmTimer = null;
},
},
- stop() { clearInterval(timer); },
- status: () => ({ enabled: config.worker.enabled, last_error: lastError, last_tick_at: lastTickAt ? new Date(lastTickAt).toISOString() : null }),
+ status: () => ({ enabled: config.worker.enabled, warming: warmTimer != null, last_error: lastError, last_tick_at: lastTickAt ? new Date(lastTickAt).toISOString() : null }),
};
};
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,113 @@
@@ -0,0 +1,113 @@
+'use strict';
+/**
+ * /hot's first page is one build shared by its readers (DEALS_HOT_TTL_MS), the same trade the
+ * front page makes with DEALS_HOME_TTL_MS: a read inside the window does not run the latest-
+ * snapshot query, a read after it gets the last copy while one background rebuild runs, and the
+ * worker can rebuild it before the window closes so a visitor never pays for it. The build carries
+ * no viewer; the layout is still rendered per request and only anonymous responses are public.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const configLib = require('../server/config');
+const { createWorker } = require('../server/worker');
+const { createPages } = require('../server/http/pages');
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+(async () => {
+ const t = await boot({ env: { DEALS_HOME_TTL_MS: '60000', DEALS_HOT_TTL_MS: '60000' } });
+ const alice = t.network.addUser('alice');
+ await t.submit(alice, { url: 'https://a.example/hot-1', title: 'Hot cached one', price: '10', currency: 'USD' });
+ const warm = await t.get('/hot');
+ assert.strictEqual(warm.status, 200);
+ assert.ok(warm.text.includes('Hot cached one'));
+ assert.match(warm.headers.get('cache-control'), /public, max-age=60/);
+
+ await check('a /hot read inside the window reuses the build, byte for byte', async () => {
+ await t.submit(alice, { url: 'https://a.example/hot-2', title: 'Hot cached two', price: '12', currency: 'USD' });
+ const r = await t.get('/hot');
+ assert.strictEqual(r.status, 200);
+ assert.strictEqual(r.text, warm.text, 'the same HTML, without the deal submitted after the build');
+ assert.ok(!r.text.includes('Hot cached two'));
+ });
+
+ await check('after the window a reader gets the last copy and one background rebuild picks up the new deal', async () => {
+ t.clock.advance(61_000);
+ const stale = await t.get('/hot');
+ assert.strictEqual(stale.status, 200);
+ assert.ok(!stale.text.includes('Hot cached two'), 'served from the last build, not a blocking rebuild');
+ let fresh = null;
+ for (let i = 0; i < 100 && !fresh; i++) {
+ await sleep(20);
+ const r = await t.get('/hot');
+ if (r.text.includes('Hot cached two')) fresh = r;
+ }
+ assert.ok(fresh, 'the background rebuild published the new deal');
+ assert.ok(fresh.text.includes('Hot cached one'));
+ });
+
+ await check('the worker rebuilds both shared pages before their windows close, so no reader pays', async () => {
+ t.clock.advance(61_000);
+ await t.ctx.worker.warmNow();
+ const hot = await t.get('/hot');
+ assert.strictEqual(hot.status, 200);
+ assert.ok(hot.text.includes('Hot cached two'), 'the warmed /hot build, not the stale copy');
+ const home = await t.get('/');
+ assert.strictEqual(home.status, 200);
+ assert.ok(home.text.includes('Hot cached two'), 'the warmed front page build');
+ });
+
+ await check('the shared build is viewer-neutral: a signed-in reader gets their own navbar and a private response', async () => {
+ const anon = await t.get('/hot');
+ const user = await t.get('/hot', { as: alice });
+ assert.strictEqual(user.status, 200);
+ assert.match(user.headers.get('cache-control'), /private/);
+ assert.match(anon.headers.get('cache-control'), /public, max-age=60/);
+ assert.ok(user.text.includes('Sign out'), 'the account controls are rendered for the viewer, not cached');
+ assert.ok(!anon.text.includes('Sign out'));
+ assert.ok(user.text.includes('Hot cached two'), 'the same deals');
+ });
+
+ await check('the warm timer is off with the worker (tests and restore drills): start() arms nothing', async () => {
+ assert.strictEqual(t.ctx.config.worker.enabled, false, 'the test harness boots with DEALS_WORKER=off');
+ t.ctx.worker.start();
+ assert.strictEqual(t.ctx.worker.status().warming, false);
+ });
+
+ await check('DEALS_HOT_TTL_MS is /hot\'s window: 60 s in production, off by default elsewhere', () => {
+ assert.strictEqual(configLib.load({ NODE_ENV: 'production' }).hotTtlMs, 60_000);
+ assert.strictEqual(configLib.load({ NODE_ENV: 'production', DEALS_HOT_TTL_MS: '15000' }).hotTtlMs, 15_000);
+ assert.strictEqual(configLib.load({ NODE_ENV: 'test' }).hotTtlMs, 0);
+ assert.strictEqual(typeof createPages, 'function', 'the routes and the shared builds live in pages.js');
+ });
+
+ await check('the worker warms at three quarters of the shortest window and stop() clears the timer', async () => {
+ const intervals = [];
+ const cleared = [];
+ const timers = {
+ setInterval: (fn, ms) => { const h = { fn, ms, unref() {} }; intervals.push(h); return h; },
+ clearInterval: (h) => cleared.push(h),
+ setTimeout: (fn, ms) => ({ fn, ms, unref() {} }),
+ };
+ let refreshes = 0;
+ const worker = createWorker({
+ config: { worker: { enabled: true, intervalMs: 10 * 60_000, hotWindowDays: 14 } },
+ store: { now: () => 0 }, offers: {}, hotness: {}, indexing: {}, listings: {}, importer: {}, limits: {},
+ log: { error() {} }, timers,
+ });
+ worker.warm([{ ttlMs: 60_000, refresh: async () => { refreshes++; } }]);
+ worker.start();
+ const warmTimer = intervals.find((h) => h.ms === 45_000);
+ assert.ok(warmTimer, 'one warm interval, at three quarters of the 60 s window');
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.strictEqual(refreshes, 1, 'the first build fills the cache before a reader asks');
+ await warmTimer.fn();
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.strictEqual(refreshes, 2, 'the interval rebuilds it before the window closes');
+ worker.stop();
+ assert.ok(cleared.includes(warmTimer), 'stop() clears the warm timer');
+ });
+
+ await t.close();
+ done();
+})().catch((err) => { console.error(err); process.exitCode = 1; });
@@ -84,5 +84,16 @@ const gate = () => { const g = {}; g.promise = new Promise((resolve) => { g.open
@@ -84,5 +84,16 @@ const gate = () => { const g = {}; g.promise = new Promise((resolve) => { g.open
assert.deepStrictEqual([await read(), await read(), await read()], [1, 2, 3]);
assert.deepStrictEqual([await read(), await read(), await read()], [1, 2, 3]);
});
});
+ await check('refresh() rebuilds before the window closes, and the next read sees it without waiting', async () => {
+ let now = 0;
+ let builds = 0;
+ const read = ttlCache(async () => ({ build: ++builds }), { ttlMs: 100, maxStaleMs: 300, now: () => now });
+ assert.deepStrictEqual(await read(), { build: 1 });
+ now = 50;
+ assert.strictEqual(read.ttlMs, 100, 'the window is readable so the worker can warm before it closes');
+ assert.deepStrictEqual(await read.refresh(), { build: 2 });
+ assert.deepStrictEqual(await read(), { build: 2 });
+ });
+
done();
done();
})().catch((err) => { console.error(err); process.exitCode = 1; });
})().catch((err) => { console.error(err); process.exitCode = 1; });