openvibers/openvibe.deals Commits

Hotness tick snapshots are kept a day, not 7 (2.68M rows, 805 MB), pruned 50,000 at a time (#81)

OP OpenVibers committed 21 days ago · 99e43be3b4 · 3 files, +34 −3

Changes

ddiff --git a/README.md b/README.md
iindex cc5a266..f7411ba 100644
---- a/README.md
++++ b/README.md
@@@ -185,7 +185,8 @@ hot = round6( score / (ageHours + 2) ^ 1.5 )
Worked examples (tested): up 3, down 1, 10 h old → 2 / 12^1.5 = **0.048113**; up 2, 0 h → **0.707107**.
Snapshots are taken on every vote change, merge, unmerge and creation, and by the worker for every
active offer of the last 14 days at **one shared `t`**, so the hot list compares numbers computed at
--the same moment.
++the same moment. The worker's tick snapshots are kept for a day, the others for 7 days, and every offer's latest
++snapshot always.
**Abuse controls (server-side, database windows shared by forms and API):**
ddiff --git a/server/domain/hotness.js b/server/domain/hotness.js
iindex b291cc5..420b2d7 100644
---- a/server/domain/hotness.js
++++ b/server/domain/hotness.js
@@@ -70,8 +70,16 @@ function createHotness({ store, reads }) {
async function tick({ windowDays = 14, t = store.now() } = {}) {
const ids = (await db.prepare(`SELECT id FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND created_at >= ?`).all(t - windowDays * 24 * HOUR)).map((r) => r.id);
await store.tx(async () => { for (const id of ids) await snapshot(id, 'tick', t); });
-- // Keep the latest snapshot per offer and everything from the last 7 days.
-- await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE computed_at < ? AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`).run(t - 7 * 24 * HOUR);
++ // Keep the latest snapshot per offer, every snapshot a vote, merge or creation made in the last 7 days, and the
++ // worker's own tick snapshots of the last day. A tick re-snapshots every active offer (about 2,700, every ten
++ // minutes), so seven days of ticks were 2.68M rows (805 MB, 2026-10-23) behind reads that only use the latest,
++ // and this delete took 1.4 s each time. At most 50,000 rows go per tick: a backlog drains over a few hours,
++ // each delete well inside the statement timeout.
++ await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE id IN (
++ SELECT id FROM deal_hotness_snapshots
++ WHERE computed_at < ? AND (reason = 'tick' OR computed_at < ?)
++ AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)
++ LIMIT 50000)`).run(t - 24 * HOUR, t - 7 * 24 * HOUR);
return ids.length;
}
ddiff --git a/test/hotness.test.js b/test/hotness.test.js
iindex 280d7db..f914aac 100644
---- a/test/hotness.test.js
++++ b/test/hotness.test.js
@@@ -86,6 +86,28 @@ const { formula } = require('../server/domain/hotness');
assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a));
});
++ await check('tick snapshots are kept a day, the others 7 days, and every offer\'s latest always', async () => {
++ const db = t.ctx.store.db;
++ const Y = await t.submit(poster, { url: 'https://a.example/keep', title: 'Offer Keep', price: '5', currency: 'USD' });
++ const count = async (where, args = []) => Number((await db.prepare(`SELECT COUNT(*) AS n FROM deal_hotness_snapshots WHERE ${where}`).get(...args)).n);
++ await t.ctx.worker.tick();
++ const firstTick = await count("reason = 'tick'");
++ assert.ok(firstTick >= 1);
++ t.clock.advance(25 * HOUR);
++ await t.ctx.worker.tick();
++ // The day-old ticks went, except where one is an offer's latest; this tick's are all there.
++ const now = Number((await db.prepare("SELECT MAX(computed_at) AS t FROM deal_hotness_snapshots WHERE reason = 'tick'").get()).t);
++ assert.strictEqual(await count("reason = 'tick' AND computed_at < ?", [now - 24 * HOUR]), 0, 'no tick older than a day');
++ assert.ok(await count("reason = 'create' AND offer_id = ?", [Y.id]) >= 1 || await count('offer_id = ?', [Y.id]) >= 1, 'the offer keeps its snapshots');
++ assert.ok(await count("reason <> 'tick'") >= 1, 'vote and creation snapshots of the week stay');
++ t.clock.advance(8 * 24 * HOUR);
++ await t.ctx.worker.tick();
++ const offers = Number((await db.prepare('SELECT COUNT(DISTINCT offer_id) AS n FROM deal_hotness_snapshots').get()).n);
++ assert.ok(offers >= 1, 'every offer still has its latest snapshot');
++ assert.strictEqual(await count('computed_at < ?', [now]), await count(`computed_at < ? AND id IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`, [now]),
++ 'past the week, only latest snapshots remain');
++ });
++
await t.close();
done();
})();