openvibers/openvibe.deals Commits
Hotness tick snapshots are kept a day, not 7 (2.68M rows, 805 MB), pruned 50,000 at a time (#81)
Changes
@@@ -185,7 +185,8 @@ hot = round6( score / (ageHours + 2) ^ 1.5 )Worked examples (tested): up 3, down 1, 10 h old → 2 / 12^1.5 = **0.048113**; up 2, 0 h → **0.707107**.Snapshots are taken on every vote change, merge, unmerge and creation, and by the worker for everyactive offer of the last 14 days at **one shared `t`**, so the hot list compares numbers computed at--the same moment.++the same moment. The worker's tick snapshots are kept for a day, the others for 7 days, and every offer's latest++snapshot always.**Abuse controls (server-side, database windows shared by forms and API):**@@@ -70,8 +70,16 @@ function createHotness({ store, reads }) {async function tick({ windowDays = 14, t = store.now() } = {}) {const ids = (await db.prepare(`SELECT id FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND created_at >= ?`).all(t - windowDays * 24 * HOUR)).map((r) => r.id);await store.tx(async () => { for (const id of ids) await snapshot(id, 'tick', t); });-- // Keep the latest snapshot per offer and everything from the last 7 days.-- await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE computed_at < ? AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`).run(t - 7 * 24 * HOUR);++ // Keep the latest snapshot per offer, every snapshot a vote, merge or creation made in the last 7 days, and the++ // worker's own tick snapshots of the last day. A tick re-snapshots every active offer (about 2,700, every ten++ // minutes), so seven days of ticks were 2.68M rows (805 MB, 2026-10-23) behind reads that only use the latest,++ // and this delete took 1.4 s each time. At most 50,000 rows go per tick: a backlog drains over a few hours,++ // each delete well inside the statement timeout.++ await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE id IN (++ SELECT id FROM deal_hotness_snapshots++ WHERE computed_at < ? AND (reason = 'tick' OR computed_at < ?)++ AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)++ LIMIT 50000)`).run(t - 24 * HOUR, t - 7 * 24 * HOUR);return ids.length;}@@@ -86,6 +86,28 @@ const { formula } = require('../server/domain/hotness');assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a));});++ await check('tick snapshots are kept a day, the others 7 days, and every offer\'s latest always', async () => {++ const db = t.ctx.store.db;++ const Y = await t.submit(poster, { url: 'https://a.example/keep', title: 'Offer Keep', price: '5', currency: 'USD' });++ const count = async (where, args = []) => Number((await db.prepare(`SELECT COUNT(*) AS n FROM deal_hotness_snapshots WHERE ${where}`).get(...args)).n);++ await t.ctx.worker.tick();++ const firstTick = await count("reason = 'tick'");++ assert.ok(firstTick >= 1);++ t.clock.advance(25 * HOUR);++ await t.ctx.worker.tick();++ // The day-old ticks went, except where one is an offer's latest; this tick's are all there.++ const now = Number((await db.prepare("SELECT MAX(computed_at) AS t FROM deal_hotness_snapshots WHERE reason = 'tick'").get()).t);++ assert.strictEqual(await count("reason = 'tick' AND computed_at < ?", [now - 24 * HOUR]), 0, 'no tick older than a day');++ assert.ok(await count("reason = 'create' AND offer_id = ?", [Y.id]) >= 1 || await count('offer_id = ?', [Y.id]) >= 1, 'the offer keeps its snapshots');++ assert.ok(await count("reason <> 'tick'") >= 1, 'vote and creation snapshots of the week stay');++ t.clock.advance(8 * 24 * HOUR);++ await t.ctx.worker.tick();++ const offers = Number((await db.prepare('SELECT COUNT(DISTINCT offer_id) AS n FROM deal_hotness_snapshots').get()).n);++ assert.ok(offers >= 1, 'every offer still has its latest snapshot');++ assert.strictEqual(await count('computed_at < ?', [now]), await count(`computed_at < ? AND id IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`, [now]),++ 'past the week, only latest snapshots remain');++ });++await t.close();done();})();
@@ -185,7 +185,8 @@ hot = round6( score / (ageHours + 2) ^ 1.5 )
@@ -185,7 +185,8 @@ hot = round6( score / (ageHours + 2) ^ 1.5 )
Worked examples (tested): up 3, down 1, 10 h old → 2 / 12^1.5 = **0.048113**; up 2, 0 h → **0.707107**.
Worked examples (tested): up 3, down 1, 10 h old → 2 / 12^1.5 = **0.048113**; up 2, 0 h → **0.707107**.
Snapshots are taken on every vote change, merge, unmerge and creation, and by the worker for every
Snapshots are taken on every vote change, merge, unmerge and creation, and by the worker for every
active offer of the last 14 days at **one shared `t`**, so the hot list compares numbers computed at
active offer of the last 14 days at **one shared `t`**, so the hot list compares numbers computed at
-the same moment.
+the same moment. The worker's tick snapshots are kept for a day, the others for 7 days, and every offer's latest
+snapshot always.
**Abuse controls (server-side, database windows shared by forms and API):**
**Abuse controls (server-side, database windows shared by forms and API):**
@@ -70,8 +70,16 @@ function createHotness({ store, reads }) {
@@ -70,8 +70,16 @@ function createHotness({ store, reads }) {
async function tick({ windowDays = 14, t = store.now() } = {}) {
async function tick({ windowDays = 14, t = store.now() } = {}) {
const ids = (await db.prepare(`SELECT id FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND created_at >= ?`).all(t - windowDays * 24 * HOUR)).map((r) => r.id);
const ids = (await db.prepare(`SELECT id FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND created_at >= ?`).all(t - windowDays * 24 * HOUR)).map((r) => r.id);
await store.tx(async () => { for (const id of ids) await snapshot(id, 'tick', t); });
await store.tx(async () => { for (const id of ids) await snapshot(id, 'tick', t); });
- // Keep the latest snapshot per offer and everything from the last 7 days.
- await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE computed_at < ? AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`).run(t - 7 * 24 * HOUR);
+ // Keep the latest snapshot per offer, every snapshot a vote, merge or creation made in the last 7 days, and the
+ // worker's own tick snapshots of the last day. A tick re-snapshots every active offer (about 2,700, every ten
+ // minutes), so seven days of ticks were 2.68M rows (805 MB, 2026-10-23) behind reads that only use the latest,
+ // and this delete took 1.4 s each time. At most 50,000 rows go per tick: a backlog drains over a few hours,
+ // each delete well inside the statement timeout.
+ await db.prepare(`DELETE FROM deal_hotness_snapshots WHERE id IN (
+ SELECT id FROM deal_hotness_snapshots
+ WHERE computed_at < ? AND (reason = 'tick' OR computed_at < ?)
+ AND id NOT IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)
+ LIMIT 50000)`).run(t - 24 * HOUR, t - 7 * 24 * HOUR);
return ids.length;
return ids.length;
}
}
@@ -86,6 +86,28 @@ const { formula } = require('../server/domain/hotness');
@@ -86,6 +86,28 @@ const { formula } = require('../server/domain/hotness');
assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a));
assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a));
});
});
+ await check('tick snapshots are kept a day, the others 7 days, and every offer\'s latest always', async () => {
+ const db = t.ctx.store.db;
+ const Y = await t.submit(poster, { url: 'https://a.example/keep', title: 'Offer Keep', price: '5', currency: 'USD' });
+ const count = async (where, args = []) => Number((await db.prepare(`SELECT COUNT(*) AS n FROM deal_hotness_snapshots WHERE ${where}`).get(...args)).n);
+ await t.ctx.worker.tick();
+ const firstTick = await count("reason = 'tick'");
+ assert.ok(firstTick >= 1);
+ t.clock.advance(25 * HOUR);
+ await t.ctx.worker.tick();
+ // The day-old ticks went, except where one is an offer's latest; this tick's are all there.
+ const now = Number((await db.prepare("SELECT MAX(computed_at) AS t FROM deal_hotness_snapshots WHERE reason = 'tick'").get()).t);
+ assert.strictEqual(await count("reason = 'tick' AND computed_at < ?", [now - 24 * HOUR]), 0, 'no tick older than a day');
+ assert.ok(await count("reason = 'create' AND offer_id = ?", [Y.id]) >= 1 || await count('offer_id = ?', [Y.id]) >= 1, 'the offer keeps its snapshots');
+ assert.ok(await count("reason <> 'tick'") >= 1, 'vote and creation snapshots of the week stay');
+ t.clock.advance(8 * 24 * HOUR);
+ await t.ctx.worker.tick();
+ const offers = Number((await db.prepare('SELECT COUNT(DISTINCT offer_id) AS n FROM deal_hotness_snapshots').get()).n);
+ assert.ok(offers >= 1, 'every offer still has its latest snapshot');
+ assert.strictEqual(await count('computed_at < ?', [now]), await count(`computed_at < ? AND id IN (SELECT MAX(id) FROM deal_hotness_snapshots GROUP BY offer_id)`, [now]),
+ 'past the week, only latest snapshots remain');
+ });
+
await t.close();
await t.close();
done();
done();
})();
})();