openvibers/openvibe.deals Commits
Hot list: each offer's latest hotness on the offer (748 ms → an index scan) (#196)
Changes
nnew file mode 100644@@@ -0,0 +1,12 @@++-- phase: expand++-- Each offer's latest hotness on the offer itself (server/domain/hotness.js snapshot() writes it with every snapshot):++-- the hot and upvoted lists read one indexed column instead of looking up every active offer's latest snapshot. On++-- 2026-11-02 the hot list's first page looked up 41,796 offers' latest among 5.8M snapshots: 748 ms, about twenty times++-- an hour. Release N-1 keeps writing snapshots only; an offer it snapshots keeps its previous hot_latest until the next++-- tick of this release (ten minutes).++ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS hot_latest double precision;++UPDATE deal_offers o SET hot_latest = h.hot++ FROM deal_hotness_snapshots h++ WHERE h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id);++CREATE INDEX IF NOT EXISTS deal_offers_hot_latest ON deal_offers ((COALESCE(hot_latest, 0)) DESC, created_at DESC, seq DESC)++ WHERE status = 'active' AND merged_into IS NULL;@@@ -41,6 +41,8 @@ function createHotness({ store, reads }) {const insert = db.prepare(`INSERT INTO deal_hotness_snapshots (offer_id, computed_at, formula, up_count, down_count, up_weight, down_weight, score, hot, first_seen_at, reason)VALUES (@offer_id, @computed_at, @formula, @up_count, @down_count, @up_weight, @down_weight, @score, @hot, @first_seen_at, @reason)`);const firstSeen = db.prepare('SELECT MIN(created_at) AS t FROM deal_offers WHERE id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb))');++ // The offer carries its latest hot (migration 0010), so the lists sort on one indexed column.++ const latest = db.prepare('UPDATE deal_offers SET hot_latest = ? WHERE id = ?');/** Inputs for a canonical offer at time t (from the database, never from a request). */async function inputs(rootId, t) {@@@ -58,6 +60,7 @@ function createHotness({ store, reads }) {up_weight: tally.upWeight, down_weight: tally.downWeight, score: r.score, hot: r.hot, first_seen_at: firstSeenAt, reason,};await insert.run(row);++ await latest.run(row.hot, rootId);return row;}@@@ -22,10 +22,10 @@ const SEARCH_RESULTS_MAX = 100; // rows per query, whatever the caller asksfunction createListings({ store }) {const { db } = store;-- const LATEST = 'h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id)';const q = {-- hot: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o LEFT JOIN deal_hotness_snapshots h ON ${LATEST}-- WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(h.hot, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),++ // An offer's latest hotness is on the offer (hot_latest, kept by hotness.js; migration 0010's index matches this order).++ hot: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o++ WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(o.hot_latest, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),newest: db.prepare(`SELECT o.* FROM deal_offers o WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),countActive: db.prepare("SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL"),byStore: db.prepare(`SELECT * FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),@@@ -52,8 +52,8 @@ function createListings({ store }) {countEnding: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND expires_at > ? AND expires_at <= ?`),picks: db.prepare(`SELECT * FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL ORDER BY created_at DESC, seq DESC LIMIT ? OFFSET ?`),countPicks: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL`),-- upvoted: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o JOIN deal_hotness_snapshots h ON ${LATEST}-- WHERE o.status = 'active' AND o.merged_into IS NULL AND h.hot > 0 ORDER BY h.hot DESC, o.created_at DESC, o.seq DESC LIMIT ?`),++ upvoted: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o++ WHERE o.status = 'active' AND o.merged_into IS NULL AND o.hot_latest > 0 ORDER BY o.hot_latest DESC, o.created_at DESC, o.seq DESC LIMIT ?`),// The latest observation of each offer (the one a card shows), when it states a price in this currency at or below the limit.under: db.prepare(`SELECT o.* FROM deal_offers o JOIN deal_price_observations p ON p.id = (SELECT id FROM deal_price_observations WHERE offer_id = o.id ORDER BY observed_at DESC, seq DESC LIMIT 1)WHERE o.status = 'active' AND o.merged_into IS NULL AND p.price_num IS NOT NULL AND p.price_num <= ? AND p.currency = ? AND p.observed_at > ?@@@ -35,6 +35,19 @@ const { formula } = require('../server/domain/hotness');assert.strictEqual(sx.hot, formula({ upWeight: 3, downWeight: 1, firstSeenAt: X.created_at, t: sx.computed_at }).hot);});++ await check('the offer carries its latest hotness, and the hot list sorts on it (migration 0010)', async () => {++ const db = t.ctx.store.db;++ for (const title of ['Offer X', 'Offer Y']) {++ const o = await db.prepare('SELECT id, hot_latest FROM deal_offers WHERE title = ?').get(title);++ const last = await t.ctx.reads.lastSnapshot(o.id);++ assert.strictEqual(Number(o.hot_latest), Number(last.hot), `${title}: hot_latest is the latest snapshot's hot`);++ }++ const rows = await t.ctx.listings.hot(50, 0);++ const hots = rows.map((r) => Number(r.hot || 0));++ assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a), 'the hot list is in hotness order');++ assert.ok(rows.some((r) => r.title === 'Offer X' && Number(r.hot) === Number(rows.find((x) => x.title === 'Offer X').hot_latest)), 'hot is hot_latest');++ });++await check('every stored snapshot recomputes to itself (GET /api/v1/offers/:id/hotness)', async () => {const rows = await t.ctx.store.db.prepare('SELECT * FROM deal_hotness_snapshots').all();assert.ok(rows.length >= 6);
new file mode 100644
new file mode 100644
@@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
+-- phase: expand
+-- Each offer's latest hotness on the offer itself (server/domain/hotness.js snapshot() writes it with every snapshot):
+-- the hot and upvoted lists read one indexed column instead of looking up every active offer's latest snapshot. On
+-- 2026-11-02 the hot list's first page looked up 41,796 offers' latest among 5.8M snapshots: 748 ms, about twenty times
+-- an hour. Release N-1 keeps writing snapshots only; an offer it snapshots keeps its previous hot_latest until the next
+-- tick of this release (ten minutes).
+ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS hot_latest double precision;
+UPDATE deal_offers o SET hot_latest = h.hot
+ FROM deal_hotness_snapshots h
+ WHERE h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id);
+CREATE INDEX IF NOT EXISTS deal_offers_hot_latest ON deal_offers ((COALESCE(hot_latest, 0)) DESC, created_at DESC, seq DESC)
+ WHERE status = 'active' AND merged_into IS NULL;
@@ -41,6 +41,8 @@ function createHotness({ store, reads }) {
@@ -41,6 +41,8 @@ function createHotness({ store, reads }) {
const insert = db.prepare(`INSERT INTO deal_hotness_snapshots (offer_id, computed_at, formula, up_count, down_count, up_weight, down_weight, score, hot, first_seen_at, reason)
const insert = db.prepare(`INSERT INTO deal_hotness_snapshots (offer_id, computed_at, formula, up_count, down_count, up_weight, down_weight, score, hot, first_seen_at, reason)
VALUES (@offer_id, @computed_at, @formula, @up_count, @down_count, @up_weight, @down_weight, @score, @hot, @first_seen_at, @reason)`);
VALUES (@offer_id, @computed_at, @formula, @up_count, @down_count, @up_weight, @down_weight, @score, @hot, @first_seen_at, @reason)`);
const firstSeen = db.prepare('SELECT MIN(created_at) AS t FROM deal_offers WHERE id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb))');
const firstSeen = db.prepare('SELECT MIN(created_at) AS t FROM deal_offers WHERE id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb))');
+ // The offer carries its latest hot (migration 0010), so the lists sort on one indexed column.
+ const latest = db.prepare('UPDATE deal_offers SET hot_latest = ? WHERE id = ?');
/** Inputs for a canonical offer at time t (from the database, never from a request). */
/** Inputs for a canonical offer at time t (from the database, never from a request). */
async function inputs(rootId, t) {
async function inputs(rootId, t) {
@@ -58,6 +60,7 @@ function createHotness({ store, reads }) {
@@ -58,6 +60,7 @@ function createHotness({ store, reads }) {
up_weight: tally.upWeight, down_weight: tally.downWeight, score: r.score, hot: r.hot, first_seen_at: firstSeenAt, reason,
up_weight: tally.upWeight, down_weight: tally.downWeight, score: r.score, hot: r.hot, first_seen_at: firstSeenAt, reason,
};
};
await insert.run(row);
await insert.run(row);
+ await latest.run(row.hot, rootId);
return row;
return row;
}
}
@@ -22,10 +22,10 @@ const SEARCH_RESULTS_MAX = 100; // rows per query, whatever the caller asks
@@ -22,10 +22,10 @@ const SEARCH_RESULTS_MAX = 100; // rows per query, whatever the caller asks
function createListings({ store }) {
function createListings({ store }) {
const { db } = store;
const { db } = store;
- const LATEST = 'h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id)';
const q = {
const q = {
- hot: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o LEFT JOIN deal_hotness_snapshots h ON ${LATEST}
- WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(h.hot, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
+ // An offer's latest hotness is on the offer (hot_latest, kept by hotness.js; migration 0010's index matches this order).
+ hot: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o
+ WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(o.hot_latest, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
newest: db.prepare(`SELECT o.* FROM deal_offers o WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
newest: db.prepare(`SELECT o.* FROM deal_offers o WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
countActive: db.prepare("SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL"),
countActive: db.prepare("SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL"),
byStore: db.prepare(`SELECT * FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
byStore: db.prepare(`SELECT * FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
@@ -52,8 +52,8 @@ function createListings({ store }) {
@@ -52,8 +52,8 @@ function createListings({ store }) {
countEnding: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND expires_at > ? AND expires_at <= ?`),
countEnding: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND expires_at > ? AND expires_at <= ?`),
picks: db.prepare(`SELECT * FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL ORDER BY created_at DESC, seq DESC LIMIT ? OFFSET ?`),
picks: db.prepare(`SELECT * FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL ORDER BY created_at DESC, seq DESC LIMIT ? OFFSET ?`),
countPicks: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL`),
countPicks: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL`),
- upvoted: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o JOIN deal_hotness_snapshots h ON ${LATEST}
- WHERE o.status = 'active' AND o.merged_into IS NULL AND h.hot > 0 ORDER BY h.hot DESC, o.created_at DESC, o.seq DESC LIMIT ?`),
+ upvoted: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o
+ WHERE o.status = 'active' AND o.merged_into IS NULL AND o.hot_latest > 0 ORDER BY o.hot_latest DESC, o.created_at DESC, o.seq DESC LIMIT ?`),
// The latest observation of each offer (the one a card shows), when it states a price in this currency at or below the limit.
// The latest observation of each offer (the one a card shows), when it states a price in this currency at or below the limit.
under: db.prepare(`SELECT o.* FROM deal_offers o JOIN deal_price_observations p ON p.id = (SELECT id FROM deal_price_observations WHERE offer_id = o.id ORDER BY observed_at DESC, seq DESC LIMIT 1)
under: db.prepare(`SELECT o.* FROM deal_offers o JOIN deal_price_observations p ON p.id = (SELECT id FROM deal_price_observations WHERE offer_id = o.id ORDER BY observed_at DESC, seq DESC LIMIT 1)
WHERE o.status = 'active' AND o.merged_into IS NULL AND p.price_num IS NOT NULL AND p.price_num <= ? AND p.currency = ? AND p.observed_at > ?
WHERE o.status = 'active' AND o.merged_into IS NULL AND p.price_num IS NOT NULL AND p.price_num <= ? AND p.currency = ? AND p.observed_at > ?
@@ -35,6 +35,19 @@ const { formula } = require('../server/domain/hotness');
@@ -35,6 +35,19 @@ const { formula } = require('../server/domain/hotness');
assert.strictEqual(sx.hot, formula({ upWeight: 3, downWeight: 1, firstSeenAt: X.created_at, t: sx.computed_at }).hot);
assert.strictEqual(sx.hot, formula({ upWeight: 3, downWeight: 1, firstSeenAt: X.created_at, t: sx.computed_at }).hot);
});
});
+ await check('the offer carries its latest hotness, and the hot list sorts on it (migration 0010)', async () => {
+ const db = t.ctx.store.db;
+ for (const title of ['Offer X', 'Offer Y']) {
+ const o = await db.prepare('SELECT id, hot_latest FROM deal_offers WHERE title = ?').get(title);
+ const last = await t.ctx.reads.lastSnapshot(o.id);
+ assert.strictEqual(Number(o.hot_latest), Number(last.hot), `${title}: hot_latest is the latest snapshot's hot`);
+ }
+ const rows = await t.ctx.listings.hot(50, 0);
+ const hots = rows.map((r) => Number(r.hot || 0));
+ assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a), 'the hot list is in hotness order');
+ assert.ok(rows.some((r) => r.title === 'Offer X' && Number(r.hot) === Number(rows.find((x) => x.title === 'Offer X').hot_latest)), 'hot is hot_latest');
+ });
+
await check('every stored snapshot recomputes to itself (GET /api/v1/offers/:id/hotness)', async () => {
await check('every stored snapshot recomputes to itself (GET /api/v1/offers/:id/hotness)', async () => {
const rows = await t.ctx.store.db.prepare('SELECT * FROM deal_hotness_snapshots').all();
const rows = await t.ctx.store.db.prepare('SELECT * FROM deal_hotness_snapshots').all();
assert.ok(rows.length >= 6);
assert.ok(rows.length >= 6);