openvibers/openvibe.deals Commits

Hot list: each offer's latest hotness on the offer (748 ms → an index scan) (#196)

OP OpenVibers committed 11 days ago · a5cfa6a8ab · 4 files, +33 −5

Changes

ddiff --git a/migrations/0010_offer_hot_latest.sql b/migrations/0010_offer_hot_latest.sql
nnew file mode 100644
iindex 0000000..c2002e3
---- /dev/null
++++ b/migrations/0010_offer_hot_latest.sql
@@@ -0,0 +1,12 @@
++-- phase: expand
++-- Each offer's latest hotness on the offer itself (server/domain/hotness.js snapshot() writes it with every snapshot):
++-- the hot and upvoted lists read one indexed column instead of looking up every active offer's latest snapshot. On
++-- 2026-11-02 the hot list's first page looked up 41,796 offers' latest among 5.8M snapshots: 748 ms, about twenty times
++-- an hour. Release N-1 keeps writing snapshots only; an offer it snapshots keeps its previous hot_latest until the next
++-- tick of this release (ten minutes).
++ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS hot_latest double precision;
++UPDATE deal_offers o SET hot_latest = h.hot
++ FROM deal_hotness_snapshots h
++ WHERE h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id);
++CREATE INDEX IF NOT EXISTS deal_offers_hot_latest ON deal_offers ((COALESCE(hot_latest, 0)) DESC, created_at DESC, seq DESC)
++ WHERE status = 'active' AND merged_into IS NULL;
ddiff --git a/server/domain/hotness.js b/server/domain/hotness.js
iindex 420b2d7..0840f42 100644
---- a/server/domain/hotness.js
++++ b/server/domain/hotness.js
@@@ -41,6 +41,8 @@ function createHotness({ store, reads }) {
const insert = db.prepare(`INSERT INTO deal_hotness_snapshots (offer_id, computed_at, formula, up_count, down_count, up_weight, down_weight, score, hot, first_seen_at, reason)
VALUES (@offer_id, @computed_at, @formula, @up_count, @down_count, @up_weight, @down_weight, @score, @hot, @first_seen_at, @reason)`);
const firstSeen = db.prepare('SELECT MIN(created_at) AS t FROM deal_offers WHERE id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb))');
++ // The offer carries its latest hot (migration 0010), so the lists sort on one indexed column.
++ const latest = db.prepare('UPDATE deal_offers SET hot_latest = ? WHERE id = ?');
/** Inputs for a canonical offer at time t (from the database, never from a request). */
async function inputs(rootId, t) {
@@@ -58,6 +60,7 @@ function createHotness({ store, reads }) {
up_weight: tally.upWeight, down_weight: tally.downWeight, score: r.score, hot: r.hot, first_seen_at: firstSeenAt, reason,
};
await insert.run(row);
++ await latest.run(row.hot, rootId);
return row;
}
ddiff --git a/server/domain/listings.js b/server/domain/listings.js
iindex 290d0b8..a3d68e5 100644
---- a/server/domain/listings.js
++++ b/server/domain/listings.js
@@@ -22,10 +22,10 @@ const SEARCH_RESULTS_MAX = 100; // rows per query, whatever the caller asks
function createListings({ store }) {
const { db } = store;
-- const LATEST = 'h.id = (SELECT MAX(id) FROM deal_hotness_snapshots WHERE offer_id = o.id)';
const q = {
-- hot: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o LEFT JOIN deal_hotness_snapshots h ON ${LATEST}
-- WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(h.hot, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
++ // An offer's latest hotness is on the offer (hot_latest, kept by hotness.js; migration 0010's index matches this order).
++ hot: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o
++ WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY COALESCE(o.hot_latest, 0) DESC, o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
newest: db.prepare(`SELECT o.* FROM deal_offers o WHERE o.status = 'active' AND o.merged_into IS NULL ORDER BY o.created_at DESC, o.seq DESC LIMIT ? OFFSET ?`),
countActive: db.prepare("SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL"),
byStore: db.prepare(`SELECT * FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
@@@ -52,8 +52,8 @@ function createListings({ store }) {
countEnding: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL AND expires_at > ? AND expires_at <= ?`),
picks: db.prepare(`SELECT * FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL ORDER BY created_at DESC, seq DESC LIMIT ? OFFSET ?`),
countPicks: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE staff_pick = true AND status = 'active' AND merged_into IS NULL`),
-- upvoted: db.prepare(`SELECT o.*, h.hot AS hot FROM deal_offers o JOIN deal_hotness_snapshots h ON ${LATEST}
-- WHERE o.status = 'active' AND o.merged_into IS NULL AND h.hot > 0 ORDER BY h.hot DESC, o.created_at DESC, o.seq DESC LIMIT ?`),
++ upvoted: db.prepare(`SELECT o.*, o.hot_latest AS hot FROM deal_offers o
++ WHERE o.status = 'active' AND o.merged_into IS NULL AND o.hot_latest > 0 ORDER BY o.hot_latest DESC, o.created_at DESC, o.seq DESC LIMIT ?`),
// The latest observation of each offer (the one a card shows), when it states a price in this currency at or below the limit.
under: db.prepare(`SELECT o.* FROM deal_offers o JOIN deal_price_observations p ON p.id = (SELECT id FROM deal_price_observations WHERE offer_id = o.id ORDER BY observed_at DESC, seq DESC LIMIT 1)
WHERE o.status = 'active' AND o.merged_into IS NULL AND p.price_num IS NOT NULL AND p.price_num <= ? AND p.currency = ? AND p.observed_at > ?
ddiff --git a/test/hotness.test.js b/test/hotness.test.js
iindex f914aac..f8cc5be 100644
---- a/test/hotness.test.js
++++ b/test/hotness.test.js
@@@ -35,6 +35,19 @@ const { formula } = require('../server/domain/hotness');
assert.strictEqual(sx.hot, formula({ upWeight: 3, downWeight: 1, firstSeenAt: X.created_at, t: sx.computed_at }).hot);
});
++ await check('the offer carries its latest hotness, and the hot list sorts on it (migration 0010)', async () => {
++ const db = t.ctx.store.db;
++ for (const title of ['Offer X', 'Offer Y']) {
++ const o = await db.prepare('SELECT id, hot_latest FROM deal_offers WHERE title = ?').get(title);
++ const last = await t.ctx.reads.lastSnapshot(o.id);
++ assert.strictEqual(Number(o.hot_latest), Number(last.hot), `${title}: hot_latest is the latest snapshot's hot`);
++ }
++ const rows = await t.ctx.listings.hot(50, 0);
++ const hots = rows.map((r) => Number(r.hot || 0));
++ assert.deepStrictEqual(hots, [...hots].sort((a, b) => b - a), 'the hot list is in hotness order');
++ assert.ok(rows.some((r) => r.title === 'Offer X' && Number(r.hot) === Number(rows.find((x) => x.title === 'Offer X').hot_latest)), 'hot is hot_latest');
++ });
++
await check('every stored snapshot recomputes to itself (GET /api/v1/offers/:id/hotness)', async () => {
const rows = await t.ctx.store.db.prepare('SELECT * FROM deal_hotness_snapshots').all();
assert.ok(rows.length >= 6);