openvibers/openvibe.media Commits
Rank object sweep moves by value per dollar within class budgets (#15)
Changes
@@@ -918,10 +918,15 @@ The sweep is step 4 of the storage sweep (`vod-storage.js runSweep`, every `sweelocal disk still needs a drain, never starts in a restore drill (and refuses under `MEDIA_DRILL` anyway).Each pass rotates the popularity counts, then:--1. **Demotes** first (at most `maxDemotionsPerSweep`): R2 copies of native objects that are no longer-- `ready`, or idle for `demoteIdleDays` and in R2 at least that long.--2. **Promotes** (at most `maxPromotionsPerSweep`), most viewed first: ready native objects of non-sandbox-- tenants within the thresholds and without a present R2 copy.++1. **Demotes** first (at most `maxDemotionsPerSweep` per class): R2 copies of native objects that are no++ longer `ready` (first), or idle for `demoteIdleDays` and in R2 at least that long, lowest value per dollar++ of keeping the copy first (its viewers over the kept 30 days, as an idle copy has none in the last 7),++ then the largest.++2. **Promotes** (at most `maxPromotionsPerSweep` per class), highest value per dollar first (then most++ viewed): ready native objects of non-sandbox tenants within the thresholds and without a present R2 copy.++ The scan pages through every eligible object (sandbox tenants filtered in the query) and keeps the best++ 500, so no class is crowded out of its budget by a most-viewed prefix.++ Budgets and minimum residency are per class (`media.storage_policy`); a refusal or back-off spends no slot.**Promotion** (`tiering.promote`): refused when the object is held, not ready, a sandbox object, or itscanonical copy is not verified on record (present, checked, not contradicted, and the object has a@@@ -7,6 +7,9 @@* is available; the canonical copy is the fallback), and loses it when it goes idle or stops being ready.* Popularity is ./popularity.js (unique viewers per UTC day, no identifiers); the thresholds and the* site-level activation gate are the revisioned policy media.object_tier (./tier-policy.js).++ * The thresholds say who may move; value per dollar (../placement/value.js, R2 Standard prices from++ * ../placement/cost-tiers.js) says who moves first: each class spends its per-sweep budget++ * (../placement/storage-policy.js) on its highest-scoring promotions and its lowest-scoring demotions.** runSweep({ trigger }) one pass: rotate the popularity counts, then promote and demote by the policy.* Step 4 of the storage sweep (server/vod/vod-storage.js runSweep), which a restore@@@ -36,11 +39,16 @@ const db = require('../db/database');const drill = require('../drill');const policy = require('./tier-policy');const storagePolicy = require('../placement/storage-policy');++const costTiers = require('../placement/cost-tiers');++const value = require('../placement/value');const popularity = require('./popularity');const MB = 1024 * 1024;const FAILED_BACKOFF_H = 6;const OUTCOMES = ['done', 'already', 'refused', 'failed', 'dry_run'];++const DAY_MS = 864e5;++const MONTH_DAYS = 30; // value.js prices a month++const LATENCY_VALUE = 1; // $ a request served from R2 is worth: no revisioned figure yet, so the score is requests per dollarconst vodStorage = () => require('../vod/vod-storage');const model = () => require('./model');@@@ -176,6 +184,7 @@ async function frame(objectId, action, ctx) {const pop = ctx.pop !== undefined ? ctx.pop : ((await popularity.stats({ ids: [obj.id], now: ctx.now })).get(obj.id) || null);const held = await model().isHeld(obj.id);const inputs = await inputsOf(obj, { locs, pop, held, settings });++ if (ctx.valuePerDollar != null) inputs.value_per_dollar = ctx.valuePerDollar;const canon = canonicalLocation(obj, locs);const trigger = ctx.trigger || 'manual';const why = ctx.reason || 'requested';@@@ -290,51 +299,117 @@ async function demote(objectId, ctx = {}) {// ── Candidates ───────────────────────────────────────────────--/** Ready native objects that pass the popularity, recency and size thresholds and have no R2 copy yet, most viewed first. */--async function promotionCandidates(settings, { now = Date.now(), appId = null, limit = 200 } = {}) {++/**++ * Value per dollar of an R2 copy of this row (../placement/value.js): its 7-day unique viewers scaled to a month++ * of requests, against R2 Standard's storage, read and retrieval prices (media.cost_tiers) and, for a promotion,++ * the class's minimum residency beyond that month (media.storage_policy), which the copy commits to. Free++ * allowances are not counted: a promotion is priced at the margin.++ */++function scoreOf(row, cls, { committing = false } = {}) {++ const p = costTiers.settings().r2.standard;++ const storage = Number(p.storagePerGbMonth) || 0;++ const residencyDays = committing ? Math.max(storagePolicy.settings().classes[cls].minResidencyMs / DAY_MS, Number(p.minResidencyDays) || 0) : 0;++ return value.score({++ // A row that carries a month of counts (the demotion scan) is read as is: an idle copy has no viewer in the++ // 7-day window, so only the longer count tells a copy last viewed 20 days ago from one never viewed.++ requests: row.unique_viewers_30d != null ? Number(row.unique_viewers_30d) || 0++ : (Number(row.unique_viewers_7d) || 0) * MONTH_DAYS / popularity.WINDOW_DAYS,++ bytes: Number(row.size_bytes) || 0,++ storageCost: storage,++ requestCost: (Number(p.classBPerMillion) || 0) / 1e6,++ retrievalCost: Number(p.retrievalPerGb) || 0,++ residencyCost: storage * Math.max(0, residencyDays - MONTH_DAYS) / MONTH_DAYS,++ latencyValue: LATENCY_VALUE,++ });++}++++const fmtScore = (v) => (Number.isFinite(v) ? Number(v.toPrecision(4)).toString() : String(v));++++const SCAN_PAGE = 500; // rows read per page of the promotion scan++const SWEEP_KEEP = 500; // top-ranked promotion candidates a sweep keeps++++/**++ * Ready native objects that pass the popularity, recency and size thresholds and have no R2 copy yet, each with++ * its class and value_per_dollar, highest value per dollar first (then most viewed). Every eligible row is scored++ * before the limit applies (a most-viewed prefix would drop smaller objects that are worth more per dollar), but++ * the scan reads SCAN_PAGE rows at a time and holds only the best `limit`, so memory stays bounded.++ * excludeSandbox drops developer-sandbox tenants in the query (their objects are never tiered).++ */++async function promotionCandidates(settings, { now = Date.now(), appId = null, limit = 200, excludeSandbox = false } = {}) {const today = popularity.dayOf(now);-- return await db.all(`SELECT o.*, p.viewers AS unique_viewers_7d, p.last_day AS last_viewed_day++ const args = [popularity.windowStart(today, popularity.WINDOW_DAYS)];++ const rank = (a, b) => b.value_per_dollar - a.value_per_dollar || Number(b.unique_viewers_7d) - Number(a.unique_viewers_7d) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);++ let best = [];++ for (let after = ''; ;) {++ const rows = await db.all(`SELECT o.*, p.viewers AS unique_viewers_7d, p.last_day AS last_viewed_dayFROM (SELECT object_id, SUM(unique_viewers)::bigint AS viewers, MAX(day) AS last_day FROM media_object_views_dailyWHERE day >= ? AND unique_viewers > 0 GROUP BY object_id) pJOIN media_objects o ON o.id = p.object_id-- WHERE o.legacy_ref IS NULL AND o.lifecycle_status = 'ready'++ WHERE o.id > ? AND o.legacy_ref IS NULL AND o.lifecycle_status = 'ready'AND p.viewers >= ? AND p.last_day >= ? AND o.size_bytes >= ? AND o.size_bytes <= ?-- AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.provider = 'r2' AND l.state = 'present')${appId ? ' AND o.app_id = ?' : ''}-- ORDER BY p.viewers DESC, o.id LIMIT ?`,-- [popularity.windowStart(today, popularity.WINDOW_DAYS), settings.promoteMinUniqueViewers7d, popularity.windowStart(today, settings.promoteRecentAccessDays),-- Math.ceil(settings.promoteMinSizeMb * MB), Math.floor(settings.promoteMaxSizeMb * MB), ...(appId ? [appId] : []), limit]);++ AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.provider = 'r2' AND l.state = 'present')${appId ? ' AND o.app_id = ?' : ''}${excludeSandbox++ ? " AND NOT EXISTS (SELECT 1 FROM apps a WHERE a.app_id = o.app_id AND a.env = 'sandbox')" : ''}++ ORDER BY o.id LIMIT ?`,++ [...args, after, settings.promoteMinUniqueViewers7d, popularity.windowStart(today, settings.promoteRecentAccessDays),++ Math.ceil(settings.promoteMinSizeMb * MB), Math.floor(settings.promoteMaxSizeMb * MB), ...(appId ? [appId] : []), SCAN_PAGE]);++ for (const r of rows) {++ r.class = storagePolicy.classOf(r);++ r.value_per_dollar = scoreOf(r, r.class, { committing: true });++ }++ best = best.concat(rows).sort(rank).slice(0, limit);++ if (rows.length < SCAN_PAGE) return best;++ after = rows[rows.length - 1].id;++ }}--/** Native objects with an R2 copy that should lose it: not ready any more, or idle. → [{ obj row, reason }] */++/**++ * Native objects with an R2 copy that should lose it: not ready any more (first, whatever their score), or idle,++ * lowest value per dollar of keeping the copy first (its viewers over the kept month, since an idle copy has none++ * in the last 7 days), then the largest. → [{ row, reason, class, value_per_dollar }]++ */async function demotionCandidates(settings, { now = Date.now(), appId = null } = {}) {const idleFrom = popularity.windowStart(popularity.dayOf(now), settings.demoteIdleDays);++ const monthFrom = popularity.windowStart(popularity.dayOf(now), popularity.KEEP_DAYS);const rows = await db.all(`SELECT o.*, l.created_at AS r2_since,-- (SELECT MAX(d.day) FROM media_object_views_daily d WHERE d.object_id = o.id AND d.unique_viewers > 0) AS last_viewed_day++ (SELECT MAX(d.day) FROM media_object_views_daily d WHERE d.object_id = o.id AND d.unique_viewers > 0) AS last_viewed_day,++ (SELECT COALESCE(SUM(d.unique_viewers), 0)::bigint FROM media_object_views_daily d WHERE d.object_id = o.id AND d.day >= ?) AS unique_viewers_30dFROM media_locations l JOIN media_objects o ON o.id = l.object_idWHERE l.provider = 'r2' AND o.legacy_ref IS NULL${appId ? ' AND o.app_id = ?' : ''}-- ORDER BY (o.lifecycle_status = 'ready'), o.id`, appId ? [appId] : []);++ ORDER BY (o.lifecycle_status = 'ready'), o.id`, appId ? [monthFrom, appId] : [monthFrom]);const out = [];for (const r of rows) {-- if (r.lifecycle_status !== 'ready') { out.push({ row: r, reason: `the object is ${r.lifecycle_status}` }); continue; }++ const cls = storagePolicy.classOf(r);++ const scored = { row: r, class: cls, value_per_dollar: scoreOf(r, cls) };++ if (r.lifecycle_status !== 'ready') { out.push({ ...scored, reason: `the object is ${r.lifecycle_status}` }); continue; }const idle = !r.last_viewed_day || r.last_viewed_day < idleFrom;const since = msOf(r.r2_since);-- const settled = !Number.isFinite(since) || now - since >= settings.demoteIdleDays * 864e5;++ const settled = !Number.isFinite(since) || now - since >= settings.demoteIdleDays * DAY_MS;if (idle && settled) {-- out.push({ row: r, reason: `no viewer since ${r.last_viewed_day || 'the kept counts began'} (idle for demoteIdleDays ${settings.demoteIdleDays}); in R2 since ${r.r2_since || 'unknown'}` });++ out.push({ ...scored, reason: `no viewer since ${r.last_viewed_day || 'the kept counts began'} (idle for demoteIdleDays ${settings.demoteIdleDays}); in R2 since ${r.r2_since || 'unknown'}` });}}++ const forced = (c) => (c.row.lifecycle_status === 'ready' ? 1 : 0);++ out.sort((a, b) => forced(a) - forced(b) || a.value_per_dollar - b.value_per_dollar++ || Number(b.row.size_bytes) - Number(a.row.size_bytes) || (a.row.id < b.row.id ? -1 : a.row.id > b.row.id ? 1 : 0));return out;}--function promoteReason(c, s) {++/** Where a move stands in its class's ranking and budget: `class video, value per dollar 1234, 1 of budget 3 (maxPromotionsPerSweep)`. */++function rankNote(cls, score, slot, budgetKey) {++ return `class ${cls}, value per dollar ${fmtScore(score)}, ${slot} of budget ${storagePolicy.budgetFor(cls)[budgetKey]} (${budgetKey}, media.storage_policy)`;++}++++function promoteReason(c, s, slot) {return `${c.unique_viewers_7d} unique viewers in 7 days >= promoteMinUniqueViewers7d ${s.promoteMinUniqueViewers7d}; last viewed ${c.last_viewed_day} `-- + `(within promoteRecentAccessDays ${s.promoteRecentAccessDays}); ${(Number(c.size_bytes) / MB).toFixed(1)} MB (promoteMinSizeMb ${s.promoteMinSizeMb} to promoteMaxSizeMb ${s.promoteMaxSizeMb})`;++ + `(within promoteRecentAccessDays ${s.promoteRecentAccessDays}); ${(Number(c.size_bytes) / MB).toFixed(1)} MB (promoteMinSizeMb ${s.promoteMinSizeMb} to promoteMaxSizeMb ${s.promoteMaxSizeMb}); `++ + rankNote(c.class, c.value_per_dollar, slot, 'maxPromotionsPerSweep');}/** What the policy would promote now, each with what (on record) would refuse it; database only. */async function candidates({ appId = null, limit = 20, now = Date.now(), settings = policy.settings() } = {}) {return (await Promise.all((await promotionCandidates(settings, { now, appId, limit: Math.min(Math.max(parseInt(limit, 10) || 20, 1), 200) })).map(async (c) => ({-- object_id: c.id, app_id: c.app_id, kind: c.kind, size_bytes: c.size_bytes, unique_viewers_7d: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day,++ object_id: c.id, app_id: c.app_id, kind: c.kind, class: c.class, value_per_dollar: c.value_per_dollar,++ size_bytes: c.size_bytes, unique_viewers_7d: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day,blocked_by: await promoteRefusal(c, { locs: await model().listLocations(c.id), held: await model().isHeld(c.id) }),}))));}@@@ -380,30 +455,31 @@ async function runSweep({ trigger = 'sweep', now = Date.now() } = {}) {// Demotions first: they free the R2 cache, and a deleted object's copy should not wait behind promotions.const demote_ = await demotionCandidates(settings, { now });out.candidates.demote = demote_.length;-- // Budgets and residency are per class (media.storage_policy); a class spends only its own budget.++ // Budgets and residency are per class (media.storage_policy); a class spends only its own budget, in the++ // candidates' value-per-dollar order. A refusal (residency, hold) or a back-off spends nothing, so the next++ // in line takes the slot.const spent = { promote: {}, demote: {} };const room = (action, cls) => (spent[action][cls] || 0) < storagePolicy.budgetFor(cls)[action === 'promote' ? 'maxPromotionsPerSweep' : 'maxDemotionsPerSweep'];-- for (const { row, reason } of demote_) {-- const cls = storagePolicy.classOf(row);++ for (const { row, reason, class: cls, value_per_dollar: score } of demote_) {if (!room('demote', cls)) continue;if (await inBackoff(row.id, 'demote')) { out.skipped_backoff++; continue; }// A copy of an object that is no longer ready is removed regardless of residency.const residency = row.lifecycle_status === 'ready' ? await residencyRefusal(row, cls, 'demote', now) : null;-- const r = await demote(row.id, { trigger, reason, settings, now, residency });++ const r = await demote(row.id, { trigger, settings, now, residency, valuePerDollar: score,++ reason: `${reason}; ${rankNote(cls, score, (spent.demote[cls] || 0) + 1, 'maxDemotionsPerSweep')}` });tally(out, { ...r, action: 'demote' });if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.demote[cls] = (spent.demote[cls] || 0) + 1;}-- const promote_ = [];-- const promoteLimit = Math.max(...storagePolicy.CLASSES.map((c) => storagePolicy.budgetFor(c).maxPromotionsPerSweep));-- for (const c of await promotionCandidates(settings, { now, limit: Math.max(50, promoteLimit * 10) })) if (!await db.isSandboxTenant(c.app_id)) promote_.push(c);++ // Every eligible candidate is ranked (not a most-viewed prefix); the best SWEEP_KEEP are kept, so no class is crowded out of its budget.++ const promote_ = await promotionCandidates(settings, { now, limit: SWEEP_KEEP, excludeSandbox: true });out.candidates.promote = promote_.length;for (const c of promote_) {-- const cls = storagePolicy.classOf(c);++ const cls = c.class;if (!room('promote', cls)) continue;if (await inBackoff(c.id, 'promote')) { out.skipped_backoff++; continue; }const residency = await residencyRefusal(c, cls, 'promote', now);-- const r = await promote(c.id, { trigger, reason: promoteReason(c, settings), settings, now, residency,++ const r = await promote(c.id, { trigger, reason: promoteReason(c, settings, (spent.promote[cls] || 0) + 1), settings, now, residency, valuePerDollar: c.value_per_dollar,pop: { unique_viewers: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day } });tally(out, { ...r, action: 'promote' });if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.promote[cls] = (spent.promote[cls] || 0) + 1;nnew file mode 100644@@@ -0,0 +1,45 @@++'use strict';++/**++ * Value per dollar of keeping one object in a tier (docs/media-fabric.md §6, decision rule): what serving its++ * expected requests from that tier is worth, against what the tier charges for it. Pure: no I/O, no clock.++ *++ * Units (one month is the horizon every figure is taken over):++ * requests expected requests served from the tier in a month (a count)++ * bytes the object's size in bytes (GB below is bytes / 1024³)++ * storageCost $ per GB-month the tier charges to keep it++ * requestCost $ per request (read operation) the tier charges++ * retrievalCost $ per GB read out of the tier (each request reads the whole object)++ * residencyCost $ per GB committed by a minimum residency beyond the month (a policy or provider minimum)++ * latencyValue $ a request is worth when the tier serves it (1 makes the score requests per dollar)++ *++ * value = requests × latencyValue++ * dollars = GB × storageCost + requests × requestCost + requests × GB × retrievalCost + GB × residencyCost++ * score = value / max(dollars, MIN_DOLLARS) (dimensionless: $ of value per $ spent)++ *++ * The benefit is per request and the cost mostly per byte, so a hotter object scores higher and, at equal demand,++ * a bigger one or one in a costlier tier scores lower: the greedy order of a size-aware cache (GDSF), which the++ * per-class knapsack takes highest first. The score never falls as requests rise (the value grows at least as++ * fast as the per-request costs) and never rises as any cost rises. A missing, negative or non-finite input++ * counts as 0; a free tier (dollars 0) is floored at MIN_DOLLARS so the score stays finite and ordered by demand.++ */++++const GB = 1024 * 1024 * 1024;++const MIN_DOLLARS = 1e-9;++++function num(v) {++ const n = Number(v);++ return Number.isFinite(n) && n > 0 ? n : 0;++}++++/** The dollars a month of keeping and serving the object in the tier costs (see the header). */++function dollars({ requests, bytes, storageCost, requestCost, retrievalCost, residencyCost } = {}) {++ const r = num(requests), gb = num(bytes) / GB;++ return gb * num(storageCost) + r * num(requestCost) + r * gb * num(retrievalCost) + gb * num(residencyCost);++}++++/** Value per dollar (see the header). → a finite number >= 0 */++function score(inputs = {}) {++ return (num(inputs.requests) * num(inputs.latencyValue)) / Math.max(dollars(inputs), MIN_DOLLARS);++}++++module.exports = { score, dollars, GB, MIN_DOLLARS };@@@ -461,19 +461,26 @@ const crypto = require('crypto');// ── 9b. Per-class budgets and residency come from media.storage_policy ──const sp = require('../server/placement/storage-policy');await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);-- const vids = [await addObject({ mime: 'video/mp4' }), await addObject({ mime: 'video/mp4' })];++ // vids[0] is the most viewed but nearly 1 MB; vids[1] has fewer viewers at 4 KB, so more value per dollar.++ const vids = [await addObject({ mime: 'video/mp4', size: 1000000 }), await addObject({ mime: 'video/mp4' })];const imgs = [await addObject({ mime: 'image/png' }), await addObject({ mime: 'image/png' })];const dl = await addObject();-- for (const o of [...vids, ...imgs, dl]) await setViews(o.id, [[0, 900]]);++ for (const o of [...vids, ...imgs, dl]) await setViews(o.id, [[0, o === vids[0] ? 950 : 900]]);++ const ranked = (await tiering.promotionCandidates(policy.settings())).filter((c) => c.class === 'video').map((c) => c.id);++ assert.deepStrictEqual(ranked, [vids[1].id, vids[0].id], 'ranked by value per dollar, not by viewers');await sp.set({ classes: { video: { maxPromotionsPerSweep: 1 }, image: { maxPromotionsPerSweep: 2, minResidencyMs: 86400000 }, download: { maxPromotionsPerSweep: 0 } } },{ reason: 'test: per-class budgets' });s = await tiering.runSweep();const promotedOf = async (list) => (await Promise.all(list.map(async (o) => !!await r2Row(o.id)))).filter(Boolean).length;assert.deepStrictEqual([await promotedOf(vids), await promotedOf(imgs), await promotedOf([dl])], [1, 2, 0], 'each class spends its own promotion budget');assert.strictEqual((await decisions(dl.id)).length, 0, 'a class with a zero budget is not tried');-- const r2Done = new Set((await Promise.all(vids.map(async (o) => (await r2Row(o.id)) ? o.id : null))).filter(Boolean));++ assert.ok(await r2Row(vids[1].id) && !await r2Row(vids[0].id), "the video budget of 1 goes to the higher value per dollar");++ assert.strictEqual((await decisions(vids[0].id)).length, 0, 'the rest of the class waits for the next sweep');assert.strictEqual(JSON.parse((await last(imgs[0].id)).inputs).class, 'image');-- assert.strictEqual(JSON.parse((await last(vids.find((o) => r2Done.has(o.id)).id)).inputs).class, 'video');++ d = await last(vids[1].id);++ assert.strictEqual(JSON.parse(d.inputs).class, 'video');++ assert.ok(JSON.parse(d.inputs).value_per_dollar > 0, d.inputs);++ assert.ok(/; class video, value per dollar [0-9.e+]+, 1 of budget 1 \(maxPromotionsPerSweep, media\.storage_policy\)$/.test(d.reason), d.reason);// Residency: the images were just promoted, so an idle demotion is suppressed and logged as refused.await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);for (const o of imgs) await ageR2(o.id, 30);@@@ -482,11 +489,58 @@ const crypto = require('crypto');d = await last(imgs[0].id);assert.deepStrictEqual([d.action, d.outcome, JSON.parse(d.inputs).class], ['demote', 'refused', 'image']);assert.ok(/minimum residency of 86400000 ms for class image/.test(d.reason), d.reason);++ assert.ok(/idle for demoteIdleDays 14\).*; class image, value per dollar 0, 1 of budget 10 \(maxDemotionsPerSweep/.test(d.reason), d.reason);await sp.set({ classes: { image: { minResidencyMs: 0 }, download: { maxPromotionsPerSweep: 3 } } }, { reason: 'test: residency off, budgets back' });s = await tiering.runSweep();assert.ok(!await r2Row(imgs[0].id) && !await r2Row(imgs[1].id), 'with no residency the idle copies leave');console.log('✅ per-class budgets and residency from media.storage_policy; every decision row shows its class; a residency-suppressed demotion logs refused');++ // ── 9c. Every eligible object is scored, not only the 1,000 most viewed ──++ // 1,001 larger, more viewed videos would fill a most-viewed prefix; the small one is worth more per dollar.++ const crowd = [];++ for (let i = 0; i < 1001; i++) {++ const id = await model.createObject({ app_id: 'live', kind: 'file', visibility: 'public', lifecycle_status: 'ready', mime_type: 'video/mp4', size_bytes: 1000000 });++ await setViews(id, [[0, 900]]);++ crowd.push(id);++ }++ const small = await addObject({ mime: 'video/mp4' });++ await setViews(small.id, [[0, 500]]);++ const all = await tiering.promotionCandidates(policy.settings(), { limit: Infinity });++ assert.ok(all.length >= 1002, `every eligible object is ranked (${all.length})`);++ assert.strictEqual(all[0].id, small.id, 'the fewest viewers but the most value per dollar ranks first');++ assert.strictEqual((await tiering.promotionCandidates(policy.settings(), { limit: 1 }))[0].id, small.id, 'the limit applies after scoring');++ await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);++ console.log('✅ promotion candidates are scored in full before the class budgets and limits apply');++++ // ── 9d. Gate off: a move residency or a hold blocks is still logged (refused) and spends no slot ──++ await policy.set({ active: false }, { actor: { type: 'service', id: 'live' }, reason: 'test: gate off with blocked moves' });++ await sp.set({ classes: { video: { maxPromotionsPerSweep: 1, minResidencyMs: 86400000 } } }, { reason: 'test: video residency, budget 1' });++ const settling = await addObject({ mime: 'video/mp4' }); await setViews(settling.id, [[0, 900]]); // the highest score, but moved an hour ago++ await db.run(`INSERT INTO media_object_tier_decisions (object_id, app_id, action, from_provider, to_provider, outcome, trigger, reason, inputs, thresholds)++ VALUES (?, 'live', 'demote', 'r2', 'local', 'done', 'manual', 'test: a recent move', '{}', '{}')`, [settling.id]);++ const heldVid = await addObject({ mime: 'video/mp4' }); await setViews(heldVid.id, [[0, 800]]);++ const heldVidHold = await model.placeHold({ object_id: heldVid.id, kind: 'admin', reason: 'test' });++ const nextVid = await addObject({ mime: 'video/mp4', size: 100000 }); await setViews(nextVid.id, [[0, 700]]);++ assert.deepStrictEqual((await tiering.promotionCandidates(policy.settings())).filter((c) => c.class === 'video').map((c) => c.id),++ [settling.id, heldVid.id, nextVid.id], 'the blocked ones rank first');++ const before = Number((await db.get('SELECT MAX(id) AS id FROM media_object_tier_decisions')).id);++ s = await tiering.runSweep();++ const written = await db.all('SELECT object_id, action, outcome FROM media_object_tier_decisions WHERE id > ? ORDER BY id', [before]);++ assert.ok(written.some((r) => r.outcome === 'dry_run') && written.some((r) => r.outcome === 'refused'), `gate off logs proposals and refusals: ${JSON.stringify(written)}`);++ assert.deepStrictEqual([(await last(settling.id)).outcome, (await last(heldVid.id)).outcome], ['refused', 'refused'], 'a blocked move is logged');++ assert.deepStrictEqual([(await decisions(settling.id)).length, (await decisions(heldVid.id)).length], [2, 1]);++ d = await last(nextVid.id);++ assert.deepStrictEqual([d.action, d.outcome], ['promote', 'dry_run']);++ assert.ok(/, 1 of budget 1 \(maxPromotionsPerSweep/.test(d.reason), 'the blocked ones spent no slot, so the next in line has it: ' + d.reason);++ assert.ok(s.refused >= 2 && s.would_promote >= 1, JSON.stringify(s));++ const direct = await tiering.promote(heldVid.id, { reason: 'asked by hand' });++ assert.deepStrictEqual([direct.outcome, direct.logged, (await decisions(heldVid.id)).length], ['refused', undefined, 2], 'a direct call is answered and logged');++ await model.releaseHold(heldVidHold.id, 'test');++ await sp.set({ classes: { video: { maxPromotionsPerSweep: 3, minResidencyMs: 0 } } }, { reason: 'test: video back' });++ await policy.set({ active: true }, { actor: { type: 'service', id: 'live' }, reason: 'test: gate on again' });++ await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);++ console.log('✅ gate off: residency- or hold-blocked moves are logged as refused and spend no slot');++// ── 10. The storage sweep runs it (step 4); a restore drill never does ──const vodSweep = await storage.runSweep();assert.ok(vodSweep.objects && vodSweep.objects.gate === true && typeof vodSweep.objects.promoted === 'number', JSON.stringify(vodSweep));nnew file mode 100644@@@ -0,0 +1,78 @@++'use strict';++// Value per dollar (F2.2, server/placement/value.js): hotter scores higher, a costlier tier lower, a free tier stays++// finite, and the score is monotone in demand and in every cost. Pure, no database.++const assert = require('assert');++const { score, dollars, GB, MIN_DOLLARS } = require('../server/placement/value');++++// R2 Standard as cost-tiers.js prices it: $0.015/GB-month, $0.36 per million reads, no retrieval fee.++const r2 = { storageCost: 0.015, requestCost: 0.36e-6, retrievalCost: 0, residencyCost: 0, latencyValue: 1 };++const base = { ...r2, requests: 1000, bytes: GB };++++// Each case: [what, higher, lower] — the first input must score strictly higher than the second.++const ordered = [++ ['hotter object scores higher', { ...base, requests: 5000 }, base],++ ['smaller object at equal demand scores higher', { ...base, bytes: GB / 10 }, base],++ ['costlier storage scores lower', base, { ...base, storageCost: 0.03 }],++ ['request fee scores lower', base, { ...base, requestCost: 4.5e-6 }],++ ['retrieval fee scores lower', base, { ...base, retrievalCost: 0.01 }],++ ['minimum residency scores lower', base, { ...base, residencyCost: 0.01 }],++ ['latency worth more scores higher', { ...base, latencyValue: 2 }, base],++];++for (const [what, hi, lo] of ordered) assert.ok(score(hi) > score(lo), `${what}: ${score(hi)} > ${score(lo)}`);++console.log(`✅ ordering: ${ordered.length} cases (hotter higher, costlier tier lower)`);++++// Exact values: [inputs, expected dollars, expected score].++const exact = [++ [base, 0.015 + 1000 * 0.36e-6, 1000 / (0.015 + 1000 * 0.36e-6)],++ [{ ...base, requests: 0 }, 0.015, 0],++ [{ ...base, latencyValue: 0 }, 0.015 + 1000 * 0.36e-6, 0],++ [{ requests: 10, bytes: 2 * GB, storageCost: 0.01, requestCost: 0, retrievalCost: 0.01, residencyCost: 0.01, latencyValue: 1 }, 0.02 + 0.2 + 0.02, 10 / 0.24],++];++for (const [inputs, d, s] of exact) {++ assert.ok(Math.abs(dollars(inputs) - d) < 1e-12, `dollars ${dollars(inputs)} != ${d}`);++ assert.ok(Math.abs(score(inputs) - s) / Math.max(s, 1) < 1e-9, `score ${score(inputs)} != ${s}`);++}++console.log('✅ exact: value = requests × latencyValue over storage + requests + retrieval + residency');++++// Zero, missing and bad costs: never a division by zero, never NaN or Infinity, never negative.++const edges = [++ ['free tier', { requests: 100, bytes: GB, storageCost: 0, requestCost: 0, retrievalCost: 0, residencyCost: 0, latencyValue: 1 }, 100 / MIN_DOLLARS],++ ['no costs given', { requests: 100, bytes: GB, latencyValue: 1 }, 100 / MIN_DOLLARS],++ ['nothing given', {}, 0],++ ['no argument', undefined, 0],++ ['negative and NaN count as 0', { requests: 100, bytes: -5, storageCost: NaN, requestCost: -1, retrievalCost: 'x', residencyCost: null, latencyValue: 1 }, 100 / MIN_DOLLARS],++ ['zero-byte object pays only per request', { ...r2, requests: 100, bytes: 0 }, 100 / (100 * 0.36e-6)],++];++for (const [what, inputs, s] of edges) {++ const got = score(inputs);++ assert.ok(Number.isFinite(got) && got >= 0, `${what}: finite and >= 0 (${got})`);++ assert.ok(Math.abs(got - s) / Math.max(s, 1) < 1e-9, `${what}: ${got} != ${s}`);++}++assert.ok(score({ ...edges[0][1], requests: 200 }) > score(edges[0][1]), 'a free tier still ranks by demand');++console.log(`✅ edges: ${edges.length} zero/missing/bad cost cases stay finite`);++++// Monotonicity: non-decreasing in requests (strictly while storage or residency costs something), non-increasing++// in every cost, across a grid of sizes and tiers.++const steps = [0, 1, 10, 100, 1e3, 1e4, 1e6];++const EPS = 1e-12; // a constant ratio (no per-byte cost) may differ in the last float bit++for (const bytes of [0, 1024, 100 * 1024 * 1024, GB, 50 * GB]) {++ for (const tier of [r2, { ...r2, storageCost: 0.00695, requestCost: 0 }, { ...r2, storageCost: 0.01, requestCost: 0.9e-6, retrievalCost: 0.01, residencyCost: 0.01 }]) {++ let prev = -1;++ for (const requests of steps) {++ const s = score({ ...tier, bytes, requests });++ assert.ok(s >= prev * (1 - EPS), `non-decreasing in requests (${bytes} B, ${requests} req): ${s} >= ${prev}`);++ if (bytes && requests > 1 && tier.storageCost) assert.ok(s > prev, `strictly increasing in requests (${bytes} B, ${requests} req)`);++ prev = s;++ }++ for (const cost of ['storageCost', 'requestCost', 'retrievalCost', 'residencyCost']) {++ let last = Infinity;++ for (const c of [0, 1e-7, 1e-4, 0.01, 1]) {++ const s = score({ ...tier, bytes, requests: 1000, [cost]: c });++ assert.ok(s <= last * (1 + EPS), `non-increasing in ${cost} (${bytes} B, ${c}): ${s} <= ${last}`);++ last = s;++ }++ }++ }++}++console.log('✅ monotone: never down as requests rise, never up as a cost rises');++console.log('\n✅ All placement value tests passed');
@@ -918,10 +918,15 @@ The sweep is step 4 of the storage sweep (`vod-storage.js runSweep`, every `swee
@@ -918,10 +918,15 @@ The sweep is step 4 of the storage sweep (`vod-storage.js runSweep`, every `swee
local disk still needs a drain, never starts in a restore drill (and refuses under `MEDIA_DRILL` anyway).
local disk still needs a drain, never starts in a restore drill (and refuses under `MEDIA_DRILL` anyway).
Each pass rotates the popularity counts, then:
Each pass rotates the popularity counts, then:
-1. **Demotes** first (at most `maxDemotionsPerSweep`): R2 copies of native objects that are no longer
- `ready`, or idle for `demoteIdleDays` and in R2 at least that long.
-2. **Promotes** (at most `maxPromotionsPerSweep`), most viewed first: ready native objects of non-sandbox
- tenants within the thresholds and without a present R2 copy.
+1. **Demotes** first (at most `maxDemotionsPerSweep` per class): R2 copies of native objects that are no
+ longer `ready` (first), or idle for `demoteIdleDays` and in R2 at least that long, lowest value per dollar
+ of keeping the copy first (its viewers over the kept 30 days, as an idle copy has none in the last 7),
+ then the largest.
+2. **Promotes** (at most `maxPromotionsPerSweep` per class), highest value per dollar first (then most
+ viewed): ready native objects of non-sandbox tenants within the thresholds and without a present R2 copy.
+ The scan pages through every eligible object (sandbox tenants filtered in the query) and keeps the best
+ 500, so no class is crowded out of its budget by a most-viewed prefix.
+ Budgets and minimum residency are per class (`media.storage_policy`); a refusal or back-off spends no slot.
**Promotion** (`tiering.promote`): refused when the object is held, not ready, a sandbox object, or its
**Promotion** (`tiering.promote`): refused when the object is held, not ready, a sandbox object, or its
canonical copy is not verified on record (present, checked, not contradicted, and the object has a
canonical copy is not verified on record (present, checked, not contradicted, and the object has a
@@ -7,6 +7,9 @@
@@ -7,6 +7,9 @@
* is available; the canonical copy is the fallback), and loses it when it goes idle or stops being ready.
* is available; the canonical copy is the fallback), and loses it when it goes idle or stops being ready.
* Popularity is ./popularity.js (unique viewers per UTC day, no identifiers); the thresholds and the
* Popularity is ./popularity.js (unique viewers per UTC day, no identifiers); the thresholds and the
* site-level activation gate are the revisioned policy media.object_tier (./tier-policy.js).
* site-level activation gate are the revisioned policy media.object_tier (./tier-policy.js).
+ * The thresholds say who may move; value per dollar (../placement/value.js, R2 Standard prices from
+ * ../placement/cost-tiers.js) says who moves first: each class spends its per-sweep budget
+ * (../placement/storage-policy.js) on its highest-scoring promotions and its lowest-scoring demotions.
*
*
* runSweep({ trigger }) one pass: rotate the popularity counts, then promote and demote by the policy.
* runSweep({ trigger }) one pass: rotate the popularity counts, then promote and demote by the policy.
* Step 4 of the storage sweep (server/vod/vod-storage.js runSweep), which a restore
* Step 4 of the storage sweep (server/vod/vod-storage.js runSweep), which a restore
@@ -36,11 +39,16 @@ const db = require('../db/database');
@@ -36,11 +39,16 @@ const db = require('../db/database');
const drill = require('../drill');
const drill = require('../drill');
const policy = require('./tier-policy');
const policy = require('./tier-policy');
const storagePolicy = require('../placement/storage-policy');
const storagePolicy = require('../placement/storage-policy');
+const costTiers = require('../placement/cost-tiers');
+const value = require('../placement/value');
const popularity = require('./popularity');
const popularity = require('./popularity');
const MB = 1024 * 1024;
const MB = 1024 * 1024;
const FAILED_BACKOFF_H = 6;
const FAILED_BACKOFF_H = 6;
const OUTCOMES = ['done', 'already', 'refused', 'failed', 'dry_run'];
const OUTCOMES = ['done', 'already', 'refused', 'failed', 'dry_run'];
+const DAY_MS = 864e5;
+const MONTH_DAYS = 30; // value.js prices a month
+const LATENCY_VALUE = 1; // $ a request served from R2 is worth: no revisioned figure yet, so the score is requests per dollar
const vodStorage = () => require('../vod/vod-storage');
const vodStorage = () => require('../vod/vod-storage');
const model = () => require('./model');
const model = () => require('./model');
@@ -176,6 +184,7 @@ async function frame(objectId, action, ctx) {
@@ -176,6 +184,7 @@ async function frame(objectId, action, ctx) {
const pop = ctx.pop !== undefined ? ctx.pop : ((await popularity.stats({ ids: [obj.id], now: ctx.now })).get(obj.id) || null);
const pop = ctx.pop !== undefined ? ctx.pop : ((await popularity.stats({ ids: [obj.id], now: ctx.now })).get(obj.id) || null);
const held = await model().isHeld(obj.id);
const held = await model().isHeld(obj.id);
const inputs = await inputsOf(obj, { locs, pop, held, settings });
const inputs = await inputsOf(obj, { locs, pop, held, settings });
+ if (ctx.valuePerDollar != null) inputs.value_per_dollar = ctx.valuePerDollar;
const canon = canonicalLocation(obj, locs);
const canon = canonicalLocation(obj, locs);
const trigger = ctx.trigger || 'manual';
const trigger = ctx.trigger || 'manual';
const why = ctx.reason || 'requested';
const why = ctx.reason || 'requested';
@@ -290,51 +299,117 @@ async function demote(objectId, ctx = {}) {
@@ -290,51 +299,117 @@ async function demote(objectId, ctx = {}) {
// ── Candidates ───────────────────────────────────────────────
// ── Candidates ───────────────────────────────────────────────
-/** Ready native objects that pass the popularity, recency and size thresholds and have no R2 copy yet, most viewed first. */
-async function promotionCandidates(settings, { now = Date.now(), appId = null, limit = 200 } = {}) {
+/**
+ * Value per dollar of an R2 copy of this row (../placement/value.js): its 7-day unique viewers scaled to a month
+ * of requests, against R2 Standard's storage, read and retrieval prices (media.cost_tiers) and, for a promotion,
+ * the class's minimum residency beyond that month (media.storage_policy), which the copy commits to. Free
+ * allowances are not counted: a promotion is priced at the margin.
+ */
+function scoreOf(row, cls, { committing = false } = {}) {
+ const p = costTiers.settings().r2.standard;
+ const storage = Number(p.storagePerGbMonth) || 0;
+ const residencyDays = committing ? Math.max(storagePolicy.settings().classes[cls].minResidencyMs / DAY_MS, Number(p.minResidencyDays) || 0) : 0;
+ return value.score({
+ // A row that carries a month of counts (the demotion scan) is read as is: an idle copy has no viewer in the
+ // 7-day window, so only the longer count tells a copy last viewed 20 days ago from one never viewed.
+ requests: row.unique_viewers_30d != null ? Number(row.unique_viewers_30d) || 0
+ : (Number(row.unique_viewers_7d) || 0) * MONTH_DAYS / popularity.WINDOW_DAYS,
+ bytes: Number(row.size_bytes) || 0,
+ storageCost: storage,
+ requestCost: (Number(p.classBPerMillion) || 0) / 1e6,
+ retrievalCost: Number(p.retrievalPerGb) || 0,
+ residencyCost: storage * Math.max(0, residencyDays - MONTH_DAYS) / MONTH_DAYS,
+ latencyValue: LATENCY_VALUE,
+ });
+}
+
+const fmtScore = (v) => (Number.isFinite(v) ? Number(v.toPrecision(4)).toString() : String(v));
+
+const SCAN_PAGE = 500; // rows read per page of the promotion scan
+const SWEEP_KEEP = 500; // top-ranked promotion candidates a sweep keeps
+
+/**
+ * Ready native objects that pass the popularity, recency and size thresholds and have no R2 copy yet, each with
+ * its class and value_per_dollar, highest value per dollar first (then most viewed). Every eligible row is scored
+ * before the limit applies (a most-viewed prefix would drop smaller objects that are worth more per dollar), but
+ * the scan reads SCAN_PAGE rows at a time and holds only the best `limit`, so memory stays bounded.
+ * excludeSandbox drops developer-sandbox tenants in the query (their objects are never tiered).
+ */
+async function promotionCandidates(settings, { now = Date.now(), appId = null, limit = 200, excludeSandbox = false } = {}) {
const today = popularity.dayOf(now);
const today = popularity.dayOf(now);
- return await db.all(`SELECT o.*, p.viewers AS unique_viewers_7d, p.last_day AS last_viewed_day
+ const args = [popularity.windowStart(today, popularity.WINDOW_DAYS)];
+ const rank = (a, b) => b.value_per_dollar - a.value_per_dollar || Number(b.unique_viewers_7d) - Number(a.unique_viewers_7d) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+ let best = [];
+ for (let after = ''; ;) {
+ const rows = await db.all(`SELECT o.*, p.viewers AS unique_viewers_7d, p.last_day AS last_viewed_day
FROM (SELECT object_id, SUM(unique_viewers)::bigint AS viewers, MAX(day) AS last_day FROM media_object_views_daily
FROM (SELECT object_id, SUM(unique_viewers)::bigint AS viewers, MAX(day) AS last_day FROM media_object_views_daily
WHERE day >= ? AND unique_viewers > 0 GROUP BY object_id) p
WHERE day >= ? AND unique_viewers > 0 GROUP BY object_id) p
JOIN media_objects o ON o.id = p.object_id
JOIN media_objects o ON o.id = p.object_id
- WHERE o.legacy_ref IS NULL AND o.lifecycle_status = 'ready'
+ WHERE o.id > ? AND o.legacy_ref IS NULL AND o.lifecycle_status = 'ready'
AND p.viewers >= ? AND p.last_day >= ? AND o.size_bytes >= ? AND o.size_bytes <= ?
AND p.viewers >= ? AND p.last_day >= ? AND o.size_bytes >= ? AND o.size_bytes <= ?
- AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.provider = 'r2' AND l.state = 'present')${appId ? ' AND o.app_id = ?' : ''}
- ORDER BY p.viewers DESC, o.id LIMIT ?`,
- [popularity.windowStart(today, popularity.WINDOW_DAYS), settings.promoteMinUniqueViewers7d, popularity.windowStart(today, settings.promoteRecentAccessDays),
- Math.ceil(settings.promoteMinSizeMb * MB), Math.floor(settings.promoteMaxSizeMb * MB), ...(appId ? [appId] : []), limit]);
+ AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.provider = 'r2' AND l.state = 'present')${appId ? ' AND o.app_id = ?' : ''}${excludeSandbox
+ ? " AND NOT EXISTS (SELECT 1 FROM apps a WHERE a.app_id = o.app_id AND a.env = 'sandbox')" : ''}
+ ORDER BY o.id LIMIT ?`,
+ [...args, after, settings.promoteMinUniqueViewers7d, popularity.windowStart(today, settings.promoteRecentAccessDays),
+ Math.ceil(settings.promoteMinSizeMb * MB), Math.floor(settings.promoteMaxSizeMb * MB), ...(appId ? [appId] : []), SCAN_PAGE]);
+ for (const r of rows) {
+ r.class = storagePolicy.classOf(r);
+ r.value_per_dollar = scoreOf(r, r.class, { committing: true });
+ }
+ best = best.concat(rows).sort(rank).slice(0, limit);
+ if (rows.length < SCAN_PAGE) return best;
+ after = rows[rows.length - 1].id;
+ }
}
}
-/** Native objects with an R2 copy that should lose it: not ready any more, or idle. → [{ obj row, reason }] */
+/**
+ * Native objects with an R2 copy that should lose it: not ready any more (first, whatever their score), or idle,
+ * lowest value per dollar of keeping the copy first (its viewers over the kept month, since an idle copy has none
+ * in the last 7 days), then the largest. → [{ row, reason, class, value_per_dollar }]
+ */
async function demotionCandidates(settings, { now = Date.now(), appId = null } = {}) {
async function demotionCandidates(settings, { now = Date.now(), appId = null } = {}) {
const idleFrom = popularity.windowStart(popularity.dayOf(now), settings.demoteIdleDays);
const idleFrom = popularity.windowStart(popularity.dayOf(now), settings.demoteIdleDays);
+ const monthFrom = popularity.windowStart(popularity.dayOf(now), popularity.KEEP_DAYS);
const rows = await db.all(`SELECT o.*, l.created_at AS r2_since,
const rows = await db.all(`SELECT o.*, l.created_at AS r2_since,
- (SELECT MAX(d.day) FROM media_object_views_daily d WHERE d.object_id = o.id AND d.unique_viewers > 0) AS last_viewed_day
+ (SELECT MAX(d.day) FROM media_object_views_daily d WHERE d.object_id = o.id AND d.unique_viewers > 0) AS last_viewed_day,
+ (SELECT COALESCE(SUM(d.unique_viewers), 0)::bigint FROM media_object_views_daily d WHERE d.object_id = o.id AND d.day >= ?) AS unique_viewers_30d
FROM media_locations l JOIN media_objects o ON o.id = l.object_id
FROM media_locations l JOIN media_objects o ON o.id = l.object_id
WHERE l.provider = 'r2' AND o.legacy_ref IS NULL${appId ? ' AND o.app_id = ?' : ''}
WHERE l.provider = 'r2' AND o.legacy_ref IS NULL${appId ? ' AND o.app_id = ?' : ''}
- ORDER BY (o.lifecycle_status = 'ready'), o.id`, appId ? [appId] : []);
+ ORDER BY (o.lifecycle_status = 'ready'), o.id`, appId ? [monthFrom, appId] : [monthFrom]);
const out = [];
const out = [];
for (const r of rows) {
for (const r of rows) {
- if (r.lifecycle_status !== 'ready') { out.push({ row: r, reason: `the object is ${r.lifecycle_status}` }); continue; }
+ const cls = storagePolicy.classOf(r);
+ const scored = { row: r, class: cls, value_per_dollar: scoreOf(r, cls) };
+ if (r.lifecycle_status !== 'ready') { out.push({ ...scored, reason: `the object is ${r.lifecycle_status}` }); continue; }
const idle = !r.last_viewed_day || r.last_viewed_day < idleFrom;
const idle = !r.last_viewed_day || r.last_viewed_day < idleFrom;
const since = msOf(r.r2_since);
const since = msOf(r.r2_since);
- const settled = !Number.isFinite(since) || now - since >= settings.demoteIdleDays * 864e5;
+ const settled = !Number.isFinite(since) || now - since >= settings.demoteIdleDays * DAY_MS;
if (idle && settled) {
if (idle && settled) {
- out.push({ row: r, reason: `no viewer since ${r.last_viewed_day || 'the kept counts began'} (idle for demoteIdleDays ${settings.demoteIdleDays}); in R2 since ${r.r2_since || 'unknown'}` });
+ out.push({ ...scored, reason: `no viewer since ${r.last_viewed_day || 'the kept counts began'} (idle for demoteIdleDays ${settings.demoteIdleDays}); in R2 since ${r.r2_since || 'unknown'}` });
}
}
}
}
+ const forced = (c) => (c.row.lifecycle_status === 'ready' ? 1 : 0);
+ out.sort((a, b) => forced(a) - forced(b) || a.value_per_dollar - b.value_per_dollar
+ || Number(b.row.size_bytes) - Number(a.row.size_bytes) || (a.row.id < b.row.id ? -1 : a.row.id > b.row.id ? 1 : 0));
return out;
return out;
}
}
-function promoteReason(c, s) {
+/** Where a move stands in its class's ranking and budget: `class video, value per dollar 1234, 1 of budget 3 (maxPromotionsPerSweep)`. */
+function rankNote(cls, score, slot, budgetKey) {
+ return `class ${cls}, value per dollar ${fmtScore(score)}, ${slot} of budget ${storagePolicy.budgetFor(cls)[budgetKey]} (${budgetKey}, media.storage_policy)`;
+}
+
+function promoteReason(c, s, slot) {
return `${c.unique_viewers_7d} unique viewers in 7 days >= promoteMinUniqueViewers7d ${s.promoteMinUniqueViewers7d}; last viewed ${c.last_viewed_day} `
return `${c.unique_viewers_7d} unique viewers in 7 days >= promoteMinUniqueViewers7d ${s.promoteMinUniqueViewers7d}; last viewed ${c.last_viewed_day} `
- + `(within promoteRecentAccessDays ${s.promoteRecentAccessDays}); ${(Number(c.size_bytes) / MB).toFixed(1)} MB (promoteMinSizeMb ${s.promoteMinSizeMb} to promoteMaxSizeMb ${s.promoteMaxSizeMb})`;
+ + `(within promoteRecentAccessDays ${s.promoteRecentAccessDays}); ${(Number(c.size_bytes) / MB).toFixed(1)} MB (promoteMinSizeMb ${s.promoteMinSizeMb} to promoteMaxSizeMb ${s.promoteMaxSizeMb}); `
+ + rankNote(c.class, c.value_per_dollar, slot, 'maxPromotionsPerSweep');
}
}
/** What the policy would promote now, each with what (on record) would refuse it; database only. */
/** What the policy would promote now, each with what (on record) would refuse it; database only. */
async function candidates({ appId = null, limit = 20, now = Date.now(), settings = policy.settings() } = {}) {
async function candidates({ appId = null, limit = 20, now = Date.now(), settings = policy.settings() } = {}) {
return (await Promise.all((await promotionCandidates(settings, { now, appId, limit: Math.min(Math.max(parseInt(limit, 10) || 20, 1), 200) })).map(async (c) => ({
return (await Promise.all((await promotionCandidates(settings, { now, appId, limit: Math.min(Math.max(parseInt(limit, 10) || 20, 1), 200) })).map(async (c) => ({
- object_id: c.id, app_id: c.app_id, kind: c.kind, size_bytes: c.size_bytes, unique_viewers_7d: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day,
+ object_id: c.id, app_id: c.app_id, kind: c.kind, class: c.class, value_per_dollar: c.value_per_dollar,
+ size_bytes: c.size_bytes, unique_viewers_7d: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day,
blocked_by: await promoteRefusal(c, { locs: await model().listLocations(c.id), held: await model().isHeld(c.id) }),
blocked_by: await promoteRefusal(c, { locs: await model().listLocations(c.id), held: await model().isHeld(c.id) }),
}))));
}))));
}
}
@@ -380,30 +455,31 @@ async function runSweep({ trigger = 'sweep', now = Date.now() } = {}) {
@@ -380,30 +455,31 @@ async function runSweep({ trigger = 'sweep', now = Date.now() } = {}) {
// Demotions first: they free the R2 cache, and a deleted object's copy should not wait behind promotions.
// Demotions first: they free the R2 cache, and a deleted object's copy should not wait behind promotions.
const demote_ = await demotionCandidates(settings, { now });
const demote_ = await demotionCandidates(settings, { now });
out.candidates.demote = demote_.length;
out.candidates.demote = demote_.length;
- // Budgets and residency are per class (media.storage_policy); a class spends only its own budget.
+ // Budgets and residency are per class (media.storage_policy); a class spends only its own budget, in the
+ // candidates' value-per-dollar order. A refusal (residency, hold) or a back-off spends nothing, so the next
+ // in line takes the slot.
const spent = { promote: {}, demote: {} };
const spent = { promote: {}, demote: {} };
const room = (action, cls) => (spent[action][cls] || 0) < storagePolicy.budgetFor(cls)[action === 'promote' ? 'maxPromotionsPerSweep' : 'maxDemotionsPerSweep'];
const room = (action, cls) => (spent[action][cls] || 0) < storagePolicy.budgetFor(cls)[action === 'promote' ? 'maxPromotionsPerSweep' : 'maxDemotionsPerSweep'];
- for (const { row, reason } of demote_) {
- const cls = storagePolicy.classOf(row);
+ for (const { row, reason, class: cls, value_per_dollar: score } of demote_) {
if (!room('demote', cls)) continue;
if (!room('demote', cls)) continue;
if (await inBackoff(row.id, 'demote')) { out.skipped_backoff++; continue; }
if (await inBackoff(row.id, 'demote')) { out.skipped_backoff++; continue; }
// A copy of an object that is no longer ready is removed regardless of residency.
// A copy of an object that is no longer ready is removed regardless of residency.
const residency = row.lifecycle_status === 'ready' ? await residencyRefusal(row, cls, 'demote', now) : null;
const residency = row.lifecycle_status === 'ready' ? await residencyRefusal(row, cls, 'demote', now) : null;
- const r = await demote(row.id, { trigger, reason, settings, now, residency });
+ const r = await demote(row.id, { trigger, settings, now, residency, valuePerDollar: score,
+ reason: `${reason}; ${rankNote(cls, score, (spent.demote[cls] || 0) + 1, 'maxDemotionsPerSweep')}` });
tally(out, { ...r, action: 'demote' });
tally(out, { ...r, action: 'demote' });
if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.demote[cls] = (spent.demote[cls] || 0) + 1;
if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.demote[cls] = (spent.demote[cls] || 0) + 1;
}
}
- const promote_ = [];
- const promoteLimit = Math.max(...storagePolicy.CLASSES.map((c) => storagePolicy.budgetFor(c).maxPromotionsPerSweep));
- for (const c of await promotionCandidates(settings, { now, limit: Math.max(50, promoteLimit * 10) })) if (!await db.isSandboxTenant(c.app_id)) promote_.push(c);
+ // Every eligible candidate is ranked (not a most-viewed prefix); the best SWEEP_KEEP are kept, so no class is crowded out of its budget.
+ const promote_ = await promotionCandidates(settings, { now, limit: SWEEP_KEEP, excludeSandbox: true });
out.candidates.promote = promote_.length;
out.candidates.promote = promote_.length;
for (const c of promote_) {
for (const c of promote_) {
- const cls = storagePolicy.classOf(c);
+ const cls = c.class;
if (!room('promote', cls)) continue;
if (!room('promote', cls)) continue;
if (await inBackoff(c.id, 'promote')) { out.skipped_backoff++; continue; }
if (await inBackoff(c.id, 'promote')) { out.skipped_backoff++; continue; }
const residency = await residencyRefusal(c, cls, 'promote', now);
const residency = await residencyRefusal(c, cls, 'promote', now);
- const r = await promote(c.id, { trigger, reason: promoteReason(c, settings), settings, now, residency,
+ const r = await promote(c.id, { trigger, reason: promoteReason(c, settings, (spent.promote[cls] || 0) + 1), settings, now, residency, valuePerDollar: c.value_per_dollar,
pop: { unique_viewers: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day } });
pop: { unique_viewers: c.unique_viewers_7d, last_viewed_day: c.last_viewed_day } });
tally(out, { ...r, action: 'promote' });
tally(out, { ...r, action: 'promote' });
if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.promote[cls] = (spent.promote[cls] || 0) + 1;
if (!r.repeat && ['done', 'failed', 'dry_run'].includes(r.outcome)) spent.promote[cls] = (spent.promote[cls] || 0) + 1;
new file mode 100644
new file mode 100644
@@ -0,0 +1,45 @@
@@ -0,0 +1,45 @@
+'use strict';
+/**
+ * Value per dollar of keeping one object in a tier (docs/media-fabric.md §6, decision rule): what serving its
+ * expected requests from that tier is worth, against what the tier charges for it. Pure: no I/O, no clock.
+ *
+ * Units (one month is the horizon every figure is taken over):
+ * requests expected requests served from the tier in a month (a count)
+ * bytes the object's size in bytes (GB below is bytes / 1024³)
+ * storageCost $ per GB-month the tier charges to keep it
+ * requestCost $ per request (read operation) the tier charges
+ * retrievalCost $ per GB read out of the tier (each request reads the whole object)
+ * residencyCost $ per GB committed by a minimum residency beyond the month (a policy or provider minimum)
+ * latencyValue $ a request is worth when the tier serves it (1 makes the score requests per dollar)
+ *
+ * value = requests × latencyValue
+ * dollars = GB × storageCost + requests × requestCost + requests × GB × retrievalCost + GB × residencyCost
+ * score = value / max(dollars, MIN_DOLLARS) (dimensionless: $ of value per $ spent)
+ *
+ * The benefit is per request and the cost mostly per byte, so a hotter object scores higher and, at equal demand,
+ * a bigger one or one in a costlier tier scores lower: the greedy order of a size-aware cache (GDSF), which the
+ * per-class knapsack takes highest first. The score never falls as requests rise (the value grows at least as
+ * fast as the per-request costs) and never rises as any cost rises. A missing, negative or non-finite input
+ * counts as 0; a free tier (dollars 0) is floored at MIN_DOLLARS so the score stays finite and ordered by demand.
+ */
+
+const GB = 1024 * 1024 * 1024;
+const MIN_DOLLARS = 1e-9;
+
+function num(v) {
+ const n = Number(v);
+ return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+/** The dollars a month of keeping and serving the object in the tier costs (see the header). */
+function dollars({ requests, bytes, storageCost, requestCost, retrievalCost, residencyCost } = {}) {
+ const r = num(requests), gb = num(bytes) / GB;
+ return gb * num(storageCost) + r * num(requestCost) + r * gb * num(retrievalCost) + gb * num(residencyCost);
+}
+
+/** Value per dollar (see the header). → a finite number >= 0 */
+function score(inputs = {}) {
+ return (num(inputs.requests) * num(inputs.latencyValue)) / Math.max(dollars(inputs), MIN_DOLLARS);
+}
+
+module.exports = { score, dollars, GB, MIN_DOLLARS };
@@ -461,19 +461,26 @@ const crypto = require('crypto');
@@ -461,19 +461,26 @@ const crypto = require('crypto');
// ── 9b. Per-class budgets and residency come from media.storage_policy ──
// ── 9b. Per-class budgets and residency come from media.storage_policy ──
const sp = require('../server/placement/storage-policy');
const sp = require('../server/placement/storage-policy');
await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
- const vids = [await addObject({ mime: 'video/mp4' }), await addObject({ mime: 'video/mp4' })];
+ // vids[0] is the most viewed but nearly 1 MB; vids[1] has fewer viewers at 4 KB, so more value per dollar.
+ const vids = [await addObject({ mime: 'video/mp4', size: 1000000 }), await addObject({ mime: 'video/mp4' })];
const imgs = [await addObject({ mime: 'image/png' }), await addObject({ mime: 'image/png' })];
const imgs = [await addObject({ mime: 'image/png' }), await addObject({ mime: 'image/png' })];
const dl = await addObject();
const dl = await addObject();
- for (const o of [...vids, ...imgs, dl]) await setViews(o.id, [[0, 900]]);
+ for (const o of [...vids, ...imgs, dl]) await setViews(o.id, [[0, o === vids[0] ? 950 : 900]]);
+ const ranked = (await tiering.promotionCandidates(policy.settings())).filter((c) => c.class === 'video').map((c) => c.id);
+ assert.deepStrictEqual(ranked, [vids[1].id, vids[0].id], 'ranked by value per dollar, not by viewers');
await sp.set({ classes: { video: { maxPromotionsPerSweep: 1 }, image: { maxPromotionsPerSweep: 2, minResidencyMs: 86400000 }, download: { maxPromotionsPerSweep: 0 } } },
await sp.set({ classes: { video: { maxPromotionsPerSweep: 1 }, image: { maxPromotionsPerSweep: 2, minResidencyMs: 86400000 }, download: { maxPromotionsPerSweep: 0 } } },
{ reason: 'test: per-class budgets' });
{ reason: 'test: per-class budgets' });
s = await tiering.runSweep();
s = await tiering.runSweep();
const promotedOf = async (list) => (await Promise.all(list.map(async (o) => !!await r2Row(o.id)))).filter(Boolean).length;
const promotedOf = async (list) => (await Promise.all(list.map(async (o) => !!await r2Row(o.id)))).filter(Boolean).length;
assert.deepStrictEqual([await promotedOf(vids), await promotedOf(imgs), await promotedOf([dl])], [1, 2, 0], 'each class spends its own promotion budget');
assert.deepStrictEqual([await promotedOf(vids), await promotedOf(imgs), await promotedOf([dl])], [1, 2, 0], 'each class spends its own promotion budget');
assert.strictEqual((await decisions(dl.id)).length, 0, 'a class with a zero budget is not tried');
assert.strictEqual((await decisions(dl.id)).length, 0, 'a class with a zero budget is not tried');
- const r2Done = new Set((await Promise.all(vids.map(async (o) => (await r2Row(o.id)) ? o.id : null))).filter(Boolean));
+ assert.ok(await r2Row(vids[1].id) && !await r2Row(vids[0].id), "the video budget of 1 goes to the higher value per dollar");
+ assert.strictEqual((await decisions(vids[0].id)).length, 0, 'the rest of the class waits for the next sweep');
assert.strictEqual(JSON.parse((await last(imgs[0].id)).inputs).class, 'image');
assert.strictEqual(JSON.parse((await last(imgs[0].id)).inputs).class, 'image');
- assert.strictEqual(JSON.parse((await last(vids.find((o) => r2Done.has(o.id)).id)).inputs).class, 'video');
+ d = await last(vids[1].id);
+ assert.strictEqual(JSON.parse(d.inputs).class, 'video');
+ assert.ok(JSON.parse(d.inputs).value_per_dollar > 0, d.inputs);
+ assert.ok(/; class video, value per dollar [0-9.e+]+, 1 of budget 1 \(maxPromotionsPerSweep, media\.storage_policy\)$/.test(d.reason), d.reason);
// Residency: the images were just promoted, so an idle demotion is suppressed and logged as refused.
// Residency: the images were just promoted, so an idle demotion is suppressed and logged as refused.
await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
for (const o of imgs) await ageR2(o.id, 30);
for (const o of imgs) await ageR2(o.id, 30);
@@ -482,11 +489,58 @@ const crypto = require('crypto');
@@ -482,11 +489,58 @@ const crypto = require('crypto');
d = await last(imgs[0].id);
d = await last(imgs[0].id);
assert.deepStrictEqual([d.action, d.outcome, JSON.parse(d.inputs).class], ['demote', 'refused', 'image']);
assert.deepStrictEqual([d.action, d.outcome, JSON.parse(d.inputs).class], ['demote', 'refused', 'image']);
assert.ok(/minimum residency of 86400000 ms for class image/.test(d.reason), d.reason);
assert.ok(/minimum residency of 86400000 ms for class image/.test(d.reason), d.reason);
+ assert.ok(/idle for demoteIdleDays 14\).*; class image, value per dollar 0, 1 of budget 10 \(maxDemotionsPerSweep/.test(d.reason), d.reason);
await sp.set({ classes: { image: { minResidencyMs: 0 }, download: { maxPromotionsPerSweep: 3 } } }, { reason: 'test: residency off, budgets back' });
await sp.set({ classes: { image: { minResidencyMs: 0 }, download: { maxPromotionsPerSweep: 3 } } }, { reason: 'test: residency off, budgets back' });
s = await tiering.runSweep();
s = await tiering.runSweep();
assert.ok(!await r2Row(imgs[0].id) && !await r2Row(imgs[1].id), 'with no residency the idle copies leave');
assert.ok(!await r2Row(imgs[0].id) && !await r2Row(imgs[1].id), 'with no residency the idle copies leave');
console.log('✅ per-class budgets and residency from media.storage_policy; every decision row shows its class; a residency-suppressed demotion logs refused');
console.log('✅ per-class budgets and residency from media.storage_policy; every decision row shows its class; a residency-suppressed demotion logs refused');
+ // ── 9c. Every eligible object is scored, not only the 1,000 most viewed ──
+ // 1,001 larger, more viewed videos would fill a most-viewed prefix; the small one is worth more per dollar.
+ const crowd = [];
+ for (let i = 0; i < 1001; i++) {
+ const id = await model.createObject({ app_id: 'live', kind: 'file', visibility: 'public', lifecycle_status: 'ready', mime_type: 'video/mp4', size_bytes: 1000000 });
+ await setViews(id, [[0, 900]]);
+ crowd.push(id);
+ }
+ const small = await addObject({ mime: 'video/mp4' });
+ await setViews(small.id, [[0, 500]]);
+ const all = await tiering.promotionCandidates(policy.settings(), { limit: Infinity });
+ assert.ok(all.length >= 1002, `every eligible object is ranked (${all.length})`);
+ assert.strictEqual(all[0].id, small.id, 'the fewest viewers but the most value per dollar ranks first');
+ assert.strictEqual((await tiering.promotionCandidates(policy.settings(), { limit: 1 }))[0].id, small.id, 'the limit applies after scoring');
+ await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
+ console.log('✅ promotion candidates are scored in full before the class budgets and limits apply');
+
+ // ── 9d. Gate off: a move residency or a hold blocks is still logged (refused) and spends no slot ──
+ await policy.set({ active: false }, { actor: { type: 'service', id: 'live' }, reason: 'test: gate off with blocked moves' });
+ await sp.set({ classes: { video: { maxPromotionsPerSweep: 1, minResidencyMs: 86400000 } } }, { reason: 'test: video residency, budget 1' });
+ const settling = await addObject({ mime: 'video/mp4' }); await setViews(settling.id, [[0, 900]]); // the highest score, but moved an hour ago
+ await db.run(`INSERT INTO media_object_tier_decisions (object_id, app_id, action, from_provider, to_provider, outcome, trigger, reason, inputs, thresholds)
+ VALUES (?, 'live', 'demote', 'r2', 'local', 'done', 'manual', 'test: a recent move', '{}', '{}')`, [settling.id]);
+ const heldVid = await addObject({ mime: 'video/mp4' }); await setViews(heldVid.id, [[0, 800]]);
+ const heldVidHold = await model.placeHold({ object_id: heldVid.id, kind: 'admin', reason: 'test' });
+ const nextVid = await addObject({ mime: 'video/mp4', size: 100000 }); await setViews(nextVid.id, [[0, 700]]);
+ assert.deepStrictEqual((await tiering.promotionCandidates(policy.settings())).filter((c) => c.class === 'video').map((c) => c.id),
+ [settling.id, heldVid.id, nextVid.id], 'the blocked ones rank first');
+ const before = Number((await db.get('SELECT MAX(id) AS id FROM media_object_tier_decisions')).id);
+ s = await tiering.runSweep();
+ const written = await db.all('SELECT object_id, action, outcome FROM media_object_tier_decisions WHERE id > ? ORDER BY id', [before]);
+ assert.ok(written.some((r) => r.outcome === 'dry_run') && written.some((r) => r.outcome === 'refused'), `gate off logs proposals and refusals: ${JSON.stringify(written)}`);
+ assert.deepStrictEqual([(await last(settling.id)).outcome, (await last(heldVid.id)).outcome], ['refused', 'refused'], 'a blocked move is logged');
+ assert.deepStrictEqual([(await decisions(settling.id)).length, (await decisions(heldVid.id)).length], [2, 1]);
+ d = await last(nextVid.id);
+ assert.deepStrictEqual([d.action, d.outcome], ['promote', 'dry_run']);
+ assert.ok(/, 1 of budget 1 \(maxPromotionsPerSweep/.test(d.reason), 'the blocked ones spent no slot, so the next in line has it: ' + d.reason);
+ assert.ok(s.refused >= 2 && s.would_promote >= 1, JSON.stringify(s));
+ const direct = await tiering.promote(heldVid.id, { reason: 'asked by hand' });
+ assert.deepStrictEqual([direct.outcome, direct.logged, (await decisions(heldVid.id)).length], ['refused', undefined, 2], 'a direct call is answered and logged');
+ await model.releaseHold(heldVidHold.id, 'test');
+ await sp.set({ classes: { video: { maxPromotionsPerSweep: 3, minResidencyMs: 0 } } }, { reason: 'test: video back' });
+ await policy.set({ active: true }, { actor: { type: 'service', id: 'live' }, reason: 'test: gate on again' });
+ await db.run('DELETE FROM media_object_views_daily WHERE object_id != ?', [cand.id]);
+ console.log('✅ gate off: residency- or hold-blocked moves are logged as refused and spend no slot');
+
// ── 10. The storage sweep runs it (step 4); a restore drill never does ──
// ── 10. The storage sweep runs it (step 4); a restore drill never does ──
const vodSweep = await storage.runSweep();
const vodSweep = await storage.runSweep();
assert.ok(vodSweep.objects && vodSweep.objects.gate === true && typeof vodSweep.objects.promoted === 'number', JSON.stringify(vodSweep));
assert.ok(vodSweep.objects && vodSweep.objects.gate === true && typeof vodSweep.objects.promoted === 'number', JSON.stringify(vodSweep));
new file mode 100644
new file mode 100644
@@ -0,0 +1,78 @@
@@ -0,0 +1,78 @@
+'use strict';
+// Value per dollar (F2.2, server/placement/value.js): hotter scores higher, a costlier tier lower, a free tier stays
+// finite, and the score is monotone in demand and in every cost. Pure, no database.
+const assert = require('assert');
+const { score, dollars, GB, MIN_DOLLARS } = require('../server/placement/value');
+
+// R2 Standard as cost-tiers.js prices it: $0.015/GB-month, $0.36 per million reads, no retrieval fee.
+const r2 = { storageCost: 0.015, requestCost: 0.36e-6, retrievalCost: 0, residencyCost: 0, latencyValue: 1 };
+const base = { ...r2, requests: 1000, bytes: GB };
+
+// Each case: [what, higher, lower] — the first input must score strictly higher than the second.
+const ordered = [
+ ['hotter object scores higher', { ...base, requests: 5000 }, base],
+ ['smaller object at equal demand scores higher', { ...base, bytes: GB / 10 }, base],
+ ['costlier storage scores lower', base, { ...base, storageCost: 0.03 }],
+ ['request fee scores lower', base, { ...base, requestCost: 4.5e-6 }],
+ ['retrieval fee scores lower', base, { ...base, retrievalCost: 0.01 }],
+ ['minimum residency scores lower', base, { ...base, residencyCost: 0.01 }],
+ ['latency worth more scores higher', { ...base, latencyValue: 2 }, base],
+];
+for (const [what, hi, lo] of ordered) assert.ok(score(hi) > score(lo), `${what}: ${score(hi)} > ${score(lo)}`);
+console.log(`✅ ordering: ${ordered.length} cases (hotter higher, costlier tier lower)`);
+
+// Exact values: [inputs, expected dollars, expected score].
+const exact = [
+ [base, 0.015 + 1000 * 0.36e-6, 1000 / (0.015 + 1000 * 0.36e-6)],
+ [{ ...base, requests: 0 }, 0.015, 0],
+ [{ ...base, latencyValue: 0 }, 0.015 + 1000 * 0.36e-6, 0],
+ [{ requests: 10, bytes: 2 * GB, storageCost: 0.01, requestCost: 0, retrievalCost: 0.01, residencyCost: 0.01, latencyValue: 1 }, 0.02 + 0.2 + 0.02, 10 / 0.24],
+];
+for (const [inputs, d, s] of exact) {
+ assert.ok(Math.abs(dollars(inputs) - d) < 1e-12, `dollars ${dollars(inputs)} != ${d}`);
+ assert.ok(Math.abs(score(inputs) - s) / Math.max(s, 1) < 1e-9, `score ${score(inputs)} != ${s}`);
+}
+console.log('✅ exact: value = requests × latencyValue over storage + requests + retrieval + residency');
+
+// Zero, missing and bad costs: never a division by zero, never NaN or Infinity, never negative.
+const edges = [
+ ['free tier', { requests: 100, bytes: GB, storageCost: 0, requestCost: 0, retrievalCost: 0, residencyCost: 0, latencyValue: 1 }, 100 / MIN_DOLLARS],
+ ['no costs given', { requests: 100, bytes: GB, latencyValue: 1 }, 100 / MIN_DOLLARS],
+ ['nothing given', {}, 0],
+ ['no argument', undefined, 0],
+ ['negative and NaN count as 0', { requests: 100, bytes: -5, storageCost: NaN, requestCost: -1, retrievalCost: 'x', residencyCost: null, latencyValue: 1 }, 100 / MIN_DOLLARS],
+ ['zero-byte object pays only per request', { ...r2, requests: 100, bytes: 0 }, 100 / (100 * 0.36e-6)],
+];
+for (const [what, inputs, s] of edges) {
+ const got = score(inputs);
+ assert.ok(Number.isFinite(got) && got >= 0, `${what}: finite and >= 0 (${got})`);
+ assert.ok(Math.abs(got - s) / Math.max(s, 1) < 1e-9, `${what}: ${got} != ${s}`);
+}
+assert.ok(score({ ...edges[0][1], requests: 200 }) > score(edges[0][1]), 'a free tier still ranks by demand');
+console.log(`✅ edges: ${edges.length} zero/missing/bad cost cases stay finite`);
+
+// Monotonicity: non-decreasing in requests (strictly while storage or residency costs something), non-increasing
+// in every cost, across a grid of sizes and tiers.
+const steps = [0, 1, 10, 100, 1e3, 1e4, 1e6];
+const EPS = 1e-12; // a constant ratio (no per-byte cost) may differ in the last float bit
+for (const bytes of [0, 1024, 100 * 1024 * 1024, GB, 50 * GB]) {
+ for (const tier of [r2, { ...r2, storageCost: 0.00695, requestCost: 0 }, { ...r2, storageCost: 0.01, requestCost: 0.9e-6, retrievalCost: 0.01, residencyCost: 0.01 }]) {
+ let prev = -1;
+ for (const requests of steps) {
+ const s = score({ ...tier, bytes, requests });
+ assert.ok(s >= prev * (1 - EPS), `non-decreasing in requests (${bytes} B, ${requests} req): ${s} >= ${prev}`);
+ if (bytes && requests > 1 && tier.storageCost) assert.ok(s > prev, `strictly increasing in requests (${bytes} B, ${requests} req)`);
+ prev = s;
+ }
+ for (const cost of ['storageCost', 'requestCost', 'retrievalCost', 'residencyCost']) {
+ let last = Infinity;
+ for (const c of [0, 1e-7, 1e-4, 0.01, 1]) {
+ const s = score({ ...tier, bytes, requests: 1000, [cost]: c });
+ assert.ok(s <= last * (1 + EPS), `non-increasing in ${cost} (${bytes} B, ${c}): ${s} <= ${last}`);
+ last = s;
+ }
+ }
+ }
+}
+console.log('✅ monotone: never down as requests rise, never up as a cost rises');
+console.log('\n✅ All placement value tests passed');