openvibers/openvibe.bench Commits

Price quality-qualified picks from Exchange (#200)

OP OpenVibers committed 2 days ago · ddc5171ac3 · 4 files, +62 −6

Changes

ddiff --git a/server/app.js b/server/app.js
iindex 0b40454..7acdd53 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -153,8 +153,10 @@ async function createApp(opts = {}) {
{ name: 'kind', in: 'query', schema: { type: 'string' } },
{ name: 'min_score', in: 'query', description: 'Minimum Bench score from 0 to 100', schema: { type: 'number', minimum: 0, maximum: 100 } },
{ name: 'max_cost', in: 'query', description: 'Maximum measured cost per solved task in USD', schema: { type: 'number', minimum: 0 } },
++ { name: 'input_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },
++ { name: 'output_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },
],
-- responses: { '200': { description: 'Measured quality and cost frontier; unknown prices stay unknown' } },
++ responses: { '200': { description: 'Measured Bench quality and costs plus the cheapest exact-match Exchange offer under the token budget; hosted picks need three Bench runs' } },
} };
paths['/api/v1/local/runs'] = { get: {
operationId: 'bench.localRuns.get', summary: 'Measured public runs for a hardware description', security: [],
ddiff --git a/server/http/pick.js b/server/http/pick.js
iindex d8636b3..e8dba6c 100644
---- a/server/http/pick.js
++++ b/server/http/pick.js
@@@ -22,6 +22,8 @@ function readControls(query) {
kind: str(query.kind).slice(0, 80),
maxCost: maxCost === '' ? null : Number(maxCost),
minScore: Number.isFinite(score) && score >= 0 && score <= 100 ? score : null,
++ inputTokens: str(query.input_tokens) === '' ? 1000 : Number(query.input_tokens),
++ outputTokens: str(query.output_tokens) === '' ? 1000 : Number(query.output_tokens),
fast: str(query.fast) === '1' || str(query.fast) === 'true',
q: str(query.q).slice(0, 200),
};
ddiff --git a/server/pick/pick.js b/server/pick/pick.js
iindex aaf2a99..e24c34b 100644
---- a/server/pick/pick.js
++++ b/server/pick/pick.js
@@@ -26,6 +26,13 @@ const MIN_RUNS = 3;
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
++function estimateExchangeTask(offer, inputTokens, outputTokens) {
++ const input = Number(offer && offer.price && offer.price.in);
++ const output = Number(offer && offer.price && offer.price.out);
++ if (!Number.isFinite(input) || !Number.isFinite(output) || input < 0 || output < 0) return null;
++ return Math.round(((inputTokens * input + outputTokens * output) / 1e6) * 1e9) / 1e9;
++}
++
function median(list) {
const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
if (!xs.length) return null;
@@@ -104,11 +111,13 @@ function whySentence(rows, kindName) {
* The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier
* with every number that made it.
*/
--async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast = false, q = '', kinds = KINDS } = {}) {
++async function answer(ctx, { kind = '', maxCost = null, minScore = null, inputTokens = 1000, outputTokens = 1000, fast = false, q = '', kinds = KINDS } = {}) {
const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };
const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);
const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));
const qualityBar = minScore == null || minScore === '' ? null : Number(minScore);
++ const inputTokenBudget = Math.min(1000000, Math.max(0, num(inputTokens, 1000)));
++ const outputTokenBudget = Math.min(1000000, Math.max(0, num(outputTokens, 1000)));
const wantFast = parsed.fast || !!fast;
const all = await ctx.bench.valueRows(kindId);
@@@ -140,6 +149,28 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
|| ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))
|| a.subject.localeCompare(b.subject));
++ const rowOffers = new Map();
++ const currentPrices = [];
++ for (const row of all) {
++ const offers = exchangeOffers.filter((offer) => [offer.model, offer.provider_model].some((model) => String(model || '').toLowerCase() === row.subject.toLowerCase()));
++ const pricedOffers = offers.map((offer) => ({
++ provider: offer.provider || null,
++ provider_model: offer.provider_model || null,
++ price: offer.price || null,
++ checked_at: offer.checked_at || offer.updated_at || null,
++ estimated_task_cost_usd: estimateExchangeTask(offer, inputTokenBudget, outputTokenBudget),
++ }));
++ rowOffers.set(row.subject, pricedOffers);
++ if (qualityBar != null && row.score >= qualityBar && row.runs >= MIN_RUNS) {
++ for (const offer of pricedOffers) if (offer.estimated_task_cost_usd != null) currentPrices.push({
++ model: row.subject, score: row.score, cost_usd: offer.estimated_task_cost_usd,
++ provider: offer.provider, provider_model: offer.provider_model, price: offer.price, checked_at: offer.checked_at,
++ });
++ }
++ }
++ const withinCurrentPrice = price == null ? currentPrices : currentPrices.filter((offer) => offer.cost_usd <= price + 1e-9);
++ withinCurrentPrice.sort((a, b) => a.cost_usd - b.cost_usd || b.score - a.score || a.model.localeCompare(b.model));
++
const rows = [];
for (const r of front) {
rows.push({
@@@ -151,8 +182,7 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
median_ms: r.median_ms,
runs: r.runs,
thin: r.runs < MIN_RUNS,
-- exchange_offers: exchangeOffers.filter((offer) => String(offer.model || '').toLowerCase() === r.subject.toLowerCase())
-- .map((offer) => ({ provider: offer.provider || null, provider_model: offer.provider_model || null, price: offer.price ?? null, checked_at: offer.checked_at || offer.updated_at || null })),
++ exchange_offers: rowOffers.get(r.subject) || [],
sample: await sampleFor(ctx, kindId, r.subject),
});
}
@@@ -181,13 +211,17 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
notes: parsed.notes || [],
},
filters: { max_cost: price == null ? null : price, min_score: qualityBar, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },
++ price_assumptions: { input_tokens: inputTokenBudget, output_tokens: outputTokenBudget, exchange_price_unit: 'USD per million tokens' },
++ cheapest_that_clears_quality_bar: qualityBar == null ? null : (withinCurrentPrice[0] || null),
++ cheapest_that_clears_quality_bar_reason: qualityBar == null ? 'Set min_score to choose the cheapest current Exchange offer among models clearing a measured Bench quality bar.'
++ : (withinCurrentPrice.length ? null : `No exact Bench model and Exchange offer match with a current input and output price was found among models with at least ${MIN_RUNS} runs within the selected cost limit.`),
rows,
also_ran: alsoRan,
why: whySentence(rows, kindInfo.name),
counts: { models: all.length, qualified: qualified.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },
-- price_source: 'frontier order uses measured cost_per_solved from Bench; current Exchange offer prices are attached when available but are not treated as per-task cost',
++ price_source: 'historical frontier uses measured Bench cost_per_solved; cheapest_that_clears_quality_bar uses current exact-match Exchange input/output prices and the stated token budgets',
generated_at: new Date().toISOString(),
};
}
--module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence };
++module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence, estimateExchangeTask };
ddiff --git a/test/pick.test.js b/test/pick.test.js
iindex 4f07618..fb0f9f0 100644
---- a/test/pick.test.js
++++ b/test/pick.test.js
@@@ -11,6 +11,7 @@ const { frontier } = require('../server/pick/frontier');
const { parsePick } = require('../server/pick/parser');
const { KINDS } = require('../server/kinds');
const { compareRuns: compareVersionRuns } = require('../server/bench/version-diff');
++const { estimateExchangeTask } = require('../server/pick/pick');
(async () => {
await check('the frontier drops the models another model beats on score and on cost', () => {
@@@ -24,6 +25,11 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);
});
++ await check('Exchange task prices use explicit input and output token budgets', () => {
++ assert.strictEqual(estimateExchangeTask({ price: { in: 2, out: 4 } }, 2000, 1000), 0.008);
++ assert.strictEqual(estimateExchangeTask({ price: { in: 2 } }, 2000, 1000), null);
++ });
++
await check('the version diff pairs identical tasks and reports a seeded confidence interval', () => {
const baseline = { id: 'old', model: 'model@1', benchmark: 'bench@1', tasks: [
{ task_id: 'a', score: 0.5 }, { task_id: 'b', score: 0.6 }, { task_id: 'held', score: 1, held_out: true },
@@@ -118,6 +124,18 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
assert.ok(json.rows.every((r) => r.score >= 99), 'every pick clears the quality floor');
});
++ await check('the cheapest qualifying hosted offer is selected using current Exchange prices', async () => {
++ const oldMarket = t.ctx.ai.market;
++ t.ctx.ai.market = async () => ({ offers: [{
++ model: 'alpha-1', provider: 'acme', provider_model: 'alpha-1', price: { in: 2, out: 4 }, checked_at: '2026-11-10',
++ }] });
++ const json = (await t.get('/api/v1/pick?kind=design.page&min_score=0&input_tokens=2000&output_tokens=1000')).json();
++ t.ctx.ai.market = oldMarket;
++ assert.strictEqual(json.cheapest_that_clears_quality_bar.model, 'alpha-1');
++ assert.strictEqual(json.cheapest_that_clears_quality_bar.cost_usd, 0.008);
++ assert.deepStrictEqual(json.price_assumptions, { input_tokens: 2000, output_tokens: 1000, exchange_price_unit: 'USD per million tokens' });
++ });
++
await check('fast takes its time cap from the run data', async () => {
const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();
assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');