openvibers/openvibe.bench Commits
Price quality-qualified picks from Exchange (#200)
Changes
@@@ -153,8 +153,10 @@ async function createApp(opts = {}) {{ name: 'kind', in: 'query', schema: { type: 'string' } },{ name: 'min_score', in: 'query', description: 'Minimum Bench score from 0 to 100', schema: { type: 'number', minimum: 0, maximum: 100 } },{ name: 'max_cost', in: 'query', description: 'Maximum measured cost per solved task in USD', schema: { type: 'number', minimum: 0 } },++ { name: 'input_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },++ { name: 'output_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },],-- responses: { '200': { description: 'Measured quality and cost frontier; unknown prices stay unknown' } },++ responses: { '200': { description: 'Measured Bench quality and costs plus the cheapest exact-match Exchange offer under the token budget; hosted picks need three Bench runs' } },} };paths['/api/v1/local/runs'] = { get: {operationId: 'bench.localRuns.get', summary: 'Measured public runs for a hardware description', security: [],@@@ -22,6 +22,8 @@ function readControls(query) {kind: str(query.kind).slice(0, 80),maxCost: maxCost === '' ? null : Number(maxCost),minScore: Number.isFinite(score) && score >= 0 && score <= 100 ? score : null,++ inputTokens: str(query.input_tokens) === '' ? 1000 : Number(query.input_tokens),++ outputTokens: str(query.output_tokens) === '' ? 1000 : Number(query.output_tokens),fast: str(query.fast) === '1' || str(query.fast) === 'true',q: str(query.q).slice(0, 200),};@@@ -26,6 +26,13 @@ const MIN_RUNS = 3;const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++function estimateExchangeTask(offer, inputTokens, outputTokens) {++ const input = Number(offer && offer.price && offer.price.in);++ const output = Number(offer && offer.price && offer.price.out);++ if (!Number.isFinite(input) || !Number.isFinite(output) || input < 0 || output < 0) return null;++ return Math.round(((inputTokens * input + outputTokens * output) / 1e6) * 1e9) / 1e9;++}++function median(list) {const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);if (!xs.length) return null;@@@ -104,11 +111,13 @@ function whySentence(rows, kindName) {* The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier* with every number that made it.*/--async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast = false, q = '', kinds = KINDS } = {}) {++async function answer(ctx, { kind = '', maxCost = null, minScore = null, inputTokens = 1000, outputTokens = 1000, fast = false, q = '', kinds = KINDS } = {}) {const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));const qualityBar = minScore == null || minScore === '' ? null : Number(minScore);++ const inputTokenBudget = Math.min(1000000, Math.max(0, num(inputTokens, 1000)));++ const outputTokenBudget = Math.min(1000000, Math.max(0, num(outputTokens, 1000)));const wantFast = parsed.fast || !!fast;const all = await ctx.bench.valueRows(kindId);@@@ -140,6 +149,28 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =|| ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))|| a.subject.localeCompare(b.subject));++ const rowOffers = new Map();++ const currentPrices = [];++ for (const row of all) {++ const offers = exchangeOffers.filter((offer) => [offer.model, offer.provider_model].some((model) => String(model || '').toLowerCase() === row.subject.toLowerCase()));++ const pricedOffers = offers.map((offer) => ({++ provider: offer.provider || null,++ provider_model: offer.provider_model || null,++ price: offer.price || null,++ checked_at: offer.checked_at || offer.updated_at || null,++ estimated_task_cost_usd: estimateExchangeTask(offer, inputTokenBudget, outputTokenBudget),++ }));++ rowOffers.set(row.subject, pricedOffers);++ if (qualityBar != null && row.score >= qualityBar && row.runs >= MIN_RUNS) {++ for (const offer of pricedOffers) if (offer.estimated_task_cost_usd != null) currentPrices.push({++ model: row.subject, score: row.score, cost_usd: offer.estimated_task_cost_usd,++ provider: offer.provider, provider_model: offer.provider_model, price: offer.price, checked_at: offer.checked_at,++ });++ }++ }++ const withinCurrentPrice = price == null ? currentPrices : currentPrices.filter((offer) => offer.cost_usd <= price + 1e-9);++ withinCurrentPrice.sort((a, b) => a.cost_usd - b.cost_usd || b.score - a.score || a.model.localeCompare(b.model));++const rows = [];for (const r of front) {rows.push({@@@ -151,8 +182,7 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =median_ms: r.median_ms,runs: r.runs,thin: r.runs < MIN_RUNS,-- exchange_offers: exchangeOffers.filter((offer) => String(offer.model || '').toLowerCase() === r.subject.toLowerCase())-- .map((offer) => ({ provider: offer.provider || null, provider_model: offer.provider_model || null, price: offer.price ?? null, checked_at: offer.checked_at || offer.updated_at || null })),++ exchange_offers: rowOffers.get(r.subject) || [],sample: await sampleFor(ctx, kindId, r.subject),});}@@@ -181,13 +211,17 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =notes: parsed.notes || [],},filters: { max_cost: price == null ? null : price, min_score: qualityBar, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },++ price_assumptions: { input_tokens: inputTokenBudget, output_tokens: outputTokenBudget, exchange_price_unit: 'USD per million tokens' },++ cheapest_that_clears_quality_bar: qualityBar == null ? null : (withinCurrentPrice[0] || null),++ cheapest_that_clears_quality_bar_reason: qualityBar == null ? 'Set min_score to choose the cheapest current Exchange offer among models clearing a measured Bench quality bar.'++ : (withinCurrentPrice.length ? null : `No exact Bench model and Exchange offer match with a current input and output price was found among models with at least ${MIN_RUNS} runs within the selected cost limit.`),rows,also_ran: alsoRan,why: whySentence(rows, kindInfo.name),counts: { models: all.length, qualified: qualified.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },-- price_source: 'frontier order uses measured cost_per_solved from Bench; current Exchange offer prices are attached when available but are not treated as per-task cost',++ price_source: 'historical frontier uses measured Bench cost_per_solved; cheapest_that_clears_quality_bar uses current exact-match Exchange input/output prices and the stated token budgets',generated_at: new Date().toISOString(),};}--module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence };++module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence, estimateExchangeTask };@@@ -11,6 +11,7 @@ const { frontier } = require('../server/pick/frontier');const { parsePick } = require('../server/pick/parser');const { KINDS } = require('../server/kinds');const { compareRuns: compareVersionRuns } = require('../server/bench/version-diff');++const { estimateExchangeTask } = require('../server/pick/pick');(async () => {await check('the frontier drops the models another model beats on score and on cost', () => {@@@ -24,6 +25,11 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-difassert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);});++ await check('Exchange task prices use explicit input and output token budgets', () => {++ assert.strictEqual(estimateExchangeTask({ price: { in: 2, out: 4 } }, 2000, 1000), 0.008);++ assert.strictEqual(estimateExchangeTask({ price: { in: 2 } }, 2000, 1000), null);++ });++await check('the version diff pairs identical tasks and reports a seeded confidence interval', () => {const baseline = { id: 'old', model: 'model@1', benchmark: 'bench@1', tasks: [{ task_id: 'a', score: 0.5 }, { task_id: 'b', score: 0.6 }, { task_id: 'held', score: 1, held_out: true },@@@ -118,6 +124,18 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-difassert.ok(json.rows.every((r) => r.score >= 99), 'every pick clears the quality floor');});++ await check('the cheapest qualifying hosted offer is selected using current Exchange prices', async () => {++ const oldMarket = t.ctx.ai.market;++ t.ctx.ai.market = async () => ({ offers: [{++ model: 'alpha-1', provider: 'acme', provider_model: 'alpha-1', price: { in: 2, out: 4 }, checked_at: '2026-11-10',++ }] });++ const json = (await t.get('/api/v1/pick?kind=design.page&min_score=0&input_tokens=2000&output_tokens=1000')).json();++ t.ctx.ai.market = oldMarket;++ assert.strictEqual(json.cheapest_that_clears_quality_bar.model, 'alpha-1');++ assert.strictEqual(json.cheapest_that_clears_quality_bar.cost_usd, 0.008);++ assert.deepStrictEqual(json.price_assumptions, { input_tokens: 2000, output_tokens: 1000, exchange_price_unit: 'USD per million tokens' });++ });++await check('fast takes its time cap from the run data', async () => {const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');
@@ -153,8 +153,10 @@ async function createApp(opts = {}) {
@@ -153,8 +153,10 @@ async function createApp(opts = {}) {
{ name: 'kind', in: 'query', schema: { type: 'string' } },
{ name: 'kind', in: 'query', schema: { type: 'string' } },
{ name: 'min_score', in: 'query', description: 'Minimum Bench score from 0 to 100', schema: { type: 'number', minimum: 0, maximum: 100 } },
{ name: 'min_score', in: 'query', description: 'Minimum Bench score from 0 to 100', schema: { type: 'number', minimum: 0, maximum: 100 } },
{ name: 'max_cost', in: 'query', description: 'Maximum measured cost per solved task in USD', schema: { type: 'number', minimum: 0 } },
{ name: 'max_cost', in: 'query', description: 'Maximum measured cost per solved task in USD', schema: { type: 'number', minimum: 0 } },
+ { name: 'input_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },
+ { name: 'output_tokens', in: 'query', description: 'Token budget for current Exchange price estimates', schema: { type: 'integer', minimum: 0, maximum: 1000000, default: 1000 } },
],
],
- responses: { '200': { description: 'Measured quality and cost frontier; unknown prices stay unknown' } },
+ responses: { '200': { description: 'Measured Bench quality and costs plus the cheapest exact-match Exchange offer under the token budget; hosted picks need three Bench runs' } },
} };
} };
paths['/api/v1/local/runs'] = { get: {
paths['/api/v1/local/runs'] = { get: {
operationId: 'bench.localRuns.get', summary: 'Measured public runs for a hardware description', security: [],
operationId: 'bench.localRuns.get', summary: 'Measured public runs for a hardware description', security: [],
@@ -22,6 +22,8 @@ function readControls(query) {
@@ -22,6 +22,8 @@ function readControls(query) {
kind: str(query.kind).slice(0, 80),
kind: str(query.kind).slice(0, 80),
maxCost: maxCost === '' ? null : Number(maxCost),
maxCost: maxCost === '' ? null : Number(maxCost),
minScore: Number.isFinite(score) && score >= 0 && score <= 100 ? score : null,
minScore: Number.isFinite(score) && score >= 0 && score <= 100 ? score : null,
+ inputTokens: str(query.input_tokens) === '' ? 1000 : Number(query.input_tokens),
+ outputTokens: str(query.output_tokens) === '' ? 1000 : Number(query.output_tokens),
fast: str(query.fast) === '1' || str(query.fast) === 'true',
fast: str(query.fast) === '1' || str(query.fast) === 'true',
q: str(query.q).slice(0, 200),
q: str(query.q).slice(0, 200),
};
};
@@ -26,6 +26,13 @@ const MIN_RUNS = 3;
@@ -26,6 +26,13 @@ const MIN_RUNS = 3;
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+function estimateExchangeTask(offer, inputTokens, outputTokens) {
+ const input = Number(offer && offer.price && offer.price.in);
+ const output = Number(offer && offer.price && offer.price.out);
+ if (!Number.isFinite(input) || !Number.isFinite(output) || input < 0 || output < 0) return null;
+ return Math.round(((inputTokens * input + outputTokens * output) / 1e6) * 1e9) / 1e9;
+}
+
function median(list) {
function median(list) {
const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
if (!xs.length) return null;
if (!xs.length) return null;
@@ -104,11 +111,13 @@ function whySentence(rows, kindName) {
@@ -104,11 +111,13 @@ function whySentence(rows, kindName) {
* The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier
* The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier
* with every number that made it.
* with every number that made it.
*/
*/
-async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast = false, q = '', kinds = KINDS } = {}) {
+async function answer(ctx, { kind = '', maxCost = null, minScore = null, inputTokens = 1000, outputTokens = 1000, fast = false, q = '', kinds = KINDS } = {}) {
const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };
const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };
const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);
const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);
const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));
const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));
const qualityBar = minScore == null || minScore === '' ? null : Number(minScore);
const qualityBar = minScore == null || minScore === '' ? null : Number(minScore);
+ const inputTokenBudget = Math.min(1000000, Math.max(0, num(inputTokens, 1000)));
+ const outputTokenBudget = Math.min(1000000, Math.max(0, num(outputTokens, 1000)));
const wantFast = parsed.fast || !!fast;
const wantFast = parsed.fast || !!fast;
const all = await ctx.bench.valueRows(kindId);
const all = await ctx.bench.valueRows(kindId);
@@ -140,6 +149,28 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
@@ -140,6 +149,28 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
|| ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))
|| ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))
|| a.subject.localeCompare(b.subject));
|| a.subject.localeCompare(b.subject));
+ const rowOffers = new Map();
+ const currentPrices = [];
+ for (const row of all) {
+ const offers = exchangeOffers.filter((offer) => [offer.model, offer.provider_model].some((model) => String(model || '').toLowerCase() === row.subject.toLowerCase()));
+ const pricedOffers = offers.map((offer) => ({
+ provider: offer.provider || null,
+ provider_model: offer.provider_model || null,
+ price: offer.price || null,
+ checked_at: offer.checked_at || offer.updated_at || null,
+ estimated_task_cost_usd: estimateExchangeTask(offer, inputTokenBudget, outputTokenBudget),
+ }));
+ rowOffers.set(row.subject, pricedOffers);
+ if (qualityBar != null && row.score >= qualityBar && row.runs >= MIN_RUNS) {
+ for (const offer of pricedOffers) if (offer.estimated_task_cost_usd != null) currentPrices.push({
+ model: row.subject, score: row.score, cost_usd: offer.estimated_task_cost_usd,
+ provider: offer.provider, provider_model: offer.provider_model, price: offer.price, checked_at: offer.checked_at,
+ });
+ }
+ }
+ const withinCurrentPrice = price == null ? currentPrices : currentPrices.filter((offer) => offer.cost_usd <= price + 1e-9);
+ withinCurrentPrice.sort((a, b) => a.cost_usd - b.cost_usd || b.score - a.score || a.model.localeCompare(b.model));
+
const rows = [];
const rows = [];
for (const r of front) {
for (const r of front) {
rows.push({
rows.push({
@@ -151,8 +182,7 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
@@ -151,8 +182,7 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
median_ms: r.median_ms,
median_ms: r.median_ms,
runs: r.runs,
runs: r.runs,
thin: r.runs < MIN_RUNS,
thin: r.runs < MIN_RUNS,
- exchange_offers: exchangeOffers.filter((offer) => String(offer.model || '').toLowerCase() === r.subject.toLowerCase())
- .map((offer) => ({ provider: offer.provider || null, provider_model: offer.provider_model || null, price: offer.price ?? null, checked_at: offer.checked_at || offer.updated_at || null })),
+ exchange_offers: rowOffers.get(r.subject) || [],
sample: await sampleFor(ctx, kindId, r.subject),
sample: await sampleFor(ctx, kindId, r.subject),
});
});
}
}
@@ -181,13 +211,17 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
@@ -181,13 +211,17 @@ async function answer(ctx, { kind = '', maxCost = null, minScore = null, fast =
notes: parsed.notes || [],
notes: parsed.notes || [],
},
},
filters: { max_cost: price == null ? null : price, min_score: qualityBar, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },
filters: { max_cost: price == null ? null : price, min_score: qualityBar, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },
+ price_assumptions: { input_tokens: inputTokenBudget, output_tokens: outputTokenBudget, exchange_price_unit: 'USD per million tokens' },
+ cheapest_that_clears_quality_bar: qualityBar == null ? null : (withinCurrentPrice[0] || null),
+ cheapest_that_clears_quality_bar_reason: qualityBar == null ? 'Set min_score to choose the cheapest current Exchange offer among models clearing a measured Bench quality bar.'
+ : (withinCurrentPrice.length ? null : `No exact Bench model and Exchange offer match with a current input and output price was found among models with at least ${MIN_RUNS} runs within the selected cost limit.`),
rows,
rows,
also_ran: alsoRan,
also_ran: alsoRan,
why: whySentence(rows, kindInfo.name),
why: whySentence(rows, kindInfo.name),
counts: { models: all.length, qualified: qualified.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },
counts: { models: all.length, qualified: qualified.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },
- price_source: 'frontier order uses measured cost_per_solved from Bench; current Exchange offer prices are attached when available but are not treated as per-task cost',
+ price_source: 'historical frontier uses measured Bench cost_per_solved; cheapest_that_clears_quality_bar uses current exact-match Exchange input/output prices and the stated token budgets',
generated_at: new Date().toISOString(),
generated_at: new Date().toISOString(),
};
};
}
}
-module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence };
+module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence, estimateExchangeTask };
@@ -11,6 +11,7 @@ const { frontier } = require('../server/pick/frontier');
@@ -11,6 +11,7 @@ const { frontier } = require('../server/pick/frontier');
const { parsePick } = require('../server/pick/parser');
const { parsePick } = require('../server/pick/parser');
const { KINDS } = require('../server/kinds');
const { KINDS } = require('../server/kinds');
const { compareRuns: compareVersionRuns } = require('../server/bench/version-diff');
const { compareRuns: compareVersionRuns } = require('../server/bench/version-diff');
+const { estimateExchangeTask } = require('../server/pick/pick');
(async () => {
(async () => {
await check('the frontier drops the models another model beats on score and on cost', () => {
await check('the frontier drops the models another model beats on score and on cost', () => {
@@ -24,6 +25,11 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
@@ -24,6 +25,11 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);
assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);
});
});
+ await check('Exchange task prices use explicit input and output token budgets', () => {
+ assert.strictEqual(estimateExchangeTask({ price: { in: 2, out: 4 } }, 2000, 1000), 0.008);
+ assert.strictEqual(estimateExchangeTask({ price: { in: 2 } }, 2000, 1000), null);
+ });
+
await check('the version diff pairs identical tasks and reports a seeded confidence interval', () => {
await check('the version diff pairs identical tasks and reports a seeded confidence interval', () => {
const baseline = { id: 'old', model: 'model@1', benchmark: 'bench@1', tasks: [
const baseline = { id: 'old', model: 'model@1', benchmark: 'bench@1', tasks: [
{ task_id: 'a', score: 0.5 }, { task_id: 'b', score: 0.6 }, { task_id: 'held', score: 1, held_out: true },
{ task_id: 'a', score: 0.5 }, { task_id: 'b', score: 0.6 }, { task_id: 'held', score: 1, held_out: true },
@@ -118,6 +124,18 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
@@ -118,6 +124,18 @@ const { compareRuns: compareVersionRuns } = require('../server/bench/version-dif
assert.ok(json.rows.every((r) => r.score >= 99), 'every pick clears the quality floor');
assert.ok(json.rows.every((r) => r.score >= 99), 'every pick clears the quality floor');
});
});
+ await check('the cheapest qualifying hosted offer is selected using current Exchange prices', async () => {
+ const oldMarket = t.ctx.ai.market;
+ t.ctx.ai.market = async () => ({ offers: [{
+ model: 'alpha-1', provider: 'acme', provider_model: 'alpha-1', price: { in: 2, out: 4 }, checked_at: '2026-11-10',
+ }] });
+ const json = (await t.get('/api/v1/pick?kind=design.page&min_score=0&input_tokens=2000&output_tokens=1000')).json();
+ t.ctx.ai.market = oldMarket;
+ assert.strictEqual(json.cheapest_that_clears_quality_bar.model, 'alpha-1');
+ assert.strictEqual(json.cheapest_that_clears_quality_bar.cost_usd, 0.008);
+ assert.deepStrictEqual(json.price_assumptions, { input_tokens: 2000, output_tokens: 1000, exchange_price_unit: 'USD per million tokens' });
+ });
+
await check('fast takes its time cap from the run data', async () => {
await check('fast takes its time cap from the run data', async () => {
const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();
const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();
assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');
assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');