openvibers/openvibe.bench Commits
B13: fresh monthly sets, the contamination flag and time-horizon estimates (#48)
Changes
nnew file mode 100644@@@ -0,0 +1,46 @@++-- phase: expand++-- B13: fresh monthly sets, the contamination flag and time-horizon estimates (spec 2026-10-30).++--++-- A task may carry the day it was added (`added_on`), whether it is held out (`held_out`, also accepted as the++-- older `holdout`) and a human time estimate (`human_minutes`) with its source. The run-task rows below copy the++-- facts at publish time, so a board can filter by month and fit a horizon without parsing every version; the++-- version-keyed table below keeps the same facts beside the closed benchmark document.++ALTER TABLE bench_run_tasks ADD COLUMN added_on date;++ALTER TABLE bench_run_tasks ADD COLUMN held_out boolean NOT NULL DEFAULT false;++ALTER TABLE bench_run_tasks ADD COLUMN human_minutes numeric(10, 1);++ALTER TABLE bench_run_tasks ADD COLUMN human_minutes_source text COLLATE "C";++++-- Editors' human-time estimates. One per person per task: a later entry replaces the earlier one, and the median of++-- three distinct people is what a task's time becomes. The row is the person's data and goes with them.++CREATE TABLE bench_task_estimates (++ id text COLLATE "C" PRIMARY KEY, -- bte_ + ULID++ benchmark_id text COLLATE "C" NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,++ benchmark_version integer NOT NULL,++ task_id text COLLATE "C" NOT NULL,++ subject text COLLATE "C" NOT NULL,++ minutes numeric(10, 1) NOT NULL,++ created_at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT bench_task_estimates_id CHECK (id ~ '^bte_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_task_estimates_version CHECK (benchmark_version >= 1),++ CONSTRAINT bench_task_estimates_minutes CHECK (minutes > 0 AND minutes <= 100000),++ FOREIGN KEY (benchmark_id, benchmark_version) REFERENCES bench_benchmark_versions (benchmark_id, version) ON DELETE CASCADE,++ UNIQUE (benchmark_id, benchmark_version, task_id, subject)++);++CREATE INDEX bench_task_estimates_by_subject ON bench_task_estimates (subject, created_at DESC);++CREATE INDEX bench_task_estimates_by_task ON bench_task_estimates (benchmark_id, benchmark_version, task_id);++++-- The task metadata itself. The published bench.benchmark@1 task shape is closed (id, prompt, files, expected,++-- holdout), so the B13 fields live beside the frozen definition, keyed by the version and task, and are merged into++-- the definition on Bench's own pages. The frozen document stays a valid contract document.++CREATE TABLE bench_task_meta (++ benchmark_id text COLLATE "C" NOT NULL,++ benchmark_version integer NOT NULL,++ task_id text COLLATE "C" NOT NULL,++ added_on date,++ held_out boolean NOT NULL DEFAULT false,++ human_minutes numeric(10, 1),++ human_minutes_source text COLLATE "C",++ PRIMARY KEY (benchmark_id, benchmark_version, task_id),++ CONSTRAINT bench_task_meta_minutes CHECK (human_minutes IS NULL OR (human_minutes > 0 AND human_minutes <= 100000)),++ FOREIGN KEY (benchmark_id, benchmark_version) REFERENCES bench_benchmark_versions (benchmark_id, version) ON DELETE CASCADE++);@@@ -163,7 +163,7 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex.br-task { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 12px; align-items: start; padding: 14px; border-radius: 14px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }.br-task p { margin: 0; line-height: 1.55; color: var(--text-secondary, #96a7c2); }.br-task.is-hidden { border-style: dashed; background: transparent; }--.br-task-id { display: inline-flex; padding: 2px 8px; border-radius: 8px; background: var(--bg-hover, rgba(255,255,255,.05)); font-size: .78rem; font-weight: 700; color: var(--text-primary, #e6edf7); white-space: nowrap; margin: 0; }++.br-task-id { display: inline-flex; justify-self: start; width: fit-content; padding: 2px 8px; border-radius: 8px; background: var(--bg-hover, rgba(255,255,255,.05)); font-size: .78rem; font-weight: 700; color: var(--text-primary, #e6edf7); white-space: nowrap; margin: 0; text-decoration: none; }.br-ptask { display: grid; gap: 10px; padding: 14px; border-radius: 16px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }.br-ptask-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; }.br-ptask-meta { margin-left: auto; font-size: .84rem; color: var(--text-muted, #7386a3); font-variant-numeric: tabular-nums; }@@@ -570,3 +570,27 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex@media (max-width: 640px) {.br-sections #br-rank .br-chart { display: none; }}++++/* ---- B13: fresh monthly sets, contamination and the time horizon ---- */++.br-since { display: flex; flex-wrap: wrap; align-items: flex-end; gap: 8px; margin: 0 0 14px; }++.br-since .br-field { margin: 0; }++.br-since .br-field input { min-height: 40px; }++.br-since-note { margin: 0 0 12px; padding: 10px 12px; border-radius: 12px; background: color-mix(in srgb, var(--room-accent) 10%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 26%, transparent); line-height: 1.5; }++.br-alltime { margin-top: 16px; border: 1px solid var(--border); border-radius: 14px; padding: 10px 14px; background: var(--bg-card); }++.br-alltime > summary { cursor: pointer; font-weight: 750; min-height: 36px; display: flex; align-items: center; }++.br-alltime > .br-table-wrap { margin-top: 8px; }++.br-task-meta { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 6px !important; }++.br-task-meta .br-chip { font-size: .72rem; }++.br-contams { list-style: none; margin: 14px 0 0; padding: 0; display: grid; gap: 8px; }++.br-contam { padding: 12px 14px; border-radius: 14px; background: color-mix(in srgb, #f59e0b 10%, var(--bg-card)); border: 1px solid color-mix(in srgb, #f59e0b 38%, var(--border)); line-height: 1.5; }++.br-contam a { color: var(--room-strong); }++.br-contam-chip { display: inline-block; margin-left: 8px; padding: 1px 7px; border-radius: 999px; font-size: .68rem; font-weight: 800; letter-spacing: .02em; color: #fde68a; background: color-mix(in srgb, #f59e0b 28%, transparent); border: 1px solid color-mix(in srgb, #f59e0b 55%, transparent); white-space: nowrap; }++.br-horizon { margin: 10px 0 0; padding: 10px 12px; border-radius: 12px; background: color-mix(in srgb, var(--room-accent) 9%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 24%, transparent); line-height: 1.5; }++.br-estimate { margin-top: 16px; }++@media (max-width: 560px) {++ .br-since { align-items: stretch; }++ .br-since .br-field { flex: 1 1 100%; }++ .br-since .bm-btn { min-height: 44px; }++ #br-tasks .br-task-meta { display: none; }++ .br-rank-table td.br-c-horizon { flex: none; order: 2; font-size: .82rem; color: var(--text-secondary); text-align: left; }++}nnew file mode 100644@@@ -0,0 +1,79 @@++'use strict';++++/**++ * The contamination flag (B13). A model that scored much better on tasks written before its release than on tasks++ * written after it may have seen the older ones in training. "Much better" means the 90% bootstrap intervals of the++ * two medians do not overlap. Two runs on each side are needed before either interval is allowed to speak; below++ * that the answer is "not enough runs yet", never a flag.++ */++const { percentile } = require('../rating/lib');++++function seededRandom(seed) {++ let a = seed >>> 0;++ return () => {++ a = (a + 0x6D2B79F5) >>> 0;++ let t = Math.imul(a ^ (a >>> 15), 1 | a);++ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;++ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;++ };++}++++const median = (values) => {++ const list = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b);++ if (!list.length) return null;++ const mid = Math.floor(list.length / 2);++ return list.length % 2 ? list[mid] : (list[mid - 1] + list[mid]) / 2;++};++++const round = (v, p = 1000) => (v == null ? null : Math.round(v * p) / p);++++/** A bootstrap percentile interval for the median of `values`. */++function medianInterval(values, { level = 0.9, bootstrap = 1000, seed = 20261030 } = {}) {++ const list = values.filter((v) => Number.isFinite(v));++ if (!list.length) return null;++ if (list.length === 1) return { median: round(list[0]), lower: round(list[0]), upper: round(list[0]), runs: 1 };++ const random = seededRandom(seed);++ const medians = [];++ for (let i = 0; i < bootstrap; i++) {++ const sample = [];++ for (let j = 0; j < list.length; j++) sample.push(list[Math.floor(random() * list.length)]);++ medians.push(median(sample));++ }++ medians.sort((a, b) => a - b);++ return {++ median: round(median(list)),++ lower: round(percentile(medians, (1 - level) / 2)),++ upper: round(percentile(medians, 1 - (1 - level) / 2)),++ runs: list.length,++ };++}++++/**++ * The flag for one model. `oldScores` and `freshScores` are one score per run, for the runs that touched a task on++ * that side of the cutoff. `cutoff`/`source` come from the repo's table; without them there is no flag.++ */++function contaminationFor({ oldScores = [], freshScores = [], cutoff = null, source = null } = {}) {++ const oldRuns = oldScores.filter((v) => Number.isFinite(v));++ const freshRuns = freshScores.filter((v) => Number.isFinite(v));++ if (!cutoff) {++ return { flag: false, available: false, reason: 'no release date', old_runs: oldRuns.length, fresh_runs: freshRuns.length };++ }++ if (oldRuns.length < 2 || freshRuns.length < 2) {++ return { flag: false, available: false, reason: 'not enough runs yet', old_runs: oldRuns.length, fresh_runs: freshRuns.length, cutoff, source };++ }++ const oldSide = medianInterval(oldRuns, { seed: 20261030 });++ const freshSide = medianInterval(freshRuns, { seed: 20261031 });++ const flag = oldSide.lower > freshSide.upper;++ return {++ flag,++ available: true,++ reason: flag ? 'the old interval is above the fresh one' : 'the intervals overlap',++ cutoff,++ source,++ old: oldSide,++ fresh: freshSide,++ gap: round(oldSide.median - freshSide.median),++ };++}++++module.exports = { contaminationFor, medianInterval, median, seededRandom, round };nnew file mode 100644@@@ -0,0 +1,118 @@++'use strict';++++/**++ * The time horizon, after METR (B13). For one model, each timed task gives a human time estimate and a success rate++ * (the mean of its task scores over the runs). A two-parameter logistic fit of success on log(human minutes) gives++ * the 50% point: the task length the model "reliably does". A bootstrap over tasks gives the 90% interval.++ *++ * Before 15 timed tasks and 3 runs, no number is shown: the fit would be noise. The interval is resampled from the++ * tasks, not the runs, because the task length is what the fit is about.++ */++const { percentile } = require('../rating/lib');++++const MIN_TASKS = 15;++const MIN_RUNS = 3;++const BOOTSTRAP = 500;++const SEED = 20261030;++++const sigmoid = (x) => (x >= 0 ? 1 / (1 + Math.exp(-x)) : (e => e / (1 + e))(Math.exp(x)));++const clamp = (p) => Math.min(0.999, Math.max(0.001, p));++const round = (v, p = 10) => (v == null ? null : Math.round(v * p) / p);++++function seededRandom(seed) {++ let a = seed >>> 0;++ return () => {++ a = (a + 0x6D2B79F5) >>> 0;++ let t = Math.imul(a ^ (a >>> 15), 1 | a);++ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;++ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;++ };++}++++/**++ * Fit intercept + slope by Newton-Raphson. `points` are { x = log(minutes), p = success 0..1, weight }.++ * No library: two parameters are two normal equations.++ */++function fitLogistic(points) {++ let b0 = 0;++ let b1 = 0;++ for (let iter = 0; iter < 80; iter++) {++ let g0 = 0; let g1 = 0;++ let h00 = 0; let h01 = 0; let h11 = 0;++ for (const point of points) {++ const w = Number.isFinite(point.weight) && point.weight > 0 ? point.weight : 1;++ const q = sigmoid(b0 + b1 * point.x);++ const r = q - point.p;++ const wt = Math.max(1e-9, q * (1 - q) * w);++ g0 += r * w;++ g1 += r * w * point.x;++ h00 += wt;++ h01 += wt * point.x;++ h11 += wt * point.x * point.x;++ }++ // A tiny ridge keeps a separable fit finite rather than throwing on a singular matrix.++ h00 += 1e-9; h11 += 1e-9;++ const det = h00 * h11 - h01 * h01;++ if (!Number.isFinite(det) || Math.abs(det) < 1e-18) break;++ const d0 = (h11 * g0 - h01 * g1) / det;++ const d1 = (-h01 * g0 + h00 * g1) / det;++ b0 -= d0; b1 -= d1;++ if (Math.max(Math.abs(d0), Math.abs(d1)) < 1e-11) break;++ }++ return { intercept: b0, slope: b1 };++}++++/** The minutes at which the fitted success is 50%, or null when the model does not improve with time. */++function horizonOf(fit) {++ if (!fit || !(fit.slope > 0) || !Number.isFinite(fit.intercept)) return null;++ return Math.exp(-fit.intercept / fit.slope);++}++++function fitPoints(groups) {++ return groups.map((g) => ({++ x: Math.log(g.minutes),++ p: clamp(Number(g.success)),++ weight: g.runs || 1,++ })).filter((p) => Number.isFinite(p.x) && Number.isFinite(p.p));++}++++/**++ * `groups`: [{ minutes, success (0..1 mean over runs), runs }]. `runs` is the number of distinct runs behind the++ * groups; `timedTasks` their count. Returns available:false with a reason below either gate.++ */++function estimateHorizon(groups = [], { runs = 0, timedTasks = groups.length, bootstrap = BOOTSTRAP, seed = SEED } = {}) {++ const gates = { timed_tasks: timedTasks, runs, need_tasks: MIN_TASKS, need_runs: MIN_RUNS };++ if (timedTasks < MIN_TASKS || runs < MIN_RUNS) {++ return { available: false, reason: 'not enough runs yet', ...gates };++ }++ const points = fitPoints(groups);++ if (points.length < MIN_TASKS) return { available: false, reason: 'not enough timed tasks', ...gates };++ const fit = fitLogistic(points);++ const horizon = horizonOf(fit);++ if (horizon == null || !Number.isFinite(horizon) || horizon <= 0) {++ return { available: false, reason: 'no 50% point yet', ...gates, slope: round(fit.slope, 1000) };++ }++ const random = seededRandom(seed);++ const samples = [];++ for (let i = 0; i < bootstrap; i++) {++ const resampled = [];++ for (let j = 0; j < points.length; j++) resampled.push(points[Math.floor(random() * points.length)]);++ const value = horizonOf(fitLogistic(resampled));++ if (value != null && Number.isFinite(value) && value > 0) samples.push(value);++ }++ samples.sort((a, b) => a - b);++ const lower = samples.length ? percentile(samples, 0.05) : horizon;++ const upper = samples.length ? percentile(samples, 0.95) : horizon;++ return {++ available: true,++ horizon_minutes: round(horizon, 1),++ ci_lower: round(lower, 1),++ ci_upper: round(upper, 1),++ ...gates,++ slope: round(fit.slope, 1000),++ method: 'logistic fit of success on log(human minutes); 90% bootstrap over tasks',++ bootstrap,++ };++}++++module.exports = { estimateHorizon, fitLogistic, horizonOf, fitPoints, MIN_TASKS, MIN_RUNS };nnew file mode 100644@@@ -0,0 +1,30 @@++'use strict';++++/**++ * When each model was released, or the training cutoff its maker states. The contamination flag needs one date per++ * model; a model not in this table gets no flag, never a guessed one. Every row names the page it comes from.++ *++ * The table is deliberately small and in the repo, so changing it is a reviewable change with a source. A provider++ * prefix ("openai:gpt-5") is stripped before the lookup.++ */++const CUTOFFS = Object.freeze([++ { model: 'gpt-5', cutoff: '2025-08-07', source: 'https://openai.com/index/introducing-gpt-5/' },++ { model: 'claude-opus-4', cutoff: '2025-05-22', source: 'https://www.anthropic.com/news/claude-4' },++ { model: 'claude-sonnet-4', cutoff: '2025-05-22', source: 'https://www.anthropic.com/news/claude-4' },++ { model: 'gemini-2.5-pro', cutoff: '2025-06-17', source: 'https://blog.google/technology/google-deepmind/google-gemini-updates-io-2025/' },++ { model: 'deepseek-v3', cutoff: '2024-12-26', source: 'https://api-docs.deepseek.com/news/news1226' },++ { model: 'kimi-k2', cutoff: '2025-07-11', source: 'https://moonshotai.github.io/Kimi-K2/' },++ { model: 'llama-4-maverick', cutoff: '2025-04-05', source: 'https://ai.meta.com/blog/llama-4-multimodal-intelligence/' },++]);++++const byModel = new Map(CUTOFFS.map((row) => [row.model.toLowerCase(), row]));++++/** The row for a model, or null: unknown means no flag. */++function cutoffFor(model) {++ const text = String(model || '').trim().toLowerCase();++ if (!text) return null;++ const bare = text.includes(':') ? text.slice(text.lastIndexOf(':') + 1) : text;++ return byModel.get(text) || byModel.get(bare) || null;++}++++module.exports = { CUTOFFS, cutoffFor };@@@ -17,6 +17,10 @@ const crypto = require('crypto');const { ids } = require('openvibe-contracts');const { contractRun, checkContract, VISIBILITIES } = require('./validate');const { rankBands } = require('../rating/style');++const { isHeldOut, addedOn, humanMinutes, humanMinutesSource } = require('./task-meta');++const { cutoffFor } = require('./model-cutoffs');++const { contaminationFor } = require('./contamination');++const { estimateHorizon } = require('./horizon');const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));@@@ -53,6 +57,51 @@ const json = (v, fallback) => {return v;};++/**++ * The published bench.benchmark@1 task shape closes its properties, so the B13 fields (added_on, held_out,++ * human_minutes and its source) live in bench_task_meta beside the frozen definition. These two helpers keep the++ * stored document valid and put the fields back on Bench's own read path.++ */++function contractTask(task) {++ const t = task && typeof task === 'object' ? task : {};++ const out = {++ id: String(t.id == null ? '' : t.id),++ prompt: String(t.prompt == null ? '' : t.prompt),++ files: Array.isArray(t.files) ? t.files : [],++ expected: t.expected === undefined ? null : t.expected,++ holdout: isHeldOut(t),++ };++ return out;++}++++function taskMetaOf(task) {++ return {++ added_on: addedOn(task),++ held_out: isHeldOut(task),++ human_minutes: humanMinutes(task),++ human_minutes_source: humanMinutesSource(task),++ };++}++++function mergeTaskMeta(definition, metaRows) {++ if (!Array.isArray(definition.tasks) || !metaRows) return definition;++ const byTask = new Map();++ for (const row of metaRows) byTask.set(String(row.task_id), row);++ return {++ ...definition,++ tasks: definition.tasks.map((task) => {++ const meta = byTask.get(String(task && task.id));++ if (!meta) return task;++ const out = { ...task };++ if (meta.added_on) out.added_on = String(meta.added_on).slice(0, 10);++ if (meta.held_out) { out.holdout = true; out.held_out = true; }++ if (meta.human_minutes != null) out.human_minutes = num(meta.human_minutes);++ if (meta.human_minutes_source) out.human_minutes_source = meta.human_minutes_source;++ return out;++ }),++ };++}++/** A share token: 256 random bits in base64url (43 characters), never guessable. */function newShareToken() {return crypto.randomBytes(32).toString('base64url');@@@ -85,7 +134,17 @@ function canViewBenchmark(benchmark, { subject = null, executor = false } = {})return true;}--function wireBenchmark(row) {++/** A working copy whose held-out prompts stay on the server, even in a listing that carries the draft. */++function wireDraft(value) {++ const draft = json(value, null);++ if (!draft || !Array.isArray(draft.tasks)) return draft;++ return {++ ...draft,++ tasks: draft.tasks.map((t) => (t && isHeldOut(t) ? { ...t, prompt: null, expected: null, files: null } : t)),++ };++}++++function wireBenchmark(row, { revealDraft = false } = {}) {if (!row) return null;return {id: row.id,@@@ -99,7 +158,7 @@ function wireBenchmark(row) {shared: Boolean(row.share_token_hash),parent: row.parent || null,latest_version: num(row.latest_version),-- draft: json(row.draft, null),++ draft: revealDraft ? json(row.draft, null) : wireDraft(row.draft),created_by: row.created_by,created_at: iso(row.created_at),updated_at: iso(row.updated_at),@@@ -168,7 +227,7 @@ function wireRun(row, tasks = null) {function wireTask(row) {if (!row) return null;-- return {++ const out = {task_id: row.task_id,score: row.score == null ? null : num(row.score),cost_usd: num(row.cost_usd),@@@ -177,6 +236,13 @@ function wireTask(row) {artifacts: (Array.isArray(json(row.artifacts, [])) ? json(row.artifacts, []) : []).map((a) => ({ kind: a.kind, url: a.url })).filter((a) => a.url),transcript_url: row.transcript_url || null,};++ if (row.added_on) out.added_on = String(row.added_on).slice(0, 10);++ if (row.held_out) out.held_out = true;++ if (row.human_minutes != null) {++ out.human_minutes = num(row.human_minutes);++ if (row.human_minutes_source) out.human_minutes_source = row.human_minutes_source;++ }++ return out;}/** The median of a list of numbers. */@@@ -187,6 +253,38 @@ function median(values) {return list.length % 2 ? list[mid] : (list[mid - 1] + list[mid]) / 2;}++const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];++++/**++ * A `since` value: "YYYY-MM" (a monthly fresh set), "YYYY-MM-DD", or a full timestamp (the B2 form, still accepted).++ * The cutoff is the first instant of the month/day. The label is what a page says beside the board.++ */++function parseSince(value) {++ const text = String(value || '').trim();++ if (!text) return null;++ let m = /^(\d{4})-(\d{2})$/.exec(text);++ if (m) {++ const month = Number(m[2]);++ if (month < 1 || month > 12) return null;++ const cutoff = `${m[1]}-${m[2]}-01`;++ return { text, cutoff, month: `${m[1]}-${m[2]}`, timestamp: `${cutoff}T00:00:00.000Z`, label: `${MONTH_NAMES[month - 1]} ${m[1]}` };++ }++ m = /^(\d{4})-(\d{2})-(\d{2})/.exec(text);++ if (m) {++ const month = Number(m[2]);++ if (month < 1 || month > 12) return null;++ const cutoff = text.slice(0, 10);++ if (Number.isNaN(Date.parse(`${cutoff}T00:00:00Z`))) return null;++ return { text, cutoff, month: `${m[1]}-${m[2]}`, timestamp: `${cutoff}T00:00:00.000Z`, label: `${MONTH_NAMES[month - 1]} ${m[1]}` };++ }++ const stamp = Date.parse(text);++ if (Number.isNaN(stamp)) return null;++ const d = new Date(stamp);++ const month = d.getUTCMonth();++ const cutoff = d.toISOString().slice(0, 10);++ return { text, cutoff, month: cutoff.slice(0, 7), timestamp: d.toISOString(), label: `${MONTH_NAMES[month]} ${d.getUTCFullYear()}` };++}++/** "bmk_x@2" → { id, version }; "bmk_x" → { id, version: null } (the newest frozen one). */function parseRef(ref) {const text = String(ref || '').trim();@@@ -220,12 +318,12 @@ function createRegistry(s, { log = console } = {}) {[id, String(owner || 'openvibe'), text, String(description || '').slice(0, 4000), kindId,JSON.stringify(list), vis, parent || null, JSON.stringify(definition || {}), String(subject || owner || 'openvibe'), s.iso()],);-- return { ok: true, benchmark: wireBenchmark(row) };++ return { ok: true, benchmark: await getBenchmark(id, { revealDraft: true }) };}-- /** The working copy: what a later freeze would publish. */-- async function getBenchmark(id) {-- return wireBenchmark(await db.maybe('SELECT * FROM bench_benchmarks WHERE id = $1', [String(id || '')]));++ /** The working copy: what a later freeze would publish. The draft's held-out prompts stay inside unless asked. */++ async function getBenchmark(id, { revealDraft = false } = {}) {++ return wireBenchmark(await db.maybe('SELECT * FROM bench_benchmarks WHERE id = $1', [String(id || '')]), { revealDraft });}async function getVersion(benchmarkId, version) {@@@ -241,9 +339,46 @@ function createRegistry(s, { log = console } = {}) {[String(benchmarkId || '')]));}++ /** The B13 metadata of the named versions, as rows keyed by version and task id. */++ async function loadTaskMeta(benchmarkId, versions) {++ if (!versions.length) return [];++ return db.many(++ `SELECT * FROM bench_task_meta WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])++ ORDER BY benchmark_version, task_id`,++ [String(benchmarkId), versions.map(num)]);++ }++++ /** The metadata map the rankings and the horizon read: stored metadata plus the base task's own holdout flag. */++ function taskMetaMap(versions, definitions, metaRows) {++ const out = new Map();++ for (const version of versions) {++ const definition = definitions.get(num(version)) || {};++ for (const task of (definition.tasks || [])) {++ if (!task || task.id == null) continue;++ out.set(`${num(version)}:${String(task.id)}`, {++ added_on: addedOn(task), held_out: isHeldOut(task),++ human_minutes: humanMinutes(task), human_minutes_source: humanMinutesSource(task),++ });++ }++ }++ for (const row of metaRows) {++ const key = `${num(row.benchmark_version)}:${String(row.task_id)}`;++ out.set(key, {++ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,++ held_out: Boolean(row.held_out),++ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),++ human_minutes_source: row.human_minutes_source || null,++ });++ }++ return out;++ }++/*** The definition of a benchmark, at a version or at its newest frozen one. Holdout tasks are withheld unless the* caller holds bench.run.publish: the executor may see them, everyone else may not.++ *++ * `version.definition` stays the closed contract document. `definition` is Bench's own view, with the B13 task++ * fields merged back in; `task_meta` is the same metadata without any prompt. A page reads `definition`.*/async function getDefinition(idOrRef, { reveal = false } = {}) {const ref = parseRef(idOrRef);@@@ -254,16 +389,49 @@ function createRegistry(s, { log = console } = {}) {? await getVersion(ref.id, ref.version): await latestVersion(ref.id);if (!version) return null;-- const definition = { ...version.definition };-- if (!reveal && Array.isArray(definition.tasks)) {-- definition.tasks = definition.tasks.map((t) => (t && t.holdout ? { ...t, prompt: null, expected: null, files: null, holdout: true } : t));++ const metaRows = await loadTaskMeta(ref.id, [version.version]);++ const base = { ...version.definition };++ const merged = mergeTaskMeta(base, metaRows);++ const hide = (definition) => {++ if (reveal || !Array.isArray(definition.tasks)) return definition;++ return {++ ...definition,++ tasks: definition.tasks.map((t) => (t && isHeldOut(t) ? { ...t, prompt: null, expected: null, files: null, holdout: true, held_out: true } : t)),++ };++ };++ return {++ benchmark,++ version: { ...version, definition: hide(base) },++ definition: hide(merged),++ task_meta: metaRows.map((row) => ({++ task_id: row.task_id,++ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,++ held_out: Boolean(row.held_out),++ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),++ human_minutes_source: row.human_minutes_source || null,++ })),++ };++ }++++ /** Write the B13 metadata of a version's tasks. The definition itself stays contract-shaped. */++ async function replaceTaskMeta(benchmarkId, version, tasks) {++ await db.exec('DELETE FROM bench_task_meta WHERE benchmark_id = $1 AND benchmark_version = $2', [String(benchmarkId), num(version)]);++ for (const task of tasks) {++ const meta = taskMetaOf(task);++ if (!meta.added_on && !meta.held_out && meta.human_minutes == null && !meta.human_minutes_source) continue;++ await db.exec(++ `INSERT INTO bench_task_meta (benchmark_id, benchmark_version, task_id, added_on, held_out, human_minutes, human_minutes_source)++ VALUES ($1, $2, $3, $4::date, $5, $6, $7)++ ON CONFLICT (benchmark_id, benchmark_version, task_id)++ DO UPDATE SET added_on = EXCLUDED.added_on, held_out = EXCLUDED.held_out,++ human_minutes = EXCLUDED.human_minutes, human_minutes_source = EXCLUDED.human_minutes_source`,++ [String(benchmarkId), num(version), String(task.id), meta.added_on, meta.held_out, meta.human_minutes, meta.human_minutes_source]);}-- return { benchmark, version: { ...version, definition }, definition };}/** Save a new draft. A version named in the body is edited in place, and only while no run names it. */async function saveDraft(id, { definition = {}, subject }) {-- const benchmark = await getBenchmark(id);++ const benchmark = await getBenchmark(id, { revealDraft: true });if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');const version = definition.version == null ? null : num(definition.version);if (version != null) {@@@ -273,10 +441,32 @@ function createRegistry(s, { log = console } = {}) {if (used > 0) {return refusal(409, 'bench.version_frozen', 'A version that a run names never changes again. Fork the benchmark instead.');}-- const next = { ...row.definition, ...definition, version };++ const tasks = Array.isArray(definition.tasks) ? definition.tasks : null;++ // A caller that read the contract-shaped definition has no B13 fields; keep the stored metadata for a++ // task it leaves unchanged rather than erasing a date or a time on every edit.++ const priorMeta = new Map((await loadTaskMeta(benchmark.id, [version])).map((m) => [String(m.task_id), m]));++ const mergedTasks = tasks ? tasks.map((task) => {++ const meta = priorMeta.get(String(task && task.id));++ if (!meta) return task;++ const out = { ...task };++ if (out.added_on == null && meta.added_on) out.added_on = String(meta.added_on).slice(0, 10);++ if (out.held_out == null && out.holdout == null && meta.held_out) out.held_out = true;++ if (out.human_minutes == null && meta.human_minutes != null) {++ out.human_minutes = num(meta.human_minutes);++ if (meta.human_minutes_source) out.human_minutes_source = meta.human_minutes_source;++ }++ return out;++ }) : null;++ const next = {++ ...row.definition,++ ...definition,++ version,++ ...(mergedTasks ? { tasks: mergedTasks.map(contractTask) } : {}),++ };await db.exec('UPDATE bench_benchmark_versions SET definition = $1::jsonb, hash = $2 WHERE benchmark_id = $3 AND version = $4',[JSON.stringify(next), hashDefinition(next), benchmark.id, version]);++ if (mergedTasks) await replaceTaskMeta(benchmark.id, version, mergedTasks);await db.exec('UPDATE bench_benchmarks SET updated_at = $1::timestamptz WHERE id = $2', [s.iso(), benchmark.id]);return { ok: true, version: await getVersion(benchmark.id, version) };}@@@ -288,9 +478,10 @@ function createRegistry(s, { log = console } = {}) {/** Freeze the working copy (or a body definition) into the next immutable version. */async function freezeVersion(id, { definition = null, subject, validate = null }) {-- const benchmark = await getBenchmark(id);++ const benchmark = await getBenchmark(id, { revealDraft: true });if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');const source = definition || benchmark.draft || {};++ const rawTasks = Array.isArray(source.tasks) ? source.tasks : [];const version = benchmark.latest_version + 1;// The published contract knows public and unlisted only. A private benchmark is frozen as unlisted (the// stricter form); the row's own visibility is what every access check reads, so its prompts stay private.@@@ -305,7 +496,7 @@ function createRegistry(s, { log = console } = {}) {visibility: frozenVisibility,version,parent: benchmark.parent,-- tasks: Array.isArray(source.tasks) ? source.tasks : [],++ tasks: rawTasks.map(contractTask),judges: Array.isArray(source.judges) ? source.judges : [],scoring: source.scoring || { method: 'mean', scale: { min: 0, max: 100 } },limits: source.limits || {},@@@ -324,15 +515,22 @@ function createRegistry(s, { log = console } = {}) {VALUES ($1, $2, $3::jsonb, $4, $5, $6::timestamptz)ON CONFLICT (benchmark_id, version) DO NOTHING`,[benchmark.id, version, JSON.stringify(doc), hashDefinition(doc), String(subject || benchmark.created_by), s.iso()]);++ await replaceTaskMeta(benchmark.id, version, rawTasks);await db.exec('UPDATE bench_benchmarks SET latest_version = $1, updated_at = $2::timestamptz WHERE id = $3',[version, s.iso(), benchmark.id]);return { ok: true, version: await getVersion(benchmark.id, version), benchmark: await getBenchmark(benchmark.id) };}/** A fork: a new benchmark of its own that records the parent it came from. */-- async function forkBenchmark(id, { version = null, title = null, owner, subject }) {-- const source = await getDefinition(id, { reveal: true });++ async function forkBenchmark(id, { version = null, title = null, owner, subject, reveal = true }) {++ const ref = version == null ? String(id || '') : `${String(id || '')}@${num(version)}`;++ const source = await getDefinition(ref, { reveal });if (!source) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');++ // A held-out prompt only goes to its owner or the executor. A fork by anyone else is refused rather than++ // copied with a blank prompt: the text never has to leave the server to make the copy.++ if (!reveal && source.definition.tasks.some((t) => isHeldOut(t))) {++ return refusal(403, 'bench.holdout_fork_refused', 'This benchmark has held-out tasks; only its owner or the executor can fork it.');++ }const parent = `${source.benchmark.id}@${source.version.version}`;const definition = {...source.definition,@@@ -411,16 +609,24 @@ function createRegistry(s, { log = console } = {}) {if (benchmark.visibility !== 'public') continue;const version = await latestVersion(benchmark.id);if (!version) continue;-- const definition = version.definition || {};++ const metaRows = await loadTaskMeta(benchmark.id, [version.version]);++ const definition = mergeTaskMeta(version.definition || {}, metaRows);lines.push(JSON.stringify({type: 'benchmark', id: benchmark.id, title: benchmark.title, description: benchmark.description,kind: benchmark.kind, tags: benchmark.tags, version: version.version, parent: benchmark.parent,created_at: benchmark.created_at,}));++ const estimates = await estimateMedians(benchmark.id, [version.version]);for (const task of (definition.tasks || [])) {-- if (!task || task.holdout) continue;++ if (!task || isHeldOut(task)) continue;const item = { type: 'benchmark_task', benchmark: `${benchmark.id}@${version.version}`, id: task.id, prompt: task.prompt };if (task.expected != null) item.expected = task.expected;++ if (addedOn(task)) item.added_on = addedOn(task);++ const timed = taskMinutes(task, `${num(version.version)}:${String(task.id)}`, estimates);++ if (timed) {++ item.human_minutes = timed.minutes;++ item.human_minutes_source = timed.source;++ }lines.push(JSON.stringify(item));}const runs = await db.many(@@@ -445,7 +651,7 @@ function createRegistry(s, { log = console } = {}) {* private benchmarks are never listed to anyone else (an unlisted one is reachable by its id; a private one only* by its owner and the executor).*/-- async function listBenchmarks({ kind = null, tag = null, q = null, limit = 50, owner = null } = {}) {++ async function listBenchmarks({ kind = null, tag = null, q = null, limit = 50, owner = null, viewer = null } = {}) {const where = [];const params = [];if (owner) { params.push(String(owner)); where.push(`(visibility = 'public' OR owner = $${params.length})`); }@@@ -459,7 +665,7 @@ function createRegistry(s, { log = console } = {}) {const sql = `SELECT * FROM bench_benchmarks ${where.length ? 'WHERE ' + where.join(' AND ') : ''}ORDER BY created_at DESC LIMIT ${Math.max(1, Math.min(200, num(limit, 50)))}`;const rows = await db.many(sql, params);-- return rows.map(wireBenchmark);++ return rows.map((row) => wireBenchmark(row, { revealDraft: viewer != null && String(row.owner) === String(viewer) }));}// ── Runs ─────────────────────────────────────────────────@@@ -489,12 +695,20 @@ function createRegistry(s, { log = console } = {}) {if (!provider) return refusal(422, 'bench.provider_required', 'A run names the provider that served the model.');const tasks = Array.isArray(doc.tasks) ? doc.tasks : [];if (!tasks.length) return refusal(422, 'bench.no_tasks', 'A run carries at least one task result.');-- const known = new Set((versionRow.definition.tasks || []).map((t) => String(t.id)));++ const metaRows = await loadTaskMeta(benchmarkId, [version]);++ const mergedDefinition = mergeTaskMeta(versionRow.definition, metaRows);++ const taskMetaById = new Map((mergedDefinition.tasks || []).map((t) => [String(t.id), t]));for (const t of tasks) {-- if (!known.has(String(t.task_id))) {++ if (!taskMetaById.has(String(t.task_id))) {return refusal(422, 'bench.unknown_task', `This benchmark version has no task "${t.task_id}".`);}}++ // A held-out task is scored only by the executor (a verified run). A submitted run, including an import or++ // a mirrored result, may name the task, but its score is dropped before anything is stored.++ const storedTasks = tasks.map((t) => {++ const meta = taskMetaById.get(String(t.task_id)) || {};++ return provenance === 'submitted' && isHeldOut(meta) ? { ...t, score: null } : t;++ });// A submitted run has no executor: whatever the body says, the provenance decides.const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };@@@ -505,7 +719,7 @@ function createRegistry(s, { log = console } = {}) {id,benchmark: `${benchmarkId}@${version}`,executor,-- tasks,++ tasks: storedTasks,score: doc.score == null ? null : num(doc.score, 0),cost_usd: num(doc.cost_usd, 0),ms: num(doc.ms, 0),@@@ -531,12 +745,15 @@ function createRegistry(s, { log = console } = {}) {JSON.stringify(doc.source || {}),doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],);-- for (const t of tasks) {++ for (const t of storedTasks) {++ const meta = taskMetaById.get(String(t.task_id)) || {};await db.exec(-- `INSERT INTO bench_run_tasks (id, run_id, task_id, score, cost_usd, ms, tokens, artifacts, transcript_url, created_at)-- VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::timestamptz)`,++ `INSERT INTO bench_run_tasks (id, run_id, task_id, score, cost_usd, ms, tokens, artifacts, transcript_url,++ added_on, held_out, human_minutes, human_minutes_source, created_at)++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::date, $11, $12, $13, $14::timestamptz)`,[s.newId('brt'), id, String(t.task_id), t.score == null ? null : num(t.score), num(t.cost_usd), num(t.ms), num(t.tokens),-- JSON.stringify(Array.isArray(t.artifacts) ? t.artifacts : []), t.transcript_url == null ? null : String(t.transcript_url), s.iso()],++ JSON.stringify(Array.isArray(t.artifacts) ? t.artifacts : []), t.transcript_url == null ? null : String(t.transcript_url),++ addedOn(meta), isHeldOut(meta), humanMinutes(meta), humanMinutesSource(meta), s.iso()],);}});@@@ -575,12 +792,47 @@ function createRegistry(s, { log = console } = {}) {};}++ /** Median of each task's editor estimates, when at least three people have entered one. */++ async function estimateMedians(benchmarkId, versions) {++ const out = new Map();++ if (!versions.length) return out;++ const rows = await db.many(++ `SELECT benchmark_version, task_id, minutes FROM bench_task_estimates++ WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])`,++ [String(benchmarkId), versions]);++ const grouped = new Map();++ for (const row of rows) {++ const key = `${num(row.benchmark_version)}:${String(row.task_id)}`;++ if (!grouped.has(key)) grouped.set(key, []);++ grouped.get(key).push(num(row.minutes));++ }++ for (const [key, list] of grouped) {++ if (list.length >= 3) out.set(key, { median: median(list), count: list.length });++ }++ return out;++ }++++ /** The minutes of a task: the author's own estimate first, else the median of three editors. */++ function taskMinutes(task, key, estimates) {++ const author = humanMinutes(task);++ if (author != null) return { minutes: author, source: humanMinutesSource(task) || 'benchmark author' };++ const estimate = estimates.get(key);++ if (estimate) return { minutes: estimate.median, source: `median of ${estimate.count} editor estimates` };++ return null;++ }++/*** The rankings for a benchmark. Verified runs only unless the caller asks for the submitted ones.* score the median score of a model* value the median score per dollar* speed the median score per minute* consistency the spread of a model's scores inside one session (the lower, the steadier)++ * horizon the 50% human-time horizon (minutes; higher is better)++ *++ * `since` is a monthly fresh set: with dated tasks, only the tasks added on or after that month are scored.++ * An older benchmark whose tasks carry no dates keeps the B2 meaning (runs created since the date), so nothing++ * that worked before stops working. Each entry also carries the contamination flag and the time horizon when++ * there is enough data for either.*/async function rankings({ benchmark, metric = 'score', harness = null, since = null, includeSubmitted = false }) {const ref = parseRef(benchmark);@@@ -590,34 +842,137 @@ function createRegistry(s, { log = console } = {}) {[ref.id]);const wanted = ref.version != null ? [ref.version] : versions.map((v) => num(v.version));if (!wanted.length) return null;++ const definitions = new Map();++ for (const version of wanted) {++ const row = await getVersion(ref.id, version);++ if (row) definitions.set(num(version), row.definition || {});++ }++ if (!definitions.size) return null;++++ const sinceParsed = since ? parseSince(since) : null;++ const metaRows = await loadTaskMeta(ref.id, wanted);++ const metaByKey = taskMetaMap(wanted, definitions, metaRows);++ const datedTasks = [...metaByKey.values()].filter((meta) => meta.added_on);++ const taskBasis = Boolean(sinceParsed && datedTasks.length);++ const selected = new Set();++ if (taskBasis) {++ for (const [key, meta] of metaByKey) {++ if (meta.added_on && meta.added_on >= sinceParsed.cutoff) selected.add(key);++ }++ }++const params = [ref.id, wanted];let sql = `SELECT * FROM bench_runs WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])`;if (!includeSubmitted) sql += " AND provenance = 'verified'";// Each filter takes the next placeholder: `since` alone is $3, not $4.if (harness) { params.push(String(harness)); sql += ` AND harness->>'name' = $${params.length}`; }-- if (since) { params.push(String(since)); sql += ` AND created_at >= $${params.length}::timestamptz`; }++ if (sinceParsed && !taskBasis) {++ params.push(sinceParsed.timestamp);++ sql += ` AND created_at >= $${params.length}::timestamptz`;++ }const rows = await db.many(sql, params);++ const runIds = rows.map((row) => String(row.id));++ const taskRows = runIds.length++ ? await db.many('SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id', [runIds])++ : [];++ const tasksByRun = new Map();++ for (const row of taskRows) {++ if (!tasksByRun.has(row.run_id)) tasksByRun.set(row.run_id, []);++ tasksByRun.get(row.run_id).push(row);++ }++ const estimates = await estimateMedians(ref.id, wanted);++++ /** The score and totals of one run, over the selected tasks when a monthly set is asked for. */++ function subsetOf(run) {++ if (!taskBasis) {++ return { score: run.score, cost_usd: run.cost_usd, ms: run.ms, tokens: run.tokens };++ }++ const picked = (tasksByRun.get(run.id) || []).filter((row) => {++ if (row.score == null) return false;++ return selected.has(`${num(run.benchmark_version)}:${String(row.task_id)}`);++ });++ if (!picked.length) return null;++ return {++ score: picked.reduce((n, row) => n + num(row.score), 0) / picked.length,++ cost_usd: picked.reduce((n, row) => n + num(row.cost_usd), 0),++ ms: picked.reduce((n, row) => n + num(row.ms), 0),++ tokens: picked.reduce((n, row) => n + num(row.tokens), 0),++ };++ }++++ /** The contamination flag: per run, the mean of its old tasks and of its fresh tasks. */++ function contaminationOf(model, list) {++ const cutoff = cutoffFor(model);++ if (!cutoff) return contaminationFor({ cutoff: null });++ const oldScores = [];++ const freshScores = [];++ for (const run of list) {++ let oldSum = 0; let oldN = 0; let freshSum = 0; let freshN = 0;++ for (const row of tasksByRun.get(run.id) || []) {++ if (row.score == null) continue;++ const meta = metaByKey.get(`${num(run.benchmark_version)}:${String(row.task_id)}`) || {};++ const day = row.added_on ? String(row.added_on).slice(0, 10) : (meta.added_on || null);++ if (!day) continue;++ if (day < cutoff.cutoff) { oldSum += num(row.score); oldN++; }++ else { freshSum += num(row.score); freshN++; }++ }++ if (oldN) oldScores.push(oldSum / oldN);++ if (freshN) freshScores.push(freshSum / freshN);++ }++ return contaminationFor({ oldScores, freshScores, cutoff: cutoff.cutoff, source: cutoff.source });++ }++++ /** The time horizon of one model over the runs in scope. */++ function horizonOfModel(list) {++ const groups = new Map();++ let timedRuns = 0;++ for (const run of list) {++ let touched = false;++ for (const row of tasksByRun.get(run.id) || []) {++ if (row.score == null) continue;++ const version = num(run.benchmark_version);++ const key = `${version}:${String(row.task_id)}`;++ const meta = metaByKey.get(key) || {};++ const timed = taskMinutes(meta, key, estimates);++ if (!timed) continue;++ if (!groups.has(key)) groups.set(key, { minutes: timed.minutes, source: timed.source, scores: [] });++ groups.get(key).scores.push(num(row.score));++ touched = true;++ }++ if (touched) timedRuns++;++ }++ const taskGroups = [...groups.values()].map((g) => ({++ minutes: g.minutes,++ success: g.scores.reduce((n, v) => n + v, 0) / g.scores.length / 100,++ runs: g.scores.length,++ source: g.source,++ }));++ return estimateHorizon(taskGroups, { runs: timedRuns, timedTasks: taskGroups.length });++ }++const byModel = new Map();for (const row of rows) {const run = wireRun(row);++ const subset = subsetOf(run);++ if (sinceParsed && taskBasis && !subset) continue;if (!byModel.has(run.model)) byModel.set(run.model, []);-- byModel.get(run.model).push(run);++ byModel.get(run.model).push({ run, subset });}const entries = [];for (const [model, list] of byModel) {-- const scores = list.map((r) => r.score);++ const scores = list.map((item) => item.subset && item.subset.score);const cleanScores = scores.filter((v) => v != null);-- const costs = list.map((r) => r.cost_usd);-- const times = list.map((r) => r.ms);++ const costs = list.map((item) => (item.subset ? item.subset.cost_usd : null));++ const times = list.map((item) => (item.subset ? item.subset.ms : null));const medScore = median(scores);const medCost = median(costs);const medMs = median(times);const sessions = new Map();-- for (const r of list) {-- if (r.score == null) continue;-- const key = r.session || 'default';++ for (const item of list) {++ if (!item.subset || item.subset.score == null) continue;++ const key = item.run.session || 'default';if (!sessions.has(key)) sessions.set(key, []);-- sessions.get(key).push(r.score);++ sessions.get(key).push(item.subset.score);}const spreads = [...sessions.values()].filter((v) => v.length > 1).map((v) => Math.max(...v) - Math.min(...v));const entry = {@@@ -636,14 +991,19 @@ function createRegistry(s, { log = console } = {}) {max_score: cleanScores.length ? Math.max(...cleanScores) : null,spread: cleanScores.length ? Math.round((Math.max(...cleanScores) - Math.min(...cleanScores)) * 1000) / 1000 : null,cost_per_solved: medScore > 0 && medCost > 0 ? Math.round((medCost / (medScore / 100)) * 1e6) / 1e6 : null,++ contamination: contaminationOf(model, list.map((item) => item.run)),++ horizon: horizonOfModel(list.map((item) => item.run)),};entries.push(entry);}-- const key = { score: 'score', value: 'value', speed: 'speed', consistency: 'consistency' }[metric];-- if (!key) return null;++ const valueOf = (entry) => {++ if (metric === 'horizon') return entry.horizon && entry.horizon.available ? entry.horizon.horizon_minutes : null;++ return entry[{ score: 'score', value: 'value', speed: 'speed', consistency: 'consistency' }[metric]];++ };++ if (!['score', 'value', 'speed', 'consistency', 'horizon'].includes(metric)) return null;const ascending = metric === 'consistency';entries.sort((a, b) => {-- const av = a[key]; const bv = b[key];++ const av = valueOf(a); const bv = valueOf(b);if (av == null && bv == null) return a.model.localeCompare(b.model);if (av == null) return 1;if (bv == null) return -1;@@@ -668,7 +1028,127 @@ function createRegistry(s, { log = console } = {}) {e.band_to = band ? band.to : null;}}-- return { benchmark: ref.id, metric, entries };++ return {++ benchmark: ref.id,++ metric,++ since: sinceParsed ? { month: sinceParsed.month, label: sinceParsed.label, basis: taskBasis ? 'tasks' : 'runs' } : null,++ entries,++ };++ }++++ // ── Human-time estimates (B13) ───────────────────────────++ const ESTIMATES_PER_HOUR = 30;++++ async function taskEstimateSummary(benchmarkId, version, taskId) {++ const rows = await db.many(++ `SELECT minutes FROM bench_task_estimates++ WHERE benchmark_id = $1 AND benchmark_version = $2 AND task_id = $3 ORDER BY created_at`,++ [String(benchmarkId), num(version), String(taskId)]);++ const values = rows.map((row) => num(row.minutes));++ return {++ count: values.length,++ needed: Math.max(0, 3 - values.length),++ median: values.length >= 3 ? Math.round(median(values) * 10) / 10 : null,++ values,++ };++ }++++ /**++ * One editor's estimate on a task. A person has one estimate per task; a later entry replaces it. More than++ * thirty entries an hour from one person is refused, so a script cannot flood the median.++ */++ async function addTaskEstimate({ benchmarkId, version, taskId, minutes, subject }) {++ if (!subject) return refusal(401, 'bench.sign_in_required', 'Sign in to estimate a task.');++ const value = Number(minutes);++ if (!Number.isFinite(value) || value <= 0 || value > 100000) {++ return refusal(422, 'bench.invalid_minutes', 'The estimate must be a positive number of minutes.');++ }++ const versionRow = await getVersion(benchmarkId, version);++ if (!versionRow) return refusal(404, 'bench.version_not_found', 'No such benchmark version.');++ const task = (versionRow.definition.tasks || []).find((t) => String(t.id) === String(taskId));++ if (!task) return refusal(404, 'bench.task_not_found', 'No such task.');++ const recent = num(await db.value(++ `SELECT COUNT(*)::int AS n FROM bench_task_estimates++ WHERE subject = $1 AND created_at >= now() - interval '1 hour'`, [String(subject)]));++ if (recent >= ESTIMATES_PER_HOUR) {++ return refusal(429, 'bench.estimate_rate_limited', 'Too many estimates in an hour. Try again later.');++ }++ const row = await db.maybe(++ `INSERT INTO bench_task_estimates (id, benchmark_id, benchmark_version, task_id, subject, minutes, created_at)++ VALUES ($1, $2, $3, $4, $5, $6, $7::timestamptz)++ ON CONFLICT (benchmark_id, benchmark_version, task_id, subject)++ DO UPDATE SET minutes = EXCLUDED.minutes, created_at = EXCLUDED.created_at++ RETURNING *`,++ [s.newId('bte'), String(benchmarkId), num(version), String(taskId), String(subject), Math.round(value * 10) / 10, s.iso()]);++ return { ok: true, estimate: { task_id: String(taskId), minutes: num(row.minutes), created_at: iso(row.created_at) }, summary: await taskEstimateSummary(benchmarkId, version, taskId) };++ }++++ /** The time horizon of one model over every timed task it ran on a public benchmark. */++ async function modelHorizon(model) {++ const rows = await db.many(++ `SELECT id, benchmark_id, benchmark_version FROM bench_runs++ WHERE model = $1 AND provenance = 'verified'++ AND benchmark_id IN (SELECT id FROM bench_benchmarks WHERE visibility = 'public')`,++ [String(model || '')]);++ if (!rows.length) return estimateHorizon([], { runs: 0, timedTasks: 0 });++ const definitions = new Map();++ for (const row of rows) {++ const key = `${row.benchmark_id}:${num(row.benchmark_version)}`;++ if (!definitions.has(key)) {++ const versionRow = await getVersion(row.benchmark_id, row.benchmark_version);++ definitions.set(key, versionRow ? versionRow.definition : null);++ }++ }++ const metaRows = await db.many(++ `SELECT m.* FROM bench_task_meta m JOIN bench_benchmarks b ON b.id = m.benchmark_id++ WHERE b.visibility = 'public'`);++ const metaByKey = new Map(metaRows.map((row) => [`${row.benchmark_id}:${num(row.benchmark_version)}:${String(row.task_id)}`, {++ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,++ held_out: Boolean(row.held_out),++ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),++ human_minutes_source: row.human_minutes_source || null,++ }]));++ const estimates = new Map();++ const estimateRows = await db.many(++ `SELECT e.benchmark_id, e.benchmark_version, e.task_id, e.minutes++ FROM bench_task_estimates e JOIN bench_benchmarks b ON b.id = e.benchmark_id++ WHERE b.visibility = 'public'`);++ const byTask = new Map();++ for (const row of estimateRows) {++ const key = `${row.benchmark_id}:${num(row.benchmark_version)}:${String(row.task_id)}`;++ if (!byTask.has(key)) byTask.set(key, []);++ byTask.get(key).push(num(row.minutes));++ }++ for (const [key, list] of byTask) if (list.length >= 3) estimates.set(key, { median: median(list), count: list.length });++++ const tasks = await db.many(++ 'SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id',++ [rows.map((row) => String(row.id))]);++ const groups = new Map();++ const runIds = new Set();++ const metaOf = (benchmarkId, version, taskId) => {++ const definition = definitions.get(`${benchmarkId}:${num(version)}`);++ return ((definition && definition.tasks) || []).find((t) => String(t.id) === String(taskId)) || {};++ };++ for (const row of tasks) {++ if (row.score == null) continue;++ const run = rows.find((r) => String(r.id) === String(row.run_id));++ if (!run) continue;++ const key = `${run.benchmark_id}:${num(run.benchmark_version)}:${String(row.task_id)}`;++ const meta = metaByKey.get(key) || metaOf(run.benchmark_id, run.benchmark_version, row.task_id);++ const timed = taskMinutes(meta, key, estimates);++ if (!timed) continue;++ if (!groups.has(key)) groups.set(key, { minutes: timed.minutes, source: timed.source, scores: [] });++ groups.get(key).scores.push(num(row.score));++ runIds.add(String(row.run_id));++ }++ const taskGroups = [...groups.values()].map((g) => ({++ minutes: g.minutes,++ success: g.scores.reduce((n, v) => n + v, 0) / g.scores.length / 100,++ runs: g.scores.length,++ source: g.source,++ }));++ return estimateHorizon(taskGroups, { runs: runIds.size, timedTasks: taskGroups.length });}/** Where a score sits among a model's runs on a benchmark: the percentile, 0-100. */@@@ -769,10 +1249,11 @@ function createRegistry(s, { log = console } = {}) {createBenchmark, getBenchmark, getVersion, latestVersion, getDefinition, saveDraft, freezeVersion,forkBenchmark, setVisibility, findShared, publicDatasetLines,listBenchmarks, publishRun, getRun, compareRuns, rankings, percentile, listRuns, modelSummary, runCounts,++ taskEstimateSummary, addTaskEstimate, modelHorizon,};}module.exports = {-- createRegistry, RegistryError, hashDefinition, canonical, median, parseRef,++ createRegistry, RegistryError, hashDefinition, canonical, median, parseRef, parseSince,canViewBenchmark, shareTokenHash, shareTokenMatches, newShareToken,};@@@ -36,6 +36,11 @@ function tasksFor(list) {files: [],expected: null,holdout: false,++ // A monthly fresh set on day one: the first task of each showcase kind was added in September, the second++ // in October. The human time is the benchmark author's own estimate.++ added_on: i === 0 ? '2026-09-18' : '2026-10-20',++ human_minutes: i === 0 ? 45 : 30,++ human_minutes_source: 'benchmark author',}));}nnew file mode 100644@@@ -0,0 +1,38 @@++'use strict';++++/**++ * The task fields B13 adds, read the same way everywhere: a date a task was added, whether it is held out, and a++ * human time estimate with its source. `holdout` is the B1 spelling; `held_out` is the B13 one. Both mean the same.++ */++++const DATE = /^(\d{4})-(\d{2})-(\d{2})$/;++++function isHeldOut(task) {++ return Boolean(task && (task.holdout === true || task.held_out === true));++}++++/** "YYYY-MM-DD", or null. The definition validates it; this is the read side. */++function addedOn(task) {++ const text = String((task && task.added_on) || '').slice(0, 10);++ return DATE.test(text) && !Number.isNaN(Date.parse(`${text}T00:00:00Z`)) ? text : null;++}++++function humanMinutes(task) {++ const n = Number(task && task.human_minutes);++ return Number.isFinite(n) && n > 0 ? Math.round(n * 10) / 10 : null;++}++++function humanMinutesSource(task) {++ const text = String((task && task.human_minutes_source) || '').trim();++ return text ? text.slice(0, 200) : null;++}++++/** The first day of a "YYYY-MM" month, or null. */++function monthStart(text) {++ const m = /^(\d{4})-(\d{2})$/.exec(String(text || ''));++ if (!m) return null;++ const month = Number(m[2]);++ return month >= 1 && month <= 12 ? `${m[1]}-${m[2]}-01` : null;++}++++module.exports = { isHeldOut, addedOn, humanMinutes, humanMinutesSource, monthStart };@@@ -30,6 +30,7 @@ const VISIBILITIES = ['public', 'unlisted', 'private'];const SCORING_METHODS = ['mean', 'weighted', 'pass_rate', 'sum'];const EXECUTOR_KINDS = ['codes', 'run', 'node', 'submitted'];const PROVENANCES = ['verified', 'submitted'];++const TASK_DATE = /^\d{4}-\d{2}-\d{2}$/;class Errors {constructor() { this.list = []; }@@@ -55,7 +56,21 @@ function validateBenchmark(doc) {if (!isObj(t)) { e.add(`/tasks/${i}`, 'must be an object'); return; }if (!isStr(t.id) || !t.id.trim()) e.add(`/tasks/${i}/id`, 'is required');if (!isStr(t.prompt) || !t.prompt.trim()) e.add(`/tasks/${i}/prompt`, 'is required');-- if (typeof t.holdout !== 'boolean') e.add(`/tasks/${i}/holdout`, 'must be true or false');++ if (t.holdout != null && typeof t.holdout !== 'boolean') e.add(`/tasks/${i}/holdout`, 'must be true or false');++ if (t.held_out != null && typeof t.held_out !== 'boolean') e.add(`/tasks/${i}/held_out`, 'must be true or false');++ if (t.holdout == null && t.held_out == null) e.add(`/tasks/${i}/holdout`, 'must be true or false');++ if (t.added_on != null) {++ const date = String(t.added_on).slice(0, 10);++ if (!TASK_DATE.test(date) || Number.isNaN(Date.parse(`${date}T00:00:00Z`))) {++ e.add(`/tasks/${i}/added_on`, 'must be a date (YYYY-MM-DD)');++ }++ }++ if (t.human_minutes != null && (!isNum(t.human_minutes) || t.human_minutes <= 0 || t.human_minutes > 100000)) {++ e.add(`/tasks/${i}/human_minutes`, 'must be a positive number of minutes');++ }++ if (t.human_minutes_source != null && (!isStr(t.human_minutes_source) || t.human_minutes_source.length > 200)) {++ e.add(`/tasks/${i}/human_minutes_source`, 'must be a string of at most 200 characters');++ }});}if (!Array.isArray(doc.judges)) e.add('/judges', 'must be an array');@@@ -13,7 +13,7 @@ const { raw } = require('../render/html');const { send } = require('../render/layout');const views = require('../views');const results = require('../views/results');--const { canViewBenchmark } = require('../bench/registry');++const { canViewBenchmark, parseSince } = require('../bench/registry');const { parsePrompts, definitionFor, TAG: PROMPT_TAG } = require('../bench/your-prompts');const { validateBenchmark } = require('../bench/validate');const { kindById } = require('../kinds');@@@ -96,10 +96,12 @@ function createPageRoutes(ctx) {const metric = METRIC_IDS.includes(String(req.query.metric)) ? String(req.query.metric) : 'score';const id = out.benchmark.id;const owner = subjectOf(req) != null && String(out.benchmark.owner) === String(subjectOf(req));++ const since = parseSince(req.query.since);const [rankings, runs] = await Promise.all([-- registry.rankings({ benchmark: id, metric }),++ registry.rankings({ benchmark: id, metric, since: since ? since.text : null }),registry.listRuns({ benchmark: id, limit: 8 }),]);++ const allTime = since ? await registry.rankings({ benchmark: id, metric }) : null;const isPublic = out.benchmark.visibility === 'public';const isPromptBenchmark = Array.isArray(out.benchmark.tags) && out.benchmark.tags.includes(PROMPT_TAG);const ownBoards = owner && isPromptBenchmark@@@ -120,7 +122,7 @@ function createPageRoutes(ctx) {body: raw(String(results.benchmarkPage({benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,rankings, metric, runs, codesRunUrl: config.codesRunUrl || null, viewer: req.viewer, baseUrl: base(),-- ownBoards, share, shared, owner,++ ownBoards, share, shared, owner, since: rankings && rankings.since, allTimeEntries: allTime && allTime.entries,}))),}, status);};@@@ -225,6 +227,48 @@ function createPageRoutes(ctx) {});});++ // One task, and the signed-in editors' human-time estimates (B13). A plain form, same-origin, rate-limited in++ // the registry per person; nothing here logs the text of a held-out task.++ const taskContext = async (req) => {++ const out = await readableBenchmark(req, req.params.ref);++ if (!out) return null;++ const version = out.version && out.version.version;++ const task = ((out.definition && out.definition.tasks) || []).find((t) => String(t.id) === String(req.params.taskId));++ if (!task) return null;++ const estimates = await registry.taskEstimateSummary(out.benchmark.id, version, task.id);++ return { out, version, task, estimates };++ };++ const taskPage = async (req, res, extra = {}) => {++ const found = await taskContext(req);++ if (!found) return notFound(req, res, 'task');++ const subject = subjectOf(req);++ return page(req, res, {++ index: false, title: `${found.task.id} on ${found.out.benchmark.title}`,++ description: `A task of ${found.out.benchmark.title}: its human-time estimate and the editors’ median.`,++ body: raw(String(results.taskPage({++ benchmark: found.out.benchmark, version: found.version, task: found.task, estimates: found.estimates,++ signedIn: Boolean(subject), loginUrl: `/auth/login?next=${encodeURIComponent(req.originalUrl)}`,++ ...extra,++ }))),++ }, extra.status || 200);++ };++ r.get('/benchmarks/:ref/tasks/:taskId', (req, res) => taskPage(req, res));++ r.post('/benchmarks/:ref/tasks/:taskId/estimates', express.urlencoded({ extended: false, limit: '8kb' }), ctx.principal.middleware, async (req, res) => {++ const subject = subjectOf(req);++ const found = await taskContext(req);++ if (!found) return notFound(req, res, 'task');++ if (!subject) return taskPage(req, res, { error: 'Sign in to add an estimate.', status: 401 });++ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');++ const body = req.body && typeof req.body === 'object' ? req.body : {};++ const version = body.version == null ? found.version : Number(body.version);++ const out = await registry.addTaskEstimate({++ benchmarkId: found.out.benchmark.id, version, taskId: found.task.id,++ minutes: body.minutes, subject,++ });++ if (!out.ok) return taskPage(req, res, { error: out.message || 'The estimate could not be saved.', status: out.status || 422 });++ return res.redirect(303, `/benchmarks/${encodeURIComponent(found.out.benchmark.id)}/tasks/${encodeURIComponent(found.task.id)}`);++ });++r.get('/benchmarks/:ref', async (req, res) => {const out = await readableBenchmark(req, req.params.ref);if (!out) return notFound(req, res, 'benchmark');@@@ -308,9 +352,10 @@ function createPageRoutes(ctx) {r.get('/models/:model', async (req, res) => {const model = String(req.params.model || '').slice(0, 160);-- const [summary, runs] = await Promise.all([++ const [summary, runs, horizon] = await Promise.all([registry.modelSummary(model),registry.listRuns({ model, publicOnly: true, limit: 12 }),++ registry.modelHorizon(model),]);const titles = {};for (const b of summary.benchmarks) titles[b.benchmark_id] = b.title;@@@ -321,7 +366,7 @@ function createPageRoutes(ctx) {index: summary.runs > 0, cache: PUBLIC_CACHE,title: model,description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,-- body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base() }))),++ body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base(), horizon }))),}, summary.runs > 0 || runs.length ? 200 : 404);});@@@ -75,6 +75,18 @@ function createRegistryApi(ctx) {});const mayRead = (req, benchmark) => canViewBenchmark(benchmark, viewerOf(req));++ /** The owner, including a service token that created the benchmark, may see its held-out task text. */++ const owns = (req, benchmark) => {++ if (!benchmark) return false;++ const person = subjectOf(req);++ if (person != null && String(benchmark.owner) === String(person)) return true;++ return req.principal.kind === 'app' && String(benchmark.owner) === String(req.principal.requester);++ };++ const maySeeTaskText = async (req, id) => {++ const bm = await registry.getBenchmark(String(id || ''));++ return mayReveal(req) || owns(req, bm);++ };++// ── Benchmarks ───────────────────────────────────────────r.post('/benchmarks', actor('bench.benchmark.write'), async (req, res) => {const b = req.body && typeof req.body === 'object' ? req.body : {};@@@ -103,7 +115,13 @@ function createRegistryApi(ctx) {try {const out = await registry.saveDraft(String(req.params.id || ''), { definition: b.definition || b, subject });if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');-- return res.json({ benchmark: out.benchmark || null, version: out.version || null, draft: out.draft || null });++ const benchmark = await registry.getBenchmark(String(req.params.id || ''));++ let version = null;++ if (out.version) {++ const seen = await registry.getDefinition(`${out.version.benchmark_id}@${out.version.version}`, { reveal: await maySeeTaskText(req, req.params.id) });++ version = seen ? seen.version : null;++ }++ return res.json({ benchmark: benchmark || null, version, draft: benchmark ? benchmark.draft : null });} catch (err) { return failed(res, err); }});@@@ -115,7 +133,8 @@ function createRegistryApi(ctx) {const out = await registry.freezeVersion(String(req.params.id || ''), { definition: body, subject, validate: validateBenchmark });if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}@${out.version.version}`);-- return res.status(201).json({ benchmark: out.benchmark, version: out.version });++ const seen = await registry.getDefinition(`${out.benchmark.id}@${out.version.version}`, { reveal: await maySeeTaskText(req, out.benchmark.id) });++ return res.status(201).json({ benchmark: out.benchmark, version: seen ? seen.version : out.version, task_meta: seen ? seen.task_meta : [] });} catch (err) { return failed(res, err); }});@@@ -123,11 +142,14 @@ function createRegistryApi(ctx) {const b = req.body && typeof req.body === 'object' ? req.body : {};const subject = subjectOf(req) || req.principal.requester;try {++ const bm = await registry.getBenchmark(String(req.params.id || ''));++ if (!bm || !mayRead(req, bm)) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');const out = await registry.forkBenchmark(String(req.params.id || ''), {version: b.version == null ? null : Number(b.version),title: b.title || null,owner: subject,subject,++ reveal: mayReveal(req) || owns(req, bm),});if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}`);@@@ -152,6 +174,7 @@ function createRegistryApi(ctx) {const mine = req.query.mine && req.principal && req.principal.kind === 'user' ? req.principal.subject : null;const list = await registry.listBenchmarks({owner: mine,++ viewer: mine,kind: req.query.kind ? String(req.query.kind) : null,tag: req.query.tag ? String(req.query.tag) : null,q: req.query.q ? String(req.query.q) : null,@@@ -170,18 +193,19 @@ function createRegistryApi(ctx) {// A private benchmark answers only its owner and the executor; a shared unlisted one only its owner by// id (the token link is the way in). Everyone else gets a 404, not a "forbidden".if (!mayRead(req, out.benchmark)) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');-- const view = viewerOf(req);-- const owner = view.subject != null && String(out.benchmark.owner) === String(view.subject);-- const reveal = mayReveal(req) || owner;++ const reveal = mayReveal(req) || owns(req, out.benchmark);++ let definition = out.version.definition;++ let taskMeta = out.task_meta;if (reveal) {const full = await registry.getDefinition(ref, { reveal: true });-- if (full) { out.definition = full.definition; out.version = full.version; }++ if (full) { definition = full.version.definition; taskMeta = full.task_meta; }}res.set('Cache-Control', out.benchmark.visibility === 'public' ? 'public, max-age=60' : 'private, no-store');return res.json({benchmark: out.benchmark,-- version: { ...out.version, definition: out.definition },-- definition: out.definition,++ version: { ...out.version, definition },++ definition,++ task_meta: taskMeta,});} catch (err) { return failed(res, err); }});@@@ -287,8 +311,8 @@ function createRegistryApi(ctx) {const benchmark = req.query.benchmark ? String(req.query.benchmark) : '';if (!benchmark) return invalid(res, 'benchmark is required: ?benchmark=bmk_…');const metric = req.query.metric ? String(req.query.metric) : 'score';-- if (!['score', 'value', 'speed', 'consistency'].includes(metric)) {-- return invalid(res, 'metric is score, value, speed or consistency');++ if (!['score', 'value', 'speed', 'consistency', 'horizon'].includes(metric)) {++ return invalid(res, 'metric is score, value, speed, consistency or horizon');}try {const bm = await registry.getBenchmark(benchmark.split('@')[0]);@@@ -37,6 +37,8 @@ const TABLES = [{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },// The results files they brought (B12): theirs, so they go with them. The bytes hang off the row and go with it.{ table: 'bench_imports', subject: 'owner', file: 'bench_imports.json', erase: 'delete' },++ // The human-time estimates they entered on a task (B13): theirs, one per task, exported and erased with them.++ { table: 'bench_task_estimates', subject: 'subject', file: 'bench_task_estimates.json', erase: 'delete' },];// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang@@@ -107,7 +109,7 @@ function create({ db, log = console } = {}) {tables: TABLES,extraExport,extraErase,-- note: 'Bench exports your tasks, rounds, judgements, private evals, benchmarks and their prompts, the runs on your private benchmarks, watched benchmarks and the pairs you have seen; the public dataset is anonymous and a rating never names a person.',++ note: 'Bench exports your tasks, rounds, judgements, private evals, benchmarks and their prompts, the runs on your private benchmarks, watched benchmarks, the pairs you have seen and the human-time estimates you entered; the public dataset is anonymous and a rating never names a person.',log,});}@@@ -58,12 +58,12 @@ ${ep('GET', '/api/v1/evals/:id/results', 'A private eval and its runs.')}</dl></section><section id="br-publish"><h2>Benchmarks and runs</h2><p class="bm-fine">A benchmark is a versioned set of tasks, judges and limits; a frozen version never changes once a run names it. A run published by a service (openvibe.codes, Run, a Node worker) is <b>verified</b>; one published with a person's token is <b>submitted</b>, labelled, and never ranked by default. Your own prompts can be pasted at <a href="/benchmarks/new">/benchmarks/new</a>: 5 to 50 of them, private by default, with an unlisted share link or a public listing when you choose.</p><dl class="bm-eps">${ep('GET', '/api/v1/benchmarks?kind=&tag=&q=&mine=1', 'The public benchmarks; <code>mine=1</code> adds your own unlisted and private ones.')}--${ep('GET', '/api/v1/benchmarks/:id[@version]', 'A definition. Held-out tasks keep their prompt for the executor and the owner; a private benchmark is a 404 to everyone else.')}++${ep('GET', '/api/v1/benchmarks/:id[@version]', 'A definition, and its <code>task_meta</code> (added_on, held_out, human_minutes and its source). Held-out tasks keep their prompt for the executor and the owner; a private benchmark is a 404 to everyone else.')}${ep('POST', '/api/v1/benchmarks', 'Create one (<code>bench.benchmark.write</code>); <code>PUT</code> saves a draft, <code>POST …/versions</code> freezes it, <code>POST …/fork</code> forks it.')}${ep('POST', '/api/v1/benchmarks/:id/share', '<code>{ visibility: "private" | "unlisted" | "public" }</code>. The owner only. An unlisted call returns a new share token once; the old link stops working.')}${ep('POST', '/api/v1/runs', 'Publish a run: <code>{ benchmark: "bmk_…@N", model, provider, harness: { name, version }, executor, tasks: [{ task_id, score, cost_usd, ms, tokens, artifacts }], score, cost_usd, ms, tokens }</code>.')}${ep('GET', '/api/v1/runs/:id', 'A run with its tasks; <code>/api/v1/compare?runs=a,b</code> puts up to four side by side.')}--${ep('GET', '/api/v1/rankings?benchmark=&metric=score|value|speed|consistency&harness=&since=', 'Medians per model over verified runs; value is score per dollar, speed is score per minute.')}++${ep('GET', '/api/v1/rankings?benchmark=&metric=score|value|speed|consistency|horizon&harness=&since=', 'Medians per model over verified runs; value is score per dollar, speed is score per minute, horizon is the 50% human-time task length. since=YYYY-MM scores only tasks added in or after that month.')}${ep('GET', '/api/v1/percentile?benchmark=&model=&score=', 'Where a score sits among a model’s verified runs.')}</dl></section><section><h2>MCP</h2><p class="bm-fine"><code>POST /mcp</code> exposes <code>bench.list_kinds</code>, <code>bench.start_round</code>, <code>bench.next_pair</code>, <code>bench.cast_judgement</code>, <code>bench.get_board</code> and <code>bench.run_eval</code>.</p></section>@@@ -37,6 +37,12 @@ function fmtTokens(v) {return String(n);}const fmtDate = (iso) => (iso ? html`<time datetime="${iso}">${String(iso).slice(0, 10)}</time>` : '—');++const MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];++function monthLabel(month) {++ const m = /^(\d{4})-(\d{2})$/.exec(String(month || ''));++ if (!m) return String(month || '');++ return `${MONTHS[Number(m[2]) - 1]} ${m[1]}`;++}function ordinal(n) {const v = Math.round(Number(n));const t = v % 100;@@@ -175,17 +181,57 @@ const METRICS = [{ id: 'value', label: 'Value', hint: 'score per dollar' },{ id: 'speed', label: 'Speed', hint: 'score per minute' },{ id: 'consistency', label: 'Steadiness', hint: 'spread inside a session, lower is steadier' },++ { id: 'horizon', label: 'Horizon', hint: 'the human-length task the model does 50% of the time, after METR' },];++function fmtMinutes(v) {++ if (v == null) return '—';++ const n = Number(v);++ if (n < 1) return `${Math.round(n * 60)} s`;++ if (n < 90) return `${Math.round(n * 10) / 10} min`;++ return `${Math.round(n / 6) / 10} h`;++}++++/** The 50% time horizon, with its bootstrap interval; never a number below the gates. */++function horizonText(h) {++ if (!h || !h.available) return '—';++ return `${fmtMinutes(h.horizon_minutes)} (${fmtMinutes(h.ci_lower)}–${fmtMinutes(h.ci_upper)})`;++}++++function horizonLine(h) {++ if (!h) return '';++ if (h.available) {++ return html`<p class="br-horizon"><b>Time horizon:</b> reliably does about ${fmtMinutes(h.horizon_minutes)} tasks. 90% interval ${fmtMinutes(h.ci_lower)}–${fmtMinutes(h.ci_upper)}, from ${plural(h.timed_tasks, 'timed task')} and ${plural(h.runs, 'run')}.</p>`;++ }++ return html`<p class="br-fine br-horizon">Time horizon: not enough runs yet (needs ${h.need_tasks || 15} timed tasks and ${h.need_runs || 3} runs).</p>`;++}++++/** The contamination badge, with both numbers and their bands. Only a real flag is ever shown. */++function contaminationList(entries) {++ const flagged = entries.filter((e) => e.contamination && e.contamination.flag);++ if (!flagged.length) return '';++ return html`<ul class="br-contams">${flagged.map((e) => {++ const c = e.contamination;++ return html`<li class="br-contam"><b>${e.model}</b> — Did much better on older tasks: possibly seen in training. Older tasks ${fmtScore(c.old.median)} (${fmtScore(c.old.lower)}–${fmtScore(c.old.upper)}), fresh tasks ${fmtScore(c.fresh.median)} (${fmtScore(c.fresh.lower)}–${fmtScore(c.fresh.upper)}). Release cutoff ${c.cutoff}${c.source ? html` · <a href="${c.source}" rel="nofollow noopener">source</a>` : ''}.</li>`;++ })}</ul>`;++}++function metricTabs(base, current) {return html`<nav class="br-tabs" aria-label="Rank by">${METRICS.map((m) => html`<a class="br-tab" href="${base}?metric=${m.id}#br-rank"${m.id === current ? raw(' aria-current="page"') : ''}${m.hint ? raw(` title="${m.hint}"`) : ''}>${m.label}</a>`)}</nav>`;}function rankingsTable(entries, metric) {if (!entries.length) return '';-- const col = { value: (e) => (e.value == null ? '—' : `${fmtScore(e.value)} / $`), speed: (e) => (e.speed == null ? '—' : `${fmtScore(e.speed)} / min`), consistency: (e) => (e.consistency == null ? '—' : `±${fmtScore(e.consistency)}`) }[metric];++ const col = {++ value: (e) => (e.value == null ? '—' : `${fmtScore(e.value)} / $`),++ speed: (e) => (e.speed == null ? '—' : `${fmtScore(e.speed)} / min`),++ consistency: (e) => (e.consistency == null ? '—' : `±${fmtScore(e.consistency)}`),++ }[metric];const bands = metric === 'score' && entries.some((e) => e.band);-- return html`<div class="br-table-wrap"><table class="br-table br-rank-table"><thead><tr><th scope="col">#</th>${bands ? html`<th scope="col">Band</th>` : ''}<th scope="col">Model</th><th scope="col">Score</th>${col ? html`<th scope="col">${METRICS.find((m) => m.id === metric).label}</th>` : ''}<th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${entries.map((e, i) => html`<tr><td class="br-rank">${i + 1}</td>${bands ? html`<td class="br-c-band"><span class="rt-band">${e.band || '—'}</span></td>` : ''}<td class="br-c-model"><a class="br-model" href="${modelHref(e.model)}">${e.model}</a></td><td class="br-c-score">${scoreBar(e.score)}</td>${col ? html`<td class="br-num br-c-metric" data-label="${METRICS.find((m) => m.id === metric).label}">${col(e)}</td>` : ''}<td class="br-num br-c-cost" data-label="cost/run">${fmtUsd(e.cost_usd)}</td><td class="br-num br-c-time" data-label="time">${fmtMs(e.ms)}</td><td class="br-num br-c-runs" data-label="runs">${e.runs}</td></tr>`)}</tbody></table></div>`;++ const isHorizon = metric === 'horizon';++ const showHorizon = isHorizon || entries.some((e) => e.horizon && (e.horizon.available || e.horizon.timed_tasks > 0));++ const metricLabel = METRICS.find((m) => m.id === metric).label;++ return html`<div class="br-table-wrap"><table class="br-table br-rank-table"><thead><tr><th scope="col">#</th>${bands ? html`<th scope="col">Band</th>` : ''}<th scope="col">Model</th><th scope="col">Score</th>${col && !isHorizon ? html`<th scope="col">${metricLabel}</th>` : ''}${showHorizon ? html`<th scope="col" class="br-num"><a href="?metric=horizon#br-rank"${isHorizon ? raw(' aria-current="page"') : ''}>Horizon</a></th>` : ''}<th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${entries.map((e, i) => html`<tr><td class="br-rank">${i + 1}</td>${bands ? html`<td class="br-c-band"><span class="rt-band">${e.band || '—'}</span></td>` : ''}<td class="br-c-model"><a class="br-model" href="${modelHref(e.model)}">${e.model}</a>${e.contamination && e.contamination.flag ? html`<span class="br-contam-chip" title="Did much better on older tasks: possibly seen in training">possibly seen in training</span>` : ''}</td><td class="br-c-score">${scoreBar(e.score)}</td>${col && !isHorizon ? html`<td class="br-num br-c-metric" data-label="${metricLabel}">${col(e)}</td>` : ''}${showHorizon ? html`<td class="br-num br-c-horizon" data-label="horizon" title="${e.horizon && e.horizon.available ? `${e.horizon.timed_tasks} timed tasks, ${e.horizon.runs} runs` : 'not enough runs yet'}">${horizonText(e.horizon)}</td>` : ''}<td class="br-num br-c-cost" data-label="cost/run">${fmtUsd(e.cost_usd)}</td><td class="br-num br-c-time" data-label="time">${fmtMs(e.ms)}</td><td class="br-num br-c-runs" data-label="runs">${e.runs}</td></tr>`)}</tbody></table></div>${contaminationList(entries)}`;}/**@@@ -338,9 +384,10 @@ ${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}</div>`;}--function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '', ownBoards = null, share = null, shared = false, owner = false }) {++function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '', ownBoards = null, share = null, shared = false, owner = false, since = null, allTimeEntries = null }) {const d = definition || {};const tasks = Array.isArray(d.tasks) ? d.tasks : [];++ const freshSets = [...new Set(tasks.map((t) => (t && t.added_on ? String(t.added_on).slice(0, 7) : null)).filter(Boolean))].sort().reverse();const shown = tasks.filter((t) => !t.holdout && t.prompt);const hidden = tasks.length - shown.length;const judges = Array.isArray(d.judges) ? d.judges : [];@@@ -376,11 +423,19 @@ ${owner ? shareBox({ benchmark, shareUrl: share && share.shareUrl, error: share${ownBoards? promptBoards(ownBoards): html`<section class="bm-section" id="br-rank" aria-labelledby="br-rank-h"><div class="bm-head-row"><h2 id="br-rank-h">Rankings</h2>${metricTabs(base, metric)}</div>++${since ? html`<p class="br-since-note" role="status"><b>On tasks since ${since.label}</b>${since.basis === 'runs' ? ' (this benchmark’s tasks carry no dates, so it filters by run date)' : ''}. <a href="${base}?metric=${metric}#br-rank">All-time board</a></p>` : ''}++<form class="br-since" method="get" action="${base}#br-rank"><input type="hidden" name="metric" value="${metric}"><label class="br-field"><span>Fresh set: on tasks since</span><input type="month" name="since" value="${since ? since.month : ''}"></label><button class="bm-btn bm-btn-small" type="submit">Show</button></form>++${freshSets.length ? html`<p class="br-fine br-fresh-sets">Fresh sets: ${freshSets.map((month, i) => html`${i ? ' · ' : ''}<a href="${base}?metric=${metric}&since=${encodeURIComponent(month)}#br-rank">${monthLabel(month)}</a>`)}. Pick one to score only the tasks added that month or later.</p>` : ''}${valueChart(entries)}--${entries.length ? rankingsTable(entries, metric) : empty('No verified runs yet', 'A run counts here once openvibe.codes, Run or a Node worker ran it and published it. Submitted runs are listed below but never ranked.')}++${entries.length ? rankingsTable(entries, metric) : empty(since ? 'No tasks in that fresh set yet' : 'No verified runs yet', since ? 'No run here scored a task added on or after that month. Pick an earlier month, or see the all-time board.' : 'A run counts here once openvibe.codes, Run or a Node worker ran it and published it. Submitted runs are listed below but never ranked.')}++${since && allTimeEntries && allTimeEntries.length ? html`<details class="br-fold br-alltime"><summary>All-time board</summary>${rankingsTable(allTimeEntries, metric)}</details>` : ''}</section>`}<section class="bm-section" id="br-tasks" aria-labelledby="br-tasks-h"><div class="bm-head-row"><h2 id="br-tasks-h">Tasks</h2><span class="bm-count">${tasks.length}</span></div>--<ol class="br-tasks">${shown.map((t, i) => html`<li class="br-task"><span class="br-task-n">${i + 1}</span><span class="br-task-id br-mono">${t.id}</span><p>${t.prompt}</p>${t.expected && t.expected.answer ? html`<p class="br-fine br-task-expected">Expected: ${String(t.expected.answer).slice(0, 300)}</p>` : ''}</li>`)}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section>++<ol class="br-tasks">${shown.map((t, i) => {++ const added = t.added_on ? String(t.added_on).slice(0, 10) : null;++ const mins = t.human_minutes == null ? null : Number(t.human_minutes);++ return html`<li class="br-task"><span class="br-task-n">${i + 1}</span><a class="br-task-id br-mono" href="${benchmarkHref(benchmark.id)}/tasks/${encodeURIComponent(t.id)}">${t.id}</a><p>${t.prompt}</p>${t.expected && t.expected.answer ? html`<p class="br-fine br-task-expected">Expected: ${String(t.expected.answer).slice(0, 300)}</p>` : ''}<p class="br-task-meta">${added ? html`<span class="br-chip">added ${added}</span>` : ''}${mins ? html`<span class="br-chip">${fmtMinutes(mins)} human estimate</span>` : ''}</p></li>`;++ })}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section><section class="bm-section" id="br-runs" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section></div>@@@ -389,6 +444,33 @@ ${benchmark.visibility === 'private' ? '' : benchmarkUseThis({ baseUrl, benchmar</div>`;}++/**++ * One task: its prompt, when it was added, its human-time estimate and (for a signed-in editor) the form that adds++ * one. The median of three distinct people becomes the task's time. A held-out task keeps its prompt on the server.++ */++function taskPage({ benchmark, version, task, estimates = { count: 0, needed: 3, median: null }, signedIn = false, loginUrl = '/auth/login', error = null, status = null }) {++ const held = Boolean(task && (task.holdout || task.held_out));++ const added = task && task.added_on ? String(task.added_on).slice(0, 10) : null;++ const author = task && task.human_minutes != null ? Number(task.human_minutes) : null;++ const source = task && task.human_minutes_source ? String(task.human_minutes_source) : (author ? 'benchmark author' : null);++ const timed = author != null ? { minutes: author, source } : (estimates.count >= 3 ? { minutes: estimates.median, source: `median of ${estimates.count} editor estimates` } : null);++ return html`<div class="bm-wrap br bm-narrow" id="br-task-page">++<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${benchmarkHref(benchmark.id)}">${benchmark.title}</a><span aria-hidden="true">/</span><span class="br-mono">${task ? task.id : 'task'}</span></nav>++<header class="bm-page-h"><p class="bm-kicker">Task ${task ? task.id : ''} · v${version || benchmark.latest_version || 0}</p><h1>${benchmark.title}</h1><p class="bm-lede">${held ? 'Held-out task: its prompt stays on the server, for the executor only.' : (task && task.prompt) || ''}</p></header>++${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}++${held ? html`<div class="bm-note" role="status"><b>Held out.</b> The prompt and expected answer never leave the server. Editors can still give it a human-time estimate.</div>` : ''}++<dl class="br-facts"><div><dt>Added</dt><dd>${added || 'not dated'}</dd></div><div><dt>Human time</dt><dd>${timed ? `${fmtMinutes(timed.minutes)} · ${timed.source}` : 'not enough estimates yet'}</dd></div><div><dt>Editors</dt><dd>${estimates.count} ${estimates.count === 1 ? 'estimate' : 'estimates'}${estimates.count < 3 ? `, ${3 - estimates.count} more for a median` : ''}</dd></div></dl>++${estimates.count >= 3 ? html`<p class="br-fine">The task time is the median of ${estimates.count} signed-in editors’ estimates: <b>${fmtMinutes(estimates.median)}</b>. It is shown here, in the open data and in the time-horizon fit.</p>` : html`<p class="br-fine">The median of three distinct people’s estimates becomes this task’s human time. No individual estimate is published.</p>`}++${signedIn++ ? html`<form class="br-form br-estimate" method="post" action="${benchmarkHref(benchmark.id)}/tasks/${encodeURIComponent(task ? task.id : '')}/estimates">++<input type="hidden" name="version" value="${version || benchmark.latest_version || 1}">++<label class="br-field"><span>Your estimate, in minutes</span><input name="minutes" type="number" min="1" max="100000" step="1" required inputmode="numeric" placeholder="e.g. 40"></label>++<div class="bm-actions"><button class="bm-btn" type="submit">${estimates.count ? 'Replace my estimate' : 'Add my estimate'}</button><a class="bm-btn bm-btn-quiet" href="${benchmarkHref(benchmark.id)}">Back to the benchmark</a></div>++</form>`++ : empty('Sign in to estimate', 'A human-time estimate is attached to your account, exported and deleted with it.', html`<a class="bm-btn bm-btn-small" href="${loginUrl}">Sign in with OpenVibe</a>`)}++</div>`;++}++/*** The run form: pick the models, the harnesses and how many times, see what it may cost, start it on openvibe.codes* with your own keys. A plain form (it works without JavaScript); the estimate above $1 asks once more.@@@ -481,11 +563,11 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta</div>`;}--function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '' }) {++function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '', horizon = null }) {const rows = (summary && summary.benchmarks) || [];return html`<div class="bm-wrap br" id="br-model"><nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>--<header class="bm-page-h"><p class="bm-kicker">Model</p><h1 class="br-mono-h">${model}</h1><p class="bm-lede">${summary && summary.runs ? `${plural(summary.runs, 'verified run')} across ${plural(rows.length, 'benchmark')}.` : 'No verified runs yet.'}</p></header>++<header class="bm-page-h"><p class="bm-kicker">Model</p><h1 class="br-mono-h">${model}</h1><p class="bm-lede">${summary && summary.runs ? `${plural(summary.runs, 'verified run')} across ${plural(rows.length, 'benchmark')}.` : 'No verified runs yet.'}</p>${horizonLine(horizon)}</header><section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}@@@ -495,6 +577,7 @@ ${modelUseThis({ baseUrl, model, rows, viewer })}module.exports = {benchmarksIndex, benchmarkPage, newBenchmarkPage, promptLeaderboard, promptBoards, shareBox,-- runForm, runPage, comparePage, modelPage, valueChart, METRICS,++ runForm, runPage, comparePage, modelPage, taskPage, valueChart, METRICS,++ fmtMinutes, horizonText, horizonLine,fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,};nnew file mode 100644@@@ -0,0 +1,298 @@++'use strict';++++/**++ * B13: fresh monthly sets, held-out tasks, the contamination flag and the time horizon.++ *++ * The rules under test: `since=YYYY-MM` scores only the tasks added in or after that month; a held-out task never++ * reaches the open data, an anonymous page or somebody else's account export, and a submitted/imported run gets no++ * score for one; the contamination maths says "flag", "no flag" or "not enough runs"; the logistic fit recovers a++ * known horizon and its bootstrap; the 15-task/3-run gates hold; editors' estimates are exported and erased with++ * the person and the form is signed-in and same-origin only.++ */++const assert = require('assert');++const contracts = require('openvibe-contracts');++const { boot, check, done } = require('./helpers/boot');++const { contaminationFor } = require('../server/bench/contamination');++const { estimateHorizon } = require('../server/bench/horizon');++const { parseSince } = require('../server/bench/registry');++++const baseTask = (id, over = {}) => ({ id, prompt: `Task ${id}: build a small page.`, files: [], expected: null, holdout: false, ...over });++++function definition(over = {}) {++ return {++ title: 'B13 arena',++ description: 'A benchmark for the fresh-set and horizon tests.',++ kind: 'design.page',++ tags: ['design'],++ visibility: 'public',++ tasks: [baseTask('t1')],++ judges: [{ type: 'rubric', version: '1.0', config: {} }],++ scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: { ms: 120000, tokens: 200000, usd: 0.5 },++ matrix: { models: [], harnesses: [] },++ ...over,++ };++}++++function runDoc(benchmark, tasks, over = {}) {++ const scores = tasks.map((t) => t.score).filter((v) => v != null);++ return {++ benchmark,++ model: 'model-a',++ route: { provider: 'acme' },++ provider: 'acme',++ harness: { name: 'claude-code', version: '3.1.0' },++ settings: { temperature: 1, effort: 'high' },++ executor: { kind: 'codes', id: 'tsk_b13' },++ tasks,++ score: scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : null,++ cost_usd: tasks.reduce((n, t) => n + (t.cost_usd || 0), 0),++ ms: tasks.reduce((n, t) => n + (t.ms || 0), 0),++ tokens: tasks.reduce((n, t) => n + (t.tokens || 0), 0),++ ...over,++ };++}++++const result = (task_id, score, extra = {}) => ({ task_id, score, cost_usd: 0.01, ms: 1000, tokens: 100, artifacts: [], ...extra });++++(async () => {++ const t = await boot();++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob');++ const carol = t.network.addUser('carol');++ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });++++ const create = async (over, as = null) => {++ const res = await t.get('/api/v1/benchmarks', as ? { as, json: definition(over) } : { bearer: publisher, json: definition(over) });++ assert.strictEqual(res.status, 201, res.text);++ const id = res.json().benchmark.id;++ const frozen = await t.get(`/api/v1/benchmarks/${id}/versions`, as ? { as, json: {} } : { bearer: publisher, json: {} });++ assert.strictEqual(frozen.status, 201, frozen.text);++ return { id, version: frozen.json().version.version };++ };++ const publish = async (ref, doc) => {++ const res = await t.get('/api/v1/runs', { bearer: publisher, json: doc });++ assert.strictEqual(res.status, 201, res.text);++ return res.json().run;++ };++ const postForm = async (path, body, as) => t.get(path, {++ as, method: 'POST', raw: new URLSearchParams(body).toString(),++ headers: { 'content-type': 'application/x-www-form-urlencoded' },++ });++++ await check('the since parser takes a month, a day or a timestamp', () => {++ assert.strictEqual(parseSince('2026-03').cutoff, '2026-03-01');++ assert.strictEqual(parseSince('2026-03').label, 'March 2026');++ assert.strictEqual(parseSince('2026-03-10').cutoff, '2026-03-10');++ assert.strictEqual(parseSince('not-a-date'), null);++ });++++ await check('since=YYYY-MM scores only the tasks added in or after that month', async () => {++ const made = await create({++ title: 'Fresh monthly set',++ tasks: [++ baseTask('old', { added_on: '2025-01-15', human_minutes: 20 }),++ baseTask('fresh', { added_on: '2026-03-10', human_minutes: 40 }),++ ],++ });++ const frozen = await t.get(`/api/v1/benchmarks/${made.id}@${made.version}`);++ assert.strictEqual(frozen.status, 200, frozen.text);++ contracts.assertValid('bench.benchmark@1', frozen.json().version.definition);++ assert.strictEqual(frozen.json().task_meta.find((m) => m.task_id === 'old').added_on, '2025-01-15', 'the date lives beside the closed document');++++ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('old', 100), result('fresh', 20)], { model: 'model-a' }));++ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('old', 80), result('fresh', 80)], { model: 'model-b' }));++++ const all = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score`);++ assert.strictEqual(all.status, 200, all.text);++ assert.strictEqual(all.json().rankings.entries.find((e) => e.model === 'model-a').score, 60, 'the all-time board uses the run score');++++ const fresh = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score&since=2026-03`);++ assert.strictEqual(fresh.status, 200, fresh.text);++ const rankings = fresh.json().rankings;++ assert.strictEqual(rankings.since.label, 'March 2026');++ assert.strictEqual(rankings.since.basis, 'tasks');++ assert.strictEqual(rankings.entries.find((e) => e.model === 'model-a').score, 20, 'only the fresh task is scored');++ assert.strictEqual(rankings.entries.find((e) => e.model === 'model-b').score, 80);++++ const page = await t.get(`/benchmarks/${made.id}?since=2026-03`);++ assert.strictEqual(page.status, 200, page.text.slice(0, 300));++ assert.ok(page.text.includes('On tasks since March 2026'), 'the page names the fresh set beside the board');++ assert.ok(page.text.includes('All-time board'), 'and offers the all-time board');++ });++++ await check('a held-out task never leaks, and a submitted run gets no score for it', async () => {++ const secret = 'SECRET_HELDOUT_PROMPT_9f2a';++ const made = await create({++ title: 'Held-out arena',++ tasks: [++ baseTask('visible', { prompt: 'A visible task.' }),++ baseTask('hidden', { prompt: secret, expected: { answer: 'SECRET_EXPECTED' }, held_out: true }),++ ],++ }, alice);++++ const data = await t.get('/api/v1/datasets/benchmarks.jsonl');++ assert.ok(data.text.includes('A visible task.'));++ assert.ok(!data.text.includes(secret), 'the open data never carries the held-out prompt');++ assert.ok(!data.text.includes('SECRET_EXPECTED'), 'nor its expected answer');++++ const api = await t.get(`/api/v1/benchmarks/${made.id}`);++ const hidden = api.json().definition.tasks.find((x) => x.id === 'hidden');++ assert.strictEqual(hidden.prompt, null);++ assert.strictEqual(hidden.holdout, true);++++ const page = await t.get(`/benchmarks/${made.id}`);++ assert.ok(!page.text.includes(secret), 'the page never carries the held-out prompt');++++ const submitted = await t.get('/api/v1/runs', {++ as: alice,++ json: runDoc(`${made.id}@${made.version}`, [result('visible', 50), result('hidden', 99)], { model: 'submitted-model' }),++ });++ assert.strictEqual(submitted.status, 201, submitted.text);++ assert.strictEqual(submitted.json().run.tasks.find((x) => x.task_id === 'hidden').score, null, 'a submitted run cannot score a held-out task');++ assert.strictEqual(submitted.json().run.tasks.find((x) => x.task_id === 'visible').score, 50);++++ const verified = await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('visible', 50), result('hidden', 99)], { model: 'verified-model' }));++ assert.strictEqual(verified.tasks.find((x) => x.task_id === 'hidden').score, 99, 'the executor may score it');++++ const fork = await t.get(`/api/v1/benchmarks/${made.id}/fork`, { as: bob, json: {} });++ assert.strictEqual(fork.status, 403, 'a fork by somebody who may not see the holdout is refused');++ assert.ok(!fork.text.includes(secret), 'and the refusal does not carry the prompt');++++ const exportPart = await t.ctx.accountData.exportPart(bob.subject);++ const blob = JSON.stringify(exportPart.files || []);++ assert.ok(!blob.includes(secret), 'somebody else\'s account export never carries the prompt');++ assert.ok(!blob.includes('SECRET_EXPECTED'));++++ const writer = t.network.serviceToken({ sub: 'svc:writer', audience: 'openvibe.bench', cap: ['bench.benchmark.write'] });++ const versioned = await t.get(`/api/v1/benchmarks/${made.id}/versions`, { bearer: writer, json: {} });++ assert.strictEqual(versioned.status, 201, versioned.text);++ assert.ok(!versioned.text.includes(secret), 'a new version response for a non-owner does not carry the prompt');++ assert.strictEqual(versioned.json().version.definition.tasks.find((x) => x.id === 'hidden').prompt, null);++ });++++ await check('the contamination flag: a flag, no flag, and not enough runs', () => {++ const cutoff = { cutoff: '2025-08-07', source: 'https://example.test/model' };++ const flagged = contaminationFor({ oldScores: [90, 92, 94], freshScores: [20, 22, 24], ...cutoff });++ assert.strictEqual(flagged.flag, true);++ assert.ok(flagged.old.median > flagged.fresh.median);++ assert.ok(flagged.old.lower > flagged.fresh.upper, 'the 90% intervals do not overlap');++ const tied = contaminationFor({ oldScores: [70, 72, 74], freshScores: [68, 70, 72], ...cutoff });++ assert.strictEqual(tied.flag, false);++ assert.strictEqual(tied.reason, 'the intervals overlap');++ const thin = contaminationFor({ oldScores: [90], freshScores: [20, 22], ...cutoff });++ assert.strictEqual(thin.flag, false);++ assert.strictEqual(thin.reason, 'not enough runs yet');++ const unknown = contaminationFor({ oldScores: [90, 92], freshScores: [20, 22] });++ assert.strictEqual(unknown.flag, false);++ assert.strictEqual(unknown.reason, 'no release date');++ });++++ await check('the logistic fit recovers a known 40-minute horizon and its bootstrap', () => {++ const groups = [];++ for (let i = 0; i < 20; i++) {++ const minutes = 2 + i * 5;++ const p = 1 / (1 + Math.exp(-(1.2 * Math.log(minutes) - 1.2 * Math.log(40))));++ groups.push({ minutes, success: p, runs: 3 });++ }++ const fit = estimateHorizon(groups, { runs: 3, timedTasks: 20 });++ assert.strictEqual(fit.available, true, JSON.stringify(fit));++ assert.ok(Math.abs(fit.horizon_minutes - 40) < 4, `the horizon is near 40: ${fit.horizon_minutes}`);++ assert.ok(fit.ci_lower <= fit.horizon_minutes && fit.horizon_minutes <= fit.ci_upper);++ assert.ok(fit.ci_lower <= 40 && 40 <= fit.ci_upper, `the 90% interval covers the true horizon: ${fit.ci_lower}-${fit.ci_upper}`);++++ const tooFewTasks = estimateHorizon(groups.slice(0, 14), { runs: 3, timedTasks: 14 });++ assert.strictEqual(tooFewTasks.available, false);++ assert.strictEqual(tooFewTasks.reason, 'not enough runs yet');++ const tooFewRuns = estimateHorizon(groups, { runs: 2, timedTasks: 20 });++ assert.strictEqual(tooFewRuns.available, false);++ assert.strictEqual(tooFewRuns.reason, 'not enough runs yet');++ });++++ await check('the board and the model page show a horizon only above 15 tasks and 3 runs', async () => {++ const tasks = [];++ const scoresForRun = [];++ for (let i = 0; i < 15; i++) {++ const minutes = 2 + i * 5;++ tasks.push(baseTask(`h${i}`, { added_on: '2026-01-10', human_minutes: minutes, human_minutes_source: 'benchmark author' }));++ const p = 1 / (1 + Math.exp(-(1.2 * Math.log(minutes) - 1.2 * Math.log(40))));++ scoresForRun.push(result(`h${i}`, p * 100));++ }++ const made = await create({ title: 'Timed arena', tasks });++ for (let run = 0; run < 3; run++) {++ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, scoresForRun.map((x) => ({ ...x })), { model: 'model-timed' }));++ }++ const board = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=horizon`);++ assert.strictEqual(board.status, 200, board.text);++ const entry = board.json().rankings.entries.find((e) => e.model === 'model-timed');++ assert.strictEqual(entry.horizon.available, true, JSON.stringify(entry.horizon));++ assert.strictEqual(entry.horizon.timed_tasks, 15);++ assert.strictEqual(entry.horizon.runs, 3);++ assert.ok(Math.abs(entry.horizon.horizon_minutes - 40) < 6, `horizon ${entry.horizon.horizon_minutes}`);++++ const page = await t.get(`/benchmarks/${made.id}?metric=horizon`);++ assert.ok(page.text.includes('br-c-horizon'), 'the board has a horizon column');++ assert.ok(page.text.includes('min'), 'the horizon is a number of minutes');++ const model = await t.get('/models/model-timed');++ assert.strictEqual(model.status, 200, model.text.slice(0, 200));++ assert.ok(model.text.includes('reliably does about'), 'the model profile has the horizon line');++ const thin = await t.get('/models/model-a');++ assert.strictEqual(thin.status, 200, thin.text.slice(0, 200));++ assert.ok(thin.text.includes('not enough runs yet'), 'below the gate the model page says so instead of a number');++ });++++ await check('editors estimate on the task page: sign-in, same-origin, median of three, account data', async () => {++ const made = await create({++ title: 'Estimate arena',++ tasks: [baseTask('timed', { added_on: '2026-02-01' })],++ });++ const path = `/benchmarks/${made.id}/tasks/timed`;++ const anon = await t.get(path);++ assert.strictEqual(anon.status, 200, anon.text.slice(0, 200));++ assert.ok(anon.text.includes('Sign in to estimate'), 'a signed-out reader is asked to sign in');++ assert.strictEqual((await postForm(`/benchmarks/${made.id}/tasks/timed/estimates`, { minutes: '30', version: String(made.version) })).status, 401);++++ const crossSite = await t.get(`/benchmarks/${made.id}/tasks/timed/estimates`, {++ as: alice, method: 'POST', raw: 'minutes=30', headers: { 'content-type': 'application/x-www-form-urlencoded', origin: 'https://evil.example' },++ });++ assert.strictEqual(crossSite.status, 403, 'a cross-site write is refused even when signed in');++++ for (const [user, minutes] of [[alice, '30'], [bob, '40'], [carol, '50']]) {++ const res = await postForm(`/benchmarks/${made.id}/tasks/timed/estimates`, { minutes, version: String(made.version) }, user);++ assert.strictEqual(res.status, 303, res.text.slice(0, 300));++ }++ const page = await t.get(path);++ assert.ok(page.text.includes('median of 3 editor estimates'), page.text.slice(0, 600));++ assert.ok(page.text.includes('40 min'), 'the median is 40 minutes');++++ const exported = await t.ctx.accountData.exportPart(alice.subject);++ const file = (exported.files || []).find((f) => f.name === 'bench_task_estimates.json');++ assert.ok(file && file.content.length === 1, 'the estimate is in the person\'s export');++ assert.strictEqual(Number(file.content[0].minutes), 30);++ const erased = await t.ctx.accountData.erase([alice.subject]);++ assert.ok(Number(erased.erased.bench_task_estimates) >= 1, 'the estimate is erased with the person');++ });++++ await check('the contamination badge is shown, with both numbers and their bands', async () => {++ const made = await create({++ title: 'Old and fresh arena',++ tasks: [++ baseTask('before', { added_on: '2024-01-10', human_minutes: 10 }),++ baseTask('after', { added_on: '2026-01-10', human_minutes: 10 }),++ ],++ });++ for (let i = 0; i < 3; i++) {++ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('before', 90 + i), result('after', 20 + i)], { model: 'gpt-5' }));++ }++ const board = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score`);++ const entry = board.json().rankings.entries.find((e) => e.model === 'gpt-5');++ assert.strictEqual(entry.contamination.flag, true, JSON.stringify(entry.contamination));++ const page = await t.get(`/benchmarks/${made.id}`);++ assert.ok(page.text.includes('Did much better on older tasks: possibly seen in training'), 'the exact badge text');++ assert.ok(page.text.includes('Older tasks'), 'both numbers');++ assert.ok(page.text.includes('fresh tasks'), 'and their bands');++ });++++ await t.close();++ done();++})().catch((err) => { console.error(err && err.stack ? err.stack : err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,46 @@
@@ -0,0 +1,46 @@
+-- phase: expand
+-- B13: fresh monthly sets, the contamination flag and time-horizon estimates (spec 2026-10-30).
+--
+-- A task may carry the day it was added (`added_on`), whether it is held out (`held_out`, also accepted as the
+-- older `holdout`) and a human time estimate (`human_minutes`) with its source. The run-task rows below copy the
+-- facts at publish time, so a board can filter by month and fit a horizon without parsing every version; the
+-- version-keyed table below keeps the same facts beside the closed benchmark document.
+ALTER TABLE bench_run_tasks ADD COLUMN added_on date;
+ALTER TABLE bench_run_tasks ADD COLUMN held_out boolean NOT NULL DEFAULT false;
+ALTER TABLE bench_run_tasks ADD COLUMN human_minutes numeric(10, 1);
+ALTER TABLE bench_run_tasks ADD COLUMN human_minutes_source text COLLATE "C";
+
+-- Editors' human-time estimates. One per person per task: a later entry replaces the earlier one, and the median of
+-- three distinct people is what a task's time becomes. The row is the person's data and goes with them.
+CREATE TABLE bench_task_estimates (
+ id text COLLATE "C" PRIMARY KEY, -- bte_ + ULID
+ benchmark_id text COLLATE "C" NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,
+ benchmark_version integer NOT NULL,
+ task_id text COLLATE "C" NOT NULL,
+ subject text COLLATE "C" NOT NULL,
+ minutes numeric(10, 1) NOT NULL,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT bench_task_estimates_id CHECK (id ~ '^bte_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_task_estimates_version CHECK (benchmark_version >= 1),
+ CONSTRAINT bench_task_estimates_minutes CHECK (minutes > 0 AND minutes <= 100000),
+ FOREIGN KEY (benchmark_id, benchmark_version) REFERENCES bench_benchmark_versions (benchmark_id, version) ON DELETE CASCADE,
+ UNIQUE (benchmark_id, benchmark_version, task_id, subject)
+);
+CREATE INDEX bench_task_estimates_by_subject ON bench_task_estimates (subject, created_at DESC);
+CREATE INDEX bench_task_estimates_by_task ON bench_task_estimates (benchmark_id, benchmark_version, task_id);
+
+-- The task metadata itself. The published bench.benchmark@1 task shape is closed (id, prompt, files, expected,
+-- holdout), so the B13 fields live beside the frozen definition, keyed by the version and task, and are merged into
+-- the definition on Bench's own pages. The frozen document stays a valid contract document.
+CREATE TABLE bench_task_meta (
+ benchmark_id text COLLATE "C" NOT NULL,
+ benchmark_version integer NOT NULL,
+ task_id text COLLATE "C" NOT NULL,
+ added_on date,
+ held_out boolean NOT NULL DEFAULT false,
+ human_minutes numeric(10, 1),
+ human_minutes_source text COLLATE "C",
+ PRIMARY KEY (benchmark_id, benchmark_version, task_id),
+ CONSTRAINT bench_task_meta_minutes CHECK (human_minutes IS NULL OR (human_minutes > 0 AND human_minutes <= 100000)),
+ FOREIGN KEY (benchmark_id, benchmark_version) REFERENCES bench_benchmark_versions (benchmark_id, version) ON DELETE CASCADE
+);
@@ -163,7 +163,7 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -163,7 +163,7 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.br-task { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 12px; align-items: start; padding: 14px; border-radius: 14px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }
.br-task { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 12px; align-items: start; padding: 14px; border-radius: 14px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }
.br-task p { margin: 0; line-height: 1.55; color: var(--text-secondary, #96a7c2); }
.br-task p { margin: 0; line-height: 1.55; color: var(--text-secondary, #96a7c2); }
.br-task.is-hidden { border-style: dashed; background: transparent; }
.br-task.is-hidden { border-style: dashed; background: transparent; }
-.br-task-id { display: inline-flex; padding: 2px 8px; border-radius: 8px; background: var(--bg-hover, rgba(255,255,255,.05)); font-size: .78rem; font-weight: 700; color: var(--text-primary, #e6edf7); white-space: nowrap; margin: 0; }
+.br-task-id { display: inline-flex; justify-self: start; width: fit-content; padding: 2px 8px; border-radius: 8px; background: var(--bg-hover, rgba(255,255,255,.05)); font-size: .78rem; font-weight: 700; color: var(--text-primary, #e6edf7); white-space: nowrap; margin: 0; text-decoration: none; }
.br-ptask { display: grid; gap: 10px; padding: 14px; border-radius: 16px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }
.br-ptask { display: grid; gap: 10px; padding: 14px; border-radius: 16px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }
.br-ptask-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; }
.br-ptask-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; }
.br-ptask-meta { margin-left: auto; font-size: .84rem; color: var(--text-muted, #7386a3); font-variant-numeric: tabular-nums; }
.br-ptask-meta { margin-left: auto; font-size: .84rem; color: var(--text-muted, #7386a3); font-variant-numeric: tabular-nums; }
@@ -570,3 +570,27 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -570,3 +570,27 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@media (max-width: 640px) {
@media (max-width: 640px) {
.br-sections #br-rank .br-chart { display: none; }
.br-sections #br-rank .br-chart { display: none; }
}
}
+
+/* ---- B13: fresh monthly sets, contamination and the time horizon ---- */
+.br-since { display: flex; flex-wrap: wrap; align-items: flex-end; gap: 8px; margin: 0 0 14px; }
+.br-since .br-field { margin: 0; }
+.br-since .br-field input { min-height: 40px; }
+.br-since-note { margin: 0 0 12px; padding: 10px 12px; border-radius: 12px; background: color-mix(in srgb, var(--room-accent) 10%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 26%, transparent); line-height: 1.5; }
+.br-alltime { margin-top: 16px; border: 1px solid var(--border); border-radius: 14px; padding: 10px 14px; background: var(--bg-card); }
+.br-alltime > summary { cursor: pointer; font-weight: 750; min-height: 36px; display: flex; align-items: center; }
+.br-alltime > .br-table-wrap { margin-top: 8px; }
+.br-task-meta { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 6px !important; }
+.br-task-meta .br-chip { font-size: .72rem; }
+.br-contams { list-style: none; margin: 14px 0 0; padding: 0; display: grid; gap: 8px; }
+.br-contam { padding: 12px 14px; border-radius: 14px; background: color-mix(in srgb, #f59e0b 10%, var(--bg-card)); border: 1px solid color-mix(in srgb, #f59e0b 38%, var(--border)); line-height: 1.5; }
+.br-contam a { color: var(--room-strong); }
+.br-contam-chip { display: inline-block; margin-left: 8px; padding: 1px 7px; border-radius: 999px; font-size: .68rem; font-weight: 800; letter-spacing: .02em; color: #fde68a; background: color-mix(in srgb, #f59e0b 28%, transparent); border: 1px solid color-mix(in srgb, #f59e0b 55%, transparent); white-space: nowrap; }
+.br-horizon { margin: 10px 0 0; padding: 10px 12px; border-radius: 12px; background: color-mix(in srgb, var(--room-accent) 9%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 24%, transparent); line-height: 1.5; }
+.br-estimate { margin-top: 16px; }
+@media (max-width: 560px) {
+ .br-since { align-items: stretch; }
+ .br-since .br-field { flex: 1 1 100%; }
+ .br-since .bm-btn { min-height: 44px; }
+ #br-tasks .br-task-meta { display: none; }
+ .br-rank-table td.br-c-horizon { flex: none; order: 2; font-size: .82rem; color: var(--text-secondary); text-align: left; }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,79 @@
@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * The contamination flag (B13). A model that scored much better on tasks written before its release than on tasks
+ * written after it may have seen the older ones in training. "Much better" means the 90% bootstrap intervals of the
+ * two medians do not overlap. Two runs on each side are needed before either interval is allowed to speak; below
+ * that the answer is "not enough runs yet", never a flag.
+ */
+const { percentile } = require('../rating/lib');
+
+function seededRandom(seed) {
+ let a = seed >>> 0;
+ return () => {
+ a = (a + 0x6D2B79F5) >>> 0;
+ let t = Math.imul(a ^ (a >>> 15), 1 | a);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+const median = (values) => {
+ const list = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
+ if (!list.length) return null;
+ const mid = Math.floor(list.length / 2);
+ return list.length % 2 ? list[mid] : (list[mid - 1] + list[mid]) / 2;
+};
+
+const round = (v, p = 1000) => (v == null ? null : Math.round(v * p) / p);
+
+/** A bootstrap percentile interval for the median of `values`. */
+function medianInterval(values, { level = 0.9, bootstrap = 1000, seed = 20261030 } = {}) {
+ const list = values.filter((v) => Number.isFinite(v));
+ if (!list.length) return null;
+ if (list.length === 1) return { median: round(list[0]), lower: round(list[0]), upper: round(list[0]), runs: 1 };
+ const random = seededRandom(seed);
+ const medians = [];
+ for (let i = 0; i < bootstrap; i++) {
+ const sample = [];
+ for (let j = 0; j < list.length; j++) sample.push(list[Math.floor(random() * list.length)]);
+ medians.push(median(sample));
+ }
+ medians.sort((a, b) => a - b);
+ return {
+ median: round(median(list)),
+ lower: round(percentile(medians, (1 - level) / 2)),
+ upper: round(percentile(medians, 1 - (1 - level) / 2)),
+ runs: list.length,
+ };
+}
+
+/**
+ * The flag for one model. `oldScores` and `freshScores` are one score per run, for the runs that touched a task on
+ * that side of the cutoff. `cutoff`/`source` come from the repo's table; without them there is no flag.
+ */
+function contaminationFor({ oldScores = [], freshScores = [], cutoff = null, source = null } = {}) {
+ const oldRuns = oldScores.filter((v) => Number.isFinite(v));
+ const freshRuns = freshScores.filter((v) => Number.isFinite(v));
+ if (!cutoff) {
+ return { flag: false, available: false, reason: 'no release date', old_runs: oldRuns.length, fresh_runs: freshRuns.length };
+ }
+ if (oldRuns.length < 2 || freshRuns.length < 2) {
+ return { flag: false, available: false, reason: 'not enough runs yet', old_runs: oldRuns.length, fresh_runs: freshRuns.length, cutoff, source };
+ }
+ const oldSide = medianInterval(oldRuns, { seed: 20261030 });
+ const freshSide = medianInterval(freshRuns, { seed: 20261031 });
+ const flag = oldSide.lower > freshSide.upper;
+ return {
+ flag,
+ available: true,
+ reason: flag ? 'the old interval is above the fresh one' : 'the intervals overlap',
+ cutoff,
+ source,
+ old: oldSide,
+ fresh: freshSide,
+ gap: round(oldSide.median - freshSide.median),
+ };
+}
+
+module.exports = { contaminationFor, medianInterval, median, seededRandom, round };
new file mode 100644
new file mode 100644
@@ -0,0 +1,118 @@
@@ -0,0 +1,118 @@
+'use strict';
+
+/**
+ * The time horizon, after METR (B13). For one model, each timed task gives a human time estimate and a success rate
+ * (the mean of its task scores over the runs). A two-parameter logistic fit of success on log(human minutes) gives
+ * the 50% point: the task length the model "reliably does". A bootstrap over tasks gives the 90% interval.
+ *
+ * Before 15 timed tasks and 3 runs, no number is shown: the fit would be noise. The interval is resampled from the
+ * tasks, not the runs, because the task length is what the fit is about.
+ */
+const { percentile } = require('../rating/lib');
+
+const MIN_TASKS = 15;
+const MIN_RUNS = 3;
+const BOOTSTRAP = 500;
+const SEED = 20261030;
+
+const sigmoid = (x) => (x >= 0 ? 1 / (1 + Math.exp(-x)) : (e => e / (1 + e))(Math.exp(x)));
+const clamp = (p) => Math.min(0.999, Math.max(0.001, p));
+const round = (v, p = 10) => (v == null ? null : Math.round(v * p) / p);
+
+function seededRandom(seed) {
+ let a = seed >>> 0;
+ return () => {
+ a = (a + 0x6D2B79F5) >>> 0;
+ let t = Math.imul(a ^ (a >>> 15), 1 | a);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+/**
+ * Fit intercept + slope by Newton-Raphson. `points` are { x = log(minutes), p = success 0..1, weight }.
+ * No library: two parameters are two normal equations.
+ */
+function fitLogistic(points) {
+ let b0 = 0;
+ let b1 = 0;
+ for (let iter = 0; iter < 80; iter++) {
+ let g0 = 0; let g1 = 0;
+ let h00 = 0; let h01 = 0; let h11 = 0;
+ for (const point of points) {
+ const w = Number.isFinite(point.weight) && point.weight > 0 ? point.weight : 1;
+ const q = sigmoid(b0 + b1 * point.x);
+ const r = q - point.p;
+ const wt = Math.max(1e-9, q * (1 - q) * w);
+ g0 += r * w;
+ g1 += r * w * point.x;
+ h00 += wt;
+ h01 += wt * point.x;
+ h11 += wt * point.x * point.x;
+ }
+ // A tiny ridge keeps a separable fit finite rather than throwing on a singular matrix.
+ h00 += 1e-9; h11 += 1e-9;
+ const det = h00 * h11 - h01 * h01;
+ if (!Number.isFinite(det) || Math.abs(det) < 1e-18) break;
+ const d0 = (h11 * g0 - h01 * g1) / det;
+ const d1 = (-h01 * g0 + h00 * g1) / det;
+ b0 -= d0; b1 -= d1;
+ if (Math.max(Math.abs(d0), Math.abs(d1)) < 1e-11) break;
+ }
+ return { intercept: b0, slope: b1 };
+}
+
+/** The minutes at which the fitted success is 50%, or null when the model does not improve with time. */
+function horizonOf(fit) {
+ if (!fit || !(fit.slope > 0) || !Number.isFinite(fit.intercept)) return null;
+ return Math.exp(-fit.intercept / fit.slope);
+}
+
+function fitPoints(groups) {
+ return groups.map((g) => ({
+ x: Math.log(g.minutes),
+ p: clamp(Number(g.success)),
+ weight: g.runs || 1,
+ })).filter((p) => Number.isFinite(p.x) && Number.isFinite(p.p));
+}
+
+/**
+ * `groups`: [{ minutes, success (0..1 mean over runs), runs }]. `runs` is the number of distinct runs behind the
+ * groups; `timedTasks` their count. Returns available:false with a reason below either gate.
+ */
+function estimateHorizon(groups = [], { runs = 0, timedTasks = groups.length, bootstrap = BOOTSTRAP, seed = SEED } = {}) {
+ const gates = { timed_tasks: timedTasks, runs, need_tasks: MIN_TASKS, need_runs: MIN_RUNS };
+ if (timedTasks < MIN_TASKS || runs < MIN_RUNS) {
+ return { available: false, reason: 'not enough runs yet', ...gates };
+ }
+ const points = fitPoints(groups);
+ if (points.length < MIN_TASKS) return { available: false, reason: 'not enough timed tasks', ...gates };
+ const fit = fitLogistic(points);
+ const horizon = horizonOf(fit);
+ if (horizon == null || !Number.isFinite(horizon) || horizon <= 0) {
+ return { available: false, reason: 'no 50% point yet', ...gates, slope: round(fit.slope, 1000) };
+ }
+ const random = seededRandom(seed);
+ const samples = [];
+ for (let i = 0; i < bootstrap; i++) {
+ const resampled = [];
+ for (let j = 0; j < points.length; j++) resampled.push(points[Math.floor(random() * points.length)]);
+ const value = horizonOf(fitLogistic(resampled));
+ if (value != null && Number.isFinite(value) && value > 0) samples.push(value);
+ }
+ samples.sort((a, b) => a - b);
+ const lower = samples.length ? percentile(samples, 0.05) : horizon;
+ const upper = samples.length ? percentile(samples, 0.95) : horizon;
+ return {
+ available: true,
+ horizon_minutes: round(horizon, 1),
+ ci_lower: round(lower, 1),
+ ci_upper: round(upper, 1),
+ ...gates,
+ slope: round(fit.slope, 1000),
+ method: 'logistic fit of success on log(human minutes); 90% bootstrap over tasks',
+ bootstrap,
+ };
+}
+
+module.exports = { estimateHorizon, fitLogistic, horizonOf, fitPoints, MIN_TASKS, MIN_RUNS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,30 @@
@@ -0,0 +1,30 @@
+'use strict';
+
+/**
+ * When each model was released, or the training cutoff its maker states. The contamination flag needs one date per
+ * model; a model not in this table gets no flag, never a guessed one. Every row names the page it comes from.
+ *
+ * The table is deliberately small and in the repo, so changing it is a reviewable change with a source. A provider
+ * prefix ("openai:gpt-5") is stripped before the lookup.
+ */
+const CUTOFFS = Object.freeze([
+ { model: 'gpt-5', cutoff: '2025-08-07', source: 'https://openai.com/index/introducing-gpt-5/' },
+ { model: 'claude-opus-4', cutoff: '2025-05-22', source: 'https://www.anthropic.com/news/claude-4' },
+ { model: 'claude-sonnet-4', cutoff: '2025-05-22', source: 'https://www.anthropic.com/news/claude-4' },
+ { model: 'gemini-2.5-pro', cutoff: '2025-06-17', source: 'https://blog.google/technology/google-deepmind/google-gemini-updates-io-2025/' },
+ { model: 'deepseek-v3', cutoff: '2024-12-26', source: 'https://api-docs.deepseek.com/news/news1226' },
+ { model: 'kimi-k2', cutoff: '2025-07-11', source: 'https://moonshotai.github.io/Kimi-K2/' },
+ { model: 'llama-4-maverick', cutoff: '2025-04-05', source: 'https://ai.meta.com/blog/llama-4-multimodal-intelligence/' },
+]);
+
+const byModel = new Map(CUTOFFS.map((row) => [row.model.toLowerCase(), row]));
+
+/** The row for a model, or null: unknown means no flag. */
+function cutoffFor(model) {
+ const text = String(model || '').trim().toLowerCase();
+ if (!text) return null;
+ const bare = text.includes(':') ? text.slice(text.lastIndexOf(':') + 1) : text;
+ return byModel.get(text) || byModel.get(bare) || null;
+}
+
+module.exports = { CUTOFFS, cutoffFor };
@@ -17,6 +17,10 @@ const crypto = require('crypto');
@@ -17,6 +17,10 @@ const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { ids } = require('openvibe-contracts');
const { contractRun, checkContract, VISIBILITIES } = require('./validate');
const { contractRun, checkContract, VISIBILITIES } = require('./validate');
const { rankBands } = require('../rating/style');
const { rankBands } = require('../rating/style');
+const { isHeldOut, addedOn, humanMinutes, humanMinutesSource } = require('./task-meta');
+const { cutoffFor } = require('./model-cutoffs');
+const { contaminationFor } = require('./contamination');
+const { estimateHorizon } = require('./horizon');
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
@@ -53,6 +57,51 @@ const json = (v, fallback) => {
@@ -53,6 +57,51 @@ const json = (v, fallback) => {
return v;
return v;
};
};
+/**
+ * The published bench.benchmark@1 task shape closes its properties, so the B13 fields (added_on, held_out,
+ * human_minutes and its source) live in bench_task_meta beside the frozen definition. These two helpers keep the
+ * stored document valid and put the fields back on Bench's own read path.
+ */
+function contractTask(task) {
+ const t = task && typeof task === 'object' ? task : {};
+ const out = {
+ id: String(t.id == null ? '' : t.id),
+ prompt: String(t.prompt == null ? '' : t.prompt),
+ files: Array.isArray(t.files) ? t.files : [],
+ expected: t.expected === undefined ? null : t.expected,
+ holdout: isHeldOut(t),
+ };
+ return out;
+}
+
+function taskMetaOf(task) {
+ return {
+ added_on: addedOn(task),
+ held_out: isHeldOut(task),
+ human_minutes: humanMinutes(task),
+ human_minutes_source: humanMinutesSource(task),
+ };
+}
+
+function mergeTaskMeta(definition, metaRows) {
+ if (!Array.isArray(definition.tasks) || !metaRows) return definition;
+ const byTask = new Map();
+ for (const row of metaRows) byTask.set(String(row.task_id), row);
+ return {
+ ...definition,
+ tasks: definition.tasks.map((task) => {
+ const meta = byTask.get(String(task && task.id));
+ if (!meta) return task;
+ const out = { ...task };
+ if (meta.added_on) out.added_on = String(meta.added_on).slice(0, 10);
+ if (meta.held_out) { out.holdout = true; out.held_out = true; }
+ if (meta.human_minutes != null) out.human_minutes = num(meta.human_minutes);
+ if (meta.human_minutes_source) out.human_minutes_source = meta.human_minutes_source;
+ return out;
+ }),
+ };
+}
+
/** A share token: 256 random bits in base64url (43 characters), never guessable. */
/** A share token: 256 random bits in base64url (43 characters), never guessable. */
function newShareToken() {
function newShareToken() {
return crypto.randomBytes(32).toString('base64url');
return crypto.randomBytes(32).toString('base64url');
@@ -85,7 +134,17 @@ function canViewBenchmark(benchmark, { subject = null, executor = false } = {})
@@ -85,7 +134,17 @@ function canViewBenchmark(benchmark, { subject = null, executor = false } = {})
return true;
return true;
}
}
-function wireBenchmark(row) {
+/** A working copy whose held-out prompts stay on the server, even in a listing that carries the draft. */
+function wireDraft(value) {
+ const draft = json(value, null);
+ if (!draft || !Array.isArray(draft.tasks)) return draft;
+ return {
+ ...draft,
+ tasks: draft.tasks.map((t) => (t && isHeldOut(t) ? { ...t, prompt: null, expected: null, files: null } : t)),
+ };
+}
+
+function wireBenchmark(row, { revealDraft = false } = {}) {
if (!row) return null;
if (!row) return null;
return {
return {
id: row.id,
id: row.id,
@@ -99,7 +158,7 @@ function wireBenchmark(row) {
@@ -99,7 +158,7 @@ function wireBenchmark(row) {
shared: Boolean(row.share_token_hash),
shared: Boolean(row.share_token_hash),
parent: row.parent || null,
parent: row.parent || null,
latest_version: num(row.latest_version),
latest_version: num(row.latest_version),
- draft: json(row.draft, null),
+ draft: revealDraft ? json(row.draft, null) : wireDraft(row.draft),
created_by: row.created_by,
created_by: row.created_by,
created_at: iso(row.created_at),
created_at: iso(row.created_at),
updated_at: iso(row.updated_at),
updated_at: iso(row.updated_at),
@@ -168,7 +227,7 @@ function wireRun(row, tasks = null) {
@@ -168,7 +227,7 @@ function wireRun(row, tasks = null) {
function wireTask(row) {
function wireTask(row) {
if (!row) return null;
if (!row) return null;
- return {
+ const out = {
task_id: row.task_id,
task_id: row.task_id,
score: row.score == null ? null : num(row.score),
score: row.score == null ? null : num(row.score),
cost_usd: num(row.cost_usd),
cost_usd: num(row.cost_usd),
@@ -177,6 +236,13 @@ function wireTask(row) {
@@ -177,6 +236,13 @@ function wireTask(row) {
artifacts: (Array.isArray(json(row.artifacts, [])) ? json(row.artifacts, []) : []).map((a) => ({ kind: a.kind, url: a.url })).filter((a) => a.url),
artifacts: (Array.isArray(json(row.artifacts, [])) ? json(row.artifacts, []) : []).map((a) => ({ kind: a.kind, url: a.url })).filter((a) => a.url),
transcript_url: row.transcript_url || null,
transcript_url: row.transcript_url || null,
};
};
+ if (row.added_on) out.added_on = String(row.added_on).slice(0, 10);
+ if (row.held_out) out.held_out = true;
+ if (row.human_minutes != null) {
+ out.human_minutes = num(row.human_minutes);
+ if (row.human_minutes_source) out.human_minutes_source = row.human_minutes_source;
+ }
+ return out;
}
}
/** The median of a list of numbers. */
/** The median of a list of numbers. */
@@ -187,6 +253,38 @@ function median(values) {
@@ -187,6 +253,38 @@ function median(values) {
return list.length % 2 ? list[mid] : (list[mid - 1] + list[mid]) / 2;
return list.length % 2 ? list[mid] : (list[mid - 1] + list[mid]) / 2;
}
}
+const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
+
+/**
+ * A `since` value: "YYYY-MM" (a monthly fresh set), "YYYY-MM-DD", or a full timestamp (the B2 form, still accepted).
+ * The cutoff is the first instant of the month/day. The label is what a page says beside the board.
+ */
+function parseSince(value) {
+ const text = String(value || '').trim();
+ if (!text) return null;
+ let m = /^(\d{4})-(\d{2})$/.exec(text);
+ if (m) {
+ const month = Number(m[2]);
+ if (month < 1 || month > 12) return null;
+ const cutoff = `${m[1]}-${m[2]}-01`;
+ return { text, cutoff, month: `${m[1]}-${m[2]}`, timestamp: `${cutoff}T00:00:00.000Z`, label: `${MONTH_NAMES[month - 1]} ${m[1]}` };
+ }
+ m = /^(\d{4})-(\d{2})-(\d{2})/.exec(text);
+ if (m) {
+ const month = Number(m[2]);
+ if (month < 1 || month > 12) return null;
+ const cutoff = text.slice(0, 10);
+ if (Number.isNaN(Date.parse(`${cutoff}T00:00:00Z`))) return null;
+ return { text, cutoff, month: `${m[1]}-${m[2]}`, timestamp: `${cutoff}T00:00:00.000Z`, label: `${MONTH_NAMES[month - 1]} ${m[1]}` };
+ }
+ const stamp = Date.parse(text);
+ if (Number.isNaN(stamp)) return null;
+ const d = new Date(stamp);
+ const month = d.getUTCMonth();
+ const cutoff = d.toISOString().slice(0, 10);
+ return { text, cutoff, month: cutoff.slice(0, 7), timestamp: d.toISOString(), label: `${MONTH_NAMES[month]} ${d.getUTCFullYear()}` };
+}
+
/** "bmk_x@2" → { id, version }; "bmk_x" → { id, version: null } (the newest frozen one). */
/** "bmk_x@2" → { id, version }; "bmk_x" → { id, version: null } (the newest frozen one). */
function parseRef(ref) {
function parseRef(ref) {
const text = String(ref || '').trim();
const text = String(ref || '').trim();
@@ -220,12 +318,12 @@ function createRegistry(s, { log = console } = {}) {
@@ -220,12 +318,12 @@ function createRegistry(s, { log = console } = {}) {
[id, String(owner || 'openvibe'), text, String(description || '').slice(0, 4000), kindId,
[id, String(owner || 'openvibe'), text, String(description || '').slice(0, 4000), kindId,
JSON.stringify(list), vis, parent || null, JSON.stringify(definition || {}), String(subject || owner || 'openvibe'), s.iso()],
JSON.stringify(list), vis, parent || null, JSON.stringify(definition || {}), String(subject || owner || 'openvibe'), s.iso()],
);
);
- return { ok: true, benchmark: wireBenchmark(row) };
+ return { ok: true, benchmark: await getBenchmark(id, { revealDraft: true }) };
}
}
- /** The working copy: what a later freeze would publish. */
- async function getBenchmark(id) {
- return wireBenchmark(await db.maybe('SELECT * FROM bench_benchmarks WHERE id = $1', [String(id || '')]));
+ /** The working copy: what a later freeze would publish. The draft's held-out prompts stay inside unless asked. */
+ async function getBenchmark(id, { revealDraft = false } = {}) {
+ return wireBenchmark(await db.maybe('SELECT * FROM bench_benchmarks WHERE id = $1', [String(id || '')]), { revealDraft });
}
}
async function getVersion(benchmarkId, version) {
async function getVersion(benchmarkId, version) {
@@ -241,9 +339,46 @@ function createRegistry(s, { log = console } = {}) {
@@ -241,9 +339,46 @@ function createRegistry(s, { log = console } = {}) {
[String(benchmarkId || '')]));
[String(benchmarkId || '')]));
}
}
+ /** The B13 metadata of the named versions, as rows keyed by version and task id. */
+ async function loadTaskMeta(benchmarkId, versions) {
+ if (!versions.length) return [];
+ return db.many(
+ `SELECT * FROM bench_task_meta WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])
+ ORDER BY benchmark_version, task_id`,
+ [String(benchmarkId), versions.map(num)]);
+ }
+
+ /** The metadata map the rankings and the horizon read: stored metadata plus the base task's own holdout flag. */
+ function taskMetaMap(versions, definitions, metaRows) {
+ const out = new Map();
+ for (const version of versions) {
+ const definition = definitions.get(num(version)) || {};
+ for (const task of (definition.tasks || [])) {
+ if (!task || task.id == null) continue;
+ out.set(`${num(version)}:${String(task.id)}`, {
+ added_on: addedOn(task), held_out: isHeldOut(task),
+ human_minutes: humanMinutes(task), human_minutes_source: humanMinutesSource(task),
+ });
+ }
+ }
+ for (const row of metaRows) {
+ const key = `${num(row.benchmark_version)}:${String(row.task_id)}`;
+ out.set(key, {
+ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,
+ held_out: Boolean(row.held_out),
+ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),
+ human_minutes_source: row.human_minutes_source || null,
+ });
+ }
+ return out;
+ }
+
/**
/**
* The definition of a benchmark, at a version or at its newest frozen one. Holdout tasks are withheld unless the
* The definition of a benchmark, at a version or at its newest frozen one. Holdout tasks are withheld unless the
* caller holds bench.run.publish: the executor may see them, everyone else may not.
* caller holds bench.run.publish: the executor may see them, everyone else may not.
+ *
+ * `version.definition` stays the closed contract document. `definition` is Bench's own view, with the B13 task
+ * fields merged back in; `task_meta` is the same metadata without any prompt. A page reads `definition`.
*/
*/
async function getDefinition(idOrRef, { reveal = false } = {}) {
async function getDefinition(idOrRef, { reveal = false } = {}) {
const ref = parseRef(idOrRef);
const ref = parseRef(idOrRef);
@@ -254,16 +389,49 @@ function createRegistry(s, { log = console } = {}) {
@@ -254,16 +389,49 @@ function createRegistry(s, { log = console } = {}) {
? await getVersion(ref.id, ref.version)
? await getVersion(ref.id, ref.version)
: await latestVersion(ref.id);
: await latestVersion(ref.id);
if (!version) return null;
if (!version) return null;
- const definition = { ...version.definition };
- if (!reveal && Array.isArray(definition.tasks)) {
- definition.tasks = definition.tasks.map((t) => (t && t.holdout ? { ...t, prompt: null, expected: null, files: null, holdout: true } : t));
+ const metaRows = await loadTaskMeta(ref.id, [version.version]);
+ const base = { ...version.definition };
+ const merged = mergeTaskMeta(base, metaRows);
+ const hide = (definition) => {
+ if (reveal || !Array.isArray(definition.tasks)) return definition;
+ return {
+ ...definition,
+ tasks: definition.tasks.map((t) => (t && isHeldOut(t) ? { ...t, prompt: null, expected: null, files: null, holdout: true, held_out: true } : t)),
+ };
+ };
+ return {
+ benchmark,
+ version: { ...version, definition: hide(base) },
+ definition: hide(merged),
+ task_meta: metaRows.map((row) => ({
+ task_id: row.task_id,
+ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,
+ held_out: Boolean(row.held_out),
+ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),
+ human_minutes_source: row.human_minutes_source || null,
+ })),
+ };
+ }
+
+ /** Write the B13 metadata of a version's tasks. The definition itself stays contract-shaped. */
+ async function replaceTaskMeta(benchmarkId, version, tasks) {
+ await db.exec('DELETE FROM bench_task_meta WHERE benchmark_id = $1 AND benchmark_version = $2', [String(benchmarkId), num(version)]);
+ for (const task of tasks) {
+ const meta = taskMetaOf(task);
+ if (!meta.added_on && !meta.held_out && meta.human_minutes == null && !meta.human_minutes_source) continue;
+ await db.exec(
+ `INSERT INTO bench_task_meta (benchmark_id, benchmark_version, task_id, added_on, held_out, human_minutes, human_minutes_source)
+ VALUES ($1, $2, $3, $4::date, $5, $6, $7)
+ ON CONFLICT (benchmark_id, benchmark_version, task_id)
+ DO UPDATE SET added_on = EXCLUDED.added_on, held_out = EXCLUDED.held_out,
+ human_minutes = EXCLUDED.human_minutes, human_minutes_source = EXCLUDED.human_minutes_source`,
+ [String(benchmarkId), num(version), String(task.id), meta.added_on, meta.held_out, meta.human_minutes, meta.human_minutes_source]);
}
}
- return { benchmark, version: { ...version, definition }, definition };
}
}
/** Save a new draft. A version named in the body is edited in place, and only while no run names it. */
/** Save a new draft. A version named in the body is edited in place, and only while no run names it. */
async function saveDraft(id, { definition = {}, subject }) {
async function saveDraft(id, { definition = {}, subject }) {
- const benchmark = await getBenchmark(id);
+ const benchmark = await getBenchmark(id, { revealDraft: true });
if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
const version = definition.version == null ? null : num(definition.version);
const version = definition.version == null ? null : num(definition.version);
if (version != null) {
if (version != null) {
@@ -273,10 +441,32 @@ function createRegistry(s, { log = console } = {}) {
@@ -273,10 +441,32 @@ function createRegistry(s, { log = console } = {}) {
if (used > 0) {
if (used > 0) {
return refusal(409, 'bench.version_frozen', 'A version that a run names never changes again. Fork the benchmark instead.');
return refusal(409, 'bench.version_frozen', 'A version that a run names never changes again. Fork the benchmark instead.');
}
}
- const next = { ...row.definition, ...definition, version };
+ const tasks = Array.isArray(definition.tasks) ? definition.tasks : null;
+ // A caller that read the contract-shaped definition has no B13 fields; keep the stored metadata for a
+ // task it leaves unchanged rather than erasing a date or a time on every edit.
+ const priorMeta = new Map((await loadTaskMeta(benchmark.id, [version])).map((m) => [String(m.task_id), m]));
+ const mergedTasks = tasks ? tasks.map((task) => {
+ const meta = priorMeta.get(String(task && task.id));
+ if (!meta) return task;
+ const out = { ...task };
+ if (out.added_on == null && meta.added_on) out.added_on = String(meta.added_on).slice(0, 10);
+ if (out.held_out == null && out.holdout == null && meta.held_out) out.held_out = true;
+ if (out.human_minutes == null && meta.human_minutes != null) {
+ out.human_minutes = num(meta.human_minutes);
+ if (meta.human_minutes_source) out.human_minutes_source = meta.human_minutes_source;
+ }
+ return out;
+ }) : null;
+ const next = {
+ ...row.definition,
+ ...definition,
+ version,
+ ...(mergedTasks ? { tasks: mergedTasks.map(contractTask) } : {}),
+ };
await db.exec(
await db.exec(
'UPDATE bench_benchmark_versions SET definition = $1::jsonb, hash = $2 WHERE benchmark_id = $3 AND version = $4',
'UPDATE bench_benchmark_versions SET definition = $1::jsonb, hash = $2 WHERE benchmark_id = $3 AND version = $4',
[JSON.stringify(next), hashDefinition(next), benchmark.id, version]);
[JSON.stringify(next), hashDefinition(next), benchmark.id, version]);
+ if (mergedTasks) await replaceTaskMeta(benchmark.id, version, mergedTasks);
await db.exec('UPDATE bench_benchmarks SET updated_at = $1::timestamptz WHERE id = $2', [s.iso(), benchmark.id]);
await db.exec('UPDATE bench_benchmarks SET updated_at = $1::timestamptz WHERE id = $2', [s.iso(), benchmark.id]);
return { ok: true, version: await getVersion(benchmark.id, version) };
return { ok: true, version: await getVersion(benchmark.id, version) };
}
}
@@ -288,9 +478,10 @@ function createRegistry(s, { log = console } = {}) {
@@ -288,9 +478,10 @@ function createRegistry(s, { log = console } = {}) {
/** Freeze the working copy (or a body definition) into the next immutable version. */
/** Freeze the working copy (or a body definition) into the next immutable version. */
async function freezeVersion(id, { definition = null, subject, validate = null }) {
async function freezeVersion(id, { definition = null, subject, validate = null }) {
- const benchmark = await getBenchmark(id);
+ const benchmark = await getBenchmark(id, { revealDraft: true });
if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
if (!benchmark) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
const source = definition || benchmark.draft || {};
const source = definition || benchmark.draft || {};
+ const rawTasks = Array.isArray(source.tasks) ? source.tasks : [];
const version = benchmark.latest_version + 1;
const version = benchmark.latest_version + 1;
// The published contract knows public and unlisted only. A private benchmark is frozen as unlisted (the
// The published contract knows public and unlisted only. A private benchmark is frozen as unlisted (the
// stricter form); the row's own visibility is what every access check reads, so its prompts stay private.
// stricter form); the row's own visibility is what every access check reads, so its prompts stay private.
@@ -305,7 +496,7 @@ function createRegistry(s, { log = console } = {}) {
@@ -305,7 +496,7 @@ function createRegistry(s, { log = console } = {}) {
visibility: frozenVisibility,
visibility: frozenVisibility,
version,
version,
parent: benchmark.parent,
parent: benchmark.parent,
- tasks: Array.isArray(source.tasks) ? source.tasks : [],
+ tasks: rawTasks.map(contractTask),
judges: Array.isArray(source.judges) ? source.judges : [],
judges: Array.isArray(source.judges) ? source.judges : [],
scoring: source.scoring || { method: 'mean', scale: { min: 0, max: 100 } },
scoring: source.scoring || { method: 'mean', scale: { min: 0, max: 100 } },
limits: source.limits || {},
limits: source.limits || {},
@@ -324,15 +515,22 @@ function createRegistry(s, { log = console } = {}) {
@@ -324,15 +515,22 @@ function createRegistry(s, { log = console } = {}) {
VALUES ($1, $2, $3::jsonb, $4, $5, $6::timestamptz)
VALUES ($1, $2, $3::jsonb, $4, $5, $6::timestamptz)
ON CONFLICT (benchmark_id, version) DO NOTHING`,
ON CONFLICT (benchmark_id, version) DO NOTHING`,
[benchmark.id, version, JSON.stringify(doc), hashDefinition(doc), String(subject || benchmark.created_by), s.iso()]);
[benchmark.id, version, JSON.stringify(doc), hashDefinition(doc), String(subject || benchmark.created_by), s.iso()]);
+ await replaceTaskMeta(benchmark.id, version, rawTasks);
await db.exec('UPDATE bench_benchmarks SET latest_version = $1, updated_at = $2::timestamptz WHERE id = $3',
await db.exec('UPDATE bench_benchmarks SET latest_version = $1, updated_at = $2::timestamptz WHERE id = $3',
[version, s.iso(), benchmark.id]);
[version, s.iso(), benchmark.id]);
return { ok: true, version: await getVersion(benchmark.id, version), benchmark: await getBenchmark(benchmark.id) };
return { ok: true, version: await getVersion(benchmark.id, version), benchmark: await getBenchmark(benchmark.id) };
}
}
/** A fork: a new benchmark of its own that records the parent it came from. */
/** A fork: a new benchmark of its own that records the parent it came from. */
- async function forkBenchmark(id, { version = null, title = null, owner, subject }) {
- const source = await getDefinition(id, { reveal: true });
+ async function forkBenchmark(id, { version = null, title = null, owner, subject, reveal = true }) {
+ const ref = version == null ? String(id || '') : `${String(id || '')}@${num(version)}`;
+ const source = await getDefinition(ref, { reveal });
if (!source) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
if (!source) return refusal(404, 'bench.benchmark_not_found', 'No such benchmark.');
+ // A held-out prompt only goes to its owner or the executor. A fork by anyone else is refused rather than
+ // copied with a blank prompt: the text never has to leave the server to make the copy.
+ if (!reveal && source.definition.tasks.some((t) => isHeldOut(t))) {
+ return refusal(403, 'bench.holdout_fork_refused', 'This benchmark has held-out tasks; only its owner or the executor can fork it.');
+ }
const parent = `${source.benchmark.id}@${source.version.version}`;
const parent = `${source.benchmark.id}@${source.version.version}`;
const definition = {
const definition = {
...source.definition,
...source.definition,
@@ -411,16 +609,24 @@ function createRegistry(s, { log = console } = {}) {
@@ -411,16 +609,24 @@ function createRegistry(s, { log = console } = {}) {
if (benchmark.visibility !== 'public') continue;
if (benchmark.visibility !== 'public') continue;
const version = await latestVersion(benchmark.id);
const version = await latestVersion(benchmark.id);
if (!version) continue;
if (!version) continue;
- const definition = version.definition || {};
+ const metaRows = await loadTaskMeta(benchmark.id, [version.version]);
+ const definition = mergeTaskMeta(version.definition || {}, metaRows);
lines.push(JSON.stringify({
lines.push(JSON.stringify({
type: 'benchmark', id: benchmark.id, title: benchmark.title, description: benchmark.description,
type: 'benchmark', id: benchmark.id, title: benchmark.title, description: benchmark.description,
kind: benchmark.kind, tags: benchmark.tags, version: version.version, parent: benchmark.parent,
kind: benchmark.kind, tags: benchmark.tags, version: version.version, parent: benchmark.parent,
created_at: benchmark.created_at,
created_at: benchmark.created_at,
}));
}));
+ const estimates = await estimateMedians(benchmark.id, [version.version]);
for (const task of (definition.tasks || [])) {
for (const task of (definition.tasks || [])) {
- if (!task || task.holdout) continue;
+ if (!task || isHeldOut(task)) continue;
const item = { type: 'benchmark_task', benchmark: `${benchmark.id}@${version.version}`, id: task.id, prompt: task.prompt };
const item = { type: 'benchmark_task', benchmark: `${benchmark.id}@${version.version}`, id: task.id, prompt: task.prompt };
if (task.expected != null) item.expected = task.expected;
if (task.expected != null) item.expected = task.expected;
+ if (addedOn(task)) item.added_on = addedOn(task);
+ const timed = taskMinutes(task, `${num(version.version)}:${String(task.id)}`, estimates);
+ if (timed) {
+ item.human_minutes = timed.minutes;
+ item.human_minutes_source = timed.source;
+ }
lines.push(JSON.stringify(item));
lines.push(JSON.stringify(item));
}
}
const runs = await db.many(
const runs = await db.many(
@@ -445,7 +651,7 @@ function createRegistry(s, { log = console } = {}) {
@@ -445,7 +651,7 @@ function createRegistry(s, { log = console } = {}) {
* private benchmarks are never listed to anyone else (an unlisted one is reachable by its id; a private one only
* private benchmarks are never listed to anyone else (an unlisted one is reachable by its id; a private one only
* by its owner and the executor).
* by its owner and the executor).
*/
*/
- async function listBenchmarks({ kind = null, tag = null, q = null, limit = 50, owner = null } = {}) {
+ async function listBenchmarks({ kind = null, tag = null, q = null, limit = 50, owner = null, viewer = null } = {}) {
const where = [];
const where = [];
const params = [];
const params = [];
if (owner) { params.push(String(owner)); where.push(`(visibility = 'public' OR owner = $${params.length})`); }
if (owner) { params.push(String(owner)); where.push(`(visibility = 'public' OR owner = $${params.length})`); }
@@ -459,7 +665,7 @@ function createRegistry(s, { log = console } = {}) {
@@ -459,7 +665,7 @@ function createRegistry(s, { log = console } = {}) {
const sql = `SELECT * FROM bench_benchmarks ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
const sql = `SELECT * FROM bench_benchmarks ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
ORDER BY created_at DESC LIMIT ${Math.max(1, Math.min(200, num(limit, 50)))}`;
ORDER BY created_at DESC LIMIT ${Math.max(1, Math.min(200, num(limit, 50)))}`;
const rows = await db.many(sql, params);
const rows = await db.many(sql, params);
- return rows.map(wireBenchmark);
+ return rows.map((row) => wireBenchmark(row, { revealDraft: viewer != null && String(row.owner) === String(viewer) }));
}
}
// ── Runs ─────────────────────────────────────────────────
// ── Runs ─────────────────────────────────────────────────
@@ -489,12 +695,20 @@ function createRegistry(s, { log = console } = {}) {
@@ -489,12 +695,20 @@ function createRegistry(s, { log = console } = {}) {
if (!provider) return refusal(422, 'bench.provider_required', 'A run names the provider that served the model.');
if (!provider) return refusal(422, 'bench.provider_required', 'A run names the provider that served the model.');
const tasks = Array.isArray(doc.tasks) ? doc.tasks : [];
const tasks = Array.isArray(doc.tasks) ? doc.tasks : [];
if (!tasks.length) return refusal(422, 'bench.no_tasks', 'A run carries at least one task result.');
if (!tasks.length) return refusal(422, 'bench.no_tasks', 'A run carries at least one task result.');
- const known = new Set((versionRow.definition.tasks || []).map((t) => String(t.id)));
+ const metaRows = await loadTaskMeta(benchmarkId, [version]);
+ const mergedDefinition = mergeTaskMeta(versionRow.definition, metaRows);
+ const taskMetaById = new Map((mergedDefinition.tasks || []).map((t) => [String(t.id), t]));
for (const t of tasks) {
for (const t of tasks) {
- if (!known.has(String(t.task_id))) {
+ if (!taskMetaById.has(String(t.task_id))) {
return refusal(422, 'bench.unknown_task', `This benchmark version has no task "${t.task_id}".`);
return refusal(422, 'bench.unknown_task', `This benchmark version has no task "${t.task_id}".`);
}
}
}
}
+ // A held-out task is scored only by the executor (a verified run). A submitted run, including an import or
+ // a mirrored result, may name the task, but its score is dropped before anything is stored.
+ const storedTasks = tasks.map((t) => {
+ const meta = taskMetaById.get(String(t.task_id)) || {};
+ return provenance === 'submitted' && isHeldOut(meta) ? { ...t, score: null } : t;
+ });
// A submitted run has no executor: whatever the body says, the provenance decides.
// A submitted run has no executor: whatever the body says, the provenance decides.
const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');
const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');
const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };
const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };
@@ -505,7 +719,7 @@ function createRegistry(s, { log = console } = {}) {
@@ -505,7 +719,7 @@ function createRegistry(s, { log = console } = {}) {
id,
id,
benchmark: `${benchmarkId}@${version}`,
benchmark: `${benchmarkId}@${version}`,
executor,
executor,
- tasks,
+ tasks: storedTasks,
score: doc.score == null ? null : num(doc.score, 0),
score: doc.score == null ? null : num(doc.score, 0),
cost_usd: num(doc.cost_usd, 0),
cost_usd: num(doc.cost_usd, 0),
ms: num(doc.ms, 0),
ms: num(doc.ms, 0),
@@ -531,12 +745,15 @@ function createRegistry(s, { log = console } = {}) {
@@ -531,12 +745,15 @@ function createRegistry(s, { log = console } = {}) {
JSON.stringify(doc.source || {}),
JSON.stringify(doc.source || {}),
doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],
doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],
);
);
- for (const t of tasks) {
+ for (const t of storedTasks) {
+ const meta = taskMetaById.get(String(t.task_id)) || {};
await db.exec(
await db.exec(
- `INSERT INTO bench_run_tasks (id, run_id, task_id, score, cost_usd, ms, tokens, artifacts, transcript_url, created_at)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::timestamptz)`,
+ `INSERT INTO bench_run_tasks (id, run_id, task_id, score, cost_usd, ms, tokens, artifacts, transcript_url,
+ added_on, held_out, human_minutes, human_minutes_source, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::date, $11, $12, $13, $14::timestamptz)`,
[s.newId('brt'), id, String(t.task_id), t.score == null ? null : num(t.score), num(t.cost_usd), num(t.ms), num(t.tokens),
[s.newId('brt'), id, String(t.task_id), t.score == null ? null : num(t.score), num(t.cost_usd), num(t.ms), num(t.tokens),
- JSON.stringify(Array.isArray(t.artifacts) ? t.artifacts : []), t.transcript_url == null ? null : String(t.transcript_url), s.iso()],
+ JSON.stringify(Array.isArray(t.artifacts) ? t.artifacts : []), t.transcript_url == null ? null : String(t.transcript_url),
+ addedOn(meta), isHeldOut(meta), humanMinutes(meta), humanMinutesSource(meta), s.iso()],
);
);
}
}
});
});
@@ -575,12 +792,47 @@ function createRegistry(s, { log = console } = {}) {
@@ -575,12 +792,47 @@ function createRegistry(s, { log = console } = {}) {
};
};
}
}
+ /** Median of each task's editor estimates, when at least three people have entered one. */
+ async function estimateMedians(benchmarkId, versions) {
+ const out = new Map();
+ if (!versions.length) return out;
+ const rows = await db.many(
+ `SELECT benchmark_version, task_id, minutes FROM bench_task_estimates
+ WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])`,
+ [String(benchmarkId), versions]);
+ const grouped = new Map();
+ for (const row of rows) {
+ const key = `${num(row.benchmark_version)}:${String(row.task_id)}`;
+ if (!grouped.has(key)) grouped.set(key, []);
+ grouped.get(key).push(num(row.minutes));
+ }
+ for (const [key, list] of grouped) {
+ if (list.length >= 3) out.set(key, { median: median(list), count: list.length });
+ }
+ return out;
+ }
+
+ /** The minutes of a task: the author's own estimate first, else the median of three editors. */
+ function taskMinutes(task, key, estimates) {
+ const author = humanMinutes(task);
+ if (author != null) return { minutes: author, source: humanMinutesSource(task) || 'benchmark author' };
+ const estimate = estimates.get(key);
+ if (estimate) return { minutes: estimate.median, source: `median of ${estimate.count} editor estimates` };
+ return null;
+ }
+
/**
/**
* The rankings for a benchmark. Verified runs only unless the caller asks for the submitted ones.
* The rankings for a benchmark. Verified runs only unless the caller asks for the submitted ones.
* score the median score of a model
* score the median score of a model
* value the median score per dollar
* value the median score per dollar
* speed the median score per minute
* speed the median score per minute
* consistency the spread of a model's scores inside one session (the lower, the steadier)
* consistency the spread of a model's scores inside one session (the lower, the steadier)
+ * horizon the 50% human-time horizon (minutes; higher is better)
+ *
+ * `since` is a monthly fresh set: with dated tasks, only the tasks added on or after that month are scored.
+ * An older benchmark whose tasks carry no dates keeps the B2 meaning (runs created since the date), so nothing
+ * that worked before stops working. Each entry also carries the contamination flag and the time horizon when
+ * there is enough data for either.
*/
*/
async function rankings({ benchmark, metric = 'score', harness = null, since = null, includeSubmitted = false }) {
async function rankings({ benchmark, metric = 'score', harness = null, since = null, includeSubmitted = false }) {
const ref = parseRef(benchmark);
const ref = parseRef(benchmark);
@@ -590,34 +842,137 @@ function createRegistry(s, { log = console } = {}) {
@@ -590,34 +842,137 @@ function createRegistry(s, { log = console } = {}) {
[ref.id]);
[ref.id]);
const wanted = ref.version != null ? [ref.version] : versions.map((v) => num(v.version));
const wanted = ref.version != null ? [ref.version] : versions.map((v) => num(v.version));
if (!wanted.length) return null;
if (!wanted.length) return null;
+ const definitions = new Map();
+ for (const version of wanted) {
+ const row = await getVersion(ref.id, version);
+ if (row) definitions.set(num(version), row.definition || {});
+ }
+ if (!definitions.size) return null;
+
+ const sinceParsed = since ? parseSince(since) : null;
+ const metaRows = await loadTaskMeta(ref.id, wanted);
+ const metaByKey = taskMetaMap(wanted, definitions, metaRows);
+ const datedTasks = [...metaByKey.values()].filter((meta) => meta.added_on);
+ const taskBasis = Boolean(sinceParsed && datedTasks.length);
+ const selected = new Set();
+ if (taskBasis) {
+ for (const [key, meta] of metaByKey) {
+ if (meta.added_on && meta.added_on >= sinceParsed.cutoff) selected.add(key);
+ }
+ }
+
const params = [ref.id, wanted];
const params = [ref.id, wanted];
let sql = `SELECT * FROM bench_runs WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])`;
let sql = `SELECT * FROM bench_runs WHERE benchmark_id = $1 AND benchmark_version = ANY($2::int[])`;
if (!includeSubmitted) sql += " AND provenance = 'verified'";
if (!includeSubmitted) sql += " AND provenance = 'verified'";
// Each filter takes the next placeholder: `since` alone is $3, not $4.
// Each filter takes the next placeholder: `since` alone is $3, not $4.
if (harness) { params.push(String(harness)); sql += ` AND harness->>'name' = $${params.length}`; }
if (harness) { params.push(String(harness)); sql += ` AND harness->>'name' = $${params.length}`; }
- if (since) { params.push(String(since)); sql += ` AND created_at >= $${params.length}::timestamptz`; }
+ if (sinceParsed && !taskBasis) {
+ params.push(sinceParsed.timestamp);
+ sql += ` AND created_at >= $${params.length}::timestamptz`;
+ }
const rows = await db.many(sql, params);
const rows = await db.many(sql, params);
+ const runIds = rows.map((row) => String(row.id));
+ const taskRows = runIds.length
+ ? await db.many('SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id', [runIds])
+ : [];
+ const tasksByRun = new Map();
+ for (const row of taskRows) {
+ if (!tasksByRun.has(row.run_id)) tasksByRun.set(row.run_id, []);
+ tasksByRun.get(row.run_id).push(row);
+ }
+ const estimates = await estimateMedians(ref.id, wanted);
+
+ /** The score and totals of one run, over the selected tasks when a monthly set is asked for. */
+ function subsetOf(run) {
+ if (!taskBasis) {
+ return { score: run.score, cost_usd: run.cost_usd, ms: run.ms, tokens: run.tokens };
+ }
+ const picked = (tasksByRun.get(run.id) || []).filter((row) => {
+ if (row.score == null) return false;
+ return selected.has(`${num(run.benchmark_version)}:${String(row.task_id)}`);
+ });
+ if (!picked.length) return null;
+ return {
+ score: picked.reduce((n, row) => n + num(row.score), 0) / picked.length,
+ cost_usd: picked.reduce((n, row) => n + num(row.cost_usd), 0),
+ ms: picked.reduce((n, row) => n + num(row.ms), 0),
+ tokens: picked.reduce((n, row) => n + num(row.tokens), 0),
+ };
+ }
+
+ /** The contamination flag: per run, the mean of its old tasks and of its fresh tasks. */
+ function contaminationOf(model, list) {
+ const cutoff = cutoffFor(model);
+ if (!cutoff) return contaminationFor({ cutoff: null });
+ const oldScores = [];
+ const freshScores = [];
+ for (const run of list) {
+ let oldSum = 0; let oldN = 0; let freshSum = 0; let freshN = 0;
+ for (const row of tasksByRun.get(run.id) || []) {
+ if (row.score == null) continue;
+ const meta = metaByKey.get(`${num(run.benchmark_version)}:${String(row.task_id)}`) || {};
+ const day = row.added_on ? String(row.added_on).slice(0, 10) : (meta.added_on || null);
+ if (!day) continue;
+ if (day < cutoff.cutoff) { oldSum += num(row.score); oldN++; }
+ else { freshSum += num(row.score); freshN++; }
+ }
+ if (oldN) oldScores.push(oldSum / oldN);
+ if (freshN) freshScores.push(freshSum / freshN);
+ }
+ return contaminationFor({ oldScores, freshScores, cutoff: cutoff.cutoff, source: cutoff.source });
+ }
+
+ /** The time horizon of one model over the runs in scope. */
+ function horizonOfModel(list) {
+ const groups = new Map();
+ let timedRuns = 0;
+ for (const run of list) {
+ let touched = false;
+ for (const row of tasksByRun.get(run.id) || []) {
+ if (row.score == null) continue;
+ const version = num(run.benchmark_version);
+ const key = `${version}:${String(row.task_id)}`;
+ const meta = metaByKey.get(key) || {};
+ const timed = taskMinutes(meta, key, estimates);
+ if (!timed) continue;
+ if (!groups.has(key)) groups.set(key, { minutes: timed.minutes, source: timed.source, scores: [] });
+ groups.get(key).scores.push(num(row.score));
+ touched = true;
+ }
+ if (touched) timedRuns++;
+ }
+ const taskGroups = [...groups.values()].map((g) => ({
+ minutes: g.minutes,
+ success: g.scores.reduce((n, v) => n + v, 0) / g.scores.length / 100,
+ runs: g.scores.length,
+ source: g.source,
+ }));
+ return estimateHorizon(taskGroups, { runs: timedRuns, timedTasks: taskGroups.length });
+ }
+
const byModel = new Map();
const byModel = new Map();
for (const row of rows) {
for (const row of rows) {
const run = wireRun(row);
const run = wireRun(row);
+ const subset = subsetOf(run);
+ if (sinceParsed && taskBasis && !subset) continue;
if (!byModel.has(run.model)) byModel.set(run.model, []);
if (!byModel.has(run.model)) byModel.set(run.model, []);
- byModel.get(run.model).push(run);
+ byModel.get(run.model).push({ run, subset });
}
}
const entries = [];
const entries = [];
for (const [model, list] of byModel) {
for (const [model, list] of byModel) {
- const scores = list.map((r) => r.score);
+ const scores = list.map((item) => item.subset && item.subset.score);
const cleanScores = scores.filter((v) => v != null);
const cleanScores = scores.filter((v) => v != null);
- const costs = list.map((r) => r.cost_usd);
- const times = list.map((r) => r.ms);
+ const costs = list.map((item) => (item.subset ? item.subset.cost_usd : null));
+ const times = list.map((item) => (item.subset ? item.subset.ms : null));
const medScore = median(scores);
const medScore = median(scores);
const medCost = median(costs);
const medCost = median(costs);
const medMs = median(times);
const medMs = median(times);
const sessions = new Map();
const sessions = new Map();
- for (const r of list) {
- if (r.score == null) continue;
- const key = r.session || 'default';
+ for (const item of list) {
+ if (!item.subset || item.subset.score == null) continue;
+ const key = item.run.session || 'default';
if (!sessions.has(key)) sessions.set(key, []);
if (!sessions.has(key)) sessions.set(key, []);
- sessions.get(key).push(r.score);
+ sessions.get(key).push(item.subset.score);
}
}
const spreads = [...sessions.values()].filter((v) => v.length > 1).map((v) => Math.max(...v) - Math.min(...v));
const spreads = [...sessions.values()].filter((v) => v.length > 1).map((v) => Math.max(...v) - Math.min(...v));
const entry = {
const entry = {
@@ -636,14 +991,19 @@ function createRegistry(s, { log = console } = {}) {
@@ -636,14 +991,19 @@ function createRegistry(s, { log = console } = {}) {
max_score: cleanScores.length ? Math.max(...cleanScores) : null,
max_score: cleanScores.length ? Math.max(...cleanScores) : null,
spread: cleanScores.length ? Math.round((Math.max(...cleanScores) - Math.min(...cleanScores)) * 1000) / 1000 : null,
spread: cleanScores.length ? Math.round((Math.max(...cleanScores) - Math.min(...cleanScores)) * 1000) / 1000 : null,
cost_per_solved: medScore > 0 && medCost > 0 ? Math.round((medCost / (medScore / 100)) * 1e6) / 1e6 : null,
cost_per_solved: medScore > 0 && medCost > 0 ? Math.round((medCost / (medScore / 100)) * 1e6) / 1e6 : null,
+ contamination: contaminationOf(model, list.map((item) => item.run)),
+ horizon: horizonOfModel(list.map((item) => item.run)),
};
};
entries.push(entry);
entries.push(entry);
}
}
- const key = { score: 'score', value: 'value', speed: 'speed', consistency: 'consistency' }[metric];
- if (!key) return null;
+ const valueOf = (entry) => {
+ if (metric === 'horizon') return entry.horizon && entry.horizon.available ? entry.horizon.horizon_minutes : null;
+ return entry[{ score: 'score', value: 'value', speed: 'speed', consistency: 'consistency' }[metric]];
+ };
+ if (!['score', 'value', 'speed', 'consistency', 'horizon'].includes(metric)) return null;
const ascending = metric === 'consistency';
const ascending = metric === 'consistency';
entries.sort((a, b) => {
entries.sort((a, b) => {
- const av = a[key]; const bv = b[key];
+ const av = valueOf(a); const bv = valueOf(b);
if (av == null && bv == null) return a.model.localeCompare(b.model);
if (av == null && bv == null) return a.model.localeCompare(b.model);
if (av == null) return 1;
if (av == null) return 1;
if (bv == null) return -1;
if (bv == null) return -1;
@@ -668,7 +1028,127 @@ function createRegistry(s, { log = console } = {}) {
@@ -668,7 +1028,127 @@ function createRegistry(s, { log = console } = {}) {
e.band_to = band ? band.to : null;
e.band_to = band ? band.to : null;
}
}
}
}
- return { benchmark: ref.id, metric, entries };
+ return {
+ benchmark: ref.id,
+ metric,
+ since: sinceParsed ? { month: sinceParsed.month, label: sinceParsed.label, basis: taskBasis ? 'tasks' : 'runs' } : null,
+ entries,
+ };
+ }
+
+ // ── Human-time estimates (B13) ───────────────────────────
+ const ESTIMATES_PER_HOUR = 30;
+
+ async function taskEstimateSummary(benchmarkId, version, taskId) {
+ const rows = await db.many(
+ `SELECT minutes FROM bench_task_estimates
+ WHERE benchmark_id = $1 AND benchmark_version = $2 AND task_id = $3 ORDER BY created_at`,
+ [String(benchmarkId), num(version), String(taskId)]);
+ const values = rows.map((row) => num(row.minutes));
+ return {
+ count: values.length,
+ needed: Math.max(0, 3 - values.length),
+ median: values.length >= 3 ? Math.round(median(values) * 10) / 10 : null,
+ values,
+ };
+ }
+
+ /**
+ * One editor's estimate on a task. A person has one estimate per task; a later entry replaces it. More than
+ * thirty entries an hour from one person is refused, so a script cannot flood the median.
+ */
+ async function addTaskEstimate({ benchmarkId, version, taskId, minutes, subject }) {
+ if (!subject) return refusal(401, 'bench.sign_in_required', 'Sign in to estimate a task.');
+ const value = Number(minutes);
+ if (!Number.isFinite(value) || value <= 0 || value > 100000) {
+ return refusal(422, 'bench.invalid_minutes', 'The estimate must be a positive number of minutes.');
+ }
+ const versionRow = await getVersion(benchmarkId, version);
+ if (!versionRow) return refusal(404, 'bench.version_not_found', 'No such benchmark version.');
+ const task = (versionRow.definition.tasks || []).find((t) => String(t.id) === String(taskId));
+ if (!task) return refusal(404, 'bench.task_not_found', 'No such task.');
+ const recent = num(await db.value(
+ `SELECT COUNT(*)::int AS n FROM bench_task_estimates
+ WHERE subject = $1 AND created_at >= now() - interval '1 hour'`, [String(subject)]));
+ if (recent >= ESTIMATES_PER_HOUR) {
+ return refusal(429, 'bench.estimate_rate_limited', 'Too many estimates in an hour. Try again later.');
+ }
+ const row = await db.maybe(
+ `INSERT INTO bench_task_estimates (id, benchmark_id, benchmark_version, task_id, subject, minutes, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7::timestamptz)
+ ON CONFLICT (benchmark_id, benchmark_version, task_id, subject)
+ DO UPDATE SET minutes = EXCLUDED.minutes, created_at = EXCLUDED.created_at
+ RETURNING *`,
+ [s.newId('bte'), String(benchmarkId), num(version), String(taskId), String(subject), Math.round(value * 10) / 10, s.iso()]);
+ return { ok: true, estimate: { task_id: String(taskId), minutes: num(row.minutes), created_at: iso(row.created_at) }, summary: await taskEstimateSummary(benchmarkId, version, taskId) };
+ }
+
+ /** The time horizon of one model over every timed task it ran on a public benchmark. */
+ async function modelHorizon(model) {
+ const rows = await db.many(
+ `SELECT id, benchmark_id, benchmark_version FROM bench_runs
+ WHERE model = $1 AND provenance = 'verified'
+ AND benchmark_id IN (SELECT id FROM bench_benchmarks WHERE visibility = 'public')`,
+ [String(model || '')]);
+ if (!rows.length) return estimateHorizon([], { runs: 0, timedTasks: 0 });
+ const definitions = new Map();
+ for (const row of rows) {
+ const key = `${row.benchmark_id}:${num(row.benchmark_version)}`;
+ if (!definitions.has(key)) {
+ const versionRow = await getVersion(row.benchmark_id, row.benchmark_version);
+ definitions.set(key, versionRow ? versionRow.definition : null);
+ }
+ }
+ const metaRows = await db.many(
+ `SELECT m.* FROM bench_task_meta m JOIN bench_benchmarks b ON b.id = m.benchmark_id
+ WHERE b.visibility = 'public'`);
+ const metaByKey = new Map(metaRows.map((row) => [`${row.benchmark_id}:${num(row.benchmark_version)}:${String(row.task_id)}`, {
+ added_on: row.added_on ? String(row.added_on).slice(0, 10) : null,
+ held_out: Boolean(row.held_out),
+ human_minutes: row.human_minutes == null ? null : num(row.human_minutes),
+ human_minutes_source: row.human_minutes_source || null,
+ }]));
+ const estimates = new Map();
+ const estimateRows = await db.many(
+ `SELECT e.benchmark_id, e.benchmark_version, e.task_id, e.minutes
+ FROM bench_task_estimates e JOIN bench_benchmarks b ON b.id = e.benchmark_id
+ WHERE b.visibility = 'public'`);
+ const byTask = new Map();
+ for (const row of estimateRows) {
+ const key = `${row.benchmark_id}:${num(row.benchmark_version)}:${String(row.task_id)}`;
+ if (!byTask.has(key)) byTask.set(key, []);
+ byTask.get(key).push(num(row.minutes));
+ }
+ for (const [key, list] of byTask) if (list.length >= 3) estimates.set(key, { median: median(list), count: list.length });
+
+ const tasks = await db.many(
+ 'SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id',
+ [rows.map((row) => String(row.id))]);
+ const groups = new Map();
+ const runIds = new Set();
+ const metaOf = (benchmarkId, version, taskId) => {
+ const definition = definitions.get(`${benchmarkId}:${num(version)}`);
+ return ((definition && definition.tasks) || []).find((t) => String(t.id) === String(taskId)) || {};
+ };
+ for (const row of tasks) {
+ if (row.score == null) continue;
+ const run = rows.find((r) => String(r.id) === String(row.run_id));
+ if (!run) continue;
+ const key = `${run.benchmark_id}:${num(run.benchmark_version)}:${String(row.task_id)}`;
+ const meta = metaByKey.get(key) || metaOf(run.benchmark_id, run.benchmark_version, row.task_id);
+ const timed = taskMinutes(meta, key, estimates);
+ if (!timed) continue;
+ if (!groups.has(key)) groups.set(key, { minutes: timed.minutes, source: timed.source, scores: [] });
+ groups.get(key).scores.push(num(row.score));
+ runIds.add(String(row.run_id));
+ }
+ const taskGroups = [...groups.values()].map((g) => ({
+ minutes: g.minutes,
+ success: g.scores.reduce((n, v) => n + v, 0) / g.scores.length / 100,
+ runs: g.scores.length,
+ source: g.source,
+ }));
+ return estimateHorizon(taskGroups, { runs: runIds.size, timedTasks: taskGroups.length });
}
}
/** Where a score sits among a model's runs on a benchmark: the percentile, 0-100. */
/** Where a score sits among a model's runs on a benchmark: the percentile, 0-100. */
@@ -769,10 +1249,11 @@ function createRegistry(s, { log = console } = {}) {
@@ -769,10 +1249,11 @@ function createRegistry(s, { log = console } = {}) {
createBenchmark, getBenchmark, getVersion, latestVersion, getDefinition, saveDraft, freezeVersion,
createBenchmark, getBenchmark, getVersion, latestVersion, getDefinition, saveDraft, freezeVersion,
forkBenchmark, setVisibility, findShared, publicDatasetLines,
forkBenchmark, setVisibility, findShared, publicDatasetLines,
listBenchmarks, publishRun, getRun, compareRuns, rankings, percentile, listRuns, modelSummary, runCounts,
listBenchmarks, publishRun, getRun, compareRuns, rankings, percentile, listRuns, modelSummary, runCounts,
+ taskEstimateSummary, addTaskEstimate, modelHorizon,
};
};
}
}
module.exports = {
module.exports = {
- createRegistry, RegistryError, hashDefinition, canonical, median, parseRef,
+ createRegistry, RegistryError, hashDefinition, canonical, median, parseRef, parseSince,
canViewBenchmark, shareTokenHash, shareTokenMatches, newShareToken,
canViewBenchmark, shareTokenHash, shareTokenMatches, newShareToken,
};
};
@@ -36,6 +36,11 @@ function tasksFor(list) {
@@ -36,6 +36,11 @@ function tasksFor(list) {
files: [],
files: [],
expected: null,
expected: null,
holdout: false,
holdout: false,
+ // A monthly fresh set on day one: the first task of each showcase kind was added in September, the second
+ // in October. The human time is the benchmark author's own estimate.
+ added_on: i === 0 ? '2026-09-18' : '2026-10-20',
+ human_minutes: i === 0 ? 45 : 30,
+ human_minutes_source: 'benchmark author',
}));
}));
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,38 @@
@@ -0,0 +1,38 @@
+'use strict';
+
+/**
+ * The task fields B13 adds, read the same way everywhere: a date a task was added, whether it is held out, and a
+ * human time estimate with its source. `holdout` is the B1 spelling; `held_out` is the B13 one. Both mean the same.
+ */
+
+const DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
+
+function isHeldOut(task) {
+ return Boolean(task && (task.holdout === true || task.held_out === true));
+}
+
+/** "YYYY-MM-DD", or null. The definition validates it; this is the read side. */
+function addedOn(task) {
+ const text = String((task && task.added_on) || '').slice(0, 10);
+ return DATE.test(text) && !Number.isNaN(Date.parse(`${text}T00:00:00Z`)) ? text : null;
+}
+
+function humanMinutes(task) {
+ const n = Number(task && task.human_minutes);
+ return Number.isFinite(n) && n > 0 ? Math.round(n * 10) / 10 : null;
+}
+
+function humanMinutesSource(task) {
+ const text = String((task && task.human_minutes_source) || '').trim();
+ return text ? text.slice(0, 200) : null;
+}
+
+/** The first day of a "YYYY-MM" month, or null. */
+function monthStart(text) {
+ const m = /^(\d{4})-(\d{2})$/.exec(String(text || ''));
+ if (!m) return null;
+ const month = Number(m[2]);
+ return month >= 1 && month <= 12 ? `${m[1]}-${m[2]}-01` : null;
+}
+
+module.exports = { isHeldOut, addedOn, humanMinutes, humanMinutesSource, monthStart };
@@ -30,6 +30,7 @@ const VISIBILITIES = ['public', 'unlisted', 'private'];
@@ -30,6 +30,7 @@ const VISIBILITIES = ['public', 'unlisted', 'private'];
const SCORING_METHODS = ['mean', 'weighted', 'pass_rate', 'sum'];
const SCORING_METHODS = ['mean', 'weighted', 'pass_rate', 'sum'];
const EXECUTOR_KINDS = ['codes', 'run', 'node', 'submitted'];
const EXECUTOR_KINDS = ['codes', 'run', 'node', 'submitted'];
const PROVENANCES = ['verified', 'submitted'];
const PROVENANCES = ['verified', 'submitted'];
+const TASK_DATE = /^\d{4}-\d{2}-\d{2}$/;
class Errors {
class Errors {
constructor() { this.list = []; }
constructor() { this.list = []; }
@@ -55,7 +56,21 @@ function validateBenchmark(doc) {
@@ -55,7 +56,21 @@ function validateBenchmark(doc) {
if (!isObj(t)) { e.add(`/tasks/${i}`, 'must be an object'); return; }
if (!isObj(t)) { e.add(`/tasks/${i}`, 'must be an object'); return; }
if (!isStr(t.id) || !t.id.trim()) e.add(`/tasks/${i}/id`, 'is required');
if (!isStr(t.id) || !t.id.trim()) e.add(`/tasks/${i}/id`, 'is required');
if (!isStr(t.prompt) || !t.prompt.trim()) e.add(`/tasks/${i}/prompt`, 'is required');
if (!isStr(t.prompt) || !t.prompt.trim()) e.add(`/tasks/${i}/prompt`, 'is required');
- if (typeof t.holdout !== 'boolean') e.add(`/tasks/${i}/holdout`, 'must be true or false');
+ if (t.holdout != null && typeof t.holdout !== 'boolean') e.add(`/tasks/${i}/holdout`, 'must be true or false');
+ if (t.held_out != null && typeof t.held_out !== 'boolean') e.add(`/tasks/${i}/held_out`, 'must be true or false');
+ if (t.holdout == null && t.held_out == null) e.add(`/tasks/${i}/holdout`, 'must be true or false');
+ if (t.added_on != null) {
+ const date = String(t.added_on).slice(0, 10);
+ if (!TASK_DATE.test(date) || Number.isNaN(Date.parse(`${date}T00:00:00Z`))) {
+ e.add(`/tasks/${i}/added_on`, 'must be a date (YYYY-MM-DD)');
+ }
+ }
+ if (t.human_minutes != null && (!isNum(t.human_minutes) || t.human_minutes <= 0 || t.human_minutes > 100000)) {
+ e.add(`/tasks/${i}/human_minutes`, 'must be a positive number of minutes');
+ }
+ if (t.human_minutes_source != null && (!isStr(t.human_minutes_source) || t.human_minutes_source.length > 200)) {
+ e.add(`/tasks/${i}/human_minutes_source`, 'must be a string of at most 200 characters');
+ }
});
});
}
}
if (!Array.isArray(doc.judges)) e.add('/judges', 'must be an array');
if (!Array.isArray(doc.judges)) e.add('/judges', 'must be an array');
@@ -13,7 +13,7 @@ const { raw } = require('../render/html');
@@ -13,7 +13,7 @@ const { raw } = require('../render/html');
const { send } = require('../render/layout');
const { send } = require('../render/layout');
const views = require('../views');
const views = require('../views');
const results = require('../views/results');
const results = require('../views/results');
-const { canViewBenchmark } = require('../bench/registry');
+const { canViewBenchmark, parseSince } = require('../bench/registry');
const { parsePrompts, definitionFor, TAG: PROMPT_TAG } = require('../bench/your-prompts');
const { parsePrompts, definitionFor, TAG: PROMPT_TAG } = require('../bench/your-prompts');
const { validateBenchmark } = require('../bench/validate');
const { validateBenchmark } = require('../bench/validate');
const { kindById } = require('../kinds');
const { kindById } = require('../kinds');
@@ -96,10 +96,12 @@ function createPageRoutes(ctx) {
@@ -96,10 +96,12 @@ function createPageRoutes(ctx) {
const metric = METRIC_IDS.includes(String(req.query.metric)) ? String(req.query.metric) : 'score';
const metric = METRIC_IDS.includes(String(req.query.metric)) ? String(req.query.metric) : 'score';
const id = out.benchmark.id;
const id = out.benchmark.id;
const owner = subjectOf(req) != null && String(out.benchmark.owner) === String(subjectOf(req));
const owner = subjectOf(req) != null && String(out.benchmark.owner) === String(subjectOf(req));
+ const since = parseSince(req.query.since);
const [rankings, runs] = await Promise.all([
const [rankings, runs] = await Promise.all([
- registry.rankings({ benchmark: id, metric }),
+ registry.rankings({ benchmark: id, metric, since: since ? since.text : null }),
registry.listRuns({ benchmark: id, limit: 8 }),
registry.listRuns({ benchmark: id, limit: 8 }),
]);
]);
+ const allTime = since ? await registry.rankings({ benchmark: id, metric }) : null;
const isPublic = out.benchmark.visibility === 'public';
const isPublic = out.benchmark.visibility === 'public';
const isPromptBenchmark = Array.isArray(out.benchmark.tags) && out.benchmark.tags.includes(PROMPT_TAG);
const isPromptBenchmark = Array.isArray(out.benchmark.tags) && out.benchmark.tags.includes(PROMPT_TAG);
const ownBoards = owner && isPromptBenchmark
const ownBoards = owner && isPromptBenchmark
@@ -120,7 +122,7 @@ function createPageRoutes(ctx) {
@@ -120,7 +122,7 @@ function createPageRoutes(ctx) {
body: raw(String(results.benchmarkPage({
body: raw(String(results.benchmarkPage({
benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,
benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,
rankings, metric, runs, codesRunUrl: config.codesRunUrl || null, viewer: req.viewer, baseUrl: base(),
rankings, metric, runs, codesRunUrl: config.codesRunUrl || null, viewer: req.viewer, baseUrl: base(),
- ownBoards, share, shared, owner,
+ ownBoards, share, shared, owner, since: rankings && rankings.since, allTimeEntries: allTime && allTime.entries,
}))),
}))),
}, status);
}, status);
};
};
@@ -225,6 +227,48 @@ function createPageRoutes(ctx) {
@@ -225,6 +227,48 @@ function createPageRoutes(ctx) {
});
});
});
});
+ // One task, and the signed-in editors' human-time estimates (B13). A plain form, same-origin, rate-limited in
+ // the registry per person; nothing here logs the text of a held-out task.
+ const taskContext = async (req) => {
+ const out = await readableBenchmark(req, req.params.ref);
+ if (!out) return null;
+ const version = out.version && out.version.version;
+ const task = ((out.definition && out.definition.tasks) || []).find((t) => String(t.id) === String(req.params.taskId));
+ if (!task) return null;
+ const estimates = await registry.taskEstimateSummary(out.benchmark.id, version, task.id);
+ return { out, version, task, estimates };
+ };
+ const taskPage = async (req, res, extra = {}) => {
+ const found = await taskContext(req);
+ if (!found) return notFound(req, res, 'task');
+ const subject = subjectOf(req);
+ return page(req, res, {
+ index: false, title: `${found.task.id} on ${found.out.benchmark.title}`,
+ description: `A task of ${found.out.benchmark.title}: its human-time estimate and the editors’ median.`,
+ body: raw(String(results.taskPage({
+ benchmark: found.out.benchmark, version: found.version, task: found.task, estimates: found.estimates,
+ signedIn: Boolean(subject), loginUrl: `/auth/login?next=${encodeURIComponent(req.originalUrl)}`,
+ ...extra,
+ }))),
+ }, extra.status || 200);
+ };
+ r.get('/benchmarks/:ref/tasks/:taskId', (req, res) => taskPage(req, res));
+ r.post('/benchmarks/:ref/tasks/:taskId/estimates', express.urlencoded({ extended: false, limit: '8kb' }), ctx.principal.middleware, async (req, res) => {
+ const subject = subjectOf(req);
+ const found = await taskContext(req);
+ if (!found) return notFound(req, res, 'task');
+ if (!subject) return taskPage(req, res, { error: 'Sign in to add an estimate.', status: 401 });
+ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
+ const body = req.body && typeof req.body === 'object' ? req.body : {};
+ const version = body.version == null ? found.version : Number(body.version);
+ const out = await registry.addTaskEstimate({
+ benchmarkId: found.out.benchmark.id, version, taskId: found.task.id,
+ minutes: body.minutes, subject,
+ });
+ if (!out.ok) return taskPage(req, res, { error: out.message || 'The estimate could not be saved.', status: out.status || 422 });
+ return res.redirect(303, `/benchmarks/${encodeURIComponent(found.out.benchmark.id)}/tasks/${encodeURIComponent(found.task.id)}`);
+ });
+
r.get('/benchmarks/:ref', async (req, res) => {
r.get('/benchmarks/:ref', async (req, res) => {
const out = await readableBenchmark(req, req.params.ref);
const out = await readableBenchmark(req, req.params.ref);
if (!out) return notFound(req, res, 'benchmark');
if (!out) return notFound(req, res, 'benchmark');
@@ -308,9 +352,10 @@ function createPageRoutes(ctx) {
@@ -308,9 +352,10 @@ function createPageRoutes(ctx) {
r.get('/models/:model', async (req, res) => {
r.get('/models/:model', async (req, res) => {
const model = String(req.params.model || '').slice(0, 160);
const model = String(req.params.model || '').slice(0, 160);
- const [summary, runs] = await Promise.all([
+ const [summary, runs, horizon] = await Promise.all([
registry.modelSummary(model),
registry.modelSummary(model),
registry.listRuns({ model, publicOnly: true, limit: 12 }),
registry.listRuns({ model, publicOnly: true, limit: 12 }),
+ registry.modelHorizon(model),
]);
]);
const titles = {};
const titles = {};
for (const b of summary.benchmarks) titles[b.benchmark_id] = b.title;
for (const b of summary.benchmarks) titles[b.benchmark_id] = b.title;
@@ -321,7 +366,7 @@ function createPageRoutes(ctx) {
@@ -321,7 +366,7 @@ function createPageRoutes(ctx) {
index: summary.runs > 0, cache: PUBLIC_CACHE,
index: summary.runs > 0, cache: PUBLIC_CACHE,
title: model,
title: model,
description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,
description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,
- body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base() }))),
+ body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base(), horizon }))),
}, summary.runs > 0 || runs.length ? 200 : 404);
}, summary.runs > 0 || runs.length ? 200 : 404);
});
});
@@ -75,6 +75,18 @@ function createRegistryApi(ctx) {
@@ -75,6 +75,18 @@ function createRegistryApi(ctx) {
});
});
const mayRead = (req, benchmark) => canViewBenchmark(benchmark, viewerOf(req));
const mayRead = (req, benchmark) => canViewBenchmark(benchmark, viewerOf(req));
+ /** The owner, including a service token that created the benchmark, may see its held-out task text. */
+ const owns = (req, benchmark) => {
+ if (!benchmark) return false;
+ const person = subjectOf(req);
+ if (person != null && String(benchmark.owner) === String(person)) return true;
+ return req.principal.kind === 'app' && String(benchmark.owner) === String(req.principal.requester);
+ };
+ const maySeeTaskText = async (req, id) => {
+ const bm = await registry.getBenchmark(String(id || ''));
+ return mayReveal(req) || owns(req, bm);
+ };
+
// ── Benchmarks ───────────────────────────────────────────
// ── Benchmarks ───────────────────────────────────────────
r.post('/benchmarks', actor('bench.benchmark.write'), async (req, res) => {
r.post('/benchmarks', actor('bench.benchmark.write'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
@@ -103,7 +115,13 @@ function createRegistryApi(ctx) {
@@ -103,7 +115,13 @@ function createRegistryApi(ctx) {
try {
try {
const out = await registry.saveDraft(String(req.params.id || ''), { definition: b.definition || b, subject });
const out = await registry.saveDraft(String(req.params.id || ''), { definition: b.definition || b, subject });
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
- return res.json({ benchmark: out.benchmark || null, version: out.version || null, draft: out.draft || null });
+ const benchmark = await registry.getBenchmark(String(req.params.id || ''));
+ let version = null;
+ if (out.version) {
+ const seen = await registry.getDefinition(`${out.version.benchmark_id}@${out.version.version}`, { reveal: await maySeeTaskText(req, req.params.id) });
+ version = seen ? seen.version : null;
+ }
+ return res.json({ benchmark: benchmark || null, version, draft: benchmark ? benchmark.draft : null });
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
});
});
@@ -115,7 +133,8 @@ function createRegistryApi(ctx) {
@@ -115,7 +133,8 @@ function createRegistryApi(ctx) {
const out = await registry.freezeVersion(String(req.params.id || ''), { definition: body, subject, validate: validateBenchmark });
const out = await registry.freezeVersion(String(req.params.id || ''), { definition: body, subject, validate: validateBenchmark });
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}@${out.version.version}`);
res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}@${out.version.version}`);
- return res.status(201).json({ benchmark: out.benchmark, version: out.version });
+ const seen = await registry.getDefinition(`${out.benchmark.id}@${out.version.version}`, { reveal: await maySeeTaskText(req, out.benchmark.id) });
+ return res.status(201).json({ benchmark: out.benchmark, version: seen ? seen.version : out.version, task_meta: seen ? seen.task_meta : [] });
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
});
});
@@ -123,11 +142,14 @@ function createRegistryApi(ctx) {
@@ -123,11 +142,14 @@ function createRegistryApi(ctx) {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
const subject = subjectOf(req) || req.principal.requester;
const subject = subjectOf(req) || req.principal.requester;
try {
try {
+ const bm = await registry.getBenchmark(String(req.params.id || ''));
+ if (!bm || !mayRead(req, bm)) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
const out = await registry.forkBenchmark(String(req.params.id || ''), {
const out = await registry.forkBenchmark(String(req.params.id || ''), {
version: b.version == null ? null : Number(b.version),
version: b.version == null ? null : Number(b.version),
title: b.title || null,
title: b.title || null,
owner: subject,
owner: subject,
subject,
subject,
+ reveal: mayReveal(req) || owns(req, bm),
});
});
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
if (!out.ok) return problem(res, out.status || 400, out.code || 'bench.invalid', out.message || 'invalid benchmark');
res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}`);
res.set('Location', `/api/v1/benchmarks/${out.benchmark.id}`);
@@ -152,6 +174,7 @@ function createRegistryApi(ctx) {
@@ -152,6 +174,7 @@ function createRegistryApi(ctx) {
const mine = req.query.mine && req.principal && req.principal.kind === 'user' ? req.principal.subject : null;
const mine = req.query.mine && req.principal && req.principal.kind === 'user' ? req.principal.subject : null;
const list = await registry.listBenchmarks({
const list = await registry.listBenchmarks({
owner: mine,
owner: mine,
+ viewer: mine,
kind: req.query.kind ? String(req.query.kind) : null,
kind: req.query.kind ? String(req.query.kind) : null,
tag: req.query.tag ? String(req.query.tag) : null,
tag: req.query.tag ? String(req.query.tag) : null,
q: req.query.q ? String(req.query.q) : null,
q: req.query.q ? String(req.query.q) : null,
@@ -170,18 +193,19 @@ function createRegistryApi(ctx) {
@@ -170,18 +193,19 @@ function createRegistryApi(ctx) {
// A private benchmark answers only its owner and the executor; a shared unlisted one only its owner by
// A private benchmark answers only its owner and the executor; a shared unlisted one only its owner by
// id (the token link is the way in). Everyone else gets a 404, not a "forbidden".
// id (the token link is the way in). Everyone else gets a 404, not a "forbidden".
if (!mayRead(req, out.benchmark)) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
if (!mayRead(req, out.benchmark)) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
- const view = viewerOf(req);
- const owner = view.subject != null && String(out.benchmark.owner) === String(view.subject);
- const reveal = mayReveal(req) || owner;
+ const reveal = mayReveal(req) || owns(req, out.benchmark);
+ let definition = out.version.definition;
+ let taskMeta = out.task_meta;
if (reveal) {
if (reveal) {
const full = await registry.getDefinition(ref, { reveal: true });
const full = await registry.getDefinition(ref, { reveal: true });
- if (full) { out.definition = full.definition; out.version = full.version; }
+ if (full) { definition = full.version.definition; taskMeta = full.task_meta; }
}
}
res.set('Cache-Control', out.benchmark.visibility === 'public' ? 'public, max-age=60' : 'private, no-store');
res.set('Cache-Control', out.benchmark.visibility === 'public' ? 'public, max-age=60' : 'private, no-store');
return res.json({
return res.json({
benchmark: out.benchmark,
benchmark: out.benchmark,
- version: { ...out.version, definition: out.definition },
- definition: out.definition,
+ version: { ...out.version, definition },
+ definition,
+ task_meta: taskMeta,
});
});
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
});
});
@@ -287,8 +311,8 @@ function createRegistryApi(ctx) {
@@ -287,8 +311,8 @@ function createRegistryApi(ctx) {
const benchmark = req.query.benchmark ? String(req.query.benchmark) : '';
const benchmark = req.query.benchmark ? String(req.query.benchmark) : '';
if (!benchmark) return invalid(res, 'benchmark is required: ?benchmark=bmk_…');
if (!benchmark) return invalid(res, 'benchmark is required: ?benchmark=bmk_…');
const metric = req.query.metric ? String(req.query.metric) : 'score';
const metric = req.query.metric ? String(req.query.metric) : 'score';
- if (!['score', 'value', 'speed', 'consistency'].includes(metric)) {
- return invalid(res, 'metric is score, value, speed or consistency');
+ if (!['score', 'value', 'speed', 'consistency', 'horizon'].includes(metric)) {
+ return invalid(res, 'metric is score, value, speed, consistency or horizon');
}
}
try {
try {
const bm = await registry.getBenchmark(benchmark.split('@')[0]);
const bm = await registry.getBenchmark(benchmark.split('@')[0]);
@@ -37,6 +37,8 @@ const TABLES = [
@@ -37,6 +37,8 @@ const TABLES = [
{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
// The results files they brought (B12): theirs, so they go with them. The bytes hang off the row and go with it.
// The results files they brought (B12): theirs, so they go with them. The bytes hang off the row and go with it.
{ table: 'bench_imports', subject: 'owner', file: 'bench_imports.json', erase: 'delete' },
{ table: 'bench_imports', subject: 'owner', file: 'bench_imports.json', erase: 'delete' },
+ // The human-time estimates they entered on a task (B13): theirs, one per task, exported and erased with them.
+ { table: 'bench_task_estimates', subject: 'subject', file: 'bench_task_estimates.json', erase: 'delete' },
];
];
// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
@@ -107,7 +109,7 @@ function create({ db, log = console } = {}) {
@@ -107,7 +109,7 @@ function create({ db, log = console } = {}) {
tables: TABLES,
tables: TABLES,
extraExport,
extraExport,
extraErase,
extraErase,
- note: 'Bench exports your tasks, rounds, judgements, private evals, benchmarks and their prompts, the runs on your private benchmarks, watched benchmarks and the pairs you have seen; the public dataset is anonymous and a rating never names a person.',
+ note: 'Bench exports your tasks, rounds, judgements, private evals, benchmarks and their prompts, the runs on your private benchmarks, watched benchmarks, the pairs you have seen and the human-time estimates you entered; the public dataset is anonymous and a rating never names a person.',
log,
log,
});
});
}
}
@@ -58,12 +58,12 @@ ${ep('GET', '/api/v1/evals/:id/results', 'A private eval and its runs.')}
@@ -58,12 +58,12 @@ ${ep('GET', '/api/v1/evals/:id/results', 'A private eval and its runs.')}
</dl></section>
</dl></section>
<section id="br-publish"><h2>Benchmarks and runs</h2><p class="bm-fine">A benchmark is a versioned set of tasks, judges and limits; a frozen version never changes once a run names it. A run published by a service (openvibe.codes, Run, a Node worker) is <b>verified</b>; one published with a person's token is <b>submitted</b>, labelled, and never ranked by default. Your own prompts can be pasted at <a href="/benchmarks/new">/benchmarks/new</a>: 5 to 50 of them, private by default, with an unlisted share link or a public listing when you choose.</p><dl class="bm-eps">
<section id="br-publish"><h2>Benchmarks and runs</h2><p class="bm-fine">A benchmark is a versioned set of tasks, judges and limits; a frozen version never changes once a run names it. A run published by a service (openvibe.codes, Run, a Node worker) is <b>verified</b>; one published with a person's token is <b>submitted</b>, labelled, and never ranked by default. Your own prompts can be pasted at <a href="/benchmarks/new">/benchmarks/new</a>: 5 to 50 of them, private by default, with an unlisted share link or a public listing when you choose.</p><dl class="bm-eps">
${ep('GET', '/api/v1/benchmarks?kind=&tag=&q=&mine=1', 'The public benchmarks; <code>mine=1</code> adds your own unlisted and private ones.')}
${ep('GET', '/api/v1/benchmarks?kind=&tag=&q=&mine=1', 'The public benchmarks; <code>mine=1</code> adds your own unlisted and private ones.')}
-${ep('GET', '/api/v1/benchmarks/:id[@version]', 'A definition. Held-out tasks keep their prompt for the executor and the owner; a private benchmark is a 404 to everyone else.')}
+${ep('GET', '/api/v1/benchmarks/:id[@version]', 'A definition, and its <code>task_meta</code> (added_on, held_out, human_minutes and its source). Held-out tasks keep their prompt for the executor and the owner; a private benchmark is a 404 to everyone else.')}
${ep('POST', '/api/v1/benchmarks', 'Create one (<code>bench.benchmark.write</code>); <code>PUT</code> saves a draft, <code>POST …/versions</code> freezes it, <code>POST …/fork</code> forks it.')}
${ep('POST', '/api/v1/benchmarks', 'Create one (<code>bench.benchmark.write</code>); <code>PUT</code> saves a draft, <code>POST …/versions</code> freezes it, <code>POST …/fork</code> forks it.')}
${ep('POST', '/api/v1/benchmarks/:id/share', '<code>{ visibility: "private" | "unlisted" | "public" }</code>. The owner only. An unlisted call returns a new share token once; the old link stops working.')}
${ep('POST', '/api/v1/benchmarks/:id/share', '<code>{ visibility: "private" | "unlisted" | "public" }</code>. The owner only. An unlisted call returns a new share token once; the old link stops working.')}
${ep('POST', '/api/v1/runs', 'Publish a run: <code>{ benchmark: "bmk_…@N", model, provider, harness: { name, version }, executor, tasks: [{ task_id, score, cost_usd, ms, tokens, artifacts }], score, cost_usd, ms, tokens }</code>.')}
${ep('POST', '/api/v1/runs', 'Publish a run: <code>{ benchmark: "bmk_…@N", model, provider, harness: { name, version }, executor, tasks: [{ task_id, score, cost_usd, ms, tokens, artifacts }], score, cost_usd, ms, tokens }</code>.')}
${ep('GET', '/api/v1/runs/:id', 'A run with its tasks; <code>/api/v1/compare?runs=a,b</code> puts up to four side by side.')}
${ep('GET', '/api/v1/runs/:id', 'A run with its tasks; <code>/api/v1/compare?runs=a,b</code> puts up to four side by side.')}
-${ep('GET', '/api/v1/rankings?benchmark=&metric=score|value|speed|consistency&harness=&since=', 'Medians per model over verified runs; value is score per dollar, speed is score per minute.')}
+${ep('GET', '/api/v1/rankings?benchmark=&metric=score|value|speed|consistency|horizon&harness=&since=', 'Medians per model over verified runs; value is score per dollar, speed is score per minute, horizon is the 50% human-time task length. since=YYYY-MM scores only tasks added in or after that month.')}
${ep('GET', '/api/v1/percentile?benchmark=&model=&score=', 'Where a score sits among a model’s verified runs.')}
${ep('GET', '/api/v1/percentile?benchmark=&model=&score=', 'Where a score sits among a model’s verified runs.')}
</dl></section>
</dl></section>
<section><h2>MCP</h2><p class="bm-fine"><code>POST /mcp</code> exposes <code>bench.list_kinds</code>, <code>bench.start_round</code>, <code>bench.next_pair</code>, <code>bench.cast_judgement</code>, <code>bench.get_board</code> and <code>bench.run_eval</code>.</p></section>
<section><h2>MCP</h2><p class="bm-fine"><code>POST /mcp</code> exposes <code>bench.list_kinds</code>, <code>bench.start_round</code>, <code>bench.next_pair</code>, <code>bench.cast_judgement</code>, <code>bench.get_board</code> and <code>bench.run_eval</code>.</p></section>
@@ -37,6 +37,12 @@ function fmtTokens(v) {
@@ -37,6 +37,12 @@ function fmtTokens(v) {
return String(n);
return String(n);
}
}
const fmtDate = (iso) => (iso ? html`<time datetime="${iso}">${String(iso).slice(0, 10)}</time>` : '—');
const fmtDate = (iso) => (iso ? html`<time datetime="${iso}">${String(iso).slice(0, 10)}</time>` : '—');
+const MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
+function monthLabel(month) {
+ const m = /^(\d{4})-(\d{2})$/.exec(String(month || ''));
+ if (!m) return String(month || '');
+ return `${MONTHS[Number(m[2]) - 1]} ${m[1]}`;
+}
function ordinal(n) {
function ordinal(n) {
const v = Math.round(Number(n));
const v = Math.round(Number(n));
const t = v % 100;
const t = v % 100;
@@ -175,17 +181,57 @@ const METRICS = [
@@ -175,17 +181,57 @@ const METRICS = [
{ id: 'value', label: 'Value', hint: 'score per dollar' },
{ id: 'value', label: 'Value', hint: 'score per dollar' },
{ id: 'speed', label: 'Speed', hint: 'score per minute' },
{ id: 'speed', label: 'Speed', hint: 'score per minute' },
{ id: 'consistency', label: 'Steadiness', hint: 'spread inside a session, lower is steadier' },
{ id: 'consistency', label: 'Steadiness', hint: 'spread inside a session, lower is steadier' },
+ { id: 'horizon', label: 'Horizon', hint: 'the human-length task the model does 50% of the time, after METR' },
];
];
+function fmtMinutes(v) {
+ if (v == null) return '—';
+ const n = Number(v);
+ if (n < 1) return `${Math.round(n * 60)} s`;
+ if (n < 90) return `${Math.round(n * 10) / 10} min`;
+ return `${Math.round(n / 6) / 10} h`;
+}
+
+/** The 50% time horizon, with its bootstrap interval; never a number below the gates. */
+function horizonText(h) {
+ if (!h || !h.available) return '—';
+ return `${fmtMinutes(h.horizon_minutes)} (${fmtMinutes(h.ci_lower)}–${fmtMinutes(h.ci_upper)})`;
+}
+
+function horizonLine(h) {
+ if (!h) return '';
+ if (h.available) {
+ return html`<p class="br-horizon"><b>Time horizon:</b> reliably does about ${fmtMinutes(h.horizon_minutes)} tasks. 90% interval ${fmtMinutes(h.ci_lower)}–${fmtMinutes(h.ci_upper)}, from ${plural(h.timed_tasks, 'timed task')} and ${plural(h.runs, 'run')}.</p>`;
+ }
+ return html`<p class="br-fine br-horizon">Time horizon: not enough runs yet (needs ${h.need_tasks || 15} timed tasks and ${h.need_runs || 3} runs).</p>`;
+}
+
+/** The contamination badge, with both numbers and their bands. Only a real flag is ever shown. */
+function contaminationList(entries) {
+ const flagged = entries.filter((e) => e.contamination && e.contamination.flag);
+ if (!flagged.length) return '';
+ return html`<ul class="br-contams">${flagged.map((e) => {
+ const c = e.contamination;
+ return html`<li class="br-contam"><b>${e.model}</b> — Did much better on older tasks: possibly seen in training. Older tasks ${fmtScore(c.old.median)} (${fmtScore(c.old.lower)}–${fmtScore(c.old.upper)}), fresh tasks ${fmtScore(c.fresh.median)} (${fmtScore(c.fresh.lower)}–${fmtScore(c.fresh.upper)}). Release cutoff ${c.cutoff}${c.source ? html` · <a href="${c.source}" rel="nofollow noopener">source</a>` : ''}.</li>`;
+ })}</ul>`;
+}
+
function metricTabs(base, current) {
function metricTabs(base, current) {
return html`<nav class="br-tabs" aria-label="Rank by">${METRICS.map((m) => html`<a class="br-tab" href="${base}?metric=${m.id}#br-rank"${m.id === current ? raw(' aria-current="page"') : ''}${m.hint ? raw(` title="${m.hint}"`) : ''}>${m.label}</a>`)}</nav>`;
return html`<nav class="br-tabs" aria-label="Rank by">${METRICS.map((m) => html`<a class="br-tab" href="${base}?metric=${m.id}#br-rank"${m.id === current ? raw(' aria-current="page"') : ''}${m.hint ? raw(` title="${m.hint}"`) : ''}>${m.label}</a>`)}</nav>`;
}
}
function rankingsTable(entries, metric) {
function rankingsTable(entries, metric) {
if (!entries.length) return '';
if (!entries.length) return '';
- const col = { value: (e) => (e.value == null ? '—' : `${fmtScore(e.value)} / $`), speed: (e) => (e.speed == null ? '—' : `${fmtScore(e.speed)} / min`), consistency: (e) => (e.consistency == null ? '—' : `±${fmtScore(e.consistency)}`) }[metric];
+ const col = {
+ value: (e) => (e.value == null ? '—' : `${fmtScore(e.value)} / $`),
+ speed: (e) => (e.speed == null ? '—' : `${fmtScore(e.speed)} / min`),
+ consistency: (e) => (e.consistency == null ? '—' : `±${fmtScore(e.consistency)}`),
+ }[metric];
const bands = metric === 'score' && entries.some((e) => e.band);
const bands = metric === 'score' && entries.some((e) => e.band);
- return html`<div class="br-table-wrap"><table class="br-table br-rank-table"><thead><tr><th scope="col">#</th>${bands ? html`<th scope="col">Band</th>` : ''}<th scope="col">Model</th><th scope="col">Score</th>${col ? html`<th scope="col">${METRICS.find((m) => m.id === metric).label}</th>` : ''}<th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${entries.map((e, i) => html`<tr><td class="br-rank">${i + 1}</td>${bands ? html`<td class="br-c-band"><span class="rt-band">${e.band || '—'}</span></td>` : ''}<td class="br-c-model"><a class="br-model" href="${modelHref(e.model)}">${e.model}</a></td><td class="br-c-score">${scoreBar(e.score)}</td>${col ? html`<td class="br-num br-c-metric" data-label="${METRICS.find((m) => m.id === metric).label}">${col(e)}</td>` : ''}<td class="br-num br-c-cost" data-label="cost/run">${fmtUsd(e.cost_usd)}</td><td class="br-num br-c-time" data-label="time">${fmtMs(e.ms)}</td><td class="br-num br-c-runs" data-label="runs">${e.runs}</td></tr>`)}</tbody></table></div>`;
+ const isHorizon = metric === 'horizon';
+ const showHorizon = isHorizon || entries.some((e) => e.horizon && (e.horizon.available || e.horizon.timed_tasks > 0));
+ const metricLabel = METRICS.find((m) => m.id === metric).label;
+ return html`<div class="br-table-wrap"><table class="br-table br-rank-table"><thead><tr><th scope="col">#</th>${bands ? html`<th scope="col">Band</th>` : ''}<th scope="col">Model</th><th scope="col">Score</th>${col && !isHorizon ? html`<th scope="col">${metricLabel}</th>` : ''}${showHorizon ? html`<th scope="col" class="br-num"><a href="?metric=horizon#br-rank"${isHorizon ? raw(' aria-current="page"') : ''}>Horizon</a></th>` : ''}<th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${entries.map((e, i) => html`<tr><td class="br-rank">${i + 1}</td>${bands ? html`<td class="br-c-band"><span class="rt-band">${e.band || '—'}</span></td>` : ''}<td class="br-c-model"><a class="br-model" href="${modelHref(e.model)}">${e.model}</a>${e.contamination && e.contamination.flag ? html`<span class="br-contam-chip" title="Did much better on older tasks: possibly seen in training">possibly seen in training</span>` : ''}</td><td class="br-c-score">${scoreBar(e.score)}</td>${col && !isHorizon ? html`<td class="br-num br-c-metric" data-label="${metricLabel}">${col(e)}</td>` : ''}${showHorizon ? html`<td class="br-num br-c-horizon" data-label="horizon" title="${e.horizon && e.horizon.available ? `${e.horizon.timed_tasks} timed tasks, ${e.horizon.runs} runs` : 'not enough runs yet'}">${horizonText(e.horizon)}</td>` : ''}<td class="br-num br-c-cost" data-label="cost/run">${fmtUsd(e.cost_usd)}</td><td class="br-num br-c-time" data-label="time">${fmtMs(e.ms)}</td><td class="br-num br-c-runs" data-label="runs">${e.runs}</td></tr>`)}</tbody></table></div>${contaminationList(entries)}`;
}
}
/**
/**
@@ -338,9 +384,10 @@ ${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}
@@ -338,9 +384,10 @@ ${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}
</div>`;
</div>`;
}
}
-function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '', ownBoards = null, share = null, shared = false, owner = false }) {
+function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '', ownBoards = null, share = null, shared = false, owner = false, since = null, allTimeEntries = null }) {
const d = definition || {};
const d = definition || {};
const tasks = Array.isArray(d.tasks) ? d.tasks : [];
const tasks = Array.isArray(d.tasks) ? d.tasks : [];
+ const freshSets = [...new Set(tasks.map((t) => (t && t.added_on ? String(t.added_on).slice(0, 7) : null)).filter(Boolean))].sort().reverse();
const shown = tasks.filter((t) => !t.holdout && t.prompt);
const shown = tasks.filter((t) => !t.holdout && t.prompt);
const hidden = tasks.length - shown.length;
const hidden = tasks.length - shown.length;
const judges = Array.isArray(d.judges) ? d.judges : [];
const judges = Array.isArray(d.judges) ? d.judges : [];
@@ -376,11 +423,19 @@ ${owner ? shareBox({ benchmark, shareUrl: share && share.shareUrl, error: share
@@ -376,11 +423,19 @@ ${owner ? shareBox({ benchmark, shareUrl: share && share.shareUrl, error: share
${ownBoards
${ownBoards
? promptBoards(ownBoards)
? promptBoards(ownBoards)
: html`<section class="bm-section" id="br-rank" aria-labelledby="br-rank-h"><div class="bm-head-row"><h2 id="br-rank-h">Rankings</h2>${metricTabs(base, metric)}</div>
: html`<section class="bm-section" id="br-rank" aria-labelledby="br-rank-h"><div class="bm-head-row"><h2 id="br-rank-h">Rankings</h2>${metricTabs(base, metric)}</div>
+${since ? html`<p class="br-since-note" role="status"><b>On tasks since ${since.label}</b>${since.basis === 'runs' ? ' (this benchmark’s tasks carry no dates, so it filters by run date)' : ''}. <a href="${base}?metric=${metric}#br-rank">All-time board</a></p>` : ''}
+<form class="br-since" method="get" action="${base}#br-rank"><input type="hidden" name="metric" value="${metric}"><label class="br-field"><span>Fresh set: on tasks since</span><input type="month" name="since" value="${since ? since.month : ''}"></label><button class="bm-btn bm-btn-small" type="submit">Show</button></form>
+${freshSets.length ? html`<p class="br-fine br-fresh-sets">Fresh sets: ${freshSets.map((month, i) => html`${i ? ' · ' : ''}<a href="${base}?metric=${metric}&since=${encodeURIComponent(month)}#br-rank">${monthLabel(month)}</a>`)}. Pick one to score only the tasks added that month or later.</p>` : ''}
${valueChart(entries)}
${valueChart(entries)}
-${entries.length ? rankingsTable(entries, metric) : empty('No verified runs yet', 'A run counts here once openvibe.codes, Run or a Node worker ran it and published it. Submitted runs are listed below but never ranked.')}
+${entries.length ? rankingsTable(entries, metric) : empty(since ? 'No tasks in that fresh set yet' : 'No verified runs yet', since ? 'No run here scored a task added on or after that month. Pick an earlier month, or see the all-time board.' : 'A run counts here once openvibe.codes, Run or a Node worker ran it and published it. Submitted runs are listed below but never ranked.')}
+${since && allTimeEntries && allTimeEntries.length ? html`<details class="br-fold br-alltime"><summary>All-time board</summary>${rankingsTable(allTimeEntries, metric)}</details>` : ''}
</section>`}
</section>`}
<section class="bm-section" id="br-tasks" aria-labelledby="br-tasks-h"><div class="bm-head-row"><h2 id="br-tasks-h">Tasks</h2><span class="bm-count">${tasks.length}</span></div>
<section class="bm-section" id="br-tasks" aria-labelledby="br-tasks-h"><div class="bm-head-row"><h2 id="br-tasks-h">Tasks</h2><span class="bm-count">${tasks.length}</span></div>
-<ol class="br-tasks">${shown.map((t, i) => html`<li class="br-task"><span class="br-task-n">${i + 1}</span><span class="br-task-id br-mono">${t.id}</span><p>${t.prompt}</p>${t.expected && t.expected.answer ? html`<p class="br-fine br-task-expected">Expected: ${String(t.expected.answer).slice(0, 300)}</p>` : ''}</li>`)}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section>
+<ol class="br-tasks">${shown.map((t, i) => {
+ const added = t.added_on ? String(t.added_on).slice(0, 10) : null;
+ const mins = t.human_minutes == null ? null : Number(t.human_minutes);
+ return html`<li class="br-task"><span class="br-task-n">${i + 1}</span><a class="br-task-id br-mono" href="${benchmarkHref(benchmark.id)}/tasks/${encodeURIComponent(t.id)}">${t.id}</a><p>${t.prompt}</p>${t.expected && t.expected.answer ? html`<p class="br-fine br-task-expected">Expected: ${String(t.expected.answer).slice(0, 300)}</p>` : ''}<p class="br-task-meta">${added ? html`<span class="br-chip">added ${added}</span>` : ''}${mins ? html`<span class="br-chip">${fmtMinutes(mins)} human estimate</span>` : ''}</p></li>`;
+ })}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section>
<section class="bm-section" id="br-runs" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>
<section class="bm-section" id="br-runs" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>
${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section>
${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section>
</div>
</div>
@@ -389,6 +444,33 @@ ${benchmark.visibility === 'private' ? '' : benchmarkUseThis({ baseUrl, benchmar
@@ -389,6 +444,33 @@ ${benchmark.visibility === 'private' ? '' : benchmarkUseThis({ baseUrl, benchmar
</div>`;
</div>`;
}
}
+/**
+ * One task: its prompt, when it was added, its human-time estimate and (for a signed-in editor) the form that adds
+ * one. The median of three distinct people becomes the task's time. A held-out task keeps its prompt on the server.
+ */
+function taskPage({ benchmark, version, task, estimates = { count: 0, needed: 3, median: null }, signedIn = false, loginUrl = '/auth/login', error = null, status = null }) {
+ const held = Boolean(task && (task.holdout || task.held_out));
+ const added = task && task.added_on ? String(task.added_on).slice(0, 10) : null;
+ const author = task && task.human_minutes != null ? Number(task.human_minutes) : null;
+ const source = task && task.human_minutes_source ? String(task.human_minutes_source) : (author ? 'benchmark author' : null);
+ const timed = author != null ? { minutes: author, source } : (estimates.count >= 3 ? { minutes: estimates.median, source: `median of ${estimates.count} editor estimates` } : null);
+ return html`<div class="bm-wrap br bm-narrow" id="br-task-page">
+<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${benchmarkHref(benchmark.id)}">${benchmark.title}</a><span aria-hidden="true">/</span><span class="br-mono">${task ? task.id : 'task'}</span></nav>
+<header class="bm-page-h"><p class="bm-kicker">Task ${task ? task.id : ''} · v${version || benchmark.latest_version || 0}</p><h1>${benchmark.title}</h1><p class="bm-lede">${held ? 'Held-out task: its prompt stays on the server, for the executor only.' : (task && task.prompt) || ''}</p></header>
+${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}
+${held ? html`<div class="bm-note" role="status"><b>Held out.</b> The prompt and expected answer never leave the server. Editors can still give it a human-time estimate.</div>` : ''}
+<dl class="br-facts"><div><dt>Added</dt><dd>${added || 'not dated'}</dd></div><div><dt>Human time</dt><dd>${timed ? `${fmtMinutes(timed.minutes)} · ${timed.source}` : 'not enough estimates yet'}</dd></div><div><dt>Editors</dt><dd>${estimates.count} ${estimates.count === 1 ? 'estimate' : 'estimates'}${estimates.count < 3 ? `, ${3 - estimates.count} more for a median` : ''}</dd></div></dl>
+${estimates.count >= 3 ? html`<p class="br-fine">The task time is the median of ${estimates.count} signed-in editors’ estimates: <b>${fmtMinutes(estimates.median)}</b>. It is shown here, in the open data and in the time-horizon fit.</p>` : html`<p class="br-fine">The median of three distinct people’s estimates becomes this task’s human time. No individual estimate is published.</p>`}
+${signedIn
+ ? html`<form class="br-form br-estimate" method="post" action="${benchmarkHref(benchmark.id)}/tasks/${encodeURIComponent(task ? task.id : '')}/estimates">
+<input type="hidden" name="version" value="${version || benchmark.latest_version || 1}">
+<label class="br-field"><span>Your estimate, in minutes</span><input name="minutes" type="number" min="1" max="100000" step="1" required inputmode="numeric" placeholder="e.g. 40"></label>
+<div class="bm-actions"><button class="bm-btn" type="submit">${estimates.count ? 'Replace my estimate' : 'Add my estimate'}</button><a class="bm-btn bm-btn-quiet" href="${benchmarkHref(benchmark.id)}">Back to the benchmark</a></div>
+</form>`
+ : empty('Sign in to estimate', 'A human-time estimate is attached to your account, exported and deleted with it.', html`<a class="bm-btn bm-btn-small" href="${loginUrl}">Sign in with OpenVibe</a>`)}
+</div>`;
+}
+
/**
/**
* The run form: pick the models, the harnesses and how many times, see what it may cost, start it on openvibe.codes
* The run form: pick the models, the harnesses and how many times, see what it may cost, start it on openvibe.codes
* with your own keys. A plain form (it works without JavaScript); the estimate above $1 asks once more.
* with your own keys. A plain form (it works without JavaScript); the estimate above $1 asks once more.
@@ -481,11 +563,11 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta
@@ -481,11 +563,11 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta
</div>`;
</div>`;
}
}
-function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '' }) {
+function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '', horizon = null }) {
const rows = (summary && summary.benchmarks) || [];
const rows = (summary && summary.benchmarks) || [];
return html`<div class="bm-wrap br" id="br-model">
return html`<div class="bm-wrap br" id="br-model">
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>
-<header class="bm-page-h"><p class="bm-kicker">Model</p><h1 class="br-mono-h">${model}</h1><p class="bm-lede">${summary && summary.runs ? `${plural(summary.runs, 'verified run')} across ${plural(rows.length, 'benchmark')}.` : 'No verified runs yet.'}</p></header>
+<header class="bm-page-h"><p class="bm-kicker">Model</p><h1 class="br-mono-h">${model}</h1><p class="bm-lede">${summary && summary.runs ? `${plural(summary.runs, 'verified run')} across ${plural(rows.length, 'benchmark')}.` : 'No verified runs yet.'}</p>${horizonLine(horizon)}</header>
<section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>
<section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>
${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>
${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>
${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}
${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}
@@ -495,6 +577,7 @@ ${modelUseThis({ baseUrl, model, rows, viewer })}
@@ -495,6 +577,7 @@ ${modelUseThis({ baseUrl, model, rows, viewer })}
module.exports = {
module.exports = {
benchmarksIndex, benchmarkPage, newBenchmarkPage, promptLeaderboard, promptBoards, shareBox,
benchmarksIndex, benchmarkPage, newBenchmarkPage, promptLeaderboard, promptBoards, shareBox,
- runForm, runPage, comparePage, modelPage, valueChart, METRICS,
+ runForm, runPage, comparePage, modelPage, taskPage, valueChart, METRICS,
+ fmtMinutes, horizonText, horizonLine,
fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,
fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,298 @@
@@ -0,0 +1,298 @@
+'use strict';
+
+/**
+ * B13: fresh monthly sets, held-out tasks, the contamination flag and the time horizon.
+ *
+ * The rules under test: `since=YYYY-MM` scores only the tasks added in or after that month; a held-out task never
+ * reaches the open data, an anonymous page or somebody else's account export, and a submitted/imported run gets no
+ * score for one; the contamination maths says "flag", "no flag" or "not enough runs"; the logistic fit recovers a
+ * known horizon and its bootstrap; the 15-task/3-run gates hold; editors' estimates are exported and erased with
+ * the person and the form is signed-in and same-origin only.
+ */
+const assert = require('assert');
+const contracts = require('openvibe-contracts');
+const { boot, check, done } = require('./helpers/boot');
+const { contaminationFor } = require('../server/bench/contamination');
+const { estimateHorizon } = require('../server/bench/horizon');
+const { parseSince } = require('../server/bench/registry');
+
+const baseTask = (id, over = {}) => ({ id, prompt: `Task ${id}: build a small page.`, files: [], expected: null, holdout: false, ...over });
+
+function definition(over = {}) {
+ return {
+ title: 'B13 arena',
+ description: 'A benchmark for the fresh-set and horizon tests.',
+ kind: 'design.page',
+ tags: ['design'],
+ visibility: 'public',
+ tasks: [baseTask('t1')],
+ judges: [{ type: 'rubric', version: '1.0', config: {} }],
+ scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: { ms: 120000, tokens: 200000, usd: 0.5 },
+ matrix: { models: [], harnesses: [] },
+ ...over,
+ };
+}
+
+function runDoc(benchmark, tasks, over = {}) {
+ const scores = tasks.map((t) => t.score).filter((v) => v != null);
+ return {
+ benchmark,
+ model: 'model-a',
+ route: { provider: 'acme' },
+ provider: 'acme',
+ harness: { name: 'claude-code', version: '3.1.0' },
+ settings: { temperature: 1, effort: 'high' },
+ executor: { kind: 'codes', id: 'tsk_b13' },
+ tasks,
+ score: scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : null,
+ cost_usd: tasks.reduce((n, t) => n + (t.cost_usd || 0), 0),
+ ms: tasks.reduce((n, t) => n + (t.ms || 0), 0),
+ tokens: tasks.reduce((n, t) => n + (t.tokens || 0), 0),
+ ...over,
+ };
+}
+
+const result = (task_id, score, extra = {}) => ({ task_id, score, cost_usd: 0.01, ms: 1000, tokens: 100, artifacts: [], ...extra });
+
+(async () => {
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob');
+ const carol = t.network.addUser('carol');
+ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });
+
+ const create = async (over, as = null) => {
+ const res = await t.get('/api/v1/benchmarks', as ? { as, json: definition(over) } : { bearer: publisher, json: definition(over) });
+ assert.strictEqual(res.status, 201, res.text);
+ const id = res.json().benchmark.id;
+ const frozen = await t.get(`/api/v1/benchmarks/${id}/versions`, as ? { as, json: {} } : { bearer: publisher, json: {} });
+ assert.strictEqual(frozen.status, 201, frozen.text);
+ return { id, version: frozen.json().version.version };
+ };
+ const publish = async (ref, doc) => {
+ const res = await t.get('/api/v1/runs', { bearer: publisher, json: doc });
+ assert.strictEqual(res.status, 201, res.text);
+ return res.json().run;
+ };
+ const postForm = async (path, body, as) => t.get(path, {
+ as, method: 'POST', raw: new URLSearchParams(body).toString(),
+ headers: { 'content-type': 'application/x-www-form-urlencoded' },
+ });
+
+ await check('the since parser takes a month, a day or a timestamp', () => {
+ assert.strictEqual(parseSince('2026-03').cutoff, '2026-03-01');
+ assert.strictEqual(parseSince('2026-03').label, 'March 2026');
+ assert.strictEqual(parseSince('2026-03-10').cutoff, '2026-03-10');
+ assert.strictEqual(parseSince('not-a-date'), null);
+ });
+
+ await check('since=YYYY-MM scores only the tasks added in or after that month', async () => {
+ const made = await create({
+ title: 'Fresh monthly set',
+ tasks: [
+ baseTask('old', { added_on: '2025-01-15', human_minutes: 20 }),
+ baseTask('fresh', { added_on: '2026-03-10', human_minutes: 40 }),
+ ],
+ });
+ const frozen = await t.get(`/api/v1/benchmarks/${made.id}@${made.version}`);
+ assert.strictEqual(frozen.status, 200, frozen.text);
+ contracts.assertValid('bench.benchmark@1', frozen.json().version.definition);
+ assert.strictEqual(frozen.json().task_meta.find((m) => m.task_id === 'old').added_on, '2025-01-15', 'the date lives beside the closed document');
+
+ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('old', 100), result('fresh', 20)], { model: 'model-a' }));
+ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('old', 80), result('fresh', 80)], { model: 'model-b' }));
+
+ const all = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score`);
+ assert.strictEqual(all.status, 200, all.text);
+ assert.strictEqual(all.json().rankings.entries.find((e) => e.model === 'model-a').score, 60, 'the all-time board uses the run score');
+
+ const fresh = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score&since=2026-03`);
+ assert.strictEqual(fresh.status, 200, fresh.text);
+ const rankings = fresh.json().rankings;
+ assert.strictEqual(rankings.since.label, 'March 2026');
+ assert.strictEqual(rankings.since.basis, 'tasks');
+ assert.strictEqual(rankings.entries.find((e) => e.model === 'model-a').score, 20, 'only the fresh task is scored');
+ assert.strictEqual(rankings.entries.find((e) => e.model === 'model-b').score, 80);
+
+ const page = await t.get(`/benchmarks/${made.id}?since=2026-03`);
+ assert.strictEqual(page.status, 200, page.text.slice(0, 300));
+ assert.ok(page.text.includes('On tasks since March 2026'), 'the page names the fresh set beside the board');
+ assert.ok(page.text.includes('All-time board'), 'and offers the all-time board');
+ });
+
+ await check('a held-out task never leaks, and a submitted run gets no score for it', async () => {
+ const secret = 'SECRET_HELDOUT_PROMPT_9f2a';
+ const made = await create({
+ title: 'Held-out arena',
+ tasks: [
+ baseTask('visible', { prompt: 'A visible task.' }),
+ baseTask('hidden', { prompt: secret, expected: { answer: 'SECRET_EXPECTED' }, held_out: true }),
+ ],
+ }, alice);
+
+ const data = await t.get('/api/v1/datasets/benchmarks.jsonl');
+ assert.ok(data.text.includes('A visible task.'));
+ assert.ok(!data.text.includes(secret), 'the open data never carries the held-out prompt');
+ assert.ok(!data.text.includes('SECRET_EXPECTED'), 'nor its expected answer');
+
+ const api = await t.get(`/api/v1/benchmarks/${made.id}`);
+ const hidden = api.json().definition.tasks.find((x) => x.id === 'hidden');
+ assert.strictEqual(hidden.prompt, null);
+ assert.strictEqual(hidden.holdout, true);
+
+ const page = await t.get(`/benchmarks/${made.id}`);
+ assert.ok(!page.text.includes(secret), 'the page never carries the held-out prompt');
+
+ const submitted = await t.get('/api/v1/runs', {
+ as: alice,
+ json: runDoc(`${made.id}@${made.version}`, [result('visible', 50), result('hidden', 99)], { model: 'submitted-model' }),
+ });
+ assert.strictEqual(submitted.status, 201, submitted.text);
+ assert.strictEqual(submitted.json().run.tasks.find((x) => x.task_id === 'hidden').score, null, 'a submitted run cannot score a held-out task');
+ assert.strictEqual(submitted.json().run.tasks.find((x) => x.task_id === 'visible').score, 50);
+
+ const verified = await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('visible', 50), result('hidden', 99)], { model: 'verified-model' }));
+ assert.strictEqual(verified.tasks.find((x) => x.task_id === 'hidden').score, 99, 'the executor may score it');
+
+ const fork = await t.get(`/api/v1/benchmarks/${made.id}/fork`, { as: bob, json: {} });
+ assert.strictEqual(fork.status, 403, 'a fork by somebody who may not see the holdout is refused');
+ assert.ok(!fork.text.includes(secret), 'and the refusal does not carry the prompt');
+
+ const exportPart = await t.ctx.accountData.exportPart(bob.subject);
+ const blob = JSON.stringify(exportPart.files || []);
+ assert.ok(!blob.includes(secret), 'somebody else\'s account export never carries the prompt');
+ assert.ok(!blob.includes('SECRET_EXPECTED'));
+
+ const writer = t.network.serviceToken({ sub: 'svc:writer', audience: 'openvibe.bench', cap: ['bench.benchmark.write'] });
+ const versioned = await t.get(`/api/v1/benchmarks/${made.id}/versions`, { bearer: writer, json: {} });
+ assert.strictEqual(versioned.status, 201, versioned.text);
+ assert.ok(!versioned.text.includes(secret), 'a new version response for a non-owner does not carry the prompt');
+ assert.strictEqual(versioned.json().version.definition.tasks.find((x) => x.id === 'hidden').prompt, null);
+ });
+
+ await check('the contamination flag: a flag, no flag, and not enough runs', () => {
+ const cutoff = { cutoff: '2025-08-07', source: 'https://example.test/model' };
+ const flagged = contaminationFor({ oldScores: [90, 92, 94], freshScores: [20, 22, 24], ...cutoff });
+ assert.strictEqual(flagged.flag, true);
+ assert.ok(flagged.old.median > flagged.fresh.median);
+ assert.ok(flagged.old.lower > flagged.fresh.upper, 'the 90% intervals do not overlap');
+ const tied = contaminationFor({ oldScores: [70, 72, 74], freshScores: [68, 70, 72], ...cutoff });
+ assert.strictEqual(tied.flag, false);
+ assert.strictEqual(tied.reason, 'the intervals overlap');
+ const thin = contaminationFor({ oldScores: [90], freshScores: [20, 22], ...cutoff });
+ assert.strictEqual(thin.flag, false);
+ assert.strictEqual(thin.reason, 'not enough runs yet');
+ const unknown = contaminationFor({ oldScores: [90, 92], freshScores: [20, 22] });
+ assert.strictEqual(unknown.flag, false);
+ assert.strictEqual(unknown.reason, 'no release date');
+ });
+
+ await check('the logistic fit recovers a known 40-minute horizon and its bootstrap', () => {
+ const groups = [];
+ for (let i = 0; i < 20; i++) {
+ const minutes = 2 + i * 5;
+ const p = 1 / (1 + Math.exp(-(1.2 * Math.log(minutes) - 1.2 * Math.log(40))));
+ groups.push({ minutes, success: p, runs: 3 });
+ }
+ const fit = estimateHorizon(groups, { runs: 3, timedTasks: 20 });
+ assert.strictEqual(fit.available, true, JSON.stringify(fit));
+ assert.ok(Math.abs(fit.horizon_minutes - 40) < 4, `the horizon is near 40: ${fit.horizon_minutes}`);
+ assert.ok(fit.ci_lower <= fit.horizon_minutes && fit.horizon_minutes <= fit.ci_upper);
+ assert.ok(fit.ci_lower <= 40 && 40 <= fit.ci_upper, `the 90% interval covers the true horizon: ${fit.ci_lower}-${fit.ci_upper}`);
+
+ const tooFewTasks = estimateHorizon(groups.slice(0, 14), { runs: 3, timedTasks: 14 });
+ assert.strictEqual(tooFewTasks.available, false);
+ assert.strictEqual(tooFewTasks.reason, 'not enough runs yet');
+ const tooFewRuns = estimateHorizon(groups, { runs: 2, timedTasks: 20 });
+ assert.strictEqual(tooFewRuns.available, false);
+ assert.strictEqual(tooFewRuns.reason, 'not enough runs yet');
+ });
+
+ await check('the board and the model page show a horizon only above 15 tasks and 3 runs', async () => {
+ const tasks = [];
+ const scoresForRun = [];
+ for (let i = 0; i < 15; i++) {
+ const minutes = 2 + i * 5;
+ tasks.push(baseTask(`h${i}`, { added_on: '2026-01-10', human_minutes: minutes, human_minutes_source: 'benchmark author' }));
+ const p = 1 / (1 + Math.exp(-(1.2 * Math.log(minutes) - 1.2 * Math.log(40))));
+ scoresForRun.push(result(`h${i}`, p * 100));
+ }
+ const made = await create({ title: 'Timed arena', tasks });
+ for (let run = 0; run < 3; run++) {
+ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, scoresForRun.map((x) => ({ ...x })), { model: 'model-timed' }));
+ }
+ const board = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=horizon`);
+ assert.strictEqual(board.status, 200, board.text);
+ const entry = board.json().rankings.entries.find((e) => e.model === 'model-timed');
+ assert.strictEqual(entry.horizon.available, true, JSON.stringify(entry.horizon));
+ assert.strictEqual(entry.horizon.timed_tasks, 15);
+ assert.strictEqual(entry.horizon.runs, 3);
+ assert.ok(Math.abs(entry.horizon.horizon_minutes - 40) < 6, `horizon ${entry.horizon.horizon_minutes}`);
+
+ const page = await t.get(`/benchmarks/${made.id}?metric=horizon`);
+ assert.ok(page.text.includes('br-c-horizon'), 'the board has a horizon column');
+ assert.ok(page.text.includes('min'), 'the horizon is a number of minutes');
+ const model = await t.get('/models/model-timed');
+ assert.strictEqual(model.status, 200, model.text.slice(0, 200));
+ assert.ok(model.text.includes('reliably does about'), 'the model profile has the horizon line');
+ const thin = await t.get('/models/model-a');
+ assert.strictEqual(thin.status, 200, thin.text.slice(0, 200));
+ assert.ok(thin.text.includes('not enough runs yet'), 'below the gate the model page says so instead of a number');
+ });
+
+ await check('editors estimate on the task page: sign-in, same-origin, median of three, account data', async () => {
+ const made = await create({
+ title: 'Estimate arena',
+ tasks: [baseTask('timed', { added_on: '2026-02-01' })],
+ });
+ const path = `/benchmarks/${made.id}/tasks/timed`;
+ const anon = await t.get(path);
+ assert.strictEqual(anon.status, 200, anon.text.slice(0, 200));
+ assert.ok(anon.text.includes('Sign in to estimate'), 'a signed-out reader is asked to sign in');
+ assert.strictEqual((await postForm(`/benchmarks/${made.id}/tasks/timed/estimates`, { minutes: '30', version: String(made.version) })).status, 401);
+
+ const crossSite = await t.get(`/benchmarks/${made.id}/tasks/timed/estimates`, {
+ as: alice, method: 'POST', raw: 'minutes=30', headers: { 'content-type': 'application/x-www-form-urlencoded', origin: 'https://evil.example' },
+ });
+ assert.strictEqual(crossSite.status, 403, 'a cross-site write is refused even when signed in');
+
+ for (const [user, minutes] of [[alice, '30'], [bob, '40'], [carol, '50']]) {
+ const res = await postForm(`/benchmarks/${made.id}/tasks/timed/estimates`, { minutes, version: String(made.version) }, user);
+ assert.strictEqual(res.status, 303, res.text.slice(0, 300));
+ }
+ const page = await t.get(path);
+ assert.ok(page.text.includes('median of 3 editor estimates'), page.text.slice(0, 600));
+ assert.ok(page.text.includes('40 min'), 'the median is 40 minutes');
+
+ const exported = await t.ctx.accountData.exportPart(alice.subject);
+ const file = (exported.files || []).find((f) => f.name === 'bench_task_estimates.json');
+ assert.ok(file && file.content.length === 1, 'the estimate is in the person\'s export');
+ assert.strictEqual(Number(file.content[0].minutes), 30);
+ const erased = await t.ctx.accountData.erase([alice.subject]);
+ assert.ok(Number(erased.erased.bench_task_estimates) >= 1, 'the estimate is erased with the person');
+ });
+
+ await check('the contamination badge is shown, with both numbers and their bands', async () => {
+ const made = await create({
+ title: 'Old and fresh arena',
+ tasks: [
+ baseTask('before', { added_on: '2024-01-10', human_minutes: 10 }),
+ baseTask('after', { added_on: '2026-01-10', human_minutes: 10 }),
+ ],
+ });
+ for (let i = 0; i < 3; i++) {
+ await publish(`${made.id}@${made.version}`, runDoc(`${made.id}@${made.version}`, [result('before', 90 + i), result('after', 20 + i)], { model: 'gpt-5' }));
+ }
+ const board = await t.get(`/api/v1/rankings?benchmark=${made.id}&metric=score`);
+ const entry = board.json().rankings.entries.find((e) => e.model === 'gpt-5');
+ assert.strictEqual(entry.contamination.flag, true, JSON.stringify(entry.contamination));
+ const page = await t.get(`/benchmarks/${made.id}`);
+ assert.ok(page.text.includes('Did much better on older tasks: possibly seen in training'), 'the exact badge text');
+ assert.ok(page.text.includes('Older tasks'), 'both numbers');
+ assert.ok(page.text.includes('fresh tasks'), 'and their bands');
+ });
+
+ await t.close();
+ done();
+})().catch((err) => { console.error(err && err.stack ? err.stack : err); process.exit(1); });