openvibers/openvibe.bench Commits
Run it yourself: a run form, and unbounded benchmarks always ask before they start (#31)
Changes
@@@ -251,3 +251,16 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex.sc-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }.sc-vote a { color: var(--room-strong); font-weight: 700; }.sc-empty { color: var(--text-muted, #7386a3); }++++/* The run form (B3): models, harnesses, how many times; large touch targets, nothing sideways on a phone */++.br-form { display: grid; gap: 18px; margin-top: 8px; }++.br-form fieldset { margin: 0; padding: 16px; border-radius: 16px; border: 1px solid var(--border, #1f2d47); background: var(--bg-card, #0f1729); display: grid; gap: 12px; min-width: 0; }++.br-form legend { padding: 0 6px; font-weight: 800; }++.br-checks { display: flex; flex-wrap: wrap; gap: 8px; }++.br-check { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0 14px; border-radius: 999px; border: 1px solid var(--border, #1f2d47); cursor: pointer; font-weight: 650; max-width: 100%; overflow-wrap: anywhere; }++.br-check:has(input:checked) { border-color: var(--room-accent); background: color-mix(in srgb, var(--room-accent) 14%, transparent); }++.br-check input { accent-color: var(--room-accent); }++.br-field { display: grid; gap: 6px; font-size: .9rem; color: var(--text-secondary, #96a7c2); }++.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }++.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }++.br-form + .br-fine { margin-top: 16px; }@@@ -31,6 +31,7 @@ const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase')const { createRenderRoutes } = require('./http/render');const { seedShowcaseTasks } = require('./bench/seed-tasks');const { createRegistry } = require('./bench/registry');++const { createLauncher } = require('./bench/launch');const { seedBenchmarks } = require('./bench/seed-benchmarks');const { createServiceReadiness } = require('./observability');const { assetVersion, send } = require('./render/layout');@@@ -78,8 +79,9 @@ async function createApp(opts = {}) {? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl }): async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Bench cannot answer account events'); });++ const launcher = createLauncher({ config, registry, fetchImpl: fetchImpl || globalThis.fetch, log });const ctx = {-- config, s, keys, sso, principal, bench, registry, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,++ config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,random: opts.random || Math.random,};nnew file mode 100644@@@ -0,0 +1,92 @@++'use strict';++++/**++ * Starting a benchmark on openvibe.codes for a signed-in person (stage B3): the one place that checks the request,++ * estimates the cost, asks for a confirmation above $1 and hands the matrix to Codes. The API++ * (POST /api/v1/benchmarks/:id/run) and the run form (/benchmarks/:id/run) both call it, so the page never promises++ * something the API refuses.++ */++const { createServiceTokenClient } = require('openvibe-sdk/auth');++++const CONFIRM_ABOVE_USD = 1;++const HARNESS_TTL_MS = 10 * 60 * 1000;++++function createLauncher({ config, registry, fetchImpl = globalThis.fetch, log = console }) {++ const codesTokens = config.oauth && config.oauth.clientSecret ? createServiceTokenClient({++ tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId,++ clientSecret: config.oauth.clientSecret, audience: 'openvibe.codes', scope: 'codes.task.start', fetch: fetchImpl,++ }) : null;++++ /**++ * The cost a run may reach: tasks × models × harnesses × repeats × the benchmark's per-task dollar limit. Null when++ * the benchmark sets no dollar limit: then nobody can say, and the run always asks for a confirmation.++ */++ function estimate(definition, { models, harnesses, repeat }) {++ const tasks = (definition && Array.isArray(definition.tasks) ? definition.tasks : []).length;++ const usd = Number(definition && definition.limits && definition.limits.usd) || 0;++ if (!(usd > 0)) return null;++ return Math.round(tasks * models.length * harnesses.length * repeat * usd * 100) / 100;++ }++++ /**++ * Start the run. Returns { status, body }: 202 with Codes' answer, 409 when the estimate needs a confirmation,++ * or a problem ({ code, detail }) for everything else.++ */++ async function launch({ principal, id, models, harnesses, repeat, confirm }) {++ const fail = (status, code, detail) => ({ status, body: { code, detail } });++ if (!principal || principal.kind !== 'user') return fail(403, 'bench.person_required', 'Sign in as the person who will run this benchmark.');++ if (!config.codesRunUrl) return fail(503, 'bench.runner_unavailable', 'Codes benchmark runs are not configured.');++ const clean = (list) => [...new Set((Array.isArray(list) ? list : []).map((x) => String(x || '').trim()).filter((x) => /^[\w.:/@-]{1,120}$/.test(x)))];++ models = clean(models); harnesses = clean(harnesses); repeat = Number(repeat);++ if (!models.length || !harnesses.length || models.length > 20 || harnesses.length > 20 || !Number.isInteger(repeat) || repeat < 1 || repeat > 20) {++ return fail(400, 'bench.invalid', 'models and harnesses must be non-empty lists (at most 20 each), and repeat must be 1–20');++ }++ const definition = await registry.getDefinition(String(id), { reveal: false });++ if (!definition) return fail(404, 'bench.benchmark_not_found', 'No such benchmark.');++ const cost = estimate(definition.definition, { models, harnesses, repeat });++ if ((cost == null || cost > CONFIRM_ABOVE_USD) && confirm !== true) {++ return { status: 409, body: { code: 'bench.confirm_cost', cost_estimate_usd: cost, confirm_required: true, ...(cost == null ? { detail: 'This benchmark sets no dollar limit per task, so its cost depends on the models.' } : {}) } };++ }++ if (!codesTokens) return fail(503, 'bench.runner_unavailable', 'Bench has no Network client credentials for Codes.');++ try {++ const url = new URL(`/api/v1/benchmarks/${encodeURIComponent(String(id))}/run`, config.codesRunUrl).toString();++ const response = await fetchImpl(url, {++ method: 'POST',++ headers: { ...(await codesTokens.authHeaders()), 'content-type': 'application/json', 'x-ov-subject': principal.subject, 'x-ov-user-token': principal.token },++ body: JSON.stringify({ benchmark: String(id), models, harnesses, repeat }),++ signal: AbortSignal.timeout(15000),++ });++ const body = await response.json().catch(() => ({}));++ if (!response.ok) {++ return fail(response.status < 500 ? response.status : 502, body.code || 'bench.runner_failed', body.detail || body.message || 'Codes could not start this benchmark.');++ }++ return { status: 202, body: { ...body, cost_estimate_usd: cost, tasks_url: new URL('/tasks', config.codesRunUrl).toString() } };++ } catch (err) {++ log.warn('[OpenVibe.Bench] starting a Codes run failed:', (err && err.message) || err);++ return fail(502, 'bench.runner_failed', 'Codes did not answer. Try again in a minute.');++ }++ }++++ // The harnesses Codes offers, for the run form: read from Codes and kept ten minutes. Unreachable, the form falls++ // back to the benchmark's own matrix.++ let harnessCache = { at: 0, list: null };++ async function harnessList() {++ if (harnessCache.list && Date.now() - harnessCache.at < HARNESS_TTL_MS) return harnessCache.list;++ if (!config.codesRunUrl) return [];++ try {++ const res = await fetchImpl(new URL('/api/v1/harnesses', config.codesRunUrl).toString(), { signal: AbortSignal.timeout(5000) });++ const body = await res.json();++ const rows = Array.isArray(body) ? body : (body.harnesses || body.offers || []);++ const list = rows.map((h) => ({ id: String(h.id || ''), label: String(h.label || h.name || h.id || '') })).filter((h) => h.id);++ harnessCache = { at: Date.now(), list };++ return list;++ } catch (err) {++ log.warn('[OpenVibe.Bench] Codes harness list unavailable:', (err && err.message) || err);++ return harnessCache.list || [];++ }++ }++++ return { launch, estimate, harnessList, CONFIRM_ABOVE_USD };++}++++module.exports = { createLauncher, CONFIRM_ABOVE_USD };@@@ -7,6 +7,7 @@const express = require('express');const seo = require('openvibe-shared/seo');const { asyncRouter } = require('./router');++const { sameOrigin } = require('./same-origin');const { createDiscoveryRoutes, homeJsonLd } = require('./discovery');const { raw } = require('../render/html');const { send } = require('../render/layout');@@@ -109,6 +110,39 @@ function createPageRoutes(ctx) {});});++ // The run form (B3): GET shows it, POST starts the run through the same launcher as the API.++ const runFormPage = async (req, res, extra = {}) => {++ const ref = String(req.params.ref || '');++ const out = await registry.getDefinition(ref, { reveal: false });++ if (!out || out.benchmark.visibility === 'private') return notFound(req, res, 'benchmark');++ if (!config.codesRunUrl) return res.redirect(303, `/benchmarks/${encodeURIComponent(out.benchmark.id)}`);++ const signedIn = req.viewer && req.viewer.kind === 'user';++ return page(req, res, {++ index: false, title: `Run ${out.benchmark.title}`,++ description: `Run ${out.benchmark.title} on openvibe.codes with your own keys.`,++ body: raw(String(results.runForm({++ benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version, signedIn,++ harnesses: signedIn ? await ctx.launcher.harnessList() : [],++ suggested: signedIn ? (((await registry.rankings({ benchmark: out.benchmark.id, metric: 'score' })) || {}).entries || []).map((e) => e.model) : [],++ loginUrl: `/auth/login?next=${encodeURIComponent(req.originalUrl)}`, ...extra,++ }))),++ }, extra.status || 200);++ };++ r.get('/benchmarks/:ref/run', (req, res) => runFormPage(req, res));++ r.post('/benchmarks/:ref/run', express.urlencoded({ extended: false, limit: '16kb' }), ctx.principal.middleware, async (req, res) => {++ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');++ const b = req.body || {};++ const list = (v) => (Array.isArray(v) ? v : v ? [v] : []);++ const models = [...list(b.models), ...String(b.more_models || '').split(',')].map((x) => x.trim()).filter(Boolean);++ const harnesses = list(b.harnesses);++ const repeat = Number(b.repeat || 1);++ const picked = { models, harnesses, repeat };++ const out = await ctx.launcher.launch({ principal: req.principal, id: req.params.ref, models, harnesses, repeat, confirm: b.confirm === '1' });++ if (out.status === 202) return res.redirect(303, out.body.tasks_url || config.codesRunUrl);++ if (out.status === 409) return runFormPage(req, res, { picked, confirm: out.body.cost_estimate_usd == null ? 'unknown' : out.body.cost_estimate_usd });++ return runFormPage(req, res, { picked, error: out.body.detail || 'The run could not start.', status: out.status === 400 ? 400 : 200 });++ });++r.get('/runs/:id', async (req, res) => {const run = await registry.getRun(String(req.params.id || ''));if (!run) return notFound(req, res, 'run');@@@ -23,15 +23,9 @@ const { http } = require('openvibe-contracts');const { asyncRouter } = require('./router');const { sameOrigin } = require('./same-origin');const { validateBenchmark, validateRun } = require('../bench/validate');--const { createServiceTokenClient } = require('openvibe-sdk/auth');function createRegistryApi(ctx) {const { config, principal, registry } = ctx;-- const fetchImpl = ctx.fetchImpl || globalThis.fetch;-- const codesTokens = config.oauth.clientSecret ? createServiceTokenClient({-- tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId,-- clientSecret: config.oauth.clientSecret, audience: 'openvibe.codes', scope: 'codes.task.start', fetch: fetchImpl,-- }) : null;const r = asyncRouter();r.use(express.json({ limit: '16mb' }));@@@ -172,34 +166,13 @@ function createRegistryApi(ctx) {});// Hand execution to Codes, which owns the person's keys and the benchmark task lifecycle.++ // Start the benchmark on openvibe.codes as the signed-in person: the launcher checks, estimates and asks for a++ // confirmation above $1 (server/bench/launch.js); the run form on the benchmark page uses the same one.r.post('/benchmarks/:id/run', actor('bench.run.publish'), async (req, res) => {-- const p = req.principal;-- if (p.kind !== 'user') return problem(res, 403, 'bench.person_required', 'Sign in as the person who will run this benchmark.');-- if (!config.codesRunUrl) return problem(res, 503, 'bench.runner_unavailable', 'Codes benchmark runs are not configured.');const b = req.body && typeof req.body === 'object' ? req.body : {};-- const models = Array.isArray(b.models) ? b.models : [];-- const harnesses = Array.isArray(b.harnesses) ? b.harnesses : [];-- const repeat = Number(b.repeat);-- if (!models.length || !harnesses.length || models.length > 20 || harnesses.length > 20 || !Number.isInteger(repeat) || repeat < 1 || repeat > 20) {-- return invalid(res, 'models and harnesses must be non-empty arrays (at most 20 each), and repeat must be 1–20');-- }-- const definition = await registry.getDefinition(String(req.params.id), { reveal: false });-- if (!definition) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');-- const costEstimate = (definition.definition.tasks || []).length * models.length * harnesses.length * repeat-- * Number(definition.definition.limits && definition.definition.limits.usd || 0);-- if (costEstimate > 1 && b.confirm !== true) return res.status(409).json({ code: 'bench.confirm_cost', cost_estimate_usd: costEstimate, confirm_required: true });-- try {-- if (!codesTokens) return problem(res, 503, 'bench.runner_unavailable', 'Bench has no Network client credentials for Codes.');-- const codesRunApi = new URL(`/api/v1/benchmarks/${encodeURIComponent(String(req.params.id))}/run`, config.codesRunUrl).toString();-- const response = await fetchImpl(codesRunApi, {-- method: 'POST', headers: { ...(await codesTokens.authHeaders()), 'content-type': 'application/json', 'x-ov-subject': p.subject, 'x-ov-user-token': p.token },-- body: JSON.stringify({ benchmark: String(req.params.id), models, harnesses, repeat }),-- signal: AbortSignal.timeout(15000),-- });-- const body = await response.json().catch(() => ({}));-- if (!response.ok) return problem(res, response.status < 500 ? response.status : 502, body.code || 'bench.runner_failed', body.detail || body.message || 'Codes could not start this benchmark.');-- return res.status(202).json(body);-- } catch (err) { return failed(res, err); }++ const out = await ctx.launcher.launch({ principal: req.principal, id: req.params.id, models: b.models, harnesses: b.harnesses, repeat: b.repeat, confirm: b.confirm === true });++ if (out.status === 202 || out.status === 409) return res.status(out.status).json(out.body);++ return problem(res, out.status, out.body.code, out.body.detail);});// ── Runs ─────────────────────────────────────────────────@@@ -211,7 +211,7 @@ function benchmarkPage({ benchmark, definition, version, rankings, metric = 'scoconst entries = (rankings && rankings.entries) || [];const base = benchmarkHref(benchmark.id);const runYourself = codesRunUrl-- ? html`<a class="bm-btn" href="${codesRunUrl}?benchmark=${encodeURIComponent(`${benchmark.id}@${version || benchmark.latest_version}`)}">Run it yourself</a>`++ ? html`<a class="bm-btn" href="${benchmarkHref(benchmark.id)}/run">Run it yourself</a>`: html`<a class="bm-btn bm-btn-quiet" href="/docs#br-publish">Publish a run</a>`;return html`<div class="bm-wrap br" id="br-benchmark"><nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>${benchmark.title}</span></nav>@@@ -231,6 +231,44 @@ ${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up he</div>`;}++/**++ * The run form: pick the models, the harnesses and how many times, see what it may cost, start it on openvibe.codes++ * with your own keys. A plain form (it works without JavaScript); the estimate above $1 asks once more.++ */++function runForm({ benchmark, definition, version, signedIn, harnesses = [], suggested = [], picked = null, confirm = null, error = null, loginUrl }) {++ const d = definition || {};++ const tasks = Array.isArray(d.tasks) ? d.tasks.length : 0;++ const usd = Number(d.limits && d.limits.usd) || 0;++ const matrix = d.matrix || {};++ const harnessList = harnesses.length ? harnesses.slice() : (matrix.harnesses || []).map((id) => ({ id, label: id }));++ // Nothing chosen yet: the benchmark's own matrix, else the first harness, so the form starts one tap from ready.++ const p = picked || { models: matrix.models || [], harnesses: (matrix.harnesses || []).length ? matrix.harnesses : harnessList.slice(0, 1).map((h) => h.id), repeat: 1 };++ // Models: the matrix, the ones already ranked here (verified runs), and whatever was typed.++ const modelList = [...new Set([...(matrix.models || []), ...suggested, ...p.models])].slice(0, 16);++ for (const id of p.harnesses) if (!harnessList.some((h) => h.id === id)) harnessList.push({ id, label: id });++ const base = benchmarkHref(benchmark.id);++ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${base}">${benchmark.title}</a><span aria-hidden="true">/</span><span>Run it</span></nav>++<header class="bm-page-h"><p class="bm-kicker">Run it yourself</p><h1>${benchmark.title}</h1><p class="bm-lede">openvibe.codes runs every task on each model and harness you pick, with your own keys, and publishes each run here as verified, whatever it scores.</p></header>`;++ if (!signedIn) {++ return html`<div class="bm-wrap br bm-narrow" id="br-run-form">${head}${empty('Sign in first', 'A run uses your own provider keys on openvibe.codes, so it starts as you.', html`<a class="bm-btn bm-btn-small" href="${loginUrl}">Sign in with OpenVibe</a>`)}</div>`;++ }++ const check = (name, value, label, on) => html`<label class="br-check"><input type="checkbox" name="${name}" value="${value}"${on ? raw(' checked') : ''}><span>${label}</span></label>`;++ return html`<div class="bm-wrap br bm-narrow" id="br-run-form">${head}++${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}++${confirm === 'unknown' ? html`<div class="bm-note" role="status">This benchmark sets no dollar limit per task, so what it costs depends on the models you pick. It runs on your own keys. Check the choices below, then confirm.</div>` : confirm != null ? html`<div class="bm-note" role="status">This run may cost up to <b>${fmtUsd(confirm)}</b> on your own keys. Check the choices below, then confirm.</div>` : ''}++<form class="br-form" method="post" action="${base}/run" data-tasks="${tasks}" data-usd="${usd}">++<fieldset><legend>Models</legend>${modelList.length ? html`<div class="br-checks">${modelList.map((m) => check('models', m, m, p.models.includes(m)))}</div>` : ''}++<label class="br-field"><span>${modelList.length ? 'More models, separated by commas' : 'Models, separated by commas'}</span><input name="more_models" type="text" autocomplete="off" placeholder="deepseek-v4.1-flash, kimi-k3"></label></fieldset>++<fieldset><legend>Harnesses</legend><div class="br-checks">${harnessList.map((h) => check('harnesses', h.id, h.label, p.harnesses.includes(h.id)))}</div></fieldset>++<label class="br-field"><span>Runs of each</span><select name="repeat">${[1, 2, 3, 5].map((n) => html`<option value="${n}"${n === Number(p.repeat) ? raw(' selected') : ''}>${n === 1 ? 'Once' : `${n} times (shows how steady it is)`}</option>`)}</select></label>++<p class="br-fine" data-estimate>${usd > 0 ? `${tasks} ${tasks === 1 ? 'task' : 'tasks'} × your picks × runs, at up to ${fmtUsd(usd)} a task. Over $1 asks you once more.` : `${tasks} ${tasks === 1 ? 'task' : 'tasks'} × your picks × runs. This benchmark sets no dollar limit per task, so you confirm before it starts.`}</p>++${confirm != null ? html`<input type="hidden" name="confirm" value="1">` : ''}++<div class="bm-actions"><button class="bm-btn" type="submit">${confirm != null ? 'Confirm and start' : 'Start on openvibe.codes'}</button><a class="bm-btn bm-btn-quiet" href="${base}">Back</a></div>++</form>++<p class="br-fine">Version v${version || benchmark.latest_version || 0}, frozen: these runs can be compared with every other run of it.</p>++</div>`;++}++function runPage({ run, benchmark, definition, pct }) {const tasksDef = new Map(((definition && definition.tasks) || []).map((t) => [t.id, t]));const env = [@@@ -296,6 +334,6 @@ ${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 i}module.exports = {-- benchmarksIndex, benchmarkPage, runPage, comparePage, modelPage, valueChart, METRICS,++ benchmarksIndex, benchmarkPage, runForm, runPage, comparePage, modelPage, valueChart, METRICS,fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,};nnew file mode 100644@@@ -0,0 +1,134 @@++'use strict';++++/**++ * Starting a benchmark on openvibe.codes (stage B3): the API and the run form share one launcher.++ *++ * Under test: a signed-out visitor is sent to sign in; the form lists Codes' harnesses; an estimate above $1 asks++ * for a confirmation before anything reaches Codes; a confirmed run reaches Codes once, as the person, with a service++ * token, and the browser lands on the person's task list; a cross-site form post is refused; the API answers the same++ * way (409 with the estimate, then 202).++ */++const assert = require('assert');++const http = require('http');++const { boot, check, done } = require('./helpers/boot');++++function fakeCodes() {++ const calls = [];++ const server = http.createServer((req, res) => {++ let body = '';++ req.on('data', (c) => { body += c; });++ req.on('end', () => {++ calls.push({ method: req.method, url: req.url, headers: req.headers, body: body ? JSON.parse(body) : null });++ res.setHeader('content-type', 'application/json');++ if (req.url === '/api/v1/harnesses') return res.end(JSON.stringify({ harnesses: [{ id: 'claude-code', label: 'Claude Code' }, { id: 'codex', label: 'Codex' }] }));++ if (/^\/api\/v1\/benchmarks\/[^/]+\/run$/.test(req.url) && req.method === 'POST') {++ res.statusCode = 202;++ return res.end(JSON.stringify({ session: 'ses_1', tasks: [{ id: 'tsk_1' }], task: { id: 'tsk_1' }, total: 1 }));++ }++ res.statusCode = 404; res.end('{}');++ });++ });++ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ url: `http://127.0.0.1:${server.address().port}`, calls, close: () => server.close() })));++}++++(async () => {++ const codes = await fakeCodes();++ const t = await boot({ env: { CODES_BENCH_RUN_URL: codes.url } });++ const alice = t.network.addUser('alice');++ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });++ const created = await t.get('/api/v1/benchmarks', { bearer: publisher, json: {++ title: 'Run form arena', description: 'For the run form tests.', kind: 'games.play', tags: [], visibility: 'public',++ tasks: [{ id: 't1', prompt: 'Make a game.', files: [], expected: null, holdout: false }, { id: 't2', prompt: 'Make another.', files: [], expected: null, holdout: false }],++ judges: [{ type: 'game_outcome', version: '1.0', config: {} }], scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: { ms: 60000, tokens: 100000, usd: 0.5 }, matrix: { models: ['model-a'], harnesses: ['codex'] },++ } });++ assert.strictEqual(created.status, 201, created.text);++ const id = created.json().benchmark.id;++ assert.strictEqual((await t.get(`/api/v1/benchmarks/${id}/versions`, { bearer: publisher, json: {} })).status, 201);++ const form = (fields) => new URLSearchParams(fields).toString();++ const post = (fields, o = {}) => t.get(`/benchmarks/${id}/run`, { as: alice, raw: form(fields), headers: { 'content-type': 'application/x-www-form-urlencoded' }, ...o });++ const runCalls = () => codes.calls.filter((c) => c.method === 'POST');++++ await check('the benchmark page links to its run form', async () => {++ const res = await t.get(`/benchmarks/${id}`);++ assert.ok(res.text.includes(`href="/benchmarks/${id}/run">Run it yourself`), 'the button opens the form, not a bare Codes link');++ });++++ await check('signed out, the form asks you to sign in', async () => {++ const res = await t.get(`/benchmarks/${id}/run`);++ assert.strictEqual(res.status, 200, res.text.slice(0, 200));++ assert.ok(res.text.includes('Sign in first'));++ assert.ok(!res.text.includes('<form class="br-form"'), 'no form for a stranger');++ });++++ await check('signed in, the form lists the benchmark matrix and Codes harnesses', async () => {++ const res = await t.get(`/benchmarks/${id}/run`, { as: alice });++ assert.ok(res.text.includes('<form class="br-form"'));++ assert.ok(res.text.includes('value="model-a" checked'), 'the matrix model is picked');++ assert.ok(res.text.includes('value="claude-code"') && res.text.includes('Codex'), "Codes' harness list");++ });++++ await check('an estimate above $1 asks once more and reaches nobody', async () => {++ const before = runCalls().length;++ const res = await post([['models', 'model-a'], ['more_models', 'model-b'], ['harnesses', 'codex'], ['repeat', '1']]);++ assert.strictEqual(res.status, 200, res.text.slice(0, 200));++ assert.ok(res.text.includes('may cost up to') && res.text.includes('$2.00'), '2 tasks × 2 models × $0.50');++ assert.ok(res.text.includes('name="confirm" value="1"') && res.text.includes('Confirm and start'));++ assert.strictEqual(runCalls().length, before, 'Codes was not called');++ });++++ await check('a confirmed run reaches Codes once, as the person, and lands on their tasks', async () => {++ const before = runCalls().length;++ const res = await post([['models', 'model-a'], ['more_models', 'model-b'], ['harnesses', 'codex'], ['repeat', '1'], ['confirm', '1']]);++ assert.strictEqual(res.status, 303, res.text.slice(0, 200));++ assert.strictEqual(res.headers.get('location'), `${codes.url}/tasks`);++ const calls = runCalls().slice(before);++ assert.strictEqual(calls.length, 1);++ assert.deepStrictEqual(calls[0].body.models, ['model-a', 'model-b']);++ assert.match(String(calls[0].headers.authorization), /^Bearer /, "Bench's own service token");++ assert.ok(calls[0].headers['x-ov-subject'], 'names the person');++ });++++ await check('nothing picked is said plainly', async () => {++ const res = await post([['repeat', '1']]);++ assert.strictEqual(res.status, 400);++ assert.ok(res.text.includes('non-empty lists'));++ });++++ await check('a cross-site form post is refused', async () => {++ const res = await post([['models', 'model-a'], ['harnesses', 'codex'], ['repeat', '1']], { origin: 'https://evil.example' });++ assert.strictEqual(res.status, 403);++ });++++ await check('the API answers the same way: 409 with the estimate, then 202', async () => {++ const body = { models: ['model-a', 'model-b'], harnesses: ['codex'], repeat: 1 };++ const ask = await t.get(`/api/v1/benchmarks/${id}/run`, { as: alice, json: body });++ assert.strictEqual(ask.status, 409, ask.text);++ assert.strictEqual(ask.json().cost_estimate_usd, 2);++ const go = await t.get(`/api/v1/benchmarks/${id}/run`, { as: alice, json: { ...body, confirm: true } });++ assert.strictEqual(go.status, 202, go.text);++ assert.strictEqual(go.json().total, 1);++ });++++ await check('a benchmark with no dollar limit always asks before it starts', async () => {++ const res2 = await t.get('/api/v1/benchmarks', { bearer: publisher, json: {++ title: 'No limit arena', description: 'No usd limit.', kind: 'games.play', tags: [], visibility: 'public',++ tasks: [{ id: 't1', prompt: 'Make a game.', files: [], expected: null, holdout: false }],++ judges: [{ type: 'game_outcome', version: '1.0', config: {} }], scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: { ms: 60000, tokens: 100000 }, matrix: { models: [], harnesses: [] },++ } });++ assert.strictEqual(res2.status, 201, res2.text);++ const nid = res2.json().benchmark.id;++ assert.strictEqual((await t.get(`/api/v1/benchmarks/${nid}/versions`, { bearer: publisher, json: {} })).status, 201);++ const before = runCalls().length;++ const ask = await t.get(`/api/v1/benchmarks/${nid}/run`, { as: alice, json: { models: ['m'], harnesses: ['codex'], repeat: 1 } });++ assert.strictEqual(ask.status, 409, ask.text);++ assert.strictEqual(ask.json().cost_estimate_usd, null, 'no estimate is invented');++ const page = await t.get(`/benchmarks/${nid}/run`, { as: alice, raw: form([['more_models', 'm'], ['harnesses', 'codex'], ['repeat', '1']]), headers: { 'content-type': 'application/x-www-form-urlencoded' } });++ assert.ok(page.text.includes('sets no dollar limit per task'), 'the form says why it asks');++ assert.strictEqual(runCalls().length, before, 'and nothing reached Codes');++ });++++ await check('stop', async () => { await t.close(); codes.close(); });++ done();++})();
@@ -251,3 +251,16 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -251,3 +251,16 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.sc-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }
.sc-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }
.sc-vote a { color: var(--room-strong); font-weight: 700; }
.sc-vote a { color: var(--room-strong); font-weight: 700; }
.sc-empty { color: var(--text-muted, #7386a3); }
.sc-empty { color: var(--text-muted, #7386a3); }
+
+/* The run form (B3): models, harnesses, how many times; large touch targets, nothing sideways on a phone */
+.br-form { display: grid; gap: 18px; margin-top: 8px; }
+.br-form fieldset { margin: 0; padding: 16px; border-radius: 16px; border: 1px solid var(--border, #1f2d47); background: var(--bg-card, #0f1729); display: grid; gap: 12px; min-width: 0; }
+.br-form legend { padding: 0 6px; font-weight: 800; }
+.br-checks { display: flex; flex-wrap: wrap; gap: 8px; }
+.br-check { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0 14px; border-radius: 999px; border: 1px solid var(--border, #1f2d47); cursor: pointer; font-weight: 650; max-width: 100%; overflow-wrap: anywhere; }
+.br-check:has(input:checked) { border-color: var(--room-accent); background: color-mix(in srgb, var(--room-accent) 14%, transparent); }
+.br-check input { accent-color: var(--room-accent); }
+.br-field { display: grid; gap: 6px; font-size: .9rem; color: var(--text-secondary, #96a7c2); }
+.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }
+.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
+.br-form + .br-fine { margin-top: 16px; }
@@ -31,6 +31,7 @@ const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase')
@@ -31,6 +31,7 @@ const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase')
const { createRenderRoutes } = require('./http/render');
const { createRenderRoutes } = require('./http/render');
const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { createRegistry } = require('./bench/registry');
const { createRegistry } = require('./bench/registry');
+const { createLauncher } = require('./bench/launch');
const { seedBenchmarks } = require('./bench/seed-benchmarks');
const { seedBenchmarks } = require('./bench/seed-benchmarks');
const { createServiceReadiness } = require('./observability');
const { createServiceReadiness } = require('./observability');
const { assetVersion, send } = require('./render/layout');
const { assetVersion, send } = require('./render/layout');
@@ -78,8 +79,9 @@ async function createApp(opts = {}) {
@@ -78,8 +79,9 @@ async function createApp(opts = {}) {
? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl })
? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl })
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Bench cannot answer account events'); });
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Bench cannot answer account events'); });
+ const launcher = createLauncher({ config, registry, fetchImpl: fetchImpl || globalThis.fetch, log });
const ctx = {
const ctx = {
- config, s, keys, sso, principal, bench, registry, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
+ config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
random: opts.random || Math.random,
random: opts.random || Math.random,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,92 @@
@@ -0,0 +1,92 @@
+'use strict';
+
+/**
+ * Starting a benchmark on openvibe.codes for a signed-in person (stage B3): the one place that checks the request,
+ * estimates the cost, asks for a confirmation above $1 and hands the matrix to Codes. The API
+ * (POST /api/v1/benchmarks/:id/run) and the run form (/benchmarks/:id/run) both call it, so the page never promises
+ * something the API refuses.
+ */
+const { createServiceTokenClient } = require('openvibe-sdk/auth');
+
+const CONFIRM_ABOVE_USD = 1;
+const HARNESS_TTL_MS = 10 * 60 * 1000;
+
+function createLauncher({ config, registry, fetchImpl = globalThis.fetch, log = console }) {
+ const codesTokens = config.oauth && config.oauth.clientSecret ? createServiceTokenClient({
+ tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId,
+ clientSecret: config.oauth.clientSecret, audience: 'openvibe.codes', scope: 'codes.task.start', fetch: fetchImpl,
+ }) : null;
+
+ /**
+ * The cost a run may reach: tasks × models × harnesses × repeats × the benchmark's per-task dollar limit. Null when
+ * the benchmark sets no dollar limit: then nobody can say, and the run always asks for a confirmation.
+ */
+ function estimate(definition, { models, harnesses, repeat }) {
+ const tasks = (definition && Array.isArray(definition.tasks) ? definition.tasks : []).length;
+ const usd = Number(definition && definition.limits && definition.limits.usd) || 0;
+ if (!(usd > 0)) return null;
+ return Math.round(tasks * models.length * harnesses.length * repeat * usd * 100) / 100;
+ }
+
+ /**
+ * Start the run. Returns { status, body }: 202 with Codes' answer, 409 when the estimate needs a confirmation,
+ * or a problem ({ code, detail }) for everything else.
+ */
+ async function launch({ principal, id, models, harnesses, repeat, confirm }) {
+ const fail = (status, code, detail) => ({ status, body: { code, detail } });
+ if (!principal || principal.kind !== 'user') return fail(403, 'bench.person_required', 'Sign in as the person who will run this benchmark.');
+ if (!config.codesRunUrl) return fail(503, 'bench.runner_unavailable', 'Codes benchmark runs are not configured.');
+ const clean = (list) => [...new Set((Array.isArray(list) ? list : []).map((x) => String(x || '').trim()).filter((x) => /^[\w.:/@-]{1,120}$/.test(x)))];
+ models = clean(models); harnesses = clean(harnesses); repeat = Number(repeat);
+ if (!models.length || !harnesses.length || models.length > 20 || harnesses.length > 20 || !Number.isInteger(repeat) || repeat < 1 || repeat > 20) {
+ return fail(400, 'bench.invalid', 'models and harnesses must be non-empty lists (at most 20 each), and repeat must be 1–20');
+ }
+ const definition = await registry.getDefinition(String(id), { reveal: false });
+ if (!definition) return fail(404, 'bench.benchmark_not_found', 'No such benchmark.');
+ const cost = estimate(definition.definition, { models, harnesses, repeat });
+ if ((cost == null || cost > CONFIRM_ABOVE_USD) && confirm !== true) {
+ return { status: 409, body: { code: 'bench.confirm_cost', cost_estimate_usd: cost, confirm_required: true, ...(cost == null ? { detail: 'This benchmark sets no dollar limit per task, so its cost depends on the models.' } : {}) } };
+ }
+ if (!codesTokens) return fail(503, 'bench.runner_unavailable', 'Bench has no Network client credentials for Codes.');
+ try {
+ const url = new URL(`/api/v1/benchmarks/${encodeURIComponent(String(id))}/run`, config.codesRunUrl).toString();
+ const response = await fetchImpl(url, {
+ method: 'POST',
+ headers: { ...(await codesTokens.authHeaders()), 'content-type': 'application/json', 'x-ov-subject': principal.subject, 'x-ov-user-token': principal.token },
+ body: JSON.stringify({ benchmark: String(id), models, harnesses, repeat }),
+ signal: AbortSignal.timeout(15000),
+ });
+ const body = await response.json().catch(() => ({}));
+ if (!response.ok) {
+ return fail(response.status < 500 ? response.status : 502, body.code || 'bench.runner_failed', body.detail || body.message || 'Codes could not start this benchmark.');
+ }
+ return { status: 202, body: { ...body, cost_estimate_usd: cost, tasks_url: new URL('/tasks', config.codesRunUrl).toString() } };
+ } catch (err) {
+ log.warn('[OpenVibe.Bench] starting a Codes run failed:', (err && err.message) || err);
+ return fail(502, 'bench.runner_failed', 'Codes did not answer. Try again in a minute.');
+ }
+ }
+
+ // The harnesses Codes offers, for the run form: read from Codes and kept ten minutes. Unreachable, the form falls
+ // back to the benchmark's own matrix.
+ let harnessCache = { at: 0, list: null };
+ async function harnessList() {
+ if (harnessCache.list && Date.now() - harnessCache.at < HARNESS_TTL_MS) return harnessCache.list;
+ if (!config.codesRunUrl) return [];
+ try {
+ const res = await fetchImpl(new URL('/api/v1/harnesses', config.codesRunUrl).toString(), { signal: AbortSignal.timeout(5000) });
+ const body = await res.json();
+ const rows = Array.isArray(body) ? body : (body.harnesses || body.offers || []);
+ const list = rows.map((h) => ({ id: String(h.id || ''), label: String(h.label || h.name || h.id || '') })).filter((h) => h.id);
+ harnessCache = { at: Date.now(), list };
+ return list;
+ } catch (err) {
+ log.warn('[OpenVibe.Bench] Codes harness list unavailable:', (err && err.message) || err);
+ return harnessCache.list || [];
+ }
+ }
+
+ return { launch, estimate, harnessList, CONFIRM_ABOVE_USD };
+}
+
+module.exports = { createLauncher, CONFIRM_ABOVE_USD };
@@ -7,6 +7,7 @@
@@ -7,6 +7,7 @@
const express = require('express');
const express = require('express');
const seo = require('openvibe-shared/seo');
const seo = require('openvibe-shared/seo');
const { asyncRouter } = require('./router');
const { asyncRouter } = require('./router');
+const { sameOrigin } = require('./same-origin');
const { createDiscoveryRoutes, homeJsonLd } = require('./discovery');
const { createDiscoveryRoutes, homeJsonLd } = require('./discovery');
const { raw } = require('../render/html');
const { raw } = require('../render/html');
const { send } = require('../render/layout');
const { send } = require('../render/layout');
@@ -109,6 +110,39 @@ function createPageRoutes(ctx) {
@@ -109,6 +110,39 @@ function createPageRoutes(ctx) {
});
});
});
});
+ // The run form (B3): GET shows it, POST starts the run through the same launcher as the API.
+ const runFormPage = async (req, res, extra = {}) => {
+ const ref = String(req.params.ref || '');
+ const out = await registry.getDefinition(ref, { reveal: false });
+ if (!out || out.benchmark.visibility === 'private') return notFound(req, res, 'benchmark');
+ if (!config.codesRunUrl) return res.redirect(303, `/benchmarks/${encodeURIComponent(out.benchmark.id)}`);
+ const signedIn = req.viewer && req.viewer.kind === 'user';
+ return page(req, res, {
+ index: false, title: `Run ${out.benchmark.title}`,
+ description: `Run ${out.benchmark.title} on openvibe.codes with your own keys.`,
+ body: raw(String(results.runForm({
+ benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version, signedIn,
+ harnesses: signedIn ? await ctx.launcher.harnessList() : [],
+ suggested: signedIn ? (((await registry.rankings({ benchmark: out.benchmark.id, metric: 'score' })) || {}).entries || []).map((e) => e.model) : [],
+ loginUrl: `/auth/login?next=${encodeURIComponent(req.originalUrl)}`, ...extra,
+ }))),
+ }, extra.status || 200);
+ };
+ r.get('/benchmarks/:ref/run', (req, res) => runFormPage(req, res));
+ r.post('/benchmarks/:ref/run', express.urlencoded({ extended: false, limit: '16kb' }), ctx.principal.middleware, async (req, res) => {
+ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
+ const b = req.body || {};
+ const list = (v) => (Array.isArray(v) ? v : v ? [v] : []);
+ const models = [...list(b.models), ...String(b.more_models || '').split(',')].map((x) => x.trim()).filter(Boolean);
+ const harnesses = list(b.harnesses);
+ const repeat = Number(b.repeat || 1);
+ const picked = { models, harnesses, repeat };
+ const out = await ctx.launcher.launch({ principal: req.principal, id: req.params.ref, models, harnesses, repeat, confirm: b.confirm === '1' });
+ if (out.status === 202) return res.redirect(303, out.body.tasks_url || config.codesRunUrl);
+ if (out.status === 409) return runFormPage(req, res, { picked, confirm: out.body.cost_estimate_usd == null ? 'unknown' : out.body.cost_estimate_usd });
+ return runFormPage(req, res, { picked, error: out.body.detail || 'The run could not start.', status: out.status === 400 ? 400 : 200 });
+ });
+
r.get('/runs/:id', async (req, res) => {
r.get('/runs/:id', async (req, res) => {
const run = await registry.getRun(String(req.params.id || ''));
const run = await registry.getRun(String(req.params.id || ''));
if (!run) return notFound(req, res, 'run');
if (!run) return notFound(req, res, 'run');
@@ -23,15 +23,9 @@ const { http } = require('openvibe-contracts');
@@ -23,15 +23,9 @@ const { http } = require('openvibe-contracts');
const { asyncRouter } = require('./router');
const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const { sameOrigin } = require('./same-origin');
const { validateBenchmark, validateRun } = require('../bench/validate');
const { validateBenchmark, validateRun } = require('../bench/validate');
-const { createServiceTokenClient } = require('openvibe-sdk/auth');
function createRegistryApi(ctx) {
function createRegistryApi(ctx) {
const { config, principal, registry } = ctx;
const { config, principal, registry } = ctx;
- const fetchImpl = ctx.fetchImpl || globalThis.fetch;
- const codesTokens = config.oauth.clientSecret ? createServiceTokenClient({
- tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId,
- clientSecret: config.oauth.clientSecret, audience: 'openvibe.codes', scope: 'codes.task.start', fetch: fetchImpl,
- }) : null;
const r = asyncRouter();
const r = asyncRouter();
r.use(express.json({ limit: '16mb' }));
r.use(express.json({ limit: '16mb' }));
@@ -172,34 +166,13 @@ function createRegistryApi(ctx) {
@@ -172,34 +166,13 @@ function createRegistryApi(ctx) {
});
});
// Hand execution to Codes, which owns the person's keys and the benchmark task lifecycle.
// Hand execution to Codes, which owns the person's keys and the benchmark task lifecycle.
+ // Start the benchmark on openvibe.codes as the signed-in person: the launcher checks, estimates and asks for a
+ // confirmation above $1 (server/bench/launch.js); the run form on the benchmark page uses the same one.
r.post('/benchmarks/:id/run', actor('bench.run.publish'), async (req, res) => {
r.post('/benchmarks/:id/run', actor('bench.run.publish'), async (req, res) => {
- const p = req.principal;
- if (p.kind !== 'user') return problem(res, 403, 'bench.person_required', 'Sign in as the person who will run this benchmark.');
- if (!config.codesRunUrl) return problem(res, 503, 'bench.runner_unavailable', 'Codes benchmark runs are not configured.');
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
- const models = Array.isArray(b.models) ? b.models : [];
- const harnesses = Array.isArray(b.harnesses) ? b.harnesses : [];
- const repeat = Number(b.repeat);
- if (!models.length || !harnesses.length || models.length > 20 || harnesses.length > 20 || !Number.isInteger(repeat) || repeat < 1 || repeat > 20) {
- return invalid(res, 'models and harnesses must be non-empty arrays (at most 20 each), and repeat must be 1–20');
- }
- const definition = await registry.getDefinition(String(req.params.id), { reveal: false });
- if (!definition) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
- const costEstimate = (definition.definition.tasks || []).length * models.length * harnesses.length * repeat
- * Number(definition.definition.limits && definition.definition.limits.usd || 0);
- if (costEstimate > 1 && b.confirm !== true) return res.status(409).json({ code: 'bench.confirm_cost', cost_estimate_usd: costEstimate, confirm_required: true });
- try {
- if (!codesTokens) return problem(res, 503, 'bench.runner_unavailable', 'Bench has no Network client credentials for Codes.');
- const codesRunApi = new URL(`/api/v1/benchmarks/${encodeURIComponent(String(req.params.id))}/run`, config.codesRunUrl).toString();
- const response = await fetchImpl(codesRunApi, {
- method: 'POST', headers: { ...(await codesTokens.authHeaders()), 'content-type': 'application/json', 'x-ov-subject': p.subject, 'x-ov-user-token': p.token },
- body: JSON.stringify({ benchmark: String(req.params.id), models, harnesses, repeat }),
- signal: AbortSignal.timeout(15000),
- });
- const body = await response.json().catch(() => ({}));
- if (!response.ok) return problem(res, response.status < 500 ? response.status : 502, body.code || 'bench.runner_failed', body.detail || body.message || 'Codes could not start this benchmark.');
- return res.status(202).json(body);
- } catch (err) { return failed(res, err); }
+ const out = await ctx.launcher.launch({ principal: req.principal, id: req.params.id, models: b.models, harnesses: b.harnesses, repeat: b.repeat, confirm: b.confirm === true });
+ if (out.status === 202 || out.status === 409) return res.status(out.status).json(out.body);
+ return problem(res, out.status, out.body.code, out.body.detail);
});
});
// ── Runs ─────────────────────────────────────────────────
// ── Runs ─────────────────────────────────────────────────
@@ -211,7 +211,7 @@ function benchmarkPage({ benchmark, definition, version, rankings, metric = 'sco
@@ -211,7 +211,7 @@ function benchmarkPage({ benchmark, definition, version, rankings, metric = 'sco
const entries = (rankings && rankings.entries) || [];
const entries = (rankings && rankings.entries) || [];
const base = benchmarkHref(benchmark.id);
const base = benchmarkHref(benchmark.id);
const runYourself = codesRunUrl
const runYourself = codesRunUrl
- ? html`<a class="bm-btn" href="${codesRunUrl}?benchmark=${encodeURIComponent(`${benchmark.id}@${version || benchmark.latest_version}`)}">Run it yourself</a>`
+ ? html`<a class="bm-btn" href="${benchmarkHref(benchmark.id)}/run">Run it yourself</a>`
: html`<a class="bm-btn bm-btn-quiet" href="/docs#br-publish">Publish a run</a>`;
: html`<a class="bm-btn bm-btn-quiet" href="/docs#br-publish">Publish a run</a>`;
return html`<div class="bm-wrap br" id="br-benchmark">
return html`<div class="bm-wrap br" id="br-benchmark">
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>${benchmark.title}</span></nav>
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>${benchmark.title}</span></nav>
@@ -231,6 +231,44 @@ ${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up he
@@ -231,6 +231,44 @@ ${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up he
</div>`;
</div>`;
}
}
+/**
+ * The run form: pick the models, the harnesses and how many times, see what it may cost, start it on openvibe.codes
+ * with your own keys. A plain form (it works without JavaScript); the estimate above $1 asks once more.
+ */
+function runForm({ benchmark, definition, version, signedIn, harnesses = [], suggested = [], picked = null, confirm = null, error = null, loginUrl }) {
+ const d = definition || {};
+ const tasks = Array.isArray(d.tasks) ? d.tasks.length : 0;
+ const usd = Number(d.limits && d.limits.usd) || 0;
+ const matrix = d.matrix || {};
+ const harnessList = harnesses.length ? harnesses.slice() : (matrix.harnesses || []).map((id) => ({ id, label: id }));
+ // Nothing chosen yet: the benchmark's own matrix, else the first harness, so the form starts one tap from ready.
+ const p = picked || { models: matrix.models || [], harnesses: (matrix.harnesses || []).length ? matrix.harnesses : harnessList.slice(0, 1).map((h) => h.id), repeat: 1 };
+ // Models: the matrix, the ones already ranked here (verified runs), and whatever was typed.
+ const modelList = [...new Set([...(matrix.models || []), ...suggested, ...p.models])].slice(0, 16);
+ for (const id of p.harnesses) if (!harnessList.some((h) => h.id === id)) harnessList.push({ id, label: id });
+ const base = benchmarkHref(benchmark.id);
+ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${base}">${benchmark.title}</a><span aria-hidden="true">/</span><span>Run it</span></nav>
+<header class="bm-page-h"><p class="bm-kicker">Run it yourself</p><h1>${benchmark.title}</h1><p class="bm-lede">openvibe.codes runs every task on each model and harness you pick, with your own keys, and publishes each run here as verified, whatever it scores.</p></header>`;
+ if (!signedIn) {
+ return html`<div class="bm-wrap br bm-narrow" id="br-run-form">${head}${empty('Sign in first', 'A run uses your own provider keys on openvibe.codes, so it starts as you.', html`<a class="bm-btn bm-btn-small" href="${loginUrl}">Sign in with OpenVibe</a>`)}</div>`;
+ }
+ const check = (name, value, label, on) => html`<label class="br-check"><input type="checkbox" name="${name}" value="${value}"${on ? raw(' checked') : ''}><span>${label}</span></label>`;
+ return html`<div class="bm-wrap br bm-narrow" id="br-run-form">${head}
+${error ? html`<div class="bm-note is-bad" role="alert">${error}</div>` : ''}
+${confirm === 'unknown' ? html`<div class="bm-note" role="status">This benchmark sets no dollar limit per task, so what it costs depends on the models you pick. It runs on your own keys. Check the choices below, then confirm.</div>` : confirm != null ? html`<div class="bm-note" role="status">This run may cost up to <b>${fmtUsd(confirm)}</b> on your own keys. Check the choices below, then confirm.</div>` : ''}
+<form class="br-form" method="post" action="${base}/run" data-tasks="${tasks}" data-usd="${usd}">
+<fieldset><legend>Models</legend>${modelList.length ? html`<div class="br-checks">${modelList.map((m) => check('models', m, m, p.models.includes(m)))}</div>` : ''}
+<label class="br-field"><span>${modelList.length ? 'More models, separated by commas' : 'Models, separated by commas'}</span><input name="more_models" type="text" autocomplete="off" placeholder="deepseek-v4.1-flash, kimi-k3"></label></fieldset>
+<fieldset><legend>Harnesses</legend><div class="br-checks">${harnessList.map((h) => check('harnesses', h.id, h.label, p.harnesses.includes(h.id)))}</div></fieldset>
+<label class="br-field"><span>Runs of each</span><select name="repeat">${[1, 2, 3, 5].map((n) => html`<option value="${n}"${n === Number(p.repeat) ? raw(' selected') : ''}>${n === 1 ? 'Once' : `${n} times (shows how steady it is)`}</option>`)}</select></label>
+<p class="br-fine" data-estimate>${usd > 0 ? `${tasks} ${tasks === 1 ? 'task' : 'tasks'} × your picks × runs, at up to ${fmtUsd(usd)} a task. Over $1 asks you once more.` : `${tasks} ${tasks === 1 ? 'task' : 'tasks'} × your picks × runs. This benchmark sets no dollar limit per task, so you confirm before it starts.`}</p>
+${confirm != null ? html`<input type="hidden" name="confirm" value="1">` : ''}
+<div class="bm-actions"><button class="bm-btn" type="submit">${confirm != null ? 'Confirm and start' : 'Start on openvibe.codes'}</button><a class="bm-btn bm-btn-quiet" href="${base}">Back</a></div>
+</form>
+<p class="br-fine">Version v${version || benchmark.latest_version || 0}, frozen: these runs can be compared with every other run of it.</p>
+</div>`;
+}
+
function runPage({ run, benchmark, definition, pct }) {
function runPage({ run, benchmark, definition, pct }) {
const tasksDef = new Map(((definition && definition.tasks) || []).map((t) => [t.id, t]));
const tasksDef = new Map(((definition && definition.tasks) || []).map((t) => [t.id, t]));
const env = [
const env = [
@@ -296,6 +334,6 @@ ${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 i
@@ -296,6 +334,6 @@ ${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 i
}
}
module.exports = {
module.exports = {
- benchmarksIndex, benchmarkPage, runPage, comparePage, modelPage, valueChart, METRICS,
+ benchmarksIndex, benchmarkPage, runForm, runPage, comparePage, modelPage, valueChart, METRICS,
fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,
fmtScore, fmtUsd, fmtMs, fmtTokens, ordinal, benchmarkCard, render,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,134 @@
@@ -0,0 +1,134 @@
+'use strict';
+
+/**
+ * Starting a benchmark on openvibe.codes (stage B3): the API and the run form share one launcher.
+ *
+ * Under test: a signed-out visitor is sent to sign in; the form lists Codes' harnesses; an estimate above $1 asks
+ * for a confirmation before anything reaches Codes; a confirmed run reaches Codes once, as the person, with a service
+ * token, and the browser lands on the person's task list; a cross-site form post is refused; the API answers the same
+ * way (409 with the estimate, then 202).
+ */
+const assert = require('assert');
+const http = require('http');
+const { boot, check, done } = require('./helpers/boot');
+
+function fakeCodes() {
+ const calls = [];
+ const server = http.createServer((req, res) => {
+ let body = '';
+ req.on('data', (c) => { body += c; });
+ req.on('end', () => {
+ calls.push({ method: req.method, url: req.url, headers: req.headers, body: body ? JSON.parse(body) : null });
+ res.setHeader('content-type', 'application/json');
+ if (req.url === '/api/v1/harnesses') return res.end(JSON.stringify({ harnesses: [{ id: 'claude-code', label: 'Claude Code' }, { id: 'codex', label: 'Codex' }] }));
+ if (/^\/api\/v1\/benchmarks\/[^/]+\/run$/.test(req.url) && req.method === 'POST') {
+ res.statusCode = 202;
+ return res.end(JSON.stringify({ session: 'ses_1', tasks: [{ id: 'tsk_1' }], task: { id: 'tsk_1' }, total: 1 }));
+ }
+ res.statusCode = 404; res.end('{}');
+ });
+ });
+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve({ url: `http://127.0.0.1:${server.address().port}`, calls, close: () => server.close() })));
+}
+
+(async () => {
+ const codes = await fakeCodes();
+ const t = await boot({ env: { CODES_BENCH_RUN_URL: codes.url } });
+ const alice = t.network.addUser('alice');
+ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });
+ const created = await t.get('/api/v1/benchmarks', { bearer: publisher, json: {
+ title: 'Run form arena', description: 'For the run form tests.', kind: 'games.play', tags: [], visibility: 'public',
+ tasks: [{ id: 't1', prompt: 'Make a game.', files: [], expected: null, holdout: false }, { id: 't2', prompt: 'Make another.', files: [], expected: null, holdout: false }],
+ judges: [{ type: 'game_outcome', version: '1.0', config: {} }], scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: { ms: 60000, tokens: 100000, usd: 0.5 }, matrix: { models: ['model-a'], harnesses: ['codex'] },
+ } });
+ assert.strictEqual(created.status, 201, created.text);
+ const id = created.json().benchmark.id;
+ assert.strictEqual((await t.get(`/api/v1/benchmarks/${id}/versions`, { bearer: publisher, json: {} })).status, 201);
+ const form = (fields) => new URLSearchParams(fields).toString();
+ const post = (fields, o = {}) => t.get(`/benchmarks/${id}/run`, { as: alice, raw: form(fields), headers: { 'content-type': 'application/x-www-form-urlencoded' }, ...o });
+ const runCalls = () => codes.calls.filter((c) => c.method === 'POST');
+
+ await check('the benchmark page links to its run form', async () => {
+ const res = await t.get(`/benchmarks/${id}`);
+ assert.ok(res.text.includes(`href="/benchmarks/${id}/run">Run it yourself`), 'the button opens the form, not a bare Codes link');
+ });
+
+ await check('signed out, the form asks you to sign in', async () => {
+ const res = await t.get(`/benchmarks/${id}/run`);
+ assert.strictEqual(res.status, 200, res.text.slice(0, 200));
+ assert.ok(res.text.includes('Sign in first'));
+ assert.ok(!res.text.includes('<form class="br-form"'), 'no form for a stranger');
+ });
+
+ await check('signed in, the form lists the benchmark matrix and Codes harnesses', async () => {
+ const res = await t.get(`/benchmarks/${id}/run`, { as: alice });
+ assert.ok(res.text.includes('<form class="br-form"'));
+ assert.ok(res.text.includes('value="model-a" checked'), 'the matrix model is picked');
+ assert.ok(res.text.includes('value="claude-code"') && res.text.includes('Codex'), "Codes' harness list");
+ });
+
+ await check('an estimate above $1 asks once more and reaches nobody', async () => {
+ const before = runCalls().length;
+ const res = await post([['models', 'model-a'], ['more_models', 'model-b'], ['harnesses', 'codex'], ['repeat', '1']]);
+ assert.strictEqual(res.status, 200, res.text.slice(0, 200));
+ assert.ok(res.text.includes('may cost up to') && res.text.includes('$2.00'), '2 tasks × 2 models × $0.50');
+ assert.ok(res.text.includes('name="confirm" value="1"') && res.text.includes('Confirm and start'));
+ assert.strictEqual(runCalls().length, before, 'Codes was not called');
+ });
+
+ await check('a confirmed run reaches Codes once, as the person, and lands on their tasks', async () => {
+ const before = runCalls().length;
+ const res = await post([['models', 'model-a'], ['more_models', 'model-b'], ['harnesses', 'codex'], ['repeat', '1'], ['confirm', '1']]);
+ assert.strictEqual(res.status, 303, res.text.slice(0, 200));
+ assert.strictEqual(res.headers.get('location'), `${codes.url}/tasks`);
+ const calls = runCalls().slice(before);
+ assert.strictEqual(calls.length, 1);
+ assert.deepStrictEqual(calls[0].body.models, ['model-a', 'model-b']);
+ assert.match(String(calls[0].headers.authorization), /^Bearer /, "Bench's own service token");
+ assert.ok(calls[0].headers['x-ov-subject'], 'names the person');
+ });
+
+ await check('nothing picked is said plainly', async () => {
+ const res = await post([['repeat', '1']]);
+ assert.strictEqual(res.status, 400);
+ assert.ok(res.text.includes('non-empty lists'));
+ });
+
+ await check('a cross-site form post is refused', async () => {
+ const res = await post([['models', 'model-a'], ['harnesses', 'codex'], ['repeat', '1']], { origin: 'https://evil.example' });
+ assert.strictEqual(res.status, 403);
+ });
+
+ await check('the API answers the same way: 409 with the estimate, then 202', async () => {
+ const body = { models: ['model-a', 'model-b'], harnesses: ['codex'], repeat: 1 };
+ const ask = await t.get(`/api/v1/benchmarks/${id}/run`, { as: alice, json: body });
+ assert.strictEqual(ask.status, 409, ask.text);
+ assert.strictEqual(ask.json().cost_estimate_usd, 2);
+ const go = await t.get(`/api/v1/benchmarks/${id}/run`, { as: alice, json: { ...body, confirm: true } });
+ assert.strictEqual(go.status, 202, go.text);
+ assert.strictEqual(go.json().total, 1);
+ });
+
+ await check('a benchmark with no dollar limit always asks before it starts', async () => {
+ const res2 = await t.get('/api/v1/benchmarks', { bearer: publisher, json: {
+ title: 'No limit arena', description: 'No usd limit.', kind: 'games.play', tags: [], visibility: 'public',
+ tasks: [{ id: 't1', prompt: 'Make a game.', files: [], expected: null, holdout: false }],
+ judges: [{ type: 'game_outcome', version: '1.0', config: {} }], scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: { ms: 60000, tokens: 100000 }, matrix: { models: [], harnesses: [] },
+ } });
+ assert.strictEqual(res2.status, 201, res2.text);
+ const nid = res2.json().benchmark.id;
+ assert.strictEqual((await t.get(`/api/v1/benchmarks/${nid}/versions`, { bearer: publisher, json: {} })).status, 201);
+ const before = runCalls().length;
+ const ask = await t.get(`/api/v1/benchmarks/${nid}/run`, { as: alice, json: { models: ['m'], harnesses: ['codex'], repeat: 1 } });
+ assert.strictEqual(ask.status, 409, ask.text);
+ assert.strictEqual(ask.json().cost_estimate_usd, null, 'no estimate is invented');
+ const page = await t.get(`/benchmarks/${nid}/run`, { as: alice, raw: form([['more_models', 'm'], ['harnesses', 'codex'], ['repeat', '1']]), headers: { 'content-type': 'application/x-www-form-urlencoded' } });
+ assert.ok(page.text.includes('sets no dollar limit per task'), 'the form says why it asks');
+ assert.strictEqual(runCalls().length, before, 'and nothing reached Codes');
+ });
+
+ await check('stop', async () => { await t.close(); codes.close(); });
+ done();
+})();