openvibers/openvibe.bench Commits
Bench B1: speak the published contracts (openvibe-contracts v0.244.0) (#25)
Changes
@@@ -13,14 +13,17 @@ capabilities. A session write must come from bench.openvibe.services itself.| `POST /api/v1/benchmarks/:id/versions` | the same | Freeze the working copy into the next immutable version. || `POST /api/v1/benchmarks/:id/fork` | the same | Fork a benchmark; the fork records its parent. || `GET /api/v1/benchmarks?kind=&tag=&q=` | anyone | The registry. |--| `GET /api/v1/benchmarks/:id[@version]` | anyone | One benchmark and its definition. Holdout tasks are withheld unless the caller holds `bench.run.publish`. |++| `GET /api/v1/benchmarks/:id[@version]` | anyone | One benchmark and its definition. Holdout tasks are withheld unless the caller holds `bench.run.publish` or owns the benchmark. |++++The documents are the published contracts, from `openvibe-contracts` v0.244.0: `bench.benchmark@1` for a++benchmark version, `bench.run@1` for a run. Every document is checked against them before it is stored.## The results| Route | Who | What ||---|---|---|| `POST /api/v1/runs` | a person, or `bench.run.publish` | Publish a run. A service token with `bench.run.publish` makes a **verified** run; a person's token makes a **submitted** one, badged and out of the default rankings. |--| `GET /api/v1/runs/:id` | anyone | One run with its per-task results. |++| `GET /api/v1/runs/:id` | anyone | One run with its per-task results. `run.document` is the published `bench.run@1` document. || `GET /api/v1/compare?runs=a,b,c` | anyone | Up to four runs side by side, per task. || `GET /api/v1/rankings?benchmark=&metric=score\|value\|speed\|consistency&harness=&since=` | anyone | The rankings for a benchmark. Verified runs only, unless `include_submitted=true`. || `GET /api/v1/percentile?benchmark=&model=&score=` | anyone | Where a score sits among that model's runs. |@@@ -56,6 +56,38 @@ published set, fork it — the fork records the parent, and the old version stay}```++## The contracts++++Bench speaks the published shapes, from `openvibe-contracts` **v0.244.0**:++++| Contract | What it is |++|---|---|++| `bench.benchmark@1` | the document a frozen version holds, exactly |++| `bench.run@1` | the document a run is, exactly |++++The service reads both from the package — there is no second copy — and checks every document against them before it++is stored, so a run that is not a `bench.run@1` run is refused with `bench.invalid` and a field-level error. A run++that is read back carries its contract document under `document`:++++```json++{++ "run": {++ "id": "brn_…",++ "provenance": "verified",++ "document": { "id": "brn_…", "benchmark": "bmk_…@1", "provenance": "verified", "…": "…" }++ }++}++```++++The capabilities the registry uses are published too: `bench.benchmark.write`, `bench.run.publish` and++`bench.run.read`.++++## The holdout++++A task with `holdout: true` keeps its prompt from everyone but two callers: the executor (a service token with++`bench.run.publish`) and the benchmark's owner. Everybody else gets the task listed with its prompt withheld, so the++set cannot be tuned to.++## Running itA run (the contract is `bench.run@1`) names the benchmark `id@version`, the model, the route, the provider, the@@@ -15,7 +15,7 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0",@@@ -1151,9 +1151,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.238.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",-- "integrity": "sha512-BQOsqor9ew75s2ElWK6g6pWk0gr5nbwzgfzMKJPBPIjnDnIkUQfNmsYw8AeexW5qeotCmfTu4lSPdqy6ZQaq5g==",++ "version": "0.244.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0",++ "integrity": "sha512-CAPdldZsfoetBHr/dso4n4W8tilqD1qo7qHdgkQstQUdlemeA6ed3EX4cy9jEEC/LTnt0UndwLUNfCCr8Y3LMQ==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -22,7 +22,7 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0",@@@ -15,6 +15,7 @@*/const crypto = require('crypto');const { ids } = require('openvibe-contracts');++const { contractRun, checkContract } = require('./validate');const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));@@@ -106,6 +107,24 @@ function wireRun(row, tasks = null) {created_at: iso(row.created_at),};if (tasks) out.tasks = tasks;++ // The published bench.run@1 document: what a caller reads back is what the contract says.++ out.document = contractRun({++ id: out.id,++ benchmark: out.benchmark,++ model: out.model,++ route: out.route,++ provider: out.provider,++ harness: out.harness,++ settings: out.settings,++ executor: out.executor,++ hardware: out.hardware,++ tasks: tasks || [],++ score: out.score,++ cost_usd: out.cost_usd,++ ms: out.ms,++ tokens: out.tokens,++ provenance: out.provenance,++ }, out.created_at);return out;}@@@ -344,6 +363,23 @@ function createRegistry(s, { log = console } = {}) {const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };const id = s.newId('brn');++ // The published contract has the last word: a run that is not a bench.run@1 document is refused.++ const contract = contractRun({++ ...doc,++ id,++ benchmark: `${benchmarkId}@${version}`,++ executor,++ tasks,++ score: num(doc.score, 0),++ cost_usd: num(doc.cost_usd, 0),++ ms: num(doc.ms, 0),++ tokens: num(doc.tokens, 0),++ provenance,++ }, s.iso());++ const contractCheck = checkContract('bench.run@1', contract);++ if (!contractCheck.valid) {++ return refusal(400, 'bench.invalid', contractCheck.errors.map((e) => e.message).join('; '));++ }const score = num(doc.score, 0);const cost = num(doc.cost_usd, 0);const ms = num(doc.ms, 0);@@@ -10,9 +10,12 @@* The rules mirror the contract's: the same required fields, the same enums, the same ranges.*/++const { assertValid } = require('openvibe-contracts');++const isStr = (v) => typeof v === 'string';const isNum = (v) => typeof v === 'number' && Number.isFinite(v);const isObj = (v) => v && typeof v === 'object' && !Array.isArray(v);++const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));/** The contract's id patterns. */const BENCHMARK_ID = /^bmk_[0-9A-HJKMNP-TV-Z]{26}$/;@@@ -102,4 +105,62 @@ function validateRun(doc) {return { valid: e.valid, errors: e.list };}--module.exports = { validateBenchmark, validateRun, Errors, BENCHMARK_ID, RUN_ID, BENCHMARK_REF, VISIBILITIES, EXECUTOR_KINDS, PROVENANCES };++++/**++ * The bench.run@1 document for a run Bench is about to store: exactly the published shape, nothing else. The extra++ * fields Bench keeps for itself (the benchmark id and version, the session) stay in the database, not in the++ * contract, so what a caller reads back is what the contract says.++ */++function contractRun(doc, createdAt) {++ const harness = (doc.harness && typeof doc.harness === 'object') ? doc.harness : {};++ const settings = (doc.settings && typeof doc.settings === 'object') ? doc.settings : {};++ const executor = (doc.executor && typeof doc.executor === 'object') ? doc.executor : {};++ const tasks = (Array.isArray(doc.tasks) ? doc.tasks : []).map((t) => {++ const out = {++ task_id: String(t.task_id || ''),++ score: num(t.score),++ cost_usd: num(t.cost_usd),++ ms: Math.round(num(t.ms)),++ tokens: Math.round(num(t.tokens)),++ artifacts: (Array.isArray(t.artifacts) ? t.artifacts : [])++ .map((a) => ({ kind: String(a.kind || 'file'), url: String(a.url || '') }))++ .filter((a) => a.url),++ };++ if (t.transcript_url) out.transcript_url = String(t.transcript_url);++ return out;++ });++ const out = {++ id: String(doc.id || ''),++ benchmark: String(doc.benchmark || ''),++ model: String(doc.model || ''),++ route: (doc.route && typeof doc.route === 'object') ? doc.route : {},++ provider: String(doc.provider || ''),++ harness: { name: String(harness.name || ''), version: String(harness.version || '') },++ settings: {++ temperature: settings.temperature == null ? null : Number(settings.temperature),++ effort: settings.effort == null ? null : String(settings.effort),++ },++ executor: { kind: String(executor.kind || ''), id: String(executor.id || '') },++ tasks,++ score: num(doc.score),++ cost_usd: num(doc.cost_usd),++ ms: Math.round(num(doc.ms)),++ tokens: Math.round(num(doc.tokens)),++ provenance: String(doc.provenance || ''),++ created_at: createdAt || new Date().toISOString(),++ };++ if (doc.hardware != null) out.hardware = doc.hardware;++ return out;++}++++/** The published contract has the last word: a document that does not match it is refused. */++function checkContract(name, doc) {++ try {++ assertValid(name, doc);++ return { valid: true, errors: [] };++ } catch (err) {++ return { valid: false, errors: [{ path: '/', message: (err && err.message) || `does not match ${name}` }] };++ }++}++++module.exports = { validateBenchmark, validateRun, contractRun, checkContract, Errors, BENCHMARK_ID, RUN_ID, BENCHMARK_REF, VISIBILITIES, EXECUTOR_KINDS, PROVENANCES };@@@ -1,11 +1,32 @@'use strict';/**-- * Bench's Contracts surface, kept as JSON in the service until the published openvibe-contracts package carries-- * the bench.* schemas and capabilities. These are the exact shapes the API, the SDK and the outbox publish, so a-- * later move into openvibe-contracts is a copy, not a redesign.++ * Bench's Contracts surface.++ *++ * openvibe-contracts v0.244.0 carries the bench.* schemas and the registry capabilities, so the service reads them++ * from the package: the published shapes are the only ones, and a pin bump is the only way they move. The list of++ * names below is checked at load time, so a wrong pin fails at boot with a clear message instead of answering with++ * a shape nobody published.++ *++ * The nine capabilities Bench has always had stay listed (the showcase kinds, rounds, entries, judgements and the++ * board), and the three the registry added join them: bench.benchmark.write, bench.run.publish and bench.run.read.*/++const { schema, capabilities } = require('openvibe-contracts');++++/** Every bench.* schema the published package must carry. */++const PUBLISHED_SCHEMAS = Object.freeze([++ 'bench.kind@1',++ 'bench.task@1',++ 'bench.round@1',++ 'bench.entry@1',++ 'bench.judgement@1',++ 'bench.rating@1',++ 'bench.benchmark@1',++ 'bench.run@1',++]);++++/** The capabilities Bench's own tokens carry. */const BENCH_CAPABILITIES = Object.freeze(['bench.kind.read','bench.task.create',@@@ -27,106 +48,36 @@ const BENCH_EVENTS = Object.freeze(['bench.rating.changed',]);--const BENCH_SCHEMAS = Object.freeze({-- 'bench.kind@1': {-- type: 'object',-- required: ['id', 'room', 'entry', 'judge', 'name', 'schema', 'metrics', 'created_at'],-- properties: {-- id: { type: 'string' },-- room: { type: 'string' },-- entry: { type: 'string' },-- judge: { type: 'string' },-- name: { type: 'string' },-- schema: { type: 'object' },-- metrics: { type: 'array', items: { type: 'string' } },-- created_at: { type: 'string', format: 'date-time' },-- },-- },-- 'bench.task@1': {-- type: 'object',-- required: ['id', 'kind_id', 'prompt', 'public', 'created_by', 'created_at'],-- properties: {-- id: { type: 'string' },-- kind_id: { type: 'string' },-- prompt: { type: 'string' },-- task_class: { type: ['string', 'null'] },-- public: { type: 'boolean' },-- created_by: { type: 'string' },-- created_at: { type: 'string', format: 'date-time' },-- },-- },-- 'bench.round@1': {-- type: 'object',-- required: ['id', 'task_id', 'kind_id', 'offers', 'status', 'seed', 'created_by', 'created_at'],-- properties: {-- id: { type: 'string' },-- task_id: { type: 'string' },-- kind_id: { type: 'string' },-- offers: { type: 'array', items: { type: 'object' } },-- status: { type: 'string', enum: ['queued', 'running', 'ready', 'finished', 'failed'] },-- seed: { type: 'integer' },-- pool_fallback: { type: 'boolean' },-- receipt: { type: ['string', 'null'] },-- created_by: { type: 'string' },-- created_at: { type: 'string', format: 'date-time' },-- },-- },-- 'bench.entry@1': {-- type: 'object',-- required: ['id', 'round_id', 'provider', 'model', 'route', 'payload', 'cost_usd', 'ms', 'tokens'],-- properties: {-- id: { type: 'string' },-- round_id: { type: 'string' },-- provider: { type: 'string' },-- model: { type: 'string' },-- route: { type: 'object' },-- payload: { type: 'object' },-- cost_usd: { type: 'number' },-- ms: { type: 'number' },-- tokens: { type: 'number' },-- error: { type: ['string', 'null'] },-- ui_check: { type: ['object', 'null'] },-- run_id: { type: ['string', 'null'] },-- },-- },-- 'bench.judgement@1': {-- type: 'object',-- required: ['id', 'round_id', 'kind_id', 'judge', 'method', 'pair', 'outcome', 'score', 'at'],-- properties: {-- id: { type: 'string' },-- round_id: { type: 'string' },-- kind_id: { type: 'string' },-- judge: { type: 'string' },-- judge_name: { type: ['string', 'null'] },-- judge_version: { type: ['string', 'null'] },-- method: { type: 'string' },-- pair: { type: ['object', 'null'] },-- outcome: { type: ['string', 'null'] },-- score: { type: ['number', 'null'] },-- vote_counts: { type: ['object', 'null'] },-- at: { type: 'string', format: 'date-time' },-- },-- },-- 'bench.rating@1': {-- type: 'object',-- required: ['kind_id', 'category', 'subject', 'elo', 'games', 'wins', 'losses', 'ties', 'win_rate', 'ci_lower', 'ci_upper'],-- properties: {-- kind_id: { type: 'string' },-- category: { type: 'string' },-- subject: { type: 'string' },-- elo: { type: 'number' },-- games: { type: 'number' },-- wins: { type: 'number' },-- losses: { type: 'number' },-- ties: { type: 'number' },-- win_rate: { type: 'number' },-- ci_lower: { type: 'number' },-- ci_upper: { type: 'number' },-- median_cost_usd: { type: ['number', 'null'] },-- median_ms: { type: ['number', 'null'] },-- provisional: { type: 'boolean' },-- },-- },++function published(name) {++ try {++ return schema(name) || null;++ } catch {++ return null;++ }++}++++const BENCH_SCHEMAS = Object.freeze(PUBLISHED_SCHEMAS.reduce((acc, name) => {++ const s = published(name);++ if (s) acc[name] = s;++ return acc;++}, {}));++++const MISSING = PUBLISHED_SCHEMAS.filter((name) => !BENCH_SCHEMAS[name]);++if (MISSING.length) {++ throw new Error(`openvibe-contracts does not carry ${MISSING.join(', ')}. Pin the release that publishes them.`);++}++++// Every capability Bench hands out must be one the published package knows, so a token is never granted a name no++// service can rely on.++const UNKNOWN_CAPABILITIES = BENCH_CAPABILITIES.filter((cap) => {++ try {++ return !capabilities.get(cap);++ } catch {++ return true;++ }});++if (UNKNOWN_CAPABILITIES.length) {++ throw new Error(`openvibe-contracts does not publish ${UNKNOWN_CAPABILITIES.join(', ')}.`);++}--module.exports = { BENCH_CAPABILITIES, BENCH_EVENTS, BENCH_SCHEMAS };++module.exports = { BENCH_CAPABILITIES, BENCH_EVENTS, BENCH_SCHEMAS, PUBLISHED_SCHEMAS };@@@ -143,7 +143,13 @@ function createRegistryApi(ctx) {r.get('/benchmarks/:id', async (req, res) => {try {const ref = String(req.params.id || '');-- const out = await registry.getDefinition(ref, { reveal: mayReveal(req) });++ // The holdout goes to the executor and to the benchmark's owner; to nobody else.++ let reveal = mayReveal(req);++ if (!reveal && req.principal.kind === 'user') {++ const owner = await registry.getBenchmark(ref.split('@')[0]);++ reveal = !!(owner && owner.owner === req.principal.subject);++ }++ const out = await registry.getDefinition(ref, { reveal });if (!out) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');res.set('Cache-Control', 'public, max-age=60');return res.json({nnew file mode 100644@@@ -0,0 +1,123 @@++'use strict';++++/**++ * The contracts pin (spec 2026-10-30, stage B1).++ *++ * Bench speaks the published shapes: openvibe-contracts v0.244.0 carries bench.benchmark@1 and bench.run@1 and the++ * three registry capabilities. These tests prove the pin is real — the package carries the shapes, Bench's own++ * capability list names only capabilities the package publishes, a frozen benchmark and a published run are the++ * published documents, and the holdout goes to the executor and to the owner and to nobody else.++ */++const assert = require('assert');++const contracts = require('openvibe-contracts');++const { boot, check, done } = require('./helpers/boot');++const { BENCH_CAPABILITIES, BENCH_SCHEMAS } = require('../server/contracts');++++const TASKS = [++ { id: 't1', prompt: 'Build a one-screen arcade game with a paddle, bricks, a ball and a score.', files: [], expected: null, holdout: false },++ { id: 't2', prompt: 'Build a snake game on a 20x20 grid.', files: [], expected: null, holdout: true },++];++++function definition() {++ return {++ title: 'The pinned set',++ description: 'A benchmark that must match bench.benchmark@1.',++ kind: 'games.play',++ tags: ['games'],++ visibility: 'public',++ tasks: TASKS,++ judges: [{ type: 'human_pairwise', version: '1.0', config: {} }],++ scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: { ms: 120000, tokens: 200000, usd: 0.5 },++ matrix: { models: ['claude-opus-4'], harnesses: ['claude-code'] },++ };++}++++function runDoc(benchmark, over = {}) {++ return {++ benchmark,++ model: 'claude-opus-4',++ route: { provider: 'anthropic' },++ provider: 'anthropic',++ harness: { name: 'claude-code', version: '1.2.0' },++ settings: { temperature: 1, effort: 'high' },++ executor: { kind: 'codes', id: 'task_1' },++ tasks: [{ task_id: 't1', score: 90, cost_usd: 0.02, ms: 40000, tokens: 8000, artifacts: [] }],++ score: 90,++ cost_usd: 0.02,++ ms: 40000,++ tokens: 8000,++ ...over,++ };++}++++(async () => {++ const t = await boot();++ const alice = t.network.addUser('alice');++ const owner = t.network.addUser('owner');++ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });++ const writer = t.network.serviceToken({ sub: 'svc:writer', audience: 'openvibe.bench', cap: ['bench.benchmark.write'] });++++ let benchmarkId;++ let version;++++ await check('the pinned package carries the registry schemas and capabilities', async () => {++ assert.ok(contracts.schema('bench.benchmark@1'), 'bench.benchmark@1 is published');++ assert.ok(contracts.schema('bench.run@1'), 'bench.run@1 is published');++ assert.ok(BENCH_SCHEMAS['bench.benchmark@1'], 'Bench reads it from the package');++ assert.ok(BENCH_SCHEMAS['bench.run@1'], 'Bench reads it from the package');++ for (const cap of ['bench.benchmark.write', 'bench.run.publish', 'bench.run.read']) {++ assert.ok(BENCH_CAPABILITIES.includes(cap), `Bench grants ${cap}`);++ assert.ok(contracts.capabilities.get(cap), `the package publishes ${cap}`);++ }++ });++++ await check('a frozen benchmark is a bench.benchmark@1 document', async () => {++ const created = await t.get('/api/v1/benchmarks', { bearer: writer, json: definition() });++ assert.strictEqual(created.status, 201, created.text);++ benchmarkId = created.json().benchmark.id;++ const frozen = await t.get(`/api/v1/benchmarks/${benchmarkId}/versions`, { bearer: writer, json: {} });++ assert.strictEqual(frozen.status, 201, frozen.text);++ version = frozen.json().version.version;++ contracts.assertValid('bench.benchmark@1', frozen.json().version.definition);++ });++++ await check('a published run is a bench.run@1 document', async () => {++ const published = await t.get('/api/v1/runs', { bearer: publisher, json: runDoc(`${benchmarkId}@${version}`) });++ assert.strictEqual(published.status, 201, published.text);++ contracts.assertValid('bench.run@1', published.json().run.document);++ assert.strictEqual(published.json().run.document.provenance, 'verified');++ const read = await t.get(`/api/v1/runs/${published.json().run.id}`);++ assert.strictEqual(read.status, 200, read.text);++ contracts.assertValid('bench.run@1', read.json().run.document);++ });++++ await check('a run that is not the published shape is refused', async () => {++ const bad = await t.get('/api/v1/runs', { bearer: publisher, json: runDoc(`${benchmarkId}@${version}`, { settings: { temperature: 9 } }) });++ assert.strictEqual(bad.status, 400, bad.text);++ assert.strictEqual(bad.json().code, 'bench.invalid');++ });++++ await check('the owner sees the holdout and a stranger does not', async () => {++ const made = await t.get('/api/v1/benchmarks', { bearer: t.network.userToken(owner), json: definition() });++ assert.strictEqual(made.status, 201, made.text);++ const id = made.json().benchmark.id;++ const frozen = await t.get(`/api/v1/benchmarks/${id}/versions`, { bearer: t.network.userToken(owner), json: {} });++ assert.strictEqual(frozen.status, 201, frozen.text);++++ const mine = await t.get(`/api/v1/benchmarks/${id}`, { bearer: t.network.userToken(owner) });++ const myTask = mine.json().definition.tasks.find((x) => x.id === 't2');++ assert.ok(myTask.prompt, 'the owner sees the holdout prompt');++++ const theirs = await t.get(`/api/v1/benchmarks/${id}`, { bearer: t.network.userToken(alice) });++ const theirTask = theirs.json().definition.tasks.find((x) => x.id === 't2');++ assert.strictEqual(theirTask.prompt, null, 'a stranger does not');++++ const public_ = await t.get(`/api/v1/benchmarks/${id}`);++ assert.strictEqual(public_.json().definition.tasks.find((x) => x.id === 't2').prompt, null, 'nobody does');++ });++++ await check('stop', async () => { await t.close(); });++++ done();++})();
@@ -13,14 +13,17 @@ capabilities. A session write must come from bench.openvibe.services itself.
@@ -13,14 +13,17 @@ capabilities. A session write must come from bench.openvibe.services itself.
| `POST /api/v1/benchmarks/:id/versions` | the same | Freeze the working copy into the next immutable version. |
| `POST /api/v1/benchmarks/:id/versions` | the same | Freeze the working copy into the next immutable version. |
| `POST /api/v1/benchmarks/:id/fork` | the same | Fork a benchmark; the fork records its parent. |
| `POST /api/v1/benchmarks/:id/fork` | the same | Fork a benchmark; the fork records its parent. |
| `GET /api/v1/benchmarks?kind=&tag=&q=` | anyone | The registry. |
| `GET /api/v1/benchmarks?kind=&tag=&q=` | anyone | The registry. |
-| `GET /api/v1/benchmarks/:id[@version]` | anyone | One benchmark and its definition. Holdout tasks are withheld unless the caller holds `bench.run.publish`. |
+| `GET /api/v1/benchmarks/:id[@version]` | anyone | One benchmark and its definition. Holdout tasks are withheld unless the caller holds `bench.run.publish` or owns the benchmark. |
+
+The documents are the published contracts, from `openvibe-contracts` v0.244.0: `bench.benchmark@1` for a
+benchmark version, `bench.run@1` for a run. Every document is checked against them before it is stored.
## The results
## The results
| Route | Who | What |
| Route | Who | What |
|---|---|---|
|---|---|---|
| `POST /api/v1/runs` | a person, or `bench.run.publish` | Publish a run. A service token with `bench.run.publish` makes a **verified** run; a person's token makes a **submitted** one, badged and out of the default rankings. |
| `POST /api/v1/runs` | a person, or `bench.run.publish` | Publish a run. A service token with `bench.run.publish` makes a **verified** run; a person's token makes a **submitted** one, badged and out of the default rankings. |
-| `GET /api/v1/runs/:id` | anyone | One run with its per-task results. |
+| `GET /api/v1/runs/:id` | anyone | One run with its per-task results. `run.document` is the published `bench.run@1` document. |
| `GET /api/v1/compare?runs=a,b,c` | anyone | Up to four runs side by side, per task. |
| `GET /api/v1/compare?runs=a,b,c` | anyone | Up to four runs side by side, per task. |
| `GET /api/v1/rankings?benchmark=&metric=score\|value\|speed\|consistency&harness=&since=` | anyone | The rankings for a benchmark. Verified runs only, unless `include_submitted=true`. |
| `GET /api/v1/rankings?benchmark=&metric=score\|value\|speed\|consistency&harness=&since=` | anyone | The rankings for a benchmark. Verified runs only, unless `include_submitted=true`. |
| `GET /api/v1/percentile?benchmark=&model=&score=` | anyone | Where a score sits among that model's runs. |
| `GET /api/v1/percentile?benchmark=&model=&score=` | anyone | Where a score sits among that model's runs. |
@@ -56,6 +56,38 @@ published set, fork it — the fork records the parent, and the old version stay
@@ -56,6 +56,38 @@ published set, fork it — the fork records the parent, and the old version stay
}
}
```
```
+## The contracts
+
+Bench speaks the published shapes, from `openvibe-contracts` **v0.244.0**:
+
+| Contract | What it is |
+|---|---|
+| `bench.benchmark@1` | the document a frozen version holds, exactly |
+| `bench.run@1` | the document a run is, exactly |
+
+The service reads both from the package — there is no second copy — and checks every document against them before it
+is stored, so a run that is not a `bench.run@1` run is refused with `bench.invalid` and a field-level error. A run
+that is read back carries its contract document under `document`:
+
+```json
+{
+ "run": {
+ "id": "brn_…",
+ "provenance": "verified",
+ "document": { "id": "brn_…", "benchmark": "bmk_…@1", "provenance": "verified", "…": "…" }
+ }
+}
+```
+
+The capabilities the registry uses are published too: `bench.benchmark.write`, `bench.run.publish` and
+`bench.run.read`.
+
+## The holdout
+
+A task with `holdout: true` keeps its prompt from everyone but two callers: the executor (a service token with
+`bench.run.publish`) and the benchmark's owner. Everybody else gets the task listed with its prompt withheld, so the
+set cannot be tuned to.
+
## Running it
## Running it
A run (the contract is `bench.run@1`) names the benchmark `id@version`, the model, the route, the provider, the
A run (the contract is `bench.run@1`) names the benchmark `id@version`, the model, the route, the provider, the
@@ -15,7 +15,7 @@
@@ -15,7 +15,7 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -1151,9 +1151,9 @@
@@ -1151,9 +1151,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.238.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",
- "integrity": "sha512-BQOsqor9ew75s2ElWK6g6pWk0gr5nbwzgfzMKJPBPIjnDnIkUQfNmsYw8AeexW5qeotCmfTu4lSPdqy6ZQaq5g==",
+ "version": "0.244.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0",
+ "integrity": "sha512-CAPdldZsfoetBHr/dso4n4W8tilqD1qo7qHdgkQstQUdlemeA6ed3EX4cy9jEEC/LTnt0UndwLUNfCCr8Y3LMQ==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -22,7 +22,7 @@
@@ -22,7 +22,7 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.238.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.244.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -15,6 +15,7 @@
@@ -15,6 +15,7 @@
*/
*/
const crypto = require('crypto');
const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { ids } = require('openvibe-contracts');
+const { contractRun, checkContract } = require('./validate');
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
@@ -106,6 +107,24 @@ function wireRun(row, tasks = null) {
@@ -106,6 +107,24 @@ function wireRun(row, tasks = null) {
created_at: iso(row.created_at),
created_at: iso(row.created_at),
};
};
if (tasks) out.tasks = tasks;
if (tasks) out.tasks = tasks;
+ // The published bench.run@1 document: what a caller reads back is what the contract says.
+ out.document = contractRun({
+ id: out.id,
+ benchmark: out.benchmark,
+ model: out.model,
+ route: out.route,
+ provider: out.provider,
+ harness: out.harness,
+ settings: out.settings,
+ executor: out.executor,
+ hardware: out.hardware,
+ tasks: tasks || [],
+ score: out.score,
+ cost_usd: out.cost_usd,
+ ms: out.ms,
+ tokens: out.tokens,
+ provenance: out.provenance,
+ }, out.created_at);
return out;
return out;
}
}
@@ -344,6 +363,23 @@ function createRegistry(s, { log = console } = {}) {
@@ -344,6 +363,23 @@ function createRegistry(s, { log = console } = {}) {
const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');
const executorKind = provenance === 'submitted' ? 'submitted' : String((doc.executor && doc.executor.kind) || 'codes');
const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };
const executor = { kind: executorKind, id: String((doc.executor && doc.executor.id) || subject || 'unknown') };
const id = s.newId('brn');
const id = s.newId('brn');
+ // The published contract has the last word: a run that is not a bench.run@1 document is refused.
+ const contract = contractRun({
+ ...doc,
+ id,
+ benchmark: `${benchmarkId}@${version}`,
+ executor,
+ tasks,
+ score: num(doc.score, 0),
+ cost_usd: num(doc.cost_usd, 0),
+ ms: num(doc.ms, 0),
+ tokens: num(doc.tokens, 0),
+ provenance,
+ }, s.iso());
+ const contractCheck = checkContract('bench.run@1', contract);
+ if (!contractCheck.valid) {
+ return refusal(400, 'bench.invalid', contractCheck.errors.map((e) => e.message).join('; '));
+ }
const score = num(doc.score, 0);
const score = num(doc.score, 0);
const cost = num(doc.cost_usd, 0);
const cost = num(doc.cost_usd, 0);
const ms = num(doc.ms, 0);
const ms = num(doc.ms, 0);
@@ -10,9 +10,12 @@
@@ -10,9 +10,12 @@
* The rules mirror the contract's: the same required fields, the same enums, the same ranges.
* The rules mirror the contract's: the same required fields, the same enums, the same ranges.
*/
*/
+const { assertValid } = require('openvibe-contracts');
+
const isStr = (v) => typeof v === 'string';
const isStr = (v) => typeof v === 'string';
const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
const isObj = (v) => v && typeof v === 'object' && !Array.isArray(v);
const isObj = (v) => v && typeof v === 'object' && !Array.isArray(v);
+const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
/** The contract's id patterns. */
/** The contract's id patterns. */
const BENCHMARK_ID = /^bmk_[0-9A-HJKMNP-TV-Z]{26}$/;
const BENCHMARK_ID = /^bmk_[0-9A-HJKMNP-TV-Z]{26}$/;
@@ -102,4 +105,62 @@ function validateRun(doc) {
@@ -102,4 +105,62 @@ function validateRun(doc) {
return { valid: e.valid, errors: e.list };
return { valid: e.valid, errors: e.list };
}
}
-module.exports = { validateBenchmark, validateRun, Errors, BENCHMARK_ID, RUN_ID, BENCHMARK_REF, VISIBILITIES, EXECUTOR_KINDS, PROVENANCES };
+
+/**
+ * The bench.run@1 document for a run Bench is about to store: exactly the published shape, nothing else. The extra
+ * fields Bench keeps for itself (the benchmark id and version, the session) stay in the database, not in the
+ * contract, so what a caller reads back is what the contract says.
+ */
+function contractRun(doc, createdAt) {
+ const harness = (doc.harness && typeof doc.harness === 'object') ? doc.harness : {};
+ const settings = (doc.settings && typeof doc.settings === 'object') ? doc.settings : {};
+ const executor = (doc.executor && typeof doc.executor === 'object') ? doc.executor : {};
+ const tasks = (Array.isArray(doc.tasks) ? doc.tasks : []).map((t) => {
+ const out = {
+ task_id: String(t.task_id || ''),
+ score: num(t.score),
+ cost_usd: num(t.cost_usd),
+ ms: Math.round(num(t.ms)),
+ tokens: Math.round(num(t.tokens)),
+ artifacts: (Array.isArray(t.artifacts) ? t.artifacts : [])
+ .map((a) => ({ kind: String(a.kind || 'file'), url: String(a.url || '') }))
+ .filter((a) => a.url),
+ };
+ if (t.transcript_url) out.transcript_url = String(t.transcript_url);
+ return out;
+ });
+ const out = {
+ id: String(doc.id || ''),
+ benchmark: String(doc.benchmark || ''),
+ model: String(doc.model || ''),
+ route: (doc.route && typeof doc.route === 'object') ? doc.route : {},
+ provider: String(doc.provider || ''),
+ harness: { name: String(harness.name || ''), version: String(harness.version || '') },
+ settings: {
+ temperature: settings.temperature == null ? null : Number(settings.temperature),
+ effort: settings.effort == null ? null : String(settings.effort),
+ },
+ executor: { kind: String(executor.kind || ''), id: String(executor.id || '') },
+ tasks,
+ score: num(doc.score),
+ cost_usd: num(doc.cost_usd),
+ ms: Math.round(num(doc.ms)),
+ tokens: Math.round(num(doc.tokens)),
+ provenance: String(doc.provenance || ''),
+ created_at: createdAt || new Date().toISOString(),
+ };
+ if (doc.hardware != null) out.hardware = doc.hardware;
+ return out;
+}
+
+/** The published contract has the last word: a document that does not match it is refused. */
+function checkContract(name, doc) {
+ try {
+ assertValid(name, doc);
+ return { valid: true, errors: [] };
+ } catch (err) {
+ return { valid: false, errors: [{ path: '/', message: (err && err.message) || `does not match ${name}` }] };
+ }
+}
+
+module.exports = { validateBenchmark, validateRun, contractRun, checkContract, Errors, BENCHMARK_ID, RUN_ID, BENCHMARK_REF, VISIBILITIES, EXECUTOR_KINDS, PROVENANCES };
@@ -1,11 +1,32 @@
@@ -1,11 +1,32 @@
'use strict';
'use strict';
/**
/**
- * Bench's Contracts surface, kept as JSON in the service until the published openvibe-contracts package carries
- * the bench.* schemas and capabilities. These are the exact shapes the API, the SDK and the outbox publish, so a
- * later move into openvibe-contracts is a copy, not a redesign.
+ * Bench's Contracts surface.
+ *
+ * openvibe-contracts v0.244.0 carries the bench.* schemas and the registry capabilities, so the service reads them
+ * from the package: the published shapes are the only ones, and a pin bump is the only way they move. The list of
+ * names below is checked at load time, so a wrong pin fails at boot with a clear message instead of answering with
+ * a shape nobody published.
+ *
+ * The nine capabilities Bench has always had stay listed (the showcase kinds, rounds, entries, judgements and the
+ * board), and the three the registry added join them: bench.benchmark.write, bench.run.publish and bench.run.read.
*/
*/
+const { schema, capabilities } = require('openvibe-contracts');
+
+/** Every bench.* schema the published package must carry. */
+const PUBLISHED_SCHEMAS = Object.freeze([
+ 'bench.kind@1',
+ 'bench.task@1',
+ 'bench.round@1',
+ 'bench.entry@1',
+ 'bench.judgement@1',
+ 'bench.rating@1',
+ 'bench.benchmark@1',
+ 'bench.run@1',
+]);
+
+/** The capabilities Bench's own tokens carry. */
const BENCH_CAPABILITIES = Object.freeze([
const BENCH_CAPABILITIES = Object.freeze([
'bench.kind.read',
'bench.kind.read',
'bench.task.create',
'bench.task.create',
@@ -27,106 +48,36 @@ const BENCH_EVENTS = Object.freeze([
@@ -27,106 +48,36 @@ const BENCH_EVENTS = Object.freeze([
'bench.rating.changed',
'bench.rating.changed',
]);
]);
-const BENCH_SCHEMAS = Object.freeze({
- 'bench.kind@1': {
- type: 'object',
- required: ['id', 'room', 'entry', 'judge', 'name', 'schema', 'metrics', 'created_at'],
- properties: {
- id: { type: 'string' },
- room: { type: 'string' },
- entry: { type: 'string' },
- judge: { type: 'string' },
- name: { type: 'string' },
- schema: { type: 'object' },
- metrics: { type: 'array', items: { type: 'string' } },
- created_at: { type: 'string', format: 'date-time' },
- },
- },
- 'bench.task@1': {
- type: 'object',
- required: ['id', 'kind_id', 'prompt', 'public', 'created_by', 'created_at'],
- properties: {
- id: { type: 'string' },
- kind_id: { type: 'string' },
- prompt: { type: 'string' },
- task_class: { type: ['string', 'null'] },
- public: { type: 'boolean' },
- created_by: { type: 'string' },
- created_at: { type: 'string', format: 'date-time' },
- },
- },
- 'bench.round@1': {
- type: 'object',
- required: ['id', 'task_id', 'kind_id', 'offers', 'status', 'seed', 'created_by', 'created_at'],
- properties: {
- id: { type: 'string' },
- task_id: { type: 'string' },
- kind_id: { type: 'string' },
- offers: { type: 'array', items: { type: 'object' } },
- status: { type: 'string', enum: ['queued', 'running', 'ready', 'finished', 'failed'] },
- seed: { type: 'integer' },
- pool_fallback: { type: 'boolean' },
- receipt: { type: ['string', 'null'] },
- created_by: { type: 'string' },
- created_at: { type: 'string', format: 'date-time' },
- },
- },
- 'bench.entry@1': {
- type: 'object',
- required: ['id', 'round_id', 'provider', 'model', 'route', 'payload', 'cost_usd', 'ms', 'tokens'],
- properties: {
- id: { type: 'string' },
- round_id: { type: 'string' },
- provider: { type: 'string' },
- model: { type: 'string' },
- route: { type: 'object' },
- payload: { type: 'object' },
- cost_usd: { type: 'number' },
- ms: { type: 'number' },
- tokens: { type: 'number' },
- error: { type: ['string', 'null'] },
- ui_check: { type: ['object', 'null'] },
- run_id: { type: ['string', 'null'] },
- },
- },
- 'bench.judgement@1': {
- type: 'object',
- required: ['id', 'round_id', 'kind_id', 'judge', 'method', 'pair', 'outcome', 'score', 'at'],
- properties: {
- id: { type: 'string' },
- round_id: { type: 'string' },
- kind_id: { type: 'string' },
- judge: { type: 'string' },
- judge_name: { type: ['string', 'null'] },
- judge_version: { type: ['string', 'null'] },
- method: { type: 'string' },
- pair: { type: ['object', 'null'] },
- outcome: { type: ['string', 'null'] },
- score: { type: ['number', 'null'] },
- vote_counts: { type: ['object', 'null'] },
- at: { type: 'string', format: 'date-time' },
- },
- },
- 'bench.rating@1': {
- type: 'object',
- required: ['kind_id', 'category', 'subject', 'elo', 'games', 'wins', 'losses', 'ties', 'win_rate', 'ci_lower', 'ci_upper'],
- properties: {
- kind_id: { type: 'string' },
- category: { type: 'string' },
- subject: { type: 'string' },
- elo: { type: 'number' },
- games: { type: 'number' },
- wins: { type: 'number' },
- losses: { type: 'number' },
- ties: { type: 'number' },
- win_rate: { type: 'number' },
- ci_lower: { type: 'number' },
- ci_upper: { type: 'number' },
- median_cost_usd: { type: ['number', 'null'] },
- median_ms: { type: ['number', 'null'] },
- provisional: { type: 'boolean' },
- },
- },
+function published(name) {
+ try {
+ return schema(name) || null;
+ } catch {
+ return null;
+ }
+}
+
+const BENCH_SCHEMAS = Object.freeze(PUBLISHED_SCHEMAS.reduce((acc, name) => {
+ const s = published(name);
+ if (s) acc[name] = s;
+ return acc;
+}, {}));
+
+const MISSING = PUBLISHED_SCHEMAS.filter((name) => !BENCH_SCHEMAS[name]);
+if (MISSING.length) {
+ throw new Error(`openvibe-contracts does not carry ${MISSING.join(', ')}. Pin the release that publishes them.`);
+}
+
+// Every capability Bench hands out must be one the published package knows, so a token is never granted a name no
+// service can rely on.
+const UNKNOWN_CAPABILITIES = BENCH_CAPABILITIES.filter((cap) => {
+ try {
+ return !capabilities.get(cap);
+ } catch {
+ return true;
+ }
});
});
+if (UNKNOWN_CAPABILITIES.length) {
+ throw new Error(`openvibe-contracts does not publish ${UNKNOWN_CAPABILITIES.join(', ')}.`);
+}
-module.exports = { BENCH_CAPABILITIES, BENCH_EVENTS, BENCH_SCHEMAS };
+module.exports = { BENCH_CAPABILITIES, BENCH_EVENTS, BENCH_SCHEMAS, PUBLISHED_SCHEMAS };
@@ -143,7 +143,13 @@ function createRegistryApi(ctx) {
@@ -143,7 +143,13 @@ function createRegistryApi(ctx) {
r.get('/benchmarks/:id', async (req, res) => {
r.get('/benchmarks/:id', async (req, res) => {
try {
try {
const ref = String(req.params.id || '');
const ref = String(req.params.id || '');
- const out = await registry.getDefinition(ref, { reveal: mayReveal(req) });
+ // The holdout goes to the executor and to the benchmark's owner; to nobody else.
+ let reveal = mayReveal(req);
+ if (!reveal && req.principal.kind === 'user') {
+ const owner = await registry.getBenchmark(ref.split('@')[0]);
+ reveal = !!(owner && owner.owner === req.principal.subject);
+ }
+ const out = await registry.getDefinition(ref, { reveal });
if (!out) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
if (!out) return problem(res, 404, 'bench.benchmark_not_found', 'No such benchmark.');
res.set('Cache-Control', 'public, max-age=60');
res.set('Cache-Control', 'public, max-age=60');
return res.json({
return res.json({
new file mode 100644
new file mode 100644
@@ -0,0 +1,123 @@
@@ -0,0 +1,123 @@
+'use strict';
+
+/**
+ * The contracts pin (spec 2026-10-30, stage B1).
+ *
+ * Bench speaks the published shapes: openvibe-contracts v0.244.0 carries bench.benchmark@1 and bench.run@1 and the
+ * three registry capabilities. These tests prove the pin is real — the package carries the shapes, Bench's own
+ * capability list names only capabilities the package publishes, a frozen benchmark and a published run are the
+ * published documents, and the holdout goes to the executor and to the owner and to nobody else.
+ */
+const assert = require('assert');
+const contracts = require('openvibe-contracts');
+const { boot, check, done } = require('./helpers/boot');
+const { BENCH_CAPABILITIES, BENCH_SCHEMAS } = require('../server/contracts');
+
+const TASKS = [
+ { id: 't1', prompt: 'Build a one-screen arcade game with a paddle, bricks, a ball and a score.', files: [], expected: null, holdout: false },
+ { id: 't2', prompt: 'Build a snake game on a 20x20 grid.', files: [], expected: null, holdout: true },
+];
+
+function definition() {
+ return {
+ title: 'The pinned set',
+ description: 'A benchmark that must match bench.benchmark@1.',
+ kind: 'games.play',
+ tags: ['games'],
+ visibility: 'public',
+ tasks: TASKS,
+ judges: [{ type: 'human_pairwise', version: '1.0', config: {} }],
+ scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: { ms: 120000, tokens: 200000, usd: 0.5 },
+ matrix: { models: ['claude-opus-4'], harnesses: ['claude-code'] },
+ };
+}
+
+function runDoc(benchmark, over = {}) {
+ return {
+ benchmark,
+ model: 'claude-opus-4',
+ route: { provider: 'anthropic' },
+ provider: 'anthropic',
+ harness: { name: 'claude-code', version: '1.2.0' },
+ settings: { temperature: 1, effort: 'high' },
+ executor: { kind: 'codes', id: 'task_1' },
+ tasks: [{ task_id: 't1', score: 90, cost_usd: 0.02, ms: 40000, tokens: 8000, artifacts: [] }],
+ score: 90,
+ cost_usd: 0.02,
+ ms: 40000,
+ tokens: 8000,
+ ...over,
+ };
+}
+
+(async () => {
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const owner = t.network.addUser('owner');
+ const publisher = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.run.publish', 'bench.benchmark.write'] });
+ const writer = t.network.serviceToken({ sub: 'svc:writer', audience: 'openvibe.bench', cap: ['bench.benchmark.write'] });
+
+ let benchmarkId;
+ let version;
+
+ await check('the pinned package carries the registry schemas and capabilities', async () => {
+ assert.ok(contracts.schema('bench.benchmark@1'), 'bench.benchmark@1 is published');
+ assert.ok(contracts.schema('bench.run@1'), 'bench.run@1 is published');
+ assert.ok(BENCH_SCHEMAS['bench.benchmark@1'], 'Bench reads it from the package');
+ assert.ok(BENCH_SCHEMAS['bench.run@1'], 'Bench reads it from the package');
+ for (const cap of ['bench.benchmark.write', 'bench.run.publish', 'bench.run.read']) {
+ assert.ok(BENCH_CAPABILITIES.includes(cap), `Bench grants ${cap}`);
+ assert.ok(contracts.capabilities.get(cap), `the package publishes ${cap}`);
+ }
+ });
+
+ await check('a frozen benchmark is a bench.benchmark@1 document', async () => {
+ const created = await t.get('/api/v1/benchmarks', { bearer: writer, json: definition() });
+ assert.strictEqual(created.status, 201, created.text);
+ benchmarkId = created.json().benchmark.id;
+ const frozen = await t.get(`/api/v1/benchmarks/${benchmarkId}/versions`, { bearer: writer, json: {} });
+ assert.strictEqual(frozen.status, 201, frozen.text);
+ version = frozen.json().version.version;
+ contracts.assertValid('bench.benchmark@1', frozen.json().version.definition);
+ });
+
+ await check('a published run is a bench.run@1 document', async () => {
+ const published = await t.get('/api/v1/runs', { bearer: publisher, json: runDoc(`${benchmarkId}@${version}`) });
+ assert.strictEqual(published.status, 201, published.text);
+ contracts.assertValid('bench.run@1', published.json().run.document);
+ assert.strictEqual(published.json().run.document.provenance, 'verified');
+ const read = await t.get(`/api/v1/runs/${published.json().run.id}`);
+ assert.strictEqual(read.status, 200, read.text);
+ contracts.assertValid('bench.run@1', read.json().run.document);
+ });
+
+ await check('a run that is not the published shape is refused', async () => {
+ const bad = await t.get('/api/v1/runs', { bearer: publisher, json: runDoc(`${benchmarkId}@${version}`, { settings: { temperature: 9 } }) });
+ assert.strictEqual(bad.status, 400, bad.text);
+ assert.strictEqual(bad.json().code, 'bench.invalid');
+ });
+
+ await check('the owner sees the holdout and a stranger does not', async () => {
+ const made = await t.get('/api/v1/benchmarks', { bearer: t.network.userToken(owner), json: definition() });
+ assert.strictEqual(made.status, 201, made.text);
+ const id = made.json().benchmark.id;
+ const frozen = await t.get(`/api/v1/benchmarks/${id}/versions`, { bearer: t.network.userToken(owner), json: {} });
+ assert.strictEqual(frozen.status, 201, frozen.text);
+
+ const mine = await t.get(`/api/v1/benchmarks/${id}`, { bearer: t.network.userToken(owner) });
+ const myTask = mine.json().definition.tasks.find((x) => x.id === 't2');
+ assert.ok(myTask.prompt, 'the owner sees the holdout prompt');
+
+ const theirs = await t.get(`/api/v1/benchmarks/${id}`, { bearer: t.network.userToken(alice) });
+ const theirTask = theirs.json().definition.tasks.find((x) => x.id === 't2');
+ assert.strictEqual(theirTask.prompt, null, 'a stranger does not');
+
+ const public_ = await t.get(`/api/v1/benchmarks/${id}`);
+ assert.strictEqual(public_.json().definition.tasks.find((x) => x.id === 't2').prompt, null, 'nobody does');
+ });
+
+ await check('stop', async () => { await t.close(); });
+
+ done();
+})();