openvibers/openvibe.bench Commits
Matches: every comparison as one record, and the model/harness factor board (#230)
Changes
@@@ -39,7 +39,7 @@ JSON under `/api/v1`; errors are `application/problem+json`. The full list is inThe public reads Report's AI pages use are `GET /api/v1/overview`, `GET /api/v1/benchmarks`, `GET /api/v1/rankings`, `GET /api/v1/runs`, `GET /api/v1/models/:model/summary` and `GET /api/v1/runs/:id`; the long list is at `/openapi.json` and in Report's method page.<!-- versions:start -->--- openvibe-contracts: v0.419.0++- openvibe-contracts: v0.422.0- openvibe-sdk: v0.161.0- openvibe-shared: v4.232.0<!-- versions:end -->@@@ -4,7 +4,7 @@"version": "0.1.0","status": "v1","summary": "The shared AI benchmark and comparison engine for every room: kinds, pinned rounds, blind judgements, elo/pareto/cost boards, an open dataset per kind, private evals and an MCP endpoint. Its public pages are OpenVibe.Report's AI section since 2026-11-08; this origin answers 301 for them and keeps the API.",-- "contracts": "openvibe-contracts v0.419.0",++ "contracts": "openvibe-contracts v0.422.0","sdk": "openvibe-sdk v0.161.0","shared": "openvibe-shared v4.232.0","updated": "2026-11-13"nnew file mode 100644@@@ -0,0 +1,20 @@++-- phase: expand++-- Every comparison on the network as one record (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md,++-- 2026-11-13): an Arena vote, a Codes race whose patch was merged, a check job's tests, a game, a Live poll. The++-- boards that rate model and harness apart (server/rating/factors.js) are fitted from these.++--++-- BODY is the whole bench.match@1 document (sides, receipts, shown order, judgements); the columns beside it are what++-- a board selects on. RECORDED_BY is the service that sent it: only that service adds judgements to it later.++CREATE TABLE IF NOT EXISTS bench_matches (++ id text COLLATE "C" PRIMARY KEY,++ source text COLLATE "C" NOT NULL,++ task_kind text COLLATE "C" NOT NULL,++ category text COLLATE "C",++ prompt_hash text COLLATE "C",++ recorded_by text COLLATE "C" NOT NULL,++ body jsonb NOT NULL,++ at timestamptz NOT NULL,++ created_at timestamptz NOT NULL DEFAULT now(),++ updated_at timestamptz NOT NULL DEFAULT now()++);++CREATE INDEX IF NOT EXISTS bench_matches_board ON bench_matches (task_kind, category, at);@@@ -15,7 +15,7 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0","pg": "^8.23.0",@@@ -1232,9 +1232,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.419.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",-- "integrity": "sha512-VxrTdeHchT8gvl2wWC9ASaId92en7ZKfJ8F/0hKzCAD0f2Y00rPrqgBWF9BNPoLGARoyQvsMbhBEhSkhxGNnjA==",++ "version": "0.422.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",++ "integrity": "sha512-xQQq7NQLM9iQve8dR4Hc2EUp75x/FZYfAsHumsmf6dGHjIy05PMX0PlYmxw5L52VXKOj+r7mRwlcehRqgMoLSw==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -22,7 +22,7 @@"express-rate-limit": "^7.5.0","helmet": "^8.1.0","iovalkey": "^0.4.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0","pg": "^8.23.0",@@@ -46,6 +46,7 @@ const BENCH_EVENTS = Object.freeze(['bench.round.finished','bench.judgement.recorded','bench.rating.changed',++ 'bench.match.recorded',]);function published(name) {@@@ -14,6 +14,8 @@* POST /rounds/:id/judgements* GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon* GET /models/:model++ * POST /matches (bench.match@1 from a service with bench.judgement.cast)++ * GET /factors?task_kind=&category=&days= model and harness apart, with intervals (bench.rating@1 factors)* GET /datasets/:kindId.jsonl* POST /evals* GET /evals/:id@@@ -414,6 +416,41 @@ function createApi(ctx) {return res.json(out);});++ // ── Matches and the factor boards (compare-arena-ecosystem.md, Contracts 0.422.0) ──++ // A service records a match it ran (the Arena's race, a Codes race, a check job, a game); the boards rate model++ // and harness apart from every match of a task kind (server/rating/factors.js), cached a few minutes.++ r.post('/matches', principal.requireCapability('bench.judgement.cast'), async (req, res) => {++ const body = { ...(req.body || {}) };++ if (!body.id) body.id = `mat_${ids.ulid(Date.now())}`;++ const checked = require('openvibe-contracts').validate('bench.match@1', body);++ if (!checked.valid) return invalid(res, `not a bench.match@1: ${checked.errors.slice(0, 3).map((e) => `${e.path} ${e.message}`).join('; ')}`);++ try {++ const out = await bench.recordMatch(body, { recordedBy: req.principal.requester });++ return res.status(out.created ? 201 : 200).json(out);++ } catch (err) { return failed(res, err); }++ });++++ const factorCache = new Map();++ r.get('/factors', async (req, res) => {++ const taskKind = String(req.query.task_kind || '');++ if (!['chat', 'code', 'design', 'image', 'speech', 'search', 'computer', 'game'].includes(taskKind)) return invalid(res, 'task_kind must be chat, code, design, image, speech, search, computer or game');++ const category = req.query.category ? String(req.query.category).slice(0, 64) : null;++ const days = [7, 30, 90, 365].includes(Number(req.query.days)) ? Number(req.query.days) : 90;++ const key = `${taskKind}|${category || ''}|${days}`;++ const hit = factorCache.get(key);++ if (hit && hit.until > Date.now()) { res.set('Cache-Control', 'public, max-age=120'); return res.json(hit.value); }++ const since = new Date(Date.now() - days * 86400000).toISOString();++ const matches = await bench.matchesFor({ taskKind, category, since });++ const { gamesOf, COVARIATES } = require('../rating/match-games');++ const games = gamesOf(matches);++ const fit = require('../rating/factors').fitFactors(games, { covariates: COVARIATES, bootstrap: games.length > 20000 ? 60 : 150 });++ const value = { task_kind: taskKind, category, days, matches: matches.length, games: games.length, style_control: true, ...fit };++ factorCache.set(key, { until: Date.now() + 5 * 60_000, value });++ if (factorCache.size > 200) factorCache.delete(factorCache.keys().next().value);++ res.set('Cache-Control', 'public, max-age=120');++ return res.json(value);++ });++return r;}nnew file mode 100644@@@ -0,0 +1,54 @@++'use strict';++++/**++ * bench.match@1 → the pairwise games fitFactors reads (compare-arena-ecosystem.md, 2026-11-13).++ *++ * A match has two to eight sides and any number of judgements. Each judgement becomes games between pairs of sides:++ *++ * winner a … h the winner beats every other side (a person, a Live poll, an implicit signal, a game)++ * winner tie every pair ties++ * winner both_bad says nothing about the order: no game++ * scores (tests passed, a rubric score, a game's points) every pair is decided by the higher score,++ * an equal score is a tie++ *++ * A judgement's weight (the voter's quality) carries into its games; a side that failed, timed out or was refused++ * still plays (it lost on its merits). The style covariates are log(1 + length) and log(1 + markdown items) of each++ * side's answer, so the fit can hold style out of the score.++ */++++const COVARIATES = 2;++const styleOf = (slot) => {++ const r = (slot && slot.receipts) || {};++ return [Math.log1p(Math.max(0, Number(r.length_chars) || 0)), Math.log1p(Math.max(0, Number(r.markdown_items) || 0))];++};++++function gamesOf(matches) {++ const games = [];++ for (const m of Array.isArray(matches) ? matches : []) {++ const slots = new Map((m.slots || []).map((s) => [s.slot, s]));++ if (slots.size < 2) continue;++ const letters = [...slots.keys()];++ const pairs = [];++ for (let i = 0; i < letters.length; i++) for (let j = i + 1; j < letters.length; j++) pairs.push([letters[i], letters[j]]);++ for (const j of m.judgements || []) {++ const weight = j.weight == null ? 1 : Math.max(0, Number(j.weight) || 0);++ if (!weight) continue;++ for (const [x, y] of pairs) {++ let outcome = null;++ if (j.winner === 'tie') outcome = 'tie';++ else if (j.winner && j.winner !== 'both_bad' && slots.has(j.winner)) outcome = j.winner === x ? 'a' : j.winner === y ? 'b' : null;++ else if (j.scores && Number.isFinite(Number(j.scores[x])) && Number.isFinite(Number(j.scores[y]))) {++ const sx = Number(j.scores[x]);++ const sy = Number(j.scores[y]);++ outcome = sx > sy ? 'a' : sx < sy ? 'b' : 'tie';++ }++ // A winner beats every other side; two losers did not play each other.++ if (!outcome) continue;++ games.push({ a: slots.get(x).entrant, b: slots.get(y).entrant, outcome, weight, fa: styleOf(slots.get(x)), fb: styleOf(slots.get(y)), judge: j.judge });++ }++ }++ }++ return games;++}++++module.exports = { gamesOf, COVARIATES };@@@ -1189,12 +1189,61 @@ function createStore(s, { log = console, random = Math.random, config = null, aireturn { eval: ev, runs };}++ // ── Matches (bench.match@1, Contracts 0.422.0): every comparison on the network as one record ──++ /**++ * Record a match. A new id is inserted; an id the same service recorded before gains the judgements it did not++ * have (a Codes race is judged by its tests first and by the merge later). Another service's id is refused.++ */++ async function recordMatch(match, { recordedBy }) {++ const m = { ...match, id: match.id || `mat_${ids.ulid(s.now())}` };++ return s.tx(async () => {++ const prev = await db.maybe('SELECT * FROM bench_matches WHERE id = $1 FOR UPDATE', [m.id]);++ let body = m;++ let created = true;++ if (prev) {++ if (prev.recorded_by !== recordedBy) throw new BenchError(409, 'bench.match_taken', 'That match id was recorded by another service.');++ const old = prev.body || {};++ // jsonb keeps keys in its own order, so a judgement is known by its fields, not by its JSON text.++ const keyOf = (j) => [j.judge, j.winner || '', j.signal || '', JSON.stringify(Object.entries(j.scores || {}).sort()),++ (j.by && j.by.type) || '', (j.by && j.by.id_hash) || '', String(j.at || '')].join('|');++ const seen = new Set((old.judgements || []).map(keyOf));++ body = { ...old, ...m, judgements: [...(old.judgements || []), ...(m.judgements || []).filter((j) => !seen.has(keyOf(j)))] };++ created = false;++ await db.exec('UPDATE bench_matches SET body = $2::jsonb, updated_at = $3::timestamptz WHERE id = $1', [m.id, JSON.stringify(body), s.iso()]);++ } else {++ await db.exec(++ `INSERT INTO bench_matches (id, source, task_kind, category, prompt_hash, recorded_by, body, at, created_at, updated_at)++ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::timestamptz, $9::timestamptz, $9::timestamptz)`,++ [m.id, m.source, m.task.kind, m.task.category || null, m.task.prompt_hash || null, recordedBy, JSON.stringify(body), m.at, s.iso()],++ );++ }++ const judges = [...new Set((body.judgements || []).map((j) => j.judge))];++ const last = (body.judgements || []).slice(-1)[0];++ const winner = last && last.winner && /^[a-h]$/.test(last.winner) ? ((body.slots || []).find((x) => x.slot === last.winner) || {}).entrant || null : null;++ await insertOutbox('bench.match.recorded', {++ match_id: body.id, source: body.source, task_kind: body.task.kind, category: body.task.category || null,++ entrants: (body.slots || []).map((x) => x.entrant), judges, winner_entrant: winner, at: s.iso(),++ });++ return { match: body, created };++ });++ }++++ /** The matches a board reads: one task kind, optionally one category, since a day. */++ async function matchesFor({ taskKind, category = null, since, limit = 50000 }) {++ const rows = await db.many(++ `SELECT body FROM bench_matches WHERE task_kind = $1 AND ($2::text IS NULL OR category = $2) AND at >= $3::timestamptz++ ORDER BY at DESC LIMIT $4`,++ [String(taskKind), category, since, Math.max(1, Math.min(200000, Number(limit) || 50000))],++ );++ return rows.map((r) => r.body);++ }++return {wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,insertOutbox, seedKinds, listKinds, getKind,createTask, getTask,createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,-- nextJudgement, castJudgement, pairwiseGames,++ nextJudgement, castJudgement, pairwiseGames, recordMatch, matchesFor,board, ratings, valueRows, modelSummary, dataset, datasetJsonl,createEval, getEval, runEval, evalResults, evalDailyCount,KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,nnew file mode 100644@@@ -0,0 +1,100 @@++'use strict';++++/**++ * Matches and the factor boards (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md): a service records the++ * comparisons it ran, the same service adds a later judgement (a Codes race judged by its tests, then by the merge),++ * nobody else can touch its match, and the code board rates model and harness apart from them.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { gamesOf } = require('../server/rating/match-games');++++const at = '2026-11-13T02:00:00Z';++const agent = (h, m) => `agent:${h}+${m}#0123abcd`;++function codeRace(a, b, winner, judge = 'test_suite') {++ return {++ source: 'codes', task: { kind: 'code', category: 'code.bugfix' },++ slots: [{ slot: 'a', entrant: a, receipts: { outcome: 'succeeded', cost_usd: 0.2 } }, { slot: 'b', entrant: b, receipts: { outcome: 'succeeded', cost_usd: 0.1 } }],++ blind: false,++ judgements: [judge === 'test_suite'++ ? { judge, scores: { a: winner === 'a' ? 1 : 0.5, b: winner === 'b' ? 1 : 0.5 }, by: { type: 'suite' }, at }++ : { judge, winner, signal: 'merged', by: { type: 'person', id_hash: '0123456789abcdef' }, at }],++ at,++ };++}++++(async () => {++ // Pure: one judgement with a winner among three sides is two games; scores decide every pair; both_bad is none.++ await check('a match becomes the pairwise games the fit reads', async () => {++ const three = { slots: [{ slot: 'a', entrant: 'model:x' }, { slot: 'b', entrant: 'model:y' }, { slot: 'c', entrant: 'model:z' }],++ judgements: [{ judge: 'human_pairwise', winner: 'b', at }, { judge: 'rubric', scores: { a: 3, b: 3, c: 1 }, at }, { judge: 'human_pairwise', winner: 'both_bad', at }] };++ const g = gamesOf([three]);++ assert.deepStrictEqual(g.map((x) => `${x.a}>${x.b}:${x.outcome}`), [++ 'model:x>model:y:b', 'model:y>model:z:a',++ 'model:x>model:y:tie', 'model:x>model:z:a', 'model:y>model:z:a',++ ]);++ });++++ const t = await boot();++ try {++ const codes = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });++ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });++ const reader = t.network.serviceToken({ sub: 'svc:compare', audience: 'openvibe.bench', cap: ['bench.board.read'] });++++ let firstId = null;++ await check('a service records a match; it gets an id and the event is queued', async () => {++ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a') });++ assert.strictEqual(r.status, 201, r.text);++ firstId = r.json().match.id;++ assert.match(firstId, /^mat_[0-9A-HJKMNP-TV-Z]{26}$/);++ const outbox = await t.store.db.many("SELECT payload FROM bench_outbox WHERE topic = 'bench.match.recorded'");++ assert.strictEqual(outbox.length, 1);++ assert.deepStrictEqual(outbox[0].payload.judges, ['test_suite']);++ });++++ await check('the same service adds the merge later; the judgements add up', async () => {++ const later = { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a', 'implicit'), id: firstId };++ const r = await t.get('/api/v1/matches', { bearer: codes, json: later });++ assert.strictEqual(r.status, 200, r.text);++ assert.deepStrictEqual(r.json().match.judgements.map((j) => j.judge), ['test_suite', 'implicit']);++ const again = await t.get('/api/v1/matches', { bearer: codes, json: later });++ assert.strictEqual(again.json().match.judgements.length, 2, 'the same judgement is not counted twice');++ });++++ await check('another service cannot touch it; a bad match and a token without the capability are refused', async () => {++ const other = await t.get('/api/v1/matches', { bearer: live, json: { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'b'), id: firstId } });++ assert.strictEqual(other.status, 409, other.text);++ const bad = await t.get('/api/v1/matches', { bearer: codes, json: { ...codeRace('model:x', 'model:y', 'a'), slots: [{ slot: 'a', entrant: 'model:x' }] } });++ assert.strictEqual(bad.status, 400, bad.text);++ const denied = await t.get('/api/v1/matches', { bearer: reader, json: codeRace('model:x', 'model:y', 'a') });++ assert.strictEqual(denied.status, 403, denied.text);++ });++++ await check('the code board rates the harnesses apart from the models', async () => {++ const models = ['gpt-5-mini', 'glm-5.3', 'deepseek-flash'];++ let n = 0;++ for (const m1 of models) for (const m2 of models) {++ for (let k = 0; k < 6; k++) {++ // Codex beats OpenCode on the same model or a weaker one, most of the time.++ const winner = (k % 3 === 2) ? 'b' : 'a';++ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', m1), agent('opencode', m2), winner) });++ assert.strictEqual(r.status, 201, r.text);++ n += 1;++ }++ }++ const board = await t.get('/api/v1/factors?task_kind=code&days=365');++ assert.strictEqual(board.status, 200, board.text);++ const b = board.json();++ assert.strictEqual(b.matches, n + 1);++ const effect = (h) => b.harnesses.find((x) => x.harness === h).effect;++ assert.ok(effect('codex') > effect('opencode'), `codex ${effect('codex')} ahead of opencode ${effect('opencode')}`);++ assert.ok(b.agents.length >= 6 && b.agents.every((x) => x.factors && x.factors.model && x.factors.harness));++ assert.match(b.method, /model \+ harness/);++ const badKind = await t.get('/api/v1/factors?task_kind=poetry');++ assert.strictEqual(badKind.status, 400);++ });++ } finally {++ await t.close();++ }++ done();++})().catch((error) => { console.error(error); process.exitCode = 1; });
@@ -39,7 +39,7 @@ JSON under `/api/v1`; errors are `application/problem+json`. The full list is in
@@ -39,7 +39,7 @@ JSON under `/api/v1`; errors are `application/problem+json`. The full list is in
The public reads Report's AI pages use are `GET /api/v1/overview`, `GET /api/v1/benchmarks`, `GET /api/v1/rankings`, `GET /api/v1/runs`, `GET /api/v1/models/:model/summary` and `GET /api/v1/runs/:id`; the long list is at `/openapi.json` and in Report's method page.
The public reads Report's AI pages use are `GET /api/v1/overview`, `GET /api/v1/benchmarks`, `GET /api/v1/rankings`, `GET /api/v1/runs`, `GET /api/v1/models/:model/summary` and `GET /api/v1/runs/:id`; the long list is at `/openapi.json` and in Report's method page.
<!-- versions:start -->
<!-- versions:start -->
-- openvibe-contracts: v0.419.0
+- openvibe-contracts: v0.422.0
- openvibe-sdk: v0.161.0
- openvibe-sdk: v0.161.0
- openvibe-shared: v4.232.0
- openvibe-shared: v4.232.0
<!-- versions:end -->
<!-- versions:end -->
@@ -4,7 +4,7 @@
@@ -4,7 +4,7 @@
"version": "0.1.0",
"version": "0.1.0",
"status": "v1",
"status": "v1",
"summary": "The shared AI benchmark and comparison engine for every room: kinds, pinned rounds, blind judgements, elo/pareto/cost boards, an open dataset per kind, private evals and an MCP endpoint. Its public pages are OpenVibe.Report's AI section since 2026-11-08; this origin answers 301 for them and keeps the API.",
"summary": "The shared AI benchmark and comparison engine for every room: kinds, pinned rounds, blind judgements, elo/pareto/cost boards, an open dataset per kind, private evals and an MCP endpoint. Its public pages are OpenVibe.Report's AI section since 2026-11-08; this origin answers 301 for them and keeps the API.",
- "contracts": "openvibe-contracts v0.419.0",
+ "contracts": "openvibe-contracts v0.422.0",
"sdk": "openvibe-sdk v0.161.0",
"sdk": "openvibe-sdk v0.161.0",
"shared": "openvibe-shared v4.232.0",
"shared": "openvibe-shared v4.232.0",
"updated": "2026-11-13"
"updated": "2026-11-13"
new file mode 100644
new file mode 100644
@@ -0,0 +1,20 @@
@@ -0,0 +1,20 @@
+-- phase: expand
+-- Every comparison on the network as one record (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md,
+-- 2026-11-13): an Arena vote, a Codes race whose patch was merged, a check job's tests, a game, a Live poll. The
+-- boards that rate model and harness apart (server/rating/factors.js) are fitted from these.
+--
+-- BODY is the whole bench.match@1 document (sides, receipts, shown order, judgements); the columns beside it are what
+-- a board selects on. RECORDED_BY is the service that sent it: only that service adds judgements to it later.
+CREATE TABLE IF NOT EXISTS bench_matches (
+ id text COLLATE "C" PRIMARY KEY,
+ source text COLLATE "C" NOT NULL,
+ task_kind text COLLATE "C" NOT NULL,
+ category text COLLATE "C",
+ prompt_hash text COLLATE "C",
+ recorded_by text COLLATE "C" NOT NULL,
+ body jsonb NOT NULL,
+ at timestamptz NOT NULL,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ updated_at timestamptz NOT NULL DEFAULT now()
+);
+CREATE INDEX IF NOT EXISTS bench_matches_board ON bench_matches (task_kind, category, at);
@@ -15,7 +15,7 @@
@@ -15,7 +15,7 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -1232,9 +1232,9 @@
@@ -1232,9 +1232,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.419.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
- "integrity": "sha512-VxrTdeHchT8gvl2wWC9ASaId92en7ZKfJ8F/0hKzCAD0f2Y00rPrqgBWF9BNPoLGARoyQvsMbhBEhSkhxGNnjA==",
+ "version": "0.422.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
+ "integrity": "sha512-xQQq7NQLM9iQve8dR4Hc2EUp75x/FZYfAsHumsmf6dGHjIy05PMX0PlYmxw5L52VXKOj+r7mRwlcehRqgMoLSw==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -22,7 +22,7 @@
@@ -22,7 +22,7 @@
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"pg": "^8.23.0",
"pg": "^8.23.0",
@@ -46,6 +46,7 @@ const BENCH_EVENTS = Object.freeze([
@@ -46,6 +46,7 @@ const BENCH_EVENTS = Object.freeze([
'bench.round.finished',
'bench.round.finished',
'bench.judgement.recorded',
'bench.judgement.recorded',
'bench.rating.changed',
'bench.rating.changed',
+ 'bench.match.recorded',
]);
]);
function published(name) {
function published(name) {
@@ -14,6 +14,8 @@
@@ -14,6 +14,8 @@
* POST /rounds/:id/judgements
* POST /rounds/:id/judgements
* GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon
* GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon
* GET /models/:model
* GET /models/:model
+ * POST /matches (bench.match@1 from a service with bench.judgement.cast)
+ * GET /factors?task_kind=&category=&days= model and harness apart, with intervals (bench.rating@1 factors)
* GET /datasets/:kindId.jsonl
* GET /datasets/:kindId.jsonl
* POST /evals
* POST /evals
* GET /evals/:id
* GET /evals/:id
@@ -414,6 +416,41 @@ function createApi(ctx) {
@@ -414,6 +416,41 @@ function createApi(ctx) {
return res.json(out);
return res.json(out);
});
});
+ // ── Matches and the factor boards (compare-arena-ecosystem.md, Contracts 0.422.0) ──
+ // A service records a match it ran (the Arena's race, a Codes race, a check job, a game); the boards rate model
+ // and harness apart from every match of a task kind (server/rating/factors.js), cached a few minutes.
+ r.post('/matches', principal.requireCapability('bench.judgement.cast'), async (req, res) => {
+ const body = { ...(req.body || {}) };
+ if (!body.id) body.id = `mat_${ids.ulid(Date.now())}`;
+ const checked = require('openvibe-contracts').validate('bench.match@1', body);
+ if (!checked.valid) return invalid(res, `not a bench.match@1: ${checked.errors.slice(0, 3).map((e) => `${e.path} ${e.message}`).join('; ')}`);
+ try {
+ const out = await bench.recordMatch(body, { recordedBy: req.principal.requester });
+ return res.status(out.created ? 201 : 200).json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ const factorCache = new Map();
+ r.get('/factors', async (req, res) => {
+ const taskKind = String(req.query.task_kind || '');
+ if (!['chat', 'code', 'design', 'image', 'speech', 'search', 'computer', 'game'].includes(taskKind)) return invalid(res, 'task_kind must be chat, code, design, image, speech, search, computer or game');
+ const category = req.query.category ? String(req.query.category).slice(0, 64) : null;
+ const days = [7, 30, 90, 365].includes(Number(req.query.days)) ? Number(req.query.days) : 90;
+ const key = `${taskKind}|${category || ''}|${days}`;
+ const hit = factorCache.get(key);
+ if (hit && hit.until > Date.now()) { res.set('Cache-Control', 'public, max-age=120'); return res.json(hit.value); }
+ const since = new Date(Date.now() - days * 86400000).toISOString();
+ const matches = await bench.matchesFor({ taskKind, category, since });
+ const { gamesOf, COVARIATES } = require('../rating/match-games');
+ const games = gamesOf(matches);
+ const fit = require('../rating/factors').fitFactors(games, { covariates: COVARIATES, bootstrap: games.length > 20000 ? 60 : 150 });
+ const value = { task_kind: taskKind, category, days, matches: matches.length, games: games.length, style_control: true, ...fit };
+ factorCache.set(key, { until: Date.now() + 5 * 60_000, value });
+ if (factorCache.size > 200) factorCache.delete(factorCache.keys().next().value);
+ res.set('Cache-Control', 'public, max-age=120');
+ return res.json(value);
+ });
+
return r;
return r;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,54 @@
@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * bench.match@1 → the pairwise games fitFactors reads (compare-arena-ecosystem.md, 2026-11-13).
+ *
+ * A match has two to eight sides and any number of judgements. Each judgement becomes games between pairs of sides:
+ *
+ * winner a … h the winner beats every other side (a person, a Live poll, an implicit signal, a game)
+ * winner tie every pair ties
+ * winner both_bad says nothing about the order: no game
+ * scores (tests passed, a rubric score, a game's points) every pair is decided by the higher score,
+ * an equal score is a tie
+ *
+ * A judgement's weight (the voter's quality) carries into its games; a side that failed, timed out or was refused
+ * still plays (it lost on its merits). The style covariates are log(1 + length) and log(1 + markdown items) of each
+ * side's answer, so the fit can hold style out of the score.
+ */
+
+const COVARIATES = 2;
+const styleOf = (slot) => {
+ const r = (slot && slot.receipts) || {};
+ return [Math.log1p(Math.max(0, Number(r.length_chars) || 0)), Math.log1p(Math.max(0, Number(r.markdown_items) || 0))];
+};
+
+function gamesOf(matches) {
+ const games = [];
+ for (const m of Array.isArray(matches) ? matches : []) {
+ const slots = new Map((m.slots || []).map((s) => [s.slot, s]));
+ if (slots.size < 2) continue;
+ const letters = [...slots.keys()];
+ const pairs = [];
+ for (let i = 0; i < letters.length; i++) for (let j = i + 1; j < letters.length; j++) pairs.push([letters[i], letters[j]]);
+ for (const j of m.judgements || []) {
+ const weight = j.weight == null ? 1 : Math.max(0, Number(j.weight) || 0);
+ if (!weight) continue;
+ for (const [x, y] of pairs) {
+ let outcome = null;
+ if (j.winner === 'tie') outcome = 'tie';
+ else if (j.winner && j.winner !== 'both_bad' && slots.has(j.winner)) outcome = j.winner === x ? 'a' : j.winner === y ? 'b' : null;
+ else if (j.scores && Number.isFinite(Number(j.scores[x])) && Number.isFinite(Number(j.scores[y]))) {
+ const sx = Number(j.scores[x]);
+ const sy = Number(j.scores[y]);
+ outcome = sx > sy ? 'a' : sx < sy ? 'b' : 'tie';
+ }
+ // A winner beats every other side; two losers did not play each other.
+ if (!outcome) continue;
+ games.push({ a: slots.get(x).entrant, b: slots.get(y).entrant, outcome, weight, fa: styleOf(slots.get(x)), fb: styleOf(slots.get(y)), judge: j.judge });
+ }
+ }
+ }
+ return games;
+}
+
+module.exports = { gamesOf, COVARIATES };
@@ -1189,12 +1189,61 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
@@ -1189,12 +1189,61 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
return { eval: ev, runs };
return { eval: ev, runs };
}
}
+ // ── Matches (bench.match@1, Contracts 0.422.0): every comparison on the network as one record ──
+ /**
+ * Record a match. A new id is inserted; an id the same service recorded before gains the judgements it did not
+ * have (a Codes race is judged by its tests first and by the merge later). Another service's id is refused.
+ */
+ async function recordMatch(match, { recordedBy }) {
+ const m = { ...match, id: match.id || `mat_${ids.ulid(s.now())}` };
+ return s.tx(async () => {
+ const prev = await db.maybe('SELECT * FROM bench_matches WHERE id = $1 FOR UPDATE', [m.id]);
+ let body = m;
+ let created = true;
+ if (prev) {
+ if (prev.recorded_by !== recordedBy) throw new BenchError(409, 'bench.match_taken', 'That match id was recorded by another service.');
+ const old = prev.body || {};
+ // jsonb keeps keys in its own order, so a judgement is known by its fields, not by its JSON text.
+ const keyOf = (j) => [j.judge, j.winner || '', j.signal || '', JSON.stringify(Object.entries(j.scores || {}).sort()),
+ (j.by && j.by.type) || '', (j.by && j.by.id_hash) || '', String(j.at || '')].join('|');
+ const seen = new Set((old.judgements || []).map(keyOf));
+ body = { ...old, ...m, judgements: [...(old.judgements || []), ...(m.judgements || []).filter((j) => !seen.has(keyOf(j)))] };
+ created = false;
+ await db.exec('UPDATE bench_matches SET body = $2::jsonb, updated_at = $3::timestamptz WHERE id = $1', [m.id, JSON.stringify(body), s.iso()]);
+ } else {
+ await db.exec(
+ `INSERT INTO bench_matches (id, source, task_kind, category, prompt_hash, recorded_by, body, at, created_at, updated_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::timestamptz, $9::timestamptz, $9::timestamptz)`,
+ [m.id, m.source, m.task.kind, m.task.category || null, m.task.prompt_hash || null, recordedBy, JSON.stringify(body), m.at, s.iso()],
+ );
+ }
+ const judges = [...new Set((body.judgements || []).map((j) => j.judge))];
+ const last = (body.judgements || []).slice(-1)[0];
+ const winner = last && last.winner && /^[a-h]$/.test(last.winner) ? ((body.slots || []).find((x) => x.slot === last.winner) || {}).entrant || null : null;
+ await insertOutbox('bench.match.recorded', {
+ match_id: body.id, source: body.source, task_kind: body.task.kind, category: body.task.category || null,
+ entrants: (body.slots || []).map((x) => x.entrant), judges, winner_entrant: winner, at: s.iso(),
+ });
+ return { match: body, created };
+ });
+ }
+
+ /** The matches a board reads: one task kind, optionally one category, since a day. */
+ async function matchesFor({ taskKind, category = null, since, limit = 50000 }) {
+ const rows = await db.many(
+ `SELECT body FROM bench_matches WHERE task_kind = $1 AND ($2::text IS NULL OR category = $2) AND at >= $3::timestamptz
+ ORDER BY at DESC LIMIT $4`,
+ [String(taskKind), category, since, Math.max(1, Math.min(200000, Number(limit) || 50000))],
+ );
+ return rows.map((r) => r.body);
+ }
+
return {
return {
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
createTask, getTask,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
- nextJudgement, castJudgement, pairwiseGames,
+ nextJudgement, castJudgement, pairwiseGames, recordMatch, matchesFor,
board, ratings, valueRows, modelSummary, dataset, datasetJsonl,
board, ratings, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
createEval, getEval, runEval, evalResults, evalDailyCount,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
new file mode 100644
new file mode 100644
@@ -0,0 +1,100 @@
@@ -0,0 +1,100 @@
+'use strict';
+
+/**
+ * Matches and the factor boards (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md): a service records the
+ * comparisons it ran, the same service adds a later judgement (a Codes race judged by its tests, then by the merge),
+ * nobody else can touch its match, and the code board rates model and harness apart from them.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { gamesOf } = require('../server/rating/match-games');
+
+const at = '2026-11-13T02:00:00Z';
+const agent = (h, m) => `agent:${h}+${m}#0123abcd`;
+function codeRace(a, b, winner, judge = 'test_suite') {
+ return {
+ source: 'codes', task: { kind: 'code', category: 'code.bugfix' },
+ slots: [{ slot: 'a', entrant: a, receipts: { outcome: 'succeeded', cost_usd: 0.2 } }, { slot: 'b', entrant: b, receipts: { outcome: 'succeeded', cost_usd: 0.1 } }],
+ blind: false,
+ judgements: [judge === 'test_suite'
+ ? { judge, scores: { a: winner === 'a' ? 1 : 0.5, b: winner === 'b' ? 1 : 0.5 }, by: { type: 'suite' }, at }
+ : { judge, winner, signal: 'merged', by: { type: 'person', id_hash: '0123456789abcdef' }, at }],
+ at,
+ };
+}
+
+(async () => {
+ // Pure: one judgement with a winner among three sides is two games; scores decide every pair; both_bad is none.
+ await check('a match becomes the pairwise games the fit reads', async () => {
+ const three = { slots: [{ slot: 'a', entrant: 'model:x' }, { slot: 'b', entrant: 'model:y' }, { slot: 'c', entrant: 'model:z' }],
+ judgements: [{ judge: 'human_pairwise', winner: 'b', at }, { judge: 'rubric', scores: { a: 3, b: 3, c: 1 }, at }, { judge: 'human_pairwise', winner: 'both_bad', at }] };
+ const g = gamesOf([three]);
+ assert.deepStrictEqual(g.map((x) => `${x.a}>${x.b}:${x.outcome}`), [
+ 'model:x>model:y:b', 'model:y>model:z:a',
+ 'model:x>model:y:tie', 'model:x>model:z:a', 'model:y>model:z:a',
+ ]);
+ });
+
+ const t = await boot();
+ try {
+ const codes = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
+ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
+ const reader = t.network.serviceToken({ sub: 'svc:compare', audience: 'openvibe.bench', cap: ['bench.board.read'] });
+
+ let firstId = null;
+ await check('a service records a match; it gets an id and the event is queued', async () => {
+ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a') });
+ assert.strictEqual(r.status, 201, r.text);
+ firstId = r.json().match.id;
+ assert.match(firstId, /^mat_[0-9A-HJKMNP-TV-Z]{26}$/);
+ const outbox = await t.store.db.many("SELECT payload FROM bench_outbox WHERE topic = 'bench.match.recorded'");
+ assert.strictEqual(outbox.length, 1);
+ assert.deepStrictEqual(outbox[0].payload.judges, ['test_suite']);
+ });
+
+ await check('the same service adds the merge later; the judgements add up', async () => {
+ const later = { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a', 'implicit'), id: firstId };
+ const r = await t.get('/api/v1/matches', { bearer: codes, json: later });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.deepStrictEqual(r.json().match.judgements.map((j) => j.judge), ['test_suite', 'implicit']);
+ const again = await t.get('/api/v1/matches', { bearer: codes, json: later });
+ assert.strictEqual(again.json().match.judgements.length, 2, 'the same judgement is not counted twice');
+ });
+
+ await check('another service cannot touch it; a bad match and a token without the capability are refused', async () => {
+ const other = await t.get('/api/v1/matches', { bearer: live, json: { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'b'), id: firstId } });
+ assert.strictEqual(other.status, 409, other.text);
+ const bad = await t.get('/api/v1/matches', { bearer: codes, json: { ...codeRace('model:x', 'model:y', 'a'), slots: [{ slot: 'a', entrant: 'model:x' }] } });
+ assert.strictEqual(bad.status, 400, bad.text);
+ const denied = await t.get('/api/v1/matches', { bearer: reader, json: codeRace('model:x', 'model:y', 'a') });
+ assert.strictEqual(denied.status, 403, denied.text);
+ });
+
+ await check('the code board rates the harnesses apart from the models', async () => {
+ const models = ['gpt-5-mini', 'glm-5.3', 'deepseek-flash'];
+ let n = 0;
+ for (const m1 of models) for (const m2 of models) {
+ for (let k = 0; k < 6; k++) {
+ // Codex beats OpenCode on the same model or a weaker one, most of the time.
+ const winner = (k % 3 === 2) ? 'b' : 'a';
+ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', m1), agent('opencode', m2), winner) });
+ assert.strictEqual(r.status, 201, r.text);
+ n += 1;
+ }
+ }
+ const board = await t.get('/api/v1/factors?task_kind=code&days=365');
+ assert.strictEqual(board.status, 200, board.text);
+ const b = board.json();
+ assert.strictEqual(b.matches, n + 1);
+ const effect = (h) => b.harnesses.find((x) => x.harness === h).effect;
+ assert.ok(effect('codex') > effect('opencode'), `codex ${effect('codex')} ahead of opencode ${effect('opencode')}`);
+ assert.ok(b.agents.length >= 6 && b.agents.every((x) => x.factors && x.factors.model && x.factors.harness));
+ assert.match(b.method, /model \+ harness/);
+ const badKind = await t.get('/api/v1/factors?task_kind=poetry');
+ assert.strictEqual(badKind.status, 400);
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})().catch((error) => { console.error(error); process.exitCode = 1; });