openvibers/openvibe.bench Commits

Matches: every comparison as one record, and the model/harness factor board (#230)

OP OpenVibers committed 10 h ago · 276ff0a65f · 10 files, +269 −8

- POST /api/v1/matches: a service with bench.judgement.cast records a bench.match@1 (an Arena race, a Codes race,
  a check job, a game). An id it recorded before gains the judgements it lacked (tests first, the merge later);
  another service's id is refused (409). Each write queues bench.match.recorded.
- GET /api/v1/factors?task_kind=&category=&days=: the factor fit (server/rating/factors.js) over the matches of a task
  kind, style-controlled, with intervals, cached five minutes.
- server/rating/match-games.js turns a match into pairwise games: a winner beats every other side, scores decide
  every pair, both_bad says nothing; the voter's weight and the answers' style come along.
- Migration 0018_matches (expand). Pins openvibe-contracts v0.422.0 (bench.match@1, the event).
- Checked by matches.test.js: the games from a three-sided match; record, add a later judgement once, refuse another
  service, a bad match and a missing capability; a code board where Codex's effect leads OpenCode's.

Changes

ddiff --git a/README.md b/README.md
iindex 4bbecd8..94d969c 100644
---- a/README.md
++++ b/README.md
@@@ -39,7 +39,7 @@ JSON under `/api/v1`; errors are `application/problem+json`. The full list is in
The public reads Report's AI pages use are `GET /api/v1/overview`, `GET /api/v1/benchmarks`, `GET /api/v1/rankings`, `GET /api/v1/runs`, `GET /api/v1/models/:model/summary` and `GET /api/v1/runs/:id`; the long list is at `/openapi.json` and in Report's method page.
<!-- versions:start -->
--- openvibe-contracts: v0.419.0
++- openvibe-contracts: v0.422.0
- openvibe-sdk: v0.161.0
- openvibe-shared: v4.232.0
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 0a2d626..2c3da80 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -4,7 +4,7 @@
"version": "0.1.0",
"status": "v1",
"summary": "The shared AI benchmark and comparison engine for every room: kinds, pinned rounds, blind judgements, elo/pareto/cost boards, an open dataset per kind, private evals and an MCP endpoint. Its public pages are OpenVibe.Report's AI section since 2026-11-08; this origin answers 301 for them and keeps the API.",
-- "contracts": "openvibe-contracts v0.419.0",
++ "contracts": "openvibe-contracts v0.422.0",
"sdk": "openvibe-sdk v0.161.0",
"shared": "openvibe-shared v4.232.0",
"updated": "2026-11-13"
ddiff --git a/migrations/0018_matches.sql b/migrations/0018_matches.sql
nnew file mode 100644
iindex 0000000..a660c27
---- /dev/null
++++ b/migrations/0018_matches.sql
@@@ -0,0 +1,20 @@
++-- phase: expand
++-- Every comparison on the network as one record (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md,
++-- 2026-11-13): an Arena vote, a Codes race whose patch was merged, a check job's tests, a game, a Live poll. The
++-- boards that rate model and harness apart (server/rating/factors.js) are fitted from these.
++--
++-- BODY is the whole bench.match@1 document (sides, receipts, shown order, judgements); the columns beside it are what
++-- a board selects on. RECORDED_BY is the service that sent it: only that service adds judgements to it later.
++CREATE TABLE IF NOT EXISTS bench_matches (
++ id text COLLATE "C" PRIMARY KEY,
++ source text COLLATE "C" NOT NULL,
++ task_kind text COLLATE "C" NOT NULL,
++ category text COLLATE "C",
++ prompt_hash text COLLATE "C",
++ recorded_by text COLLATE "C" NOT NULL,
++ body jsonb NOT NULL,
++ at timestamptz NOT NULL,
++ created_at timestamptz NOT NULL DEFAULT now(),
++ updated_at timestamptz NOT NULL DEFAULT now()
++);
++CREATE INDEX IF NOT EXISTS bench_matches_board ON bench_matches (task_kind, category, at);
ddiff --git a/package-lock.json b/package-lock.json
iindex 0c92faf..8604368 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -15,7 +15,7 @@
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"pg": "^8.23.0",
@@@ -1232,9 +1232,9 @@
}
},
"node_modules/openvibe-contracts": {
-- "version": "0.419.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
-- "integrity": "sha512-VxrTdeHchT8gvl2wWC9ASaId92en7ZKfJ8F/0hKzCAD0f2Y00rPrqgBWF9BNPoLGARoyQvsMbhBEhSkhxGNnjA==",
++ "version": "0.422.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
++ "integrity": "sha512-xQQq7NQLM9iQve8dR4Hc2EUp75x/FZYfAsHumsmf6dGHjIy05PMX0PlYmxw5L52VXKOj+r7mRwlcehRqgMoLSw==",
"license": "AGPL-3.0-only",
"dependencies": {
"ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 9cbb46c..e68f7f8 100644
---- a/package.json
++++ b/package.json
@@@ -22,7 +22,7 @@
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.422.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.232.0",
"pg": "^8.23.0",
ddiff --git a/server/contracts.js b/server/contracts.js
iindex 9937913..74fb34e 100644
---- a/server/contracts.js
++++ b/server/contracts.js
@@@ -46,6 +46,7 @@ const BENCH_EVENTS = Object.freeze([
'bench.round.finished',
'bench.judgement.recorded',
'bench.rating.changed',
++ 'bench.match.recorded',
]);
function published(name) {
ddiff --git a/server/http/api.js b/server/http/api.js
iindex c171539..8b7e077 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -14,6 +14,8 @@
* POST /rounds/:id/judgements
* GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon
* GET /models/:model
++ * POST /matches (bench.match@1 from a service with bench.judgement.cast)
++ * GET /factors?task_kind=&category=&days= model and harness apart, with intervals (bench.rating@1 factors)
* GET /datasets/:kindId.jsonl
* POST /evals
* GET /evals/:id
@@@ -414,6 +416,41 @@ function createApi(ctx) {
return res.json(out);
});
++ // ── Matches and the factor boards (compare-arena-ecosystem.md, Contracts 0.422.0) ──
++ // A service records a match it ran (the Arena's race, a Codes race, a check job, a game); the boards rate model
++ // and harness apart from every match of a task kind (server/rating/factors.js), cached a few minutes.
++ r.post('/matches', principal.requireCapability('bench.judgement.cast'), async (req, res) => {
++ const body = { ...(req.body || {}) };
++ if (!body.id) body.id = `mat_${ids.ulid(Date.now())}`;
++ const checked = require('openvibe-contracts').validate('bench.match@1', body);
++ if (!checked.valid) return invalid(res, `not a bench.match@1: ${checked.errors.slice(0, 3).map((e) => `${e.path} ${e.message}`).join('; ')}`);
++ try {
++ const out = await bench.recordMatch(body, { recordedBy: req.principal.requester });
++ return res.status(out.created ? 201 : 200).json(out);
++ } catch (err) { return failed(res, err); }
++ });
++
++ const factorCache = new Map();
++ r.get('/factors', async (req, res) => {
++ const taskKind = String(req.query.task_kind || '');
++ if (!['chat', 'code', 'design', 'image', 'speech', 'search', 'computer', 'game'].includes(taskKind)) return invalid(res, 'task_kind must be chat, code, design, image, speech, search, computer or game');
++ const category = req.query.category ? String(req.query.category).slice(0, 64) : null;
++ const days = [7, 30, 90, 365].includes(Number(req.query.days)) ? Number(req.query.days) : 90;
++ const key = `${taskKind}|${category || ''}|${days}`;
++ const hit = factorCache.get(key);
++ if (hit && hit.until > Date.now()) { res.set('Cache-Control', 'public, max-age=120'); return res.json(hit.value); }
++ const since = new Date(Date.now() - days * 86400000).toISOString();
++ const matches = await bench.matchesFor({ taskKind, category, since });
++ const { gamesOf, COVARIATES } = require('../rating/match-games');
++ const games = gamesOf(matches);
++ const fit = require('../rating/factors').fitFactors(games, { covariates: COVARIATES, bootstrap: games.length > 20000 ? 60 : 150 });
++ const value = { task_kind: taskKind, category, days, matches: matches.length, games: games.length, style_control: true, ...fit };
++ factorCache.set(key, { until: Date.now() + 5 * 60_000, value });
++ if (factorCache.size > 200) factorCache.delete(factorCache.keys().next().value);
++ res.set('Cache-Control', 'public, max-age=120');
++ return res.json(value);
++ });
++
return r;
}
ddiff --git a/server/rating/match-games.js b/server/rating/match-games.js
nnew file mode 100644
iindex 0000000..7c64e29
---- /dev/null
++++ b/server/rating/match-games.js
@@@ -0,0 +1,54 @@
++'use strict';
++
++/**
++ * bench.match@1 → the pairwise games fitFactors reads (compare-arena-ecosystem.md, 2026-11-13).
++ *
++ * A match has two to eight sides and any number of judgements. Each judgement becomes games between pairs of sides:
++ *
++ * winner a … h the winner beats every other side (a person, a Live poll, an implicit signal, a game)
++ * winner tie every pair ties
++ * winner both_bad says nothing about the order: no game
++ * scores (tests passed, a rubric score, a game's points) every pair is decided by the higher score,
++ * an equal score is a tie
++ *
++ * A judgement's weight (the voter's quality) carries into its games; a side that failed, timed out or was refused
++ * still plays (it lost on its merits). The style covariates are log(1 + length) and log(1 + markdown items) of each
++ * side's answer, so the fit can hold style out of the score.
++ */
++
++const COVARIATES = 2;
++const styleOf = (slot) => {
++ const r = (slot && slot.receipts) || {};
++ return [Math.log1p(Math.max(0, Number(r.length_chars) || 0)), Math.log1p(Math.max(0, Number(r.markdown_items) || 0))];
++};
++
++function gamesOf(matches) {
++ const games = [];
++ for (const m of Array.isArray(matches) ? matches : []) {
++ const slots = new Map((m.slots || []).map((s) => [s.slot, s]));
++ if (slots.size < 2) continue;
++ const letters = [...slots.keys()];
++ const pairs = [];
++ for (let i = 0; i < letters.length; i++) for (let j = i + 1; j < letters.length; j++) pairs.push([letters[i], letters[j]]);
++ for (const j of m.judgements || []) {
++ const weight = j.weight == null ? 1 : Math.max(0, Number(j.weight) || 0);
++ if (!weight) continue;
++ for (const [x, y] of pairs) {
++ let outcome = null;
++ if (j.winner === 'tie') outcome = 'tie';
++ else if (j.winner && j.winner !== 'both_bad' && slots.has(j.winner)) outcome = j.winner === x ? 'a' : j.winner === y ? 'b' : null;
++ else if (j.scores && Number.isFinite(Number(j.scores[x])) && Number.isFinite(Number(j.scores[y]))) {
++ const sx = Number(j.scores[x]);
++ const sy = Number(j.scores[y]);
++ outcome = sx > sy ? 'a' : sx < sy ? 'b' : 'tie';
++ }
++ // A winner beats every other side; two losers did not play each other.
++ if (!outcome) continue;
++ games.push({ a: slots.get(x).entrant, b: slots.get(y).entrant, outcome, weight, fa: styleOf(slots.get(x)), fb: styleOf(slots.get(y)), judge: j.judge });
++ }
++ }
++ }
++ return games;
++}
++
++module.exports = { gamesOf, COVARIATES };
ddiff --git a/server/store.js b/server/store.js
iindex 699b370..a100b37 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -1189,12 +1189,61 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
return { eval: ev, runs };
}
++ // ── Matches (bench.match@1, Contracts 0.422.0): every comparison on the network as one record ──
++ /**
++ * Record a match. A new id is inserted; an id the same service recorded before gains the judgements it did not
++ * have (a Codes race is judged by its tests first and by the merge later). Another service's id is refused.
++ */
++ async function recordMatch(match, { recordedBy }) {
++ const m = { ...match, id: match.id || `mat_${ids.ulid(s.now())}` };
++ return s.tx(async () => {
++ const prev = await db.maybe('SELECT * FROM bench_matches WHERE id = $1 FOR UPDATE', [m.id]);
++ let body = m;
++ let created = true;
++ if (prev) {
++ if (prev.recorded_by !== recordedBy) throw new BenchError(409, 'bench.match_taken', 'That match id was recorded by another service.');
++ const old = prev.body || {};
++ // jsonb keeps keys in its own order, so a judgement is known by its fields, not by its JSON text.
++ const keyOf = (j) => [j.judge, j.winner || '', j.signal || '', JSON.stringify(Object.entries(j.scores || {}).sort()),
++ (j.by && j.by.type) || '', (j.by && j.by.id_hash) || '', String(j.at || '')].join('|');
++ const seen = new Set((old.judgements || []).map(keyOf));
++ body = { ...old, ...m, judgements: [...(old.judgements || []), ...(m.judgements || []).filter((j) => !seen.has(keyOf(j)))] };
++ created = false;
++ await db.exec('UPDATE bench_matches SET body = $2::jsonb, updated_at = $3::timestamptz WHERE id = $1', [m.id, JSON.stringify(body), s.iso()]);
++ } else {
++ await db.exec(
++ `INSERT INTO bench_matches (id, source, task_kind, category, prompt_hash, recorded_by, body, at, created_at, updated_at)
++ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::timestamptz, $9::timestamptz, $9::timestamptz)`,
++ [m.id, m.source, m.task.kind, m.task.category || null, m.task.prompt_hash || null, recordedBy, JSON.stringify(body), m.at, s.iso()],
++ );
++ }
++ const judges = [...new Set((body.judgements || []).map((j) => j.judge))];
++ const last = (body.judgements || []).slice(-1)[0];
++ const winner = last && last.winner && /^[a-h]$/.test(last.winner) ? ((body.slots || []).find((x) => x.slot === last.winner) || {}).entrant || null : null;
++ await insertOutbox('bench.match.recorded', {
++ match_id: body.id, source: body.source, task_kind: body.task.kind, category: body.task.category || null,
++ entrants: (body.slots || []).map((x) => x.entrant), judges, winner_entrant: winner, at: s.iso(),
++ });
++ return { match: body, created };
++ });
++ }
++
++ /** The matches a board reads: one task kind, optionally one category, since a day. */
++ async function matchesFor({ taskKind, category = null, since, limit = 50000 }) {
++ const rows = await db.many(
++ `SELECT body FROM bench_matches WHERE task_kind = $1 AND ($2::text IS NULL OR category = $2) AND at >= $3::timestamptz
++ ORDER BY at DESC LIMIT $4`,
++ [String(taskKind), category, since, Math.max(1, Math.min(200000, Number(limit) || 50000))],
++ );
++ return rows.map((r) => r.body);
++ }
++
return {
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
-- nextJudgement, castJudgement, pairwiseGames,
++ nextJudgement, castJudgement, pairwiseGames, recordMatch, matchesFor,
board, ratings, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
ddiff --git a/test/matches.test.js b/test/matches.test.js
nnew file mode 100644
iindex 0000000..d10b688
---- /dev/null
++++ b/test/matches.test.js
@@@ -0,0 +1,100 @@
++'use strict';
++
++/**
++ * Matches and the factor boards (bench.match@1, Contracts 0.422.0, compare-arena-ecosystem.md): a service records the
++ * comparisons it ran, the same service adds a later judgement (a Codes race judged by its tests, then by the merge),
++ * nobody else can touch its match, and the code board rates model and harness apart from them.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++const { gamesOf } = require('../server/rating/match-games');
++
++const at = '2026-11-13T02:00:00Z';
++const agent = (h, m) => `agent:${h}+${m}#0123abcd`;
++function codeRace(a, b, winner, judge = 'test_suite') {
++ return {
++ source: 'codes', task: { kind: 'code', category: 'code.bugfix' },
++ slots: [{ slot: 'a', entrant: a, receipts: { outcome: 'succeeded', cost_usd: 0.2 } }, { slot: 'b', entrant: b, receipts: { outcome: 'succeeded', cost_usd: 0.1 } }],
++ blind: false,
++ judgements: [judge === 'test_suite'
++ ? { judge, scores: { a: winner === 'a' ? 1 : 0.5, b: winner === 'b' ? 1 : 0.5 }, by: { type: 'suite' }, at }
++ : { judge, winner, signal: 'merged', by: { type: 'person', id_hash: '0123456789abcdef' }, at }],
++ at,
++ };
++}
++
++(async () => {
++ // Pure: one judgement with a winner among three sides is two games; scores decide every pair; both_bad is none.
++ await check('a match becomes the pairwise games the fit reads', async () => {
++ const three = { slots: [{ slot: 'a', entrant: 'model:x' }, { slot: 'b', entrant: 'model:y' }, { slot: 'c', entrant: 'model:z' }],
++ judgements: [{ judge: 'human_pairwise', winner: 'b', at }, { judge: 'rubric', scores: { a: 3, b: 3, c: 1 }, at }, { judge: 'human_pairwise', winner: 'both_bad', at }] };
++ const g = gamesOf([three]);
++ assert.deepStrictEqual(g.map((x) => `${x.a}>${x.b}:${x.outcome}`), [
++ 'model:x>model:y:b', 'model:y>model:z:a',
++ 'model:x>model:y:tie', 'model:x>model:z:a', 'model:y>model:z:a',
++ ]);
++ });
++
++ const t = await boot();
++ try {
++ const codes = t.network.serviceToken({ sub: 'svc:codes', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
++ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
++ const reader = t.network.serviceToken({ sub: 'svc:compare', audience: 'openvibe.bench', cap: ['bench.board.read'] });
++
++ let firstId = null;
++ await check('a service records a match; it gets an id and the event is queued', async () => {
++ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a') });
++ assert.strictEqual(r.status, 201, r.text);
++ firstId = r.json().match.id;
++ assert.match(firstId, /^mat_[0-9A-HJKMNP-TV-Z]{26}$/);
++ const outbox = await t.store.db.many("SELECT payload FROM bench_outbox WHERE topic = 'bench.match.recorded'");
++ assert.strictEqual(outbox.length, 1);
++ assert.deepStrictEqual(outbox[0].payload.judges, ['test_suite']);
++ });
++
++ await check('the same service adds the merge later; the judgements add up', async () => {
++ const later = { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'a', 'implicit'), id: firstId };
++ const r = await t.get('/api/v1/matches', { bearer: codes, json: later });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.deepStrictEqual(r.json().match.judgements.map((j) => j.judge), ['test_suite', 'implicit']);
++ const again = await t.get('/api/v1/matches', { bearer: codes, json: later });
++ assert.strictEqual(again.json().match.judgements.length, 2, 'the same judgement is not counted twice');
++ });
++
++ await check('another service cannot touch it; a bad match and a token without the capability are refused', async () => {
++ const other = await t.get('/api/v1/matches', { bearer: live, json: { ...codeRace(agent('codex', 'gpt-5-mini'), agent('opencode', 'glm-5.3'), 'b'), id: firstId } });
++ assert.strictEqual(other.status, 409, other.text);
++ const bad = await t.get('/api/v1/matches', { bearer: codes, json: { ...codeRace('model:x', 'model:y', 'a'), slots: [{ slot: 'a', entrant: 'model:x' }] } });
++ assert.strictEqual(bad.status, 400, bad.text);
++ const denied = await t.get('/api/v1/matches', { bearer: reader, json: codeRace('model:x', 'model:y', 'a') });
++ assert.strictEqual(denied.status, 403, denied.text);
++ });
++
++ await check('the code board rates the harnesses apart from the models', async () => {
++ const models = ['gpt-5-mini', 'glm-5.3', 'deepseek-flash'];
++ let n = 0;
++ for (const m1 of models) for (const m2 of models) {
++ for (let k = 0; k < 6; k++) {
++ // Codex beats OpenCode on the same model or a weaker one, most of the time.
++ const winner = (k % 3 === 2) ? 'b' : 'a';
++ const r = await t.get('/api/v1/matches', { bearer: codes, json: codeRace(agent('codex', m1), agent('opencode', m2), winner) });
++ assert.strictEqual(r.status, 201, r.text);
++ n += 1;
++ }
++ }
++ const board = await t.get('/api/v1/factors?task_kind=code&days=365');
++ assert.strictEqual(board.status, 200, board.text);
++ const b = board.json();
++ assert.strictEqual(b.matches, n + 1);
++ const effect = (h) => b.harnesses.find((x) => x.harness === h).effect;
++ assert.ok(effect('codex') > effect('opencode'), `codex ${effect('codex')} ahead of opencode ${effect('opencode')}`);
++ assert.ok(b.agents.length >= 6 && b.agents.every((x) => x.factors && x.factors.model && x.factors.harness));
++ assert.match(b.method, /model \+ harness/);
++ const badKind = await t.get('/api/v1/factors?task_kind=poetry');
++ assert.strictEqual(badKind.status, 400);
++ });
++ } finally {
++ await t.close();
++ }
++ done();
++})().catch((error) => { console.error(error); process.exitCode = 1; });