openvibers/openvibe.bench Commits

The Exchange arena: first-party rounds and votes, and ratings by task category (#130)

OP OpenVibers committed 7 days ago · 667300fc15 · 11 files, +381 −15

* The Exchange arena: a first-party round and vote API, and task-category ratings

- New kind `chat.arena` (room exchange, entry model_answer, human_pairwise, elo/cost/ms).
- `POST /api/v1/arena/rounds`: OpenVibe.Exchange only (service:exchange, bench.round.create). Builds one
  round from the prompt and the two answers it already has, with the service as creator so the person can
  vote on it. The arena's rounds skip a person's daily quota and have BENCH_ARENA_ROUNDS_DAILY (2000).
- `POST /api/v1/arena/rounds/:id/votes`: service:exchange + bench.judgement.cast, with
  `X-OV-Subject: usr_…` naming the person. The judgement is theirs (`human_pairwise`), one per round
  (a second is 409) and the round is revealed. The service can never vote on its own round.
- A judgement now also updates the task's `task_class` category beside `all` and the voter kind;
  `GET /api/v1/boards/:kindId?category=code` reads it and lists only the models that played in it.
- Docs page and CHANGELOG. Three hardcoded kind counts in tests now derive from KINDS.

Checked: `OV_ALLOW_SUITE=1 npm test` — 41/41 files, including the new test/arena.test.js (build, reveal,
double-vote refusal, wrong service, task category on the board).

* The arena refuses a model against itself

An arena round whose two answers name the same model could never move a rating,
so it is refused (422) rather than recorded and left without effect.

Checked: node test/arena.test.js.

Changes

ddiff --git a/CHANGELOG.md b/CHANGELOG.md
iindex 2189d86..987835c 100644
---- a/CHANGELOG.md
++++ b/CHANGELOG.md
@@@ -1,5 +1,16 @@
# Changelog
++## Unreleased
++
++- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:
++ `POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, and
++ `POST /api/v1/arena/rounds/:id/votes` records the person's own `human_pairwise` judgement from
++ `X-OV-Subject`. The service owns the round, so the person can vote on it; a second vote is 409 and the round
++ is revealed. The arena's rounds skip a person's daily quota and have their own daily cap.
++- **Task-category ratings.** A judgement now also updates the task's `task_class` category (code, writing, chat,
++ reasoning, translation, analysis) beside `all` and the voter kind, and `GET /api/v1/boards/:kindId?category=…`
++ reads it. A category board lists only the models that played in it.
++
## 0.1.0 — 2026-10-28
- OpenVibe.Bench v1: the shared AI benchmark and comparison engine (plan 2026-10-28 §4.12).
ddiff --git a/server/app.js b/server/app.js
iindex ee81bec..49b5143 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -30,6 +30,7 @@ const { createRegistryApi } = require('./http/registry');
const { createVariantEvidenceApi } = require('./http/variant-evidence');
const { createPageRoutes } = require('./http/pages');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
++const { createArenaRoutes } = require('./http/arena');
const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
const { createPickPages, createPickRoutes } = require('./http/pick');
const { createBadgeRoutes } = require('./http/badge');
@@@ -241,6 +242,7 @@ async function createApp(opts = {}) {
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createVariantEvidenceApi(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
++ app.use('/api/v1/arena', createArenaRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1', createPickRoutes(ctx));
app.use(require('./http/mirror').createMirrorRoutes(ctx));
ddiff --git a/server/config.js b/server/config.js
iindex 3700679..f654765 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -88,6 +88,9 @@ function load(env = process.env) {
pollMs: Math.max(250, int(env.BENCH_POLL_MS, 20_000)),
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
++ // The Arena's own cap: its rounds skip a person's daily quota (the Exchange already paces its
++ // people), so the day's total is the guard.
++ arenaRoundsDaily: Math.max(1, int(env.BENCH_ARENA_ROUNDS_DAILY, 2000)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
// Battles: how many votes one person and one network may cast.
battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),
ddiff --git a/server/http/arena.js b/server/http/arena.js
nnew file mode 100644
iindex 0000000..bf9db14
---- /dev/null
++++ b/server/http/arena.js
@@@ -0,0 +1,94 @@
++'use strict';
++
++/**
++ * The Exchange arena's own API, for OpenVibe.Exchange and nobody else.
++ *
++ * The Arena is a blind pair in Exchange's chat: a person's prompt, two models' answers side by side, a vote,
++ * then both names and their receipts. Each vote is a Bench human_pairwise judgement owned by that person, and
++ * each round belongs to the service so the person can vote on it.
++ *
++ * POST /api/v1/arena/rounds (service:exchange, bench.round.create)
++ * { prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }
++ * → 201 { round: { id, status }, category, entries: [{ id, side, provider, model }] }
++ *
++ * POST /api/v1/arena/rounds/:id/votes (service:exchange, bench.judgement.cast)
++ * x-ov-subject: the person's Network subject
++ * { outcome: left|right|tie|both_bad, pair: { left, right } }
++ * → 201 { judgement, revealed }
++ *
++ * A second vote by the same person is 409 (Bench's one-vote rule), and the service that built the round can
++ * never cast the vote itself: the judgement belongs to the person named in x-ov-subject.
++ */
++const express = require('express');
++const { http, ids } = require('openvibe-contracts');
++const { asyncRouter } = require('./router');
++
++const SERVICE = 'service:exchange';
++const ARENA_KIND = 'chat.arena';
++const OUTCOMES = new Set(['left', 'right', 'tie', 'both_bad']);
++
++function createArenaRoutes(ctx) {
++ const { bench, principal, log } = ctx;
++ const r = asyncRouter();
++ r.use(express.json({ limit: '128kb' }));
++ r.use((_req, res, next) => { res.set('Cache-Control', 'no-store'); next(); });
++ r.use(principal.middleware);
++
++ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });
++
++ const fromExchange = (req, res, next) => {
++ if (req.principal && req.principal.kind === 'app' && req.principal.requester === SERVICE) return next();
++ return problem(res, 403, 'bench.arena_service_only', 'The arena is built by OpenVibe.Exchange.');
++ };
++ const failed = (res, err) => {
++ if (err && err.status) return problem(res, err.status, err.code || 'bench.invalid', err.detail || 'The arena request was not accepted.');
++ log.warn('[OpenVibe.Bench] arena failed:', (err && err.message) || err);
++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
++ };
++
++ r.post('/rounds', principal.requireCapability('bench.round.create'), fromExchange, async (req, res) => {
++ const b = req.body && typeof req.body === 'object' ? req.body : {};
++ const prompt = String(b.prompt == null ? '' : b.prompt).replace(/\s+/g, ' ').trim();
++ if (!prompt) return problem(res, 422, 'bench.prompt_required', 'A round needs the prompt the models answered.');
++ try {
++ const out = await bench.createArenaRound({
++ prompt,
++ category: b.category == null ? null : String(b.category),
++ entries: Array.isArray(b.entries) ? b.entries : [],
++ });
++ res.set('Location', `/api/v1/arena/rounds/${out.round.id}`);
++ return res.status(201).json({
++ round: { id: out.round.id, status: out.round.status, kind_id: ARENA_KIND },
++ category: b.category == null ? null : String(b.category).slice(0, 40),
++ entries: out.entries,
++ });
++ } catch (err) { return failed(res, err); }
++ });
++
++ r.post('/rounds/:id/votes', principal.requireCapability('bench.judgement.cast'), fromExchange, async (req, res) => {
++ const b = req.body && typeof req.body === 'object' ? req.body : {};
++ const subject = String(req.headers['x-ov-subject'] || '');
++ if (!ids.isSubjectId('user', subject)) {
++ return problem(res, 422, 'bench.invalid_subject', 'x-ov-subject must be the Network subject of the person who voted.');
++ }
++ const outcome = String(b.outcome || '');
++ if (!OUTCOMES.has(outcome)) return problem(res, 422, 'bench.invalid_outcome', 'The outcome must be left, right, tie or both_bad.');
++ const pair = b.pair && typeof b.pair === 'object' ? { left: String(b.pair.left || ''), right: String(b.pair.right || '') } : null;
++ if (!pair || !pair.left || !pair.right || pair.left === pair.right) {
++ return problem(res, 422, 'bench.invalid_pair', 'A vote names the two entries it compares.');
++ }
++ try {
++ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });
++ if (!round || round.kind_id !== ARENA_KIND) return problem(res, 404, 'bench.round_not_found', 'No such arena round.');
++ if (round.created_by !== SERVICE) return problem(res, 403, 'bench.arena_service_only', 'That round was not built by OpenVibe.Exchange.');
++ const out = await bench.castJudgement({
++ roundId: round.id, judge: 'human_pairwise', pair, outcome, subject,
++ });
++ return res.status(201).json({ judgement: out.judgement, revealed: out.revealed });
++ } catch (err) { return failed(res, err); }
++ });
++
++ return r;
++}
++
++module.exports = { createArenaRoutes, SERVICE, ARENA_KIND };
ddiff --git a/server/kinds.js b/server/kinds.js
iindex 8a5837d..bdd90d7 100644
---- a/server/kinds.js
++++ b/server/kinds.js
@@@ -23,6 +23,7 @@ const KINDS = Object.freeze([
{ id: 'codes.review', room: 'codes', entry: 'review', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'review', payload: { review: 'string' }, prompt: { diff: 'string', intent: 'string' }, rubric: ['Find specific correctness or security defects', 'Cite the affected lines and explain impact', 'Avoid speculative or duplicate findings'], seeds: [{ diff: 'A cache key changes from userId to a constant', intent: 'Review for cross-user data exposure' }] } },
{ id: 'surf.desk', room: 'surf', entry: 'agent_on_desk', judge: 'human_pairwise', name: 'Agents on desks', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'agent_on_desk', payload: { desk: 'object' } } },
{ id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' }, prompt: { conversation: 'array', latest_user_message: 'string' }, rubric: ['Answer the latest request accurately', 'Use prior turns when relevant', 'Be clear and avoid invented facts'], seeds: [{ conversation: [{ role: 'user', content: 'I have 20 minutes for dinner.' }], latest_user_message: 'Suggest a quick vegetarian meal.' }] } },
++ { id: 'chat.arena', room: 'exchange', entry: 'model_answer', judge: 'human_pairwise', name: 'Exchange arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { answer: 'string' }, prompt: { text: 'string' } } },
{ id: 'speech.tts', room: 'speech', entry: 'audio', judge: 'human_pairwise', name: 'Text to speech voices', metrics: ['elo', 'cost_usd'], schema: { entry: 'audio', payload: { audio_url: 'string' } } },
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
ddiff --git a/server/store.js b/server/store.js
iindex 9381d3a..579d48b 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -20,6 +20,9 @@ const CATEGORY_DEFAULT = 'all';
// The rating categories: everyone ('all'), the signed-in people ('people') and the anonymous voters ('anon').
const CATEGORY_PEOPLE = 'people';
const CATEGORY_ANON = 'anon';
++// The Arena's kind and the first-party service that builds its rounds (OpenVibe.Exchange).
++const ARENA_KIND = 'chat.arena';
++const ARENA_SERVICE = 'service:exchange';
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
@@@ -119,12 +122,26 @@ function kindOfVoter(voterKind, subject) {
return 'service';
}
--/** The rating categories a vote is counted in: everyone, and the kind of voter it came from. */
--function categoriesOf(voterKind) {
++/**
++ * The rating categories a vote is counted in: everyone, the kind of voter it came from, and the task's own
++ * category (code, writing, chat, reasoning…) when the room classified the prompt. One vote updates each of
++ * them, so a board can show all votes, only people's, or only one kind of task.
++ */
++function categoriesOf(voterKind, taskCategory = null) {
const out = [CATEGORY_DEFAULT];
if (voterKind === 'person') out.push(CATEGORY_PEOPLE);
else if (voterKind === 'anon') out.push(CATEGORY_ANON);
-- return out;
++ const task = normalizeCategory(taskCategory);
++ if (task) out.push(task);
++ return [...new Set(out)];
++}
++
++/** A task category is a short slug; 'all', 'people' and 'anon' are the board's own, never a task's. */
++function normalizeCategory(value) {
++ const slug = String(value == null ? '' : value).toLowerCase()
++ .replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
++ if (!slug || slug === CATEGORY_DEFAULT || slug === CATEGORY_PEOPLE || slug === CATEGORY_ANON) return null;
++ return slug;
}
function wireRating(row) {
@@@ -302,6 +319,69 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return { ok: true, round: await getRound(id) };
}
++ /**
++ * One Arena round for OpenVibe.Exchange: a person's prompt, two or three answers from different models, and
++ * a round this service owns. The Exchange's rounds spend no AI and no person's daily quota here; they have
++ * their own daily cap. The voter is the person who asked, so the creator rule still keeps this service from
++ * voting on the round it built.
++ */
++ async function createArenaRound({ prompt, category = null, entries = [] } = {}) {
++ const list = Array.isArray(entries) ? entries.slice(0, 3) : [];
++ if (list.length < 2) throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs two answers.');
++ const clean = list.map((e) => ({
++ provider: String((e && e.provider) || '').slice(0, 80),
++ model: String((e && e.model) || '').slice(0, 120),
++ answer: String((e && e.answer) == null ? '' : e.answer).slice(0, 16000),
++ costUsd: num(e && e.cost_usd),
++ ms: num(e && e.ms),
++ tokens: num(e && e.tokens),
++ runId: e && e.run_id ? String(e.run_id).slice(0, 120) : null,
++ }));
++ if (clean.some((e) => !e.provider || !e.model || !e.answer.trim())) {
++ throw new BenchError(422, 'bench.invalid_entries', 'Every answer needs a provider, a model and text.');
++ }
++ // One model against itself is not a comparison: it could never move a rating.
++ if (new Set(clean.map((e) => e.model)).size < 2) {
++ throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs answers from two different models.');
++ }
++
++ const cap = config ? config.bench.arenaRoundsDaily : 2000;
++ const used = num(await db.value('SELECT COUNT(*)::int AS n FROM bench_rounds WHERE kind_id = $1 AND created_at >= $2::timestamptz',
++ [ARENA_KIND, dayStart()]));
++ if (used >= cap) throw new BenchError(429, 'bench.arena_limit', 'The arena has reached its rounds for today.');
++
++ const taskOut = await createTask({
++ kindId: ARENA_KIND, prompt, taskClass: normalizeCategory(category), public: true, subject: ARENA_SERVICE,
++ });
++ if (!taskOut.ok) throw new BenchError(taskOut.status || 400, taskOut.code || 'bench.invalid', taskOut.message || 'invalid task');
++ const built = await createRound({
++ taskId: taskOut.task.id,
++ offers: clean.map((e) => ({ provider: e.provider, model: e.model })),
++ subject: ARENA_SERVICE,
++ bypassQuota: true,
++ });
++ if (!built.ok) throw new BenchError(built.status || 400, built.code || 'bench.invalid', built.message || 'invalid round');
++
++ const roundId = built.round.id;
++ const made = [];
++ try {
++ for (const e of clean) {
++ made.push(await addEntry({
++ roundId, provider: e.provider, model: e.model, payload: { answer: e.answer },
++ costUsd: e.costUsd, ms: e.ms, tokens: e.tokens, runId: e.runId,
++ }));
++ }
++ } catch (err) {
++ await db.exec('DELETE FROM bench_rounds WHERE id = $1', [roundId]);
++ throw err;
++ }
++ const sides = ['left', 'right', 'third'];
++ return {
++ round: await getRound(roundId, { viewerSubject: ARENA_SERVICE }),
++ entries: made.map((e, i) => ({ id: e.id, side: sides[i], provider: e.provider, model: e.model })),
++ };
++ }
++
async function getRound(id, { viewerSubject = null } = {}) {
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1', [String(id || '')]);
if (!r) return null;
@@@ -477,7 +557,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
);
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
-- await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject));
++ const task = await getTask(r.task_id);
++ await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject),
++ task && task.task_class);
}
await insertOutbox('bench.judgement.recorded', {
judgement_id: id, round_id: r.id, kind_id: r.kind_id, judge, method: judge, outcome: outcome || null, score: score == null ? null : num(score),
@@@ -490,14 +572,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
}
}
-- async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person') {
++ async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person', taskCategory = null) {
if (outcome === 'both_bad') return;
if (!left || !right || left.model === right.model) return;
const models = [left.model, right.model];
const now = s.iso();
-- // The vote is counted twice: once for everyone ('all') and once for the kind of voter it came from
-- // ('people' or 'anon'), so a board can show the signed-in votes alone and still count the rest.
-- for (const category of categoriesOf(voterKind)) {
++ // The vote is counted for everyone ('all'), for the kind of voter it came from ('people' or 'anon')
++ // and for the task's own category (code, writing, chat, reasoning…), so a board can show the votes
++ // for one kind of work beside the whole.
++ for (const category of categoriesOf(voterKind, taskCategory)) {
for (const model of models) {
await db.exec(
`INSERT INTO bench_ratings (kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, updated_at)
@@@ -554,7 +637,10 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const subjects = await subjectsFor(kindId);
const ratings = await db.many('SELECT * FROM bench_ratings WHERE kind_id = $1 AND category = $2', [kindId, category]);
const by = new Map(ratings.map((r) => [r.subject, wireRating(r)]));
-- const rows = subjects.map((subject) => {
++ // A task category only lists the models that played in it: a model with no game in this kind of
++ // work has no place on its board, rather than a baseline score it never earned.
++ const taskCategory = ![CATEGORY_DEFAULT, CATEGORY_PEOPLE, CATEGORY_ANON].includes(category);
++ const rows = (taskCategory ? subjects.filter((subject) => by.has(subject)) : subjects).map((subject) => {
const r = by.get(subject);
const games = r ? r.games : 0;
return {
@@@ -611,7 +697,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const c = String(category == null ? '' : category).trim().toLowerCase();
if (c === 'people' || c === 'person') return CATEGORY_PEOPLE;
if (c === 'anon' || c === 'anonymous') return CATEGORY_ANON;
-- return CATEGORY_DEFAULT;
++ // A task category (code, writing, chat, reasoning…) is its own board.
++ return normalizeCategory(c) || CATEGORY_DEFAULT;
}
/** How many votes of each kind of voter a board holds. */
@@@ -847,7 +934,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
-- createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
++ createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
board, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
ddiff --git a/server/views/index.js b/server/views/index.js
iindex c3fd932..1fe213d 100644
---- a/server/views/index.js
++++ b/server/views/index.js
@@@ -57,7 +57,7 @@ ${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}
${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}
${ep('GET', '/api/v1/tasks/:id', 'One task.')}
${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}
--${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds.')}
++${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&category=code', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds. <code>category</code> reads one task class (code, writing, chat, reasoning, translation, analysis), which a vote updates beside the whole.')}
${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}
${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}
${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}
@@@ -66,6 +66,8 @@ ${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its vi
${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}
${ep('POST', '/api/v1/tasks/:id/rounds', '<code>{ offers?, seed? }</code>. Pins a reproducible round; the job generates entries.')}
${ep('POST', '/api/v1/rounds/:id/entries', 'A room submits outside entries (a game match, a test-suite run) with a service token carrying bench.entry.create.')}
++${ep('POST', '/api/v1/arena/rounds', '<code>{ prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }</code>. OpenVibe.Exchange only: one blind arena round from two answers, owned by the service so the person can vote on it.')}
++${ep('POST', '/api/v1/arena/rounds/:id/votes', '<code>{ outcome: left|right|tie|both_bad, pair: { left, right } }</code> with <code>X-OV-Subject: usr_…</code>. OpenVibe.Exchange casts the person’s own human_pairwise judgement; a second vote is 409 and the round is revealed.')}
${ep('GET', '/api/v1/judgements/next?kind=&judge=', 'The next blind pair for this person, or for an anonymous voter (the signed cookie, or the X-OV-Voter header a relaying service sends).')}
${ep('POST', '/api/v1/rounds/:id/judgements', '<code>{ judge?, pair?, outcome?, score?, vote_counts? }</code>. Human judges are people; automatic/test/game judges and audience_poll are services with bench.judgement.cast. An audience_poll supplies { left, right, tie, voters }; it changes Elo as one game regardless of audience size. A zero-voter poll is recorded without changing ratings. A service on the relay allow-list may cast the vote of an anonymous voter it names in <code>X-OV-Voter: anon:&lt;id&gt;</code>.')}
${ep('POST', '/api/v1/evals', 'Create a private task suite (never in the public dataset).')}
ddiff --git a/test/api.test.js b/test/api.test.js
iindex 4b0e017..a0f4918 100644
---- a/test/api.test.js
++++ b/test/api.test.js
@@@ -17,7 +17,7 @@ const { boot, check, done } = require('./helpers/boot');
await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {
const r = await t.get('/api/v1/kinds');
assert.strictEqual(r.status, 200, r.text);
-- assert.strictEqual(r.json().kinds.length, 25);
++ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);
const ids = r.json().kinds.map((k) => k.id);
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
assert.ok(ids.includes(want), `missing kind ${want}`);
ddiff --git a/test/arena.test.js b/test/arena.test.js
nnew file mode 100644
iindex 0000000..b861ed3
---- /dev/null
++++ b/test/arena.test.js
@@@ -0,0 +1,166 @@
++'use strict';
++/**
++ * The Exchange arena: the service builds a blind round from two answers, one person votes once, the names are
++ * revealed, and the vote lands in Bench as that person's own human_pairwise judgement — under the whole board,
++ * the people's board, and the board for the prompt's task category.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++
++(async () => {
++ const t = await boot();
++ const alice = t.network.addUser('alice');
++ const bob = t.network.addUser('bob');
++ const exchange = t.network.serviceToken({
++ sub: 'svc:exchange', audience: 'openvibe.bench',
++ cap: ['bench.round.create', 'bench.judgement.cast', 'bench.board.read'],
++ });
++ const other = t.network.serviceToken({
++ sub: 'svc:other', audience: 'openvibe.bench',
++ cap: ['bench.round.create', 'bench.judgement.cast'],
++ });
++
++ const ENTRIES = [
++ { provider: 'deepseek', model: 'deepseek/deepseek-chat@v1', answer: 'A mutex is a lock; a semaphore is a counter.', cost_usd: 0.0001, ms: 400, tokens: 40, run_id: 'run_left' },
++ { provider: 'novita', model: 'qwen/qwen3-32b@v1', answer: 'A semaphore can count; a mutex cannot.', cost_usd: 0.0002, ms: 520, tokens: 42, run_id: 'run_right' },
++ ];
++
++ let roundId;
++ let pair;
++
++ try {
++ await check('the Exchange builds a blind arena round from two answers', async () => {
++ const r = await t.get('/api/v1/arena/rounds', {
++ bearer: exchange,
++ json: { prompt: 'Explain the difference between a mutex and a semaphore', category: 'code', entries: ENTRIES },
++ });
++ assert.strictEqual(r.status, 201, r.text);
++ const body = r.json();
++ roundId = body.round.id;
++ assert.match(roundId, /^brd_/);
++ assert.strictEqual(body.round.status, 'finished', 'two answers finish the round');
++ assert.strictEqual(body.category, 'code');
++ assert.deepStrictEqual(body.entries.map((e) => e.side), ['left', 'right']);
++ assert.deepStrictEqual(body.entries.map((e) => e.model), ENTRIES.map((e) => e.model));
++ pair = { left: body.entries[0].id, right: body.entries[1].id };
++
++ const open = await t.get(`/api/v1/rounds/${roundId}`);
++ assert.strictEqual(open.status, 200, open.text);
++ assert.strictEqual(open.json().round.blind, true, 'blind before a vote');
++ assert.strictEqual(open.json().round.entries[0].model, undefined, 'no name before a vote');
++ });
++
++ await check('a service that is not Exchange cannot build or vote', async () => {
++ const make = await t.get('/api/v1/arena/rounds', {
++ bearer: other, json: { prompt: 'hi', category: 'chat', entries: ENTRIES },
++ });
++ assert.strictEqual(make.status, 403, make.text);
++ const vote = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: other, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'left', pair },
++ });
++ assert.strictEqual(vote.status, 403, vote.text);
++ });
++
++ await check('the vote names the person, and the person is not the round’s creator', async () => {
++ const missing = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: exchange, json: { outcome: 'left', pair },
++ });
++ assert.strictEqual(missing.status, 422, missing.text);
++ const bad = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'sideways', pair },
++ });
++ assert.strictEqual(bad.status, 422, bad.text);
++ });
++
++ await check('the vote is a human_pairwise judgement for that person, and the names are revealed', async () => {
++ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
++ json: { outcome: 'left', pair },
++ });
++ assert.strictEqual(r.status, 201, r.text);
++ const body = r.json();
++ assert.strictEqual(body.judgement.method, 'human_pairwise');
++ assert.strictEqual(body.judgement.outcome, 'left');
++ assert.strictEqual(body.judgement.round_id, roundId);
++ assert.ok(body.revealed && body.revealed.left.model && body.revealed.right.model, 'both names after the vote');
++
++ const row = await t.ctx.s.db.maybe('SELECT subject, method, outcome FROM bench_judgements WHERE round_id = $1', [roundId]);
++ assert.strictEqual(row.subject, bob.subject, 'the judgement belongs to the person');
++ assert.strictEqual(row.method, 'human_pairwise');
++ assert.strictEqual(row.outcome, 'left');
++
++ const open = await t.get(`/api/v1/rounds/${roundId}`, { bearer: t.network.userToken(bob) });
++ assert.strictEqual(open.json().round.blind, undefined, 'revealed to the person who voted');
++ assert.ok(open.json().round.entries.every((e) => e.model), 'both models are named');
++ });
++
++ await check('one vote per person per round: the second is 409', async () => {
++ const again = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
++ json: { outcome: 'right', pair },
++ });
++ assert.strictEqual(again.status, 409, again.text);
++ assert.strictEqual(again.json().code, 'bench.already_judged');
++ });
++
++ await check('another person votes on the same round', async () => {
++ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': alice.subject },
++ json: { outcome: 'right', pair },
++ });
++ assert.strictEqual(r.status, 201, r.text);
++ });
++
++ await check('the task category has its own board, beside all and people', async () => {
++ const board = await t.get('/api/v1/boards/chat.arena?category=code');
++ assert.strictEqual(board.status, 200, board.text);
++ const out = board.json().board;
++ assert.strictEqual(out.category, 'code');
++ assert.strictEqual(out.rows.length, 2, 'only the models that played in this category');
++ for (const row of out.rows) {
++ assert.strictEqual(row.games, 2, `${row.subject} played both votes`);
++ assert.ok(row.elo > 900 && row.elo < 1100, `${row.subject} has an Elo`);
++ }
++ const all = await t.get('/api/v1/boards/chat.arena');
++ assert.strictEqual(all.json().board.category, 'all');
++ const people = await t.get('/api/v1/boards/chat.arena?voters=people');
++ assert.strictEqual(people.json().board.category, 'people');
++ });
++
++ await check('a vote names two entries of the round', async () => {
++ const make = await t.get('/api/v1/arena/rounds', {
++ bearer: exchange, json: { prompt: 'Write a haiku about rain', category: 'writing', entries: ENTRIES },
++ });
++ const id = make.json().round.id;
++ const bad = await t.get(`/api/v1/arena/rounds/${id}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
++ json: { outcome: 'left', pair: { left: pair.left, right: pair.right } },
++ });
++ assert.strictEqual(bad.status, 422, bad.text);
++ const good = await t.get(`/api/v1/arena/rounds/${id}/votes`, {
++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
++ json: { outcome: 'tie', pair: { left: make.json().entries[0].id, right: make.json().entries[1].id } },
++ });
++ assert.strictEqual(good.status, 201, good.text);
++ assert.strictEqual(good.json().judgement.outcome, 'tie');
++ });
++
++ await check('a round needs two different answers', async () => {
++ const one = await t.get('/api/v1/arena/rounds', {
++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0]] },
++ });
++ assert.strictEqual(one.status, 422, one.text);
++ const blank = await t.get('/api/v1/arena/rounds', {
++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [{ provider: 'x', model: 'y', answer: ' ' }, ENTRIES[1]] },
++ });
++ assert.strictEqual(blank.status, 422, blank.text);
++ const same = await t.get('/api/v1/arena/rounds', {
++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0], { ...ENTRIES[1], model: ENTRIES[0].model }] },
++ });
++ assert.strictEqual(same.status, 422, same.text);
++ assert.ok(same.json().detail.includes('two different models'));
++ });
++ } finally { await t.close(); }
++
++ done();
++})();
ddiff --git a/test/mcp.test.js b/test/mcp.test.js
iindex a5e2c18..5f33c96 100644
---- a/test/mcp.test.js
++++ b/test/mcp.test.js
@@@ -58,7 +58,7 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(param
await check('bench.list_kinds lists every kind Bench runs', async () => {
const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });
assert.strictEqual(r.json().result.isError, false);
-- assert.strictEqual(r.json().result.structuredContent.kinds.length, 25);
++ assert.strictEqual(r.json().result.structuredContent.kinds.length, require('../server/kinds').KINDS.length);
});
await check('bench.get_board renders a board without a person', async () => {
ddiff --git a/test/showcase.test.js b/test/showcase.test.js
iindex 08b4be7..deaf01e 100644
---- a/test/showcase.test.js
++++ b/test/showcase.test.js
@@@ -28,7 +28,7 @@ const HOSTILE = [
const r = await t.get('/api/v1/kinds');
assert.strictEqual(r.status, 200, r.text);
const byId = new Map(r.json().kinds.map((k) => [k.id, k]));
-- assert.strictEqual(r.json().kinds.length, 25);
++ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);
const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];
for (const id of want) {
const k = byId.get(id);