openvibers/openvibe.bench Commits
The Exchange arena: first-party rounds and votes, and ratings by task category (#130)
Changes
@@@ -1,5 +1,16 @@# Changelog++## Unreleased++++- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:++ `POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, and++ `POST /api/v1/arena/rounds/:id/votes` records the person's own `human_pairwise` judgement from++ `X-OV-Subject`. The service owns the round, so the person can vote on it; a second vote is 409 and the round++ is revealed. The arena's rounds skip a person's daily quota and have their own daily cap.++- **Task-category ratings.** A judgement now also updates the task's `task_class` category (code, writing, chat,++ reasoning, translation, analysis) beside `all` and the voter kind, and `GET /api/v1/boards/:kindId?category=…`++ reads it. A category board lists only the models that played in it.++## 0.1.0 — 2026-10-28- OpenVibe.Bench v1: the shared AI benchmark and comparison engine (plan 2026-10-28 §4.12).@@@ -30,6 +30,7 @@ const { createRegistryApi } = require('./http/registry');const { createVariantEvidenceApi } = require('./http/variant-evidence');const { createPageRoutes } = require('./http/pages');const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');++const { createArenaRoutes } = require('./http/arena');const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');const { createPickPages, createPickRoutes } = require('./http/pick');const { createBadgeRoutes } = require('./http/badge');@@@ -241,6 +242,7 @@ async function createApp(opts = {}) {app.use('/api/v1', createRegistryApi(ctx));app.use('/api/v1', createVariantEvidenceApi(ctx));app.use('/api/v1/showcase', createShowcaseRoutes(ctx));++ app.use('/api/v1/arena', createArenaRoutes(ctx));app.use('/api/v1/gallery', createGalleryRoutes(ctx));app.use('/api/v1', createPickRoutes(ctx));app.use(require('./http/mirror').createMirrorRoutes(ctx));@@@ -88,6 +88,9 @@ function load(env = process.env) {pollMs: Math.max(250, int(env.BENCH_POLL_MS, 20_000)),maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),++ // The Arena's own cap: its rounds skip a person's daily quota (the Exchange already paces its++ // people), so the day's total is the guard.++ arenaRoundsDaily: Math.max(1, int(env.BENCH_ARENA_ROUNDS_DAILY, 2000)),provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),// Battles: how many votes one person and one network may cast.battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),nnew file mode 100644@@@ -0,0 +1,94 @@++'use strict';++++/**++ * The Exchange arena's own API, for OpenVibe.Exchange and nobody else.++ *++ * The Arena is a blind pair in Exchange's chat: a person's prompt, two models' answers side by side, a vote,++ * then both names and their receipts. Each vote is a Bench human_pairwise judgement owned by that person, and++ * each round belongs to the service so the person can vote on it.++ *++ * POST /api/v1/arena/rounds (service:exchange, bench.round.create)++ * { prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }++ * → 201 { round: { id, status }, category, entries: [{ id, side, provider, model }] }++ *++ * POST /api/v1/arena/rounds/:id/votes (service:exchange, bench.judgement.cast)++ * x-ov-subject: the person's Network subject++ * { outcome: left|right|tie|both_bad, pair: { left, right } }++ * → 201 { judgement, revealed }++ *++ * A second vote by the same person is 409 (Bench's one-vote rule), and the service that built the round can++ * never cast the vote itself: the judgement belongs to the person named in x-ov-subject.++ */++const express = require('express');++const { http, ids } = require('openvibe-contracts');++const { asyncRouter } = require('./router');++++const SERVICE = 'service:exchange';++const ARENA_KIND = 'chat.arena';++const OUTCOMES = new Set(['left', 'right', 'tie', 'both_bad']);++++function createArenaRoutes(ctx) {++ const { bench, principal, log } = ctx;++ const r = asyncRouter();++ r.use(express.json({ limit: '128kb' }));++ r.use((_req, res, next) => { res.set('Cache-Control', 'no-store'); next(); });++ r.use(principal.middleware);++++ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });++++ const fromExchange = (req, res, next) => {++ if (req.principal && req.principal.kind === 'app' && req.principal.requester === SERVICE) return next();++ return problem(res, 403, 'bench.arena_service_only', 'The arena is built by OpenVibe.Exchange.');++ };++ const failed = (res, err) => {++ if (err && err.status) return problem(res, err.status, err.code || 'bench.invalid', err.detail || 'The arena request was not accepted.');++ log.warn('[OpenVibe.Bench] arena failed:', (err && err.message) || err);++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');++ };++++ r.post('/rounds', principal.requireCapability('bench.round.create'), fromExchange, async (req, res) => {++ const b = req.body && typeof req.body === 'object' ? req.body : {};++ const prompt = String(b.prompt == null ? '' : b.prompt).replace(/\s+/g, ' ').trim();++ if (!prompt) return problem(res, 422, 'bench.prompt_required', 'A round needs the prompt the models answered.');++ try {++ const out = await bench.createArenaRound({++ prompt,++ category: b.category == null ? null : String(b.category),++ entries: Array.isArray(b.entries) ? b.entries : [],++ });++ res.set('Location', `/api/v1/arena/rounds/${out.round.id}`);++ return res.status(201).json({++ round: { id: out.round.id, status: out.round.status, kind_id: ARENA_KIND },++ category: b.category == null ? null : String(b.category).slice(0, 40),++ entries: out.entries,++ });++ } catch (err) { return failed(res, err); }++ });++++ r.post('/rounds/:id/votes', principal.requireCapability('bench.judgement.cast'), fromExchange, async (req, res) => {++ const b = req.body && typeof req.body === 'object' ? req.body : {};++ const subject = String(req.headers['x-ov-subject'] || '');++ if (!ids.isSubjectId('user', subject)) {++ return problem(res, 422, 'bench.invalid_subject', 'x-ov-subject must be the Network subject of the person who voted.');++ }++ const outcome = String(b.outcome || '');++ if (!OUTCOMES.has(outcome)) return problem(res, 422, 'bench.invalid_outcome', 'The outcome must be left, right, tie or both_bad.');++ const pair = b.pair && typeof b.pair === 'object' ? { left: String(b.pair.left || ''), right: String(b.pair.right || '') } : null;++ if (!pair || !pair.left || !pair.right || pair.left === pair.right) {++ return problem(res, 422, 'bench.invalid_pair', 'A vote names the two entries it compares.');++ }++ try {++ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });++ if (!round || round.kind_id !== ARENA_KIND) return problem(res, 404, 'bench.round_not_found', 'No such arena round.');++ if (round.created_by !== SERVICE) return problem(res, 403, 'bench.arena_service_only', 'That round was not built by OpenVibe.Exchange.');++ const out = await bench.castJudgement({++ roundId: round.id, judge: 'human_pairwise', pair, outcome, subject,++ });++ return res.status(201).json({ judgement: out.judgement, revealed: out.revealed });++ } catch (err) { return failed(res, err); }++ });++++ return r;++}++++module.exports = { createArenaRoutes, SERVICE, ARENA_KIND };@@@ -23,6 +23,7 @@ const KINDS = Object.freeze([{ id: 'codes.review', room: 'codes', entry: 'review', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'review', payload: { review: 'string' }, prompt: { diff: 'string', intent: 'string' }, rubric: ['Find specific correctness or security defects', 'Cite the affected lines and explain impact', 'Avoid speculative or duplicate findings'], seeds: [{ diff: 'A cache key changes from userId to a constant', intent: 'Review for cross-user data exposure' }] } },{ id: 'surf.desk', room: 'surf', entry: 'agent_on_desk', judge: 'human_pairwise', name: 'Agents on desks', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'agent_on_desk', payload: { desk: 'object' } } },{ id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' }, prompt: { conversation: 'array', latest_user_message: 'string' }, rubric: ['Answer the latest request accurately', 'Use prior turns when relevant', 'Be clear and avoid invented facts'], seeds: [{ conversation: [{ role: 'user', content: 'I have 20 minutes for dinner.' }], latest_user_message: 'Suggest a quick vegetarian meal.' }] } },++ { id: 'chat.arena', room: 'exchange', entry: 'model_answer', judge: 'human_pairwise', name: 'Exchange arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { answer: 'string' }, prompt: { text: 'string' } } },{ id: 'speech.tts', room: 'speech', entry: 'audio', judge: 'human_pairwise', name: 'Text to speech voices', metrics: ['elo', 'cost_usd'], schema: { entry: 'audio', payload: { audio_url: 'string' } } },{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },@@@ -20,6 +20,9 @@ const CATEGORY_DEFAULT = 'all';// The rating categories: everyone ('all'), the signed-in people ('people') and the anonymous voters ('anon').const CATEGORY_PEOPLE = 'people';const CATEGORY_ANON = 'anon';++// The Arena's kind and the first-party service that builds its rounds (OpenVibe.Exchange).++const ARENA_KIND = 'chat.arena';++const ARENA_SERVICE = 'service:exchange';const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));@@@ -119,12 +122,26 @@ function kindOfVoter(voterKind, subject) {return 'service';}--/** The rating categories a vote is counted in: everyone, and the kind of voter it came from. */--function categoriesOf(voterKind) {++/**++ * The rating categories a vote is counted in: everyone, the kind of voter it came from, and the task's own++ * category (code, writing, chat, reasoning…) when the room classified the prompt. One vote updates each of++ * them, so a board can show all votes, only people's, or only one kind of task.++ */++function categoriesOf(voterKind, taskCategory = null) {const out = [CATEGORY_DEFAULT];if (voterKind === 'person') out.push(CATEGORY_PEOPLE);else if (voterKind === 'anon') out.push(CATEGORY_ANON);-- return out;++ const task = normalizeCategory(taskCategory);++ if (task) out.push(task);++ return [...new Set(out)];++}++++/** A task category is a short slug; 'all', 'people' and 'anon' are the board's own, never a task's. */++function normalizeCategory(value) {++ const slug = String(value == null ? '' : value).toLowerCase()++ .replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);++ if (!slug || slug === CATEGORY_DEFAULT || slug === CATEGORY_PEOPLE || slug === CATEGORY_ANON) return null;++ return slug;}function wireRating(row) {@@@ -302,6 +319,69 @@ function createStore(s, { log = console, random = Math.random, config = null } =return { ok: true, round: await getRound(id) };}++ /**++ * One Arena round for OpenVibe.Exchange: a person's prompt, two or three answers from different models, and++ * a round this service owns. The Exchange's rounds spend no AI and no person's daily quota here; they have++ * their own daily cap. The voter is the person who asked, so the creator rule still keeps this service from++ * voting on the round it built.++ */++ async function createArenaRound({ prompt, category = null, entries = [] } = {}) {++ const list = Array.isArray(entries) ? entries.slice(0, 3) : [];++ if (list.length < 2) throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs two answers.');++ const clean = list.map((e) => ({++ provider: String((e && e.provider) || '').slice(0, 80),++ model: String((e && e.model) || '').slice(0, 120),++ answer: String((e && e.answer) == null ? '' : e.answer).slice(0, 16000),++ costUsd: num(e && e.cost_usd),++ ms: num(e && e.ms),++ tokens: num(e && e.tokens),++ runId: e && e.run_id ? String(e.run_id).slice(0, 120) : null,++ }));++ if (clean.some((e) => !e.provider || !e.model || !e.answer.trim())) {++ throw new BenchError(422, 'bench.invalid_entries', 'Every answer needs a provider, a model and text.');++ }++ // One model against itself is not a comparison: it could never move a rating.++ if (new Set(clean.map((e) => e.model)).size < 2) {++ throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs answers from two different models.');++ }++++ const cap = config ? config.bench.arenaRoundsDaily : 2000;++ const used = num(await db.value('SELECT COUNT(*)::int AS n FROM bench_rounds WHERE kind_id = $1 AND created_at >= $2::timestamptz',++ [ARENA_KIND, dayStart()]));++ if (used >= cap) throw new BenchError(429, 'bench.arena_limit', 'The arena has reached its rounds for today.');++++ const taskOut = await createTask({++ kindId: ARENA_KIND, prompt, taskClass: normalizeCategory(category), public: true, subject: ARENA_SERVICE,++ });++ if (!taskOut.ok) throw new BenchError(taskOut.status || 400, taskOut.code || 'bench.invalid', taskOut.message || 'invalid task');++ const built = await createRound({++ taskId: taskOut.task.id,++ offers: clean.map((e) => ({ provider: e.provider, model: e.model })),++ subject: ARENA_SERVICE,++ bypassQuota: true,++ });++ if (!built.ok) throw new BenchError(built.status || 400, built.code || 'bench.invalid', built.message || 'invalid round');++++ const roundId = built.round.id;++ const made = [];++ try {++ for (const e of clean) {++ made.push(await addEntry({++ roundId, provider: e.provider, model: e.model, payload: { answer: e.answer },++ costUsd: e.costUsd, ms: e.ms, tokens: e.tokens, runId: e.runId,++ }));++ }++ } catch (err) {++ await db.exec('DELETE FROM bench_rounds WHERE id = $1', [roundId]);++ throw err;++ }++ const sides = ['left', 'right', 'third'];++ return {++ round: await getRound(roundId, { viewerSubject: ARENA_SERVICE }),++ entries: made.map((e, i) => ({ id: e.id, side: sides[i], provider: e.provider, model: e.model })),++ };++ }++async function getRound(id, { viewerSubject = null } = {}) {const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1', [String(id || '')]);if (!r) return null;@@@ -477,7 +557,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =);// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {-- await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject));++ const task = await getTask(r.task_id);++ await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject),++ task && task.task_class);}await insertOutbox('bench.judgement.recorded', {judgement_id: id, round_id: r.id, kind_id: r.kind_id, judge, method: judge, outcome: outcome || null, score: score == null ? null : num(score),@@@ -490,14 +572,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =}}-- async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person') {++ async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person', taskCategory = null) {if (outcome === 'both_bad') return;if (!left || !right || left.model === right.model) return;const models = [left.model, right.model];const now = s.iso();-- // The vote is counted twice: once for everyone ('all') and once for the kind of voter it came from-- // ('people' or 'anon'), so a board can show the signed-in votes alone and still count the rest.-- for (const category of categoriesOf(voterKind)) {++ // The vote is counted for everyone ('all'), for the kind of voter it came from ('people' or 'anon')++ // and for the task's own category (code, writing, chat, reasoning…), so a board can show the votes++ // for one kind of work beside the whole.++ for (const category of categoriesOf(voterKind, taskCategory)) {for (const model of models) {await db.exec(`INSERT INTO bench_ratings (kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, updated_at)@@@ -554,7 +637,10 @@ function createStore(s, { log = console, random = Math.random, config = null } =const subjects = await subjectsFor(kindId);const ratings = await db.many('SELECT * FROM bench_ratings WHERE kind_id = $1 AND category = $2', [kindId, category]);const by = new Map(ratings.map((r) => [r.subject, wireRating(r)]));-- const rows = subjects.map((subject) => {++ // A task category only lists the models that played in it: a model with no game in this kind of++ // work has no place on its board, rather than a baseline score it never earned.++ const taskCategory = ![CATEGORY_DEFAULT, CATEGORY_PEOPLE, CATEGORY_ANON].includes(category);++ const rows = (taskCategory ? subjects.filter((subject) => by.has(subject)) : subjects).map((subject) => {const r = by.get(subject);const games = r ? r.games : 0;return {@@@ -611,7 +697,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =const c = String(category == null ? '' : category).trim().toLowerCase();if (c === 'people' || c === 'person') return CATEGORY_PEOPLE;if (c === 'anon' || c === 'anonymous') return CATEGORY_ANON;-- return CATEGORY_DEFAULT;++ // A task category (code, writing, chat, reasoning…) is its own board.++ return normalizeCategory(c) || CATEGORY_DEFAULT;}/** How many votes of each kind of voter a board holds. */@@@ -847,7 +934,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,insertOutbox, seedKinds, listKinds, getKind,createTask, getTask,-- createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,++ createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,nextJudgement, castJudgement, pairwiseGames,board, valueRows, modelSummary, dataset, datasetJsonl,createEval, getEval, runEval, evalResults, evalDailyCount,@@@ -57,7 +57,7 @@ ${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}${ep('GET', '/api/v1/tasks/:id', 'One task.')}${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}--${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds.')}++${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&category=code', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds. <code>category</code> reads one task class (code, writing, chat, reasoning, translation, analysis), which a vote updates beside the whole.')}${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}@@@ -66,6 +66,8 @@ ${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its vi${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}${ep('POST', '/api/v1/tasks/:id/rounds', '<code>{ offers?, seed? }</code>. Pins a reproducible round; the job generates entries.')}${ep('POST', '/api/v1/rounds/:id/entries', 'A room submits outside entries (a game match, a test-suite run) with a service token carrying bench.entry.create.')}++${ep('POST', '/api/v1/arena/rounds', '<code>{ prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }</code>. OpenVibe.Exchange only: one blind arena round from two answers, owned by the service so the person can vote on it.')}++${ep('POST', '/api/v1/arena/rounds/:id/votes', '<code>{ outcome: left|right|tie|both_bad, pair: { left, right } }</code> with <code>X-OV-Subject: usr_…</code>. OpenVibe.Exchange casts the person’s own human_pairwise judgement; a second vote is 409 and the round is revealed.')}${ep('GET', '/api/v1/judgements/next?kind=&judge=', 'The next blind pair for this person, or for an anonymous voter (the signed cookie, or the X-OV-Voter header a relaying service sends).')}${ep('POST', '/api/v1/rounds/:id/judgements', '<code>{ judge?, pair?, outcome?, score?, vote_counts? }</code>. Human judges are people; automatic/test/game judges and audience_poll are services with bench.judgement.cast. An audience_poll supplies { left, right, tie, voters }; it changes Elo as one game regardless of audience size. A zero-voter poll is recorded without changing ratings. A service on the relay allow-list may cast the vote of an anonymous voter it names in <code>X-OV-Voter: anon:<id></code>.')}${ep('POST', '/api/v1/evals', 'Create a private task suite (never in the public dataset).')}@@@ -17,7 +17,7 @@ const { boot, check, done } = require('./helpers/boot');await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {const r = await t.get('/api/v1/kinds');assert.strictEqual(r.status, 200, r.text);-- assert.strictEqual(r.json().kinds.length, 25);++ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);const ids = r.json().kinds.map((k) => k.id);for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {assert.ok(ids.includes(want), `missing kind ${want}`);nnew file mode 100644@@@ -0,0 +1,166 @@++'use strict';++/**++ * The Exchange arena: the service builds a blind round from two answers, one person votes once, the names are++ * revealed, and the vote lands in Bench as that person's own human_pairwise judgement — under the whole board,++ * the people's board, and the board for the prompt's task category.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++(async () => {++ const t = await boot();++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob');++ const exchange = t.network.serviceToken({++ sub: 'svc:exchange', audience: 'openvibe.bench',++ cap: ['bench.round.create', 'bench.judgement.cast', 'bench.board.read'],++ });++ const other = t.network.serviceToken({++ sub: 'svc:other', audience: 'openvibe.bench',++ cap: ['bench.round.create', 'bench.judgement.cast'],++ });++++ const ENTRIES = [++ { provider: 'deepseek', model: 'deepseek/deepseek-chat@v1', answer: 'A mutex is a lock; a semaphore is a counter.', cost_usd: 0.0001, ms: 400, tokens: 40, run_id: 'run_left' },++ { provider: 'novita', model: 'qwen/qwen3-32b@v1', answer: 'A semaphore can count; a mutex cannot.', cost_usd: 0.0002, ms: 520, tokens: 42, run_id: 'run_right' },++ ];++++ let roundId;++ let pair;++++ try {++ await check('the Exchange builds a blind arena round from two answers', async () => {++ const r = await t.get('/api/v1/arena/rounds', {++ bearer: exchange,++ json: { prompt: 'Explain the difference between a mutex and a semaphore', category: 'code', entries: ENTRIES },++ });++ assert.strictEqual(r.status, 201, r.text);++ const body = r.json();++ roundId = body.round.id;++ assert.match(roundId, /^brd_/);++ assert.strictEqual(body.round.status, 'finished', 'two answers finish the round');++ assert.strictEqual(body.category, 'code');++ assert.deepStrictEqual(body.entries.map((e) => e.side), ['left', 'right']);++ assert.deepStrictEqual(body.entries.map((e) => e.model), ENTRIES.map((e) => e.model));++ pair = { left: body.entries[0].id, right: body.entries[1].id };++++ const open = await t.get(`/api/v1/rounds/${roundId}`);++ assert.strictEqual(open.status, 200, open.text);++ assert.strictEqual(open.json().round.blind, true, 'blind before a vote');++ assert.strictEqual(open.json().round.entries[0].model, undefined, 'no name before a vote');++ });++++ await check('a service that is not Exchange cannot build or vote', async () => {++ const make = await t.get('/api/v1/arena/rounds', {++ bearer: other, json: { prompt: 'hi', category: 'chat', entries: ENTRIES },++ });++ assert.strictEqual(make.status, 403, make.text);++ const vote = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: other, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'left', pair },++ });++ assert.strictEqual(vote.status, 403, vote.text);++ });++++ await check('the vote names the person, and the person is not the round’s creator', async () => {++ const missing = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: exchange, json: { outcome: 'left', pair },++ });++ assert.strictEqual(missing.status, 422, missing.text);++ const bad = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'sideways', pair },++ });++ assert.strictEqual(bad.status, 422, bad.text);++ });++++ await check('the vote is a human_pairwise judgement for that person, and the names are revealed', async () => {++ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },++ json: { outcome: 'left', pair },++ });++ assert.strictEqual(r.status, 201, r.text);++ const body = r.json();++ assert.strictEqual(body.judgement.method, 'human_pairwise');++ assert.strictEqual(body.judgement.outcome, 'left');++ assert.strictEqual(body.judgement.round_id, roundId);++ assert.ok(body.revealed && body.revealed.left.model && body.revealed.right.model, 'both names after the vote');++++ const row = await t.ctx.s.db.maybe('SELECT subject, method, outcome FROM bench_judgements WHERE round_id = $1', [roundId]);++ assert.strictEqual(row.subject, bob.subject, 'the judgement belongs to the person');++ assert.strictEqual(row.method, 'human_pairwise');++ assert.strictEqual(row.outcome, 'left');++++ const open = await t.get(`/api/v1/rounds/${roundId}`, { bearer: t.network.userToken(bob) });++ assert.strictEqual(open.json().round.blind, undefined, 'revealed to the person who voted');++ assert.ok(open.json().round.entries.every((e) => e.model), 'both models are named');++ });++++ await check('one vote per person per round: the second is 409', async () => {++ const again = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },++ json: { outcome: 'right', pair },++ });++ assert.strictEqual(again.status, 409, again.text);++ assert.strictEqual(again.json().code, 'bench.already_judged');++ });++++ await check('another person votes on the same round', async () => {++ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': alice.subject },++ json: { outcome: 'right', pair },++ });++ assert.strictEqual(r.status, 201, r.text);++ });++++ await check('the task category has its own board, beside all and people', async () => {++ const board = await t.get('/api/v1/boards/chat.arena?category=code');++ assert.strictEqual(board.status, 200, board.text);++ const out = board.json().board;++ assert.strictEqual(out.category, 'code');++ assert.strictEqual(out.rows.length, 2, 'only the models that played in this category');++ for (const row of out.rows) {++ assert.strictEqual(row.games, 2, `${row.subject} played both votes`);++ assert.ok(row.elo > 900 && row.elo < 1100, `${row.subject} has an Elo`);++ }++ const all = await t.get('/api/v1/boards/chat.arena');++ assert.strictEqual(all.json().board.category, 'all');++ const people = await t.get('/api/v1/boards/chat.arena?voters=people');++ assert.strictEqual(people.json().board.category, 'people');++ });++++ await check('a vote names two entries of the round', async () => {++ const make = await t.get('/api/v1/arena/rounds', {++ bearer: exchange, json: { prompt: 'Write a haiku about rain', category: 'writing', entries: ENTRIES },++ });++ const id = make.json().round.id;++ const bad = await t.get(`/api/v1/arena/rounds/${id}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },++ json: { outcome: 'left', pair: { left: pair.left, right: pair.right } },++ });++ assert.strictEqual(bad.status, 422, bad.text);++ const good = await t.get(`/api/v1/arena/rounds/${id}/votes`, {++ bearer: exchange, headers: { 'x-ov-subject': bob.subject },++ json: { outcome: 'tie', pair: { left: make.json().entries[0].id, right: make.json().entries[1].id } },++ });++ assert.strictEqual(good.status, 201, good.text);++ assert.strictEqual(good.json().judgement.outcome, 'tie');++ });++++ await check('a round needs two different answers', async () => {++ const one = await t.get('/api/v1/arena/rounds', {++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0]] },++ });++ assert.strictEqual(one.status, 422, one.text);++ const blank = await t.get('/api/v1/arena/rounds', {++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [{ provider: 'x', model: 'y', answer: ' ' }, ENTRIES[1]] },++ });++ assert.strictEqual(blank.status, 422, blank.text);++ const same = await t.get('/api/v1/arena/rounds', {++ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0], { ...ENTRIES[1], model: ENTRIES[0].model }] },++ });++ assert.strictEqual(same.status, 422, same.text);++ assert.ok(same.json().detail.includes('two different models'));++ });++ } finally { await t.close(); }++++ done();++})();@@@ -58,7 +58,7 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(paramawait check('bench.list_kinds lists every kind Bench runs', async () => {const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });assert.strictEqual(r.json().result.isError, false);-- assert.strictEqual(r.json().result.structuredContent.kinds.length, 25);++ assert.strictEqual(r.json().result.structuredContent.kinds.length, require('../server/kinds').KINDS.length);});await check('bench.get_board renders a board without a person', async () => {@@@ -28,7 +28,7 @@ const HOSTILE = [const r = await t.get('/api/v1/kinds');assert.strictEqual(r.status, 200, r.text);const byId = new Map(r.json().kinds.map((k) => [k.id, k]));-- assert.strictEqual(r.json().kinds.length, 25);++ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];for (const id of want) {const k = byId.get(id);
@@ -1,5 +1,16 @@
@@ -1,5 +1,16 @@
# Changelog
# Changelog
+## Unreleased
+
+- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:
+ `POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, and
+ `POST /api/v1/arena/rounds/:id/votes` records the person's own `human_pairwise` judgement from
+ `X-OV-Subject`. The service owns the round, so the person can vote on it; a second vote is 409 and the round
+ is revealed. The arena's rounds skip a person's daily quota and have their own daily cap.
+- **Task-category ratings.** A judgement now also updates the task's `task_class` category (code, writing, chat,
+ reasoning, translation, analysis) beside `all` and the voter kind, and `GET /api/v1/boards/:kindId?category=…`
+ reads it. A category board lists only the models that played in it.
+
## 0.1.0 — 2026-10-28
## 0.1.0 — 2026-10-28
- OpenVibe.Bench v1: the shared AI benchmark and comparison engine (plan 2026-10-28 §4.12).
- OpenVibe.Bench v1: the shared AI benchmark and comparison engine (plan 2026-10-28 §4.12).
@@ -30,6 +30,7 @@ const { createRegistryApi } = require('./http/registry');
@@ -30,6 +30,7 @@ const { createRegistryApi } = require('./http/registry');
const { createVariantEvidenceApi } = require('./http/variant-evidence');
const { createVariantEvidenceApi } = require('./http/variant-evidence');
const { createPageRoutes } = require('./http/pages');
const { createPageRoutes } = require('./http/pages');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
+const { createArenaRoutes } = require('./http/arena');
const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
const { createPickPages, createPickRoutes } = require('./http/pick');
const { createPickPages, createPickRoutes } = require('./http/pick');
const { createBadgeRoutes } = require('./http/badge');
const { createBadgeRoutes } = require('./http/badge');
@@ -241,6 +242,7 @@ async function createApp(opts = {}) {
@@ -241,6 +242,7 @@ async function createApp(opts = {}) {
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createVariantEvidenceApi(ctx));
app.use('/api/v1', createVariantEvidenceApi(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
+ app.use('/api/v1/arena', createArenaRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1', createPickRoutes(ctx));
app.use('/api/v1', createPickRoutes(ctx));
app.use(require('./http/mirror').createMirrorRoutes(ctx));
app.use(require('./http/mirror').createMirrorRoutes(ctx));
@@ -88,6 +88,9 @@ function load(env = process.env) {
@@ -88,6 +88,9 @@ function load(env = process.env) {
pollMs: Math.max(250, int(env.BENCH_POLL_MS, 20_000)),
pollMs: Math.max(250, int(env.BENCH_POLL_MS, 20_000)),
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
+ // The Arena's own cap: its rounds skip a person's daily quota (the Exchange already paces its
+ // people), so the day's total is the guard.
+ arenaRoundsDaily: Math.max(1, int(env.BENCH_ARENA_ROUNDS_DAILY, 2000)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
// Battles: how many votes one person and one network may cast.
// Battles: how many votes one person and one network may cast.
battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),
battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),
new file mode 100644
new file mode 100644
@@ -0,0 +1,94 @@
@@ -0,0 +1,94 @@
+'use strict';
+
+/**
+ * The Exchange arena's own API, for OpenVibe.Exchange and nobody else.
+ *
+ * The Arena is a blind pair in Exchange's chat: a person's prompt, two models' answers side by side, a vote,
+ * then both names and their receipts. Each vote is a Bench human_pairwise judgement owned by that person, and
+ * each round belongs to the service so the person can vote on it.
+ *
+ * POST /api/v1/arena/rounds (service:exchange, bench.round.create)
+ * { prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }
+ * → 201 { round: { id, status }, category, entries: [{ id, side, provider, model }] }
+ *
+ * POST /api/v1/arena/rounds/:id/votes (service:exchange, bench.judgement.cast)
+ * x-ov-subject: the person's Network subject
+ * { outcome: left|right|tie|both_bad, pair: { left, right } }
+ * → 201 { judgement, revealed }
+ *
+ * A second vote by the same person is 409 (Bench's one-vote rule), and the service that built the round can
+ * never cast the vote itself: the judgement belongs to the person named in x-ov-subject.
+ */
+const express = require('express');
+const { http, ids } = require('openvibe-contracts');
+const { asyncRouter } = require('./router');
+
+const SERVICE = 'service:exchange';
+const ARENA_KIND = 'chat.arena';
+const OUTCOMES = new Set(['left', 'right', 'tie', 'both_bad']);
+
+function createArenaRoutes(ctx) {
+ const { bench, principal, log } = ctx;
+ const r = asyncRouter();
+ r.use(express.json({ limit: '128kb' }));
+ r.use((_req, res, next) => { res.set('Cache-Control', 'no-store'); next(); });
+ r.use(principal.middleware);
+
+ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });
+
+ const fromExchange = (req, res, next) => {
+ if (req.principal && req.principal.kind === 'app' && req.principal.requester === SERVICE) return next();
+ return problem(res, 403, 'bench.arena_service_only', 'The arena is built by OpenVibe.Exchange.');
+ };
+ const failed = (res, err) => {
+ if (err && err.status) return problem(res, err.status, err.code || 'bench.invalid', err.detail || 'The arena request was not accepted.');
+ log.warn('[OpenVibe.Bench] arena failed:', (err && err.message) || err);
+ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
+ };
+
+ r.post('/rounds', principal.requireCapability('bench.round.create'), fromExchange, async (req, res) => {
+ const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const prompt = String(b.prompt == null ? '' : b.prompt).replace(/\s+/g, ' ').trim();
+ if (!prompt) return problem(res, 422, 'bench.prompt_required', 'A round needs the prompt the models answered.');
+ try {
+ const out = await bench.createArenaRound({
+ prompt,
+ category: b.category == null ? null : String(b.category),
+ entries: Array.isArray(b.entries) ? b.entries : [],
+ });
+ res.set('Location', `/api/v1/arena/rounds/${out.round.id}`);
+ return res.status(201).json({
+ round: { id: out.round.id, status: out.round.status, kind_id: ARENA_KIND },
+ category: b.category == null ? null : String(b.category).slice(0, 40),
+ entries: out.entries,
+ });
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.post('/rounds/:id/votes', principal.requireCapability('bench.judgement.cast'), fromExchange, async (req, res) => {
+ const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const subject = String(req.headers['x-ov-subject'] || '');
+ if (!ids.isSubjectId('user', subject)) {
+ return problem(res, 422, 'bench.invalid_subject', 'x-ov-subject must be the Network subject of the person who voted.');
+ }
+ const outcome = String(b.outcome || '');
+ if (!OUTCOMES.has(outcome)) return problem(res, 422, 'bench.invalid_outcome', 'The outcome must be left, right, tie or both_bad.');
+ const pair = b.pair && typeof b.pair === 'object' ? { left: String(b.pair.left || ''), right: String(b.pair.right || '') } : null;
+ if (!pair || !pair.left || !pair.right || pair.left === pair.right) {
+ return problem(res, 422, 'bench.invalid_pair', 'A vote names the two entries it compares.');
+ }
+ try {
+ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });
+ if (!round || round.kind_id !== ARENA_KIND) return problem(res, 404, 'bench.round_not_found', 'No such arena round.');
+ if (round.created_by !== SERVICE) return problem(res, 403, 'bench.arena_service_only', 'That round was not built by OpenVibe.Exchange.');
+ const out = await bench.castJudgement({
+ roundId: round.id, judge: 'human_pairwise', pair, outcome, subject,
+ });
+ return res.status(201).json({ judgement: out.judgement, revealed: out.revealed });
+ } catch (err) { return failed(res, err); }
+ });
+
+ return r;
+}
+
+module.exports = { createArenaRoutes, SERVICE, ARENA_KIND };
@@ -23,6 +23,7 @@ const KINDS = Object.freeze([
@@ -23,6 +23,7 @@ const KINDS = Object.freeze([
{ id: 'codes.review', room: 'codes', entry: 'review', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'review', payload: { review: 'string' }, prompt: { diff: 'string', intent: 'string' }, rubric: ['Find specific correctness or security defects', 'Cite the affected lines and explain impact', 'Avoid speculative or duplicate findings'], seeds: [{ diff: 'A cache key changes from userId to a constant', intent: 'Review for cross-user data exposure' }] } },
{ id: 'codes.review', room: 'codes', entry: 'review', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'review', payload: { review: 'string' }, prompt: { diff: 'string', intent: 'string' }, rubric: ['Find specific correctness or security defects', 'Cite the affected lines and explain impact', 'Avoid speculative or duplicate findings'], seeds: [{ diff: 'A cache key changes from userId to a constant', intent: 'Review for cross-user data exposure' }] } },
{ id: 'surf.desk', room: 'surf', entry: 'agent_on_desk', judge: 'human_pairwise', name: 'Agents on desks', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'agent_on_desk', payload: { desk: 'object' } } },
{ id: 'surf.desk', room: 'surf', entry: 'agent_on_desk', judge: 'human_pairwise', name: 'Agents on desks', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'agent_on_desk', payload: { desk: 'object' } } },
{ id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' }, prompt: { conversation: 'array', latest_user_message: 'string' }, rubric: ['Answer the latest request accurately', 'Use prior turns when relevant', 'Be clear and avoid invented facts'], seeds: [{ conversation: [{ role: 'user', content: 'I have 20 minutes for dinner.' }], latest_user_message: 'Suggest a quick vegetarian meal.' }] } },
{ id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' }, prompt: { conversation: 'array', latest_user_message: 'string' }, rubric: ['Answer the latest request accurately', 'Use prior turns when relevant', 'Be clear and avoid invented facts'], seeds: [{ conversation: [{ role: 'user', content: 'I have 20 minutes for dinner.' }], latest_user_message: 'Suggest a quick vegetarian meal.' }] } },
+ { id: 'chat.arena', room: 'exchange', entry: 'model_answer', judge: 'human_pairwise', name: 'Exchange arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { answer: 'string' }, prompt: { text: 'string' } } },
{ id: 'speech.tts', room: 'speech', entry: 'audio', judge: 'human_pairwise', name: 'Text to speech voices', metrics: ['elo', 'cost_usd'], schema: { entry: 'audio', payload: { audio_url: 'string' } } },
{ id: 'speech.tts', room: 'speech', entry: 'audio', judge: 'human_pairwise', name: 'Text to speech voices', metrics: ['elo', 'cost_usd'], schema: { entry: 'audio', payload: { audio_url: 'string' } } },
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
@@ -20,6 +20,9 @@ const CATEGORY_DEFAULT = 'all';
@@ -20,6 +20,9 @@ const CATEGORY_DEFAULT = 'all';
// The rating categories: everyone ('all'), the signed-in people ('people') and the anonymous voters ('anon').
// The rating categories: everyone ('all'), the signed-in people ('people') and the anonymous voters ('anon').
const CATEGORY_PEOPLE = 'people';
const CATEGORY_PEOPLE = 'people';
const CATEGORY_ANON = 'anon';
const CATEGORY_ANON = 'anon';
+// The Arena's kind and the first-party service that builds its rounds (OpenVibe.Exchange).
+const ARENA_KIND = 'chat.arena';
+const ARENA_SERVICE = 'service:exchange';
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
@@ -119,12 +122,26 @@ function kindOfVoter(voterKind, subject) {
@@ -119,12 +122,26 @@ function kindOfVoter(voterKind, subject) {
return 'service';
return 'service';
}
}
-/** The rating categories a vote is counted in: everyone, and the kind of voter it came from. */
-function categoriesOf(voterKind) {
+/**
+ * The rating categories a vote is counted in: everyone, the kind of voter it came from, and the task's own
+ * category (code, writing, chat, reasoning…) when the room classified the prompt. One vote updates each of
+ * them, so a board can show all votes, only people's, or only one kind of task.
+ */
+function categoriesOf(voterKind, taskCategory = null) {
const out = [CATEGORY_DEFAULT];
const out = [CATEGORY_DEFAULT];
if (voterKind === 'person') out.push(CATEGORY_PEOPLE);
if (voterKind === 'person') out.push(CATEGORY_PEOPLE);
else if (voterKind === 'anon') out.push(CATEGORY_ANON);
else if (voterKind === 'anon') out.push(CATEGORY_ANON);
- return out;
+ const task = normalizeCategory(taskCategory);
+ if (task) out.push(task);
+ return [...new Set(out)];
+}
+
+/** A task category is a short slug; 'all', 'people' and 'anon' are the board's own, never a task's. */
+function normalizeCategory(value) {
+ const slug = String(value == null ? '' : value).toLowerCase()
+ .replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
+ if (!slug || slug === CATEGORY_DEFAULT || slug === CATEGORY_PEOPLE || slug === CATEGORY_ANON) return null;
+ return slug;
}
}
function wireRating(row) {
function wireRating(row) {
@@ -302,6 +319,69 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -302,6 +319,69 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return { ok: true, round: await getRound(id) };
return { ok: true, round: await getRound(id) };
}
}
+ /**
+ * One Arena round for OpenVibe.Exchange: a person's prompt, two or three answers from different models, and
+ * a round this service owns. The Exchange's rounds spend no AI and no person's daily quota here; they have
+ * their own daily cap. The voter is the person who asked, so the creator rule still keeps this service from
+ * voting on the round it built.
+ */
+ async function createArenaRound({ prompt, category = null, entries = [] } = {}) {
+ const list = Array.isArray(entries) ? entries.slice(0, 3) : [];
+ if (list.length < 2) throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs two answers.');
+ const clean = list.map((e) => ({
+ provider: String((e && e.provider) || '').slice(0, 80),
+ model: String((e && e.model) || '').slice(0, 120),
+ answer: String((e && e.answer) == null ? '' : e.answer).slice(0, 16000),
+ costUsd: num(e && e.cost_usd),
+ ms: num(e && e.ms),
+ tokens: num(e && e.tokens),
+ runId: e && e.run_id ? String(e.run_id).slice(0, 120) : null,
+ }));
+ if (clean.some((e) => !e.provider || !e.model || !e.answer.trim())) {
+ throw new BenchError(422, 'bench.invalid_entries', 'Every answer needs a provider, a model and text.');
+ }
+ // One model against itself is not a comparison: it could never move a rating.
+ if (new Set(clean.map((e) => e.model)).size < 2) {
+ throw new BenchError(422, 'bench.invalid_entries', 'An arena round needs answers from two different models.');
+ }
+
+ const cap = config ? config.bench.arenaRoundsDaily : 2000;
+ const used = num(await db.value('SELECT COUNT(*)::int AS n FROM bench_rounds WHERE kind_id = $1 AND created_at >= $2::timestamptz',
+ [ARENA_KIND, dayStart()]));
+ if (used >= cap) throw new BenchError(429, 'bench.arena_limit', 'The arena has reached its rounds for today.');
+
+ const taskOut = await createTask({
+ kindId: ARENA_KIND, prompt, taskClass: normalizeCategory(category), public: true, subject: ARENA_SERVICE,
+ });
+ if (!taskOut.ok) throw new BenchError(taskOut.status || 400, taskOut.code || 'bench.invalid', taskOut.message || 'invalid task');
+ const built = await createRound({
+ taskId: taskOut.task.id,
+ offers: clean.map((e) => ({ provider: e.provider, model: e.model })),
+ subject: ARENA_SERVICE,
+ bypassQuota: true,
+ });
+ if (!built.ok) throw new BenchError(built.status || 400, built.code || 'bench.invalid', built.message || 'invalid round');
+
+ const roundId = built.round.id;
+ const made = [];
+ try {
+ for (const e of clean) {
+ made.push(await addEntry({
+ roundId, provider: e.provider, model: e.model, payload: { answer: e.answer },
+ costUsd: e.costUsd, ms: e.ms, tokens: e.tokens, runId: e.runId,
+ }));
+ }
+ } catch (err) {
+ await db.exec('DELETE FROM bench_rounds WHERE id = $1', [roundId]);
+ throw err;
+ }
+ const sides = ['left', 'right', 'third'];
+ return {
+ round: await getRound(roundId, { viewerSubject: ARENA_SERVICE }),
+ entries: made.map((e, i) => ({ id: e.id, side: sides[i], provider: e.provider, model: e.model })),
+ };
+ }
+
async function getRound(id, { viewerSubject = null } = {}) {
async function getRound(id, { viewerSubject = null } = {}) {
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1', [String(id || '')]);
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1', [String(id || '')]);
if (!r) return null;
if (!r) return null;
@@ -477,7 +557,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -477,7 +557,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
);
);
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
- await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject));
+ const task = await getTask(r.task_id);
+ await applyPairwiseRating(r.id, r.kind_id, left, right, outcome, kindOfVoter(voterKind, subject),
+ task && task.task_class);
}
}
await insertOutbox('bench.judgement.recorded', {
await insertOutbox('bench.judgement.recorded', {
judgement_id: id, round_id: r.id, kind_id: r.kind_id, judge, method: judge, outcome: outcome || null, score: score == null ? null : num(score),
judgement_id: id, round_id: r.id, kind_id: r.kind_id, judge, method: judge, outcome: outcome || null, score: score == null ? null : num(score),
@@ -490,14 +572,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -490,14 +572,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
}
}
}
}
- async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person') {
+ async function applyPairwiseRating(roundId, kindId, left, right, outcome, voterKind = 'person', taskCategory = null) {
if (outcome === 'both_bad') return;
if (outcome === 'both_bad') return;
if (!left || !right || left.model === right.model) return;
if (!left || !right || left.model === right.model) return;
const models = [left.model, right.model];
const models = [left.model, right.model];
const now = s.iso();
const now = s.iso();
- // The vote is counted twice: once for everyone ('all') and once for the kind of voter it came from
- // ('people' or 'anon'), so a board can show the signed-in votes alone and still count the rest.
- for (const category of categoriesOf(voterKind)) {
+ // The vote is counted for everyone ('all'), for the kind of voter it came from ('people' or 'anon')
+ // and for the task's own category (code, writing, chat, reasoning…), so a board can show the votes
+ // for one kind of work beside the whole.
+ for (const category of categoriesOf(voterKind, taskCategory)) {
for (const model of models) {
for (const model of models) {
await db.exec(
await db.exec(
`INSERT INTO bench_ratings (kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, updated_at)
`INSERT INTO bench_ratings (kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, updated_at)
@@ -554,7 +637,10 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -554,7 +637,10 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const subjects = await subjectsFor(kindId);
const subjects = await subjectsFor(kindId);
const ratings = await db.many('SELECT * FROM bench_ratings WHERE kind_id = $1 AND category = $2', [kindId, category]);
const ratings = await db.many('SELECT * FROM bench_ratings WHERE kind_id = $1 AND category = $2', [kindId, category]);
const by = new Map(ratings.map((r) => [r.subject, wireRating(r)]));
const by = new Map(ratings.map((r) => [r.subject, wireRating(r)]));
- const rows = subjects.map((subject) => {
+ // A task category only lists the models that played in it: a model with no game in this kind of
+ // work has no place on its board, rather than a baseline score it never earned.
+ const taskCategory = ![CATEGORY_DEFAULT, CATEGORY_PEOPLE, CATEGORY_ANON].includes(category);
+ const rows = (taskCategory ? subjects.filter((subject) => by.has(subject)) : subjects).map((subject) => {
const r = by.get(subject);
const r = by.get(subject);
const games = r ? r.games : 0;
const games = r ? r.games : 0;
return {
return {
@@ -611,7 +697,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -611,7 +697,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const c = String(category == null ? '' : category).trim().toLowerCase();
const c = String(category == null ? '' : category).trim().toLowerCase();
if (c === 'people' || c === 'person') return CATEGORY_PEOPLE;
if (c === 'people' || c === 'person') return CATEGORY_PEOPLE;
if (c === 'anon' || c === 'anonymous') return CATEGORY_ANON;
if (c === 'anon' || c === 'anonymous') return CATEGORY_ANON;
- return CATEGORY_DEFAULT;
+ // A task category (code, writing, chat, reasoning…) is its own board.
+ return normalizeCategory(c) || CATEGORY_DEFAULT;
}
}
/** How many votes of each kind of voter a board holds. */
/** How many votes of each kind of voter a board holds. */
@@ -847,7 +934,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -847,7 +934,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
createTask, getTask,
- createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
+ createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
nextJudgement, castJudgement, pairwiseGames,
board, valueRows, modelSummary, dataset, datasetJsonl,
board, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
createEval, getEval, runEval, evalResults, evalDailyCount,
@@ -57,7 +57,7 @@ ${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}
@@ -57,7 +57,7 @@ ${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}
${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}
${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}
${ep('GET', '/api/v1/tasks/:id', 'One task.')}
${ep('GET', '/api/v1/tasks/:id', 'One task.')}
${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}
${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}
-${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds.')}
+${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&category=code', 'A rendered board. <code>voters=people</code> shows the signed-in votes alone, <code>voters=anon</code> the anonymous ones; every board says how many votes of each kind it holds. <code>category</code> reads one task class (code, writing, chat, reasoning, translation, analysis), which a vote updates beside the whole.')}
${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}
${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}
${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}
${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}
${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}
${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}
@@ -66,6 +66,8 @@ ${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its vi
@@ -66,6 +66,8 @@ ${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its vi
${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}
${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}
${ep('POST', '/api/v1/tasks/:id/rounds', '<code>{ offers?, seed? }</code>. Pins a reproducible round; the job generates entries.')}
${ep('POST', '/api/v1/tasks/:id/rounds', '<code>{ offers?, seed? }</code>. Pins a reproducible round; the job generates entries.')}
${ep('POST', '/api/v1/rounds/:id/entries', 'A room submits outside entries (a game match, a test-suite run) with a service token carrying bench.entry.create.')}
${ep('POST', '/api/v1/rounds/:id/entries', 'A room submits outside entries (a game match, a test-suite run) with a service token carrying bench.entry.create.')}
+${ep('POST', '/api/v1/arena/rounds', '<code>{ prompt, category?, entries: [{ provider, model, answer, cost_usd?, ms?, tokens?, run_id? }] }</code>. OpenVibe.Exchange only: one blind arena round from two answers, owned by the service so the person can vote on it.')}
+${ep('POST', '/api/v1/arena/rounds/:id/votes', '<code>{ outcome: left|right|tie|both_bad, pair: { left, right } }</code> with <code>X-OV-Subject: usr_…</code>. OpenVibe.Exchange casts the person’s own human_pairwise judgement; a second vote is 409 and the round is revealed.')}
${ep('GET', '/api/v1/judgements/next?kind=&judge=', 'The next blind pair for this person, or for an anonymous voter (the signed cookie, or the X-OV-Voter header a relaying service sends).')}
${ep('GET', '/api/v1/judgements/next?kind=&judge=', 'The next blind pair for this person, or for an anonymous voter (the signed cookie, or the X-OV-Voter header a relaying service sends).')}
${ep('POST', '/api/v1/rounds/:id/judgements', '<code>{ judge?, pair?, outcome?, score?, vote_counts? }</code>. Human judges are people; automatic/test/game judges and audience_poll are services with bench.judgement.cast. An audience_poll supplies { left, right, tie, voters }; it changes Elo as one game regardless of audience size. A zero-voter poll is recorded without changing ratings. A service on the relay allow-list may cast the vote of an anonymous voter it names in <code>X-OV-Voter: anon:<id></code>.')}
${ep('POST', '/api/v1/rounds/:id/judgements', '<code>{ judge?, pair?, outcome?, score?, vote_counts? }</code>. Human judges are people; automatic/test/game judges and audience_poll are services with bench.judgement.cast. An audience_poll supplies { left, right, tie, voters }; it changes Elo as one game regardless of audience size. A zero-voter poll is recorded without changing ratings. A service on the relay allow-list may cast the vote of an anonymous voter it names in <code>X-OV-Voter: anon:<id></code>.')}
${ep('POST', '/api/v1/evals', 'Create a private task suite (never in the public dataset).')}
${ep('POST', '/api/v1/evals', 'Create a private task suite (never in the public dataset).')}
@@ -17,7 +17,7 @@ const { boot, check, done } = require('./helpers/boot');
@@ -17,7 +17,7 @@ const { boot, check, done } = require('./helpers/boot');
await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {
await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {
const r = await t.get('/api/v1/kinds');
const r = await t.get('/api/v1/kinds');
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
- assert.strictEqual(r.json().kinds.length, 25);
+ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);
const ids = r.json().kinds.map((k) => k.id);
const ids = r.json().kinds.map((k) => k.id);
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
assert.ok(ids.includes(want), `missing kind ${want}`);
assert.ok(ids.includes(want), `missing kind ${want}`);
new file mode 100644
new file mode 100644
@@ -0,0 +1,166 @@
@@ -0,0 +1,166 @@
+'use strict';
+/**
+ * The Exchange arena: the service builds a blind round from two answers, one person votes once, the names are
+ * revealed, and the vote lands in Bench as that person's own human_pairwise judgement — under the whole board,
+ * the people's board, and the board for the prompt's task category.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+(async () => {
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob');
+ const exchange = t.network.serviceToken({
+ sub: 'svc:exchange', audience: 'openvibe.bench',
+ cap: ['bench.round.create', 'bench.judgement.cast', 'bench.board.read'],
+ });
+ const other = t.network.serviceToken({
+ sub: 'svc:other', audience: 'openvibe.bench',
+ cap: ['bench.round.create', 'bench.judgement.cast'],
+ });
+
+ const ENTRIES = [
+ { provider: 'deepseek', model: 'deepseek/deepseek-chat@v1', answer: 'A mutex is a lock; a semaphore is a counter.', cost_usd: 0.0001, ms: 400, tokens: 40, run_id: 'run_left' },
+ { provider: 'novita', model: 'qwen/qwen3-32b@v1', answer: 'A semaphore can count; a mutex cannot.', cost_usd: 0.0002, ms: 520, tokens: 42, run_id: 'run_right' },
+ ];
+
+ let roundId;
+ let pair;
+
+ try {
+ await check('the Exchange builds a blind arena round from two answers', async () => {
+ const r = await t.get('/api/v1/arena/rounds', {
+ bearer: exchange,
+ json: { prompt: 'Explain the difference between a mutex and a semaphore', category: 'code', entries: ENTRIES },
+ });
+ assert.strictEqual(r.status, 201, r.text);
+ const body = r.json();
+ roundId = body.round.id;
+ assert.match(roundId, /^brd_/);
+ assert.strictEqual(body.round.status, 'finished', 'two answers finish the round');
+ assert.strictEqual(body.category, 'code');
+ assert.deepStrictEqual(body.entries.map((e) => e.side), ['left', 'right']);
+ assert.deepStrictEqual(body.entries.map((e) => e.model), ENTRIES.map((e) => e.model));
+ pair = { left: body.entries[0].id, right: body.entries[1].id };
+
+ const open = await t.get(`/api/v1/rounds/${roundId}`);
+ assert.strictEqual(open.status, 200, open.text);
+ assert.strictEqual(open.json().round.blind, true, 'blind before a vote');
+ assert.strictEqual(open.json().round.entries[0].model, undefined, 'no name before a vote');
+ });
+
+ await check('a service that is not Exchange cannot build or vote', async () => {
+ const make = await t.get('/api/v1/arena/rounds', {
+ bearer: other, json: { prompt: 'hi', category: 'chat', entries: ENTRIES },
+ });
+ assert.strictEqual(make.status, 403, make.text);
+ const vote = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: other, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'left', pair },
+ });
+ assert.strictEqual(vote.status, 403, vote.text);
+ });
+
+ await check('the vote names the person, and the person is not the round’s creator', async () => {
+ const missing = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: exchange, json: { outcome: 'left', pair },
+ });
+ assert.strictEqual(missing.status, 422, missing.text);
+ const bad = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': bob.subject }, json: { outcome: 'sideways', pair },
+ });
+ assert.strictEqual(bad.status, 422, bad.text);
+ });
+
+ await check('the vote is a human_pairwise judgement for that person, and the names are revealed', async () => {
+ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
+ json: { outcome: 'left', pair },
+ });
+ assert.strictEqual(r.status, 201, r.text);
+ const body = r.json();
+ assert.strictEqual(body.judgement.method, 'human_pairwise');
+ assert.strictEqual(body.judgement.outcome, 'left');
+ assert.strictEqual(body.judgement.round_id, roundId);
+ assert.ok(body.revealed && body.revealed.left.model && body.revealed.right.model, 'both names after the vote');
+
+ const row = await t.ctx.s.db.maybe('SELECT subject, method, outcome FROM bench_judgements WHERE round_id = $1', [roundId]);
+ assert.strictEqual(row.subject, bob.subject, 'the judgement belongs to the person');
+ assert.strictEqual(row.method, 'human_pairwise');
+ assert.strictEqual(row.outcome, 'left');
+
+ const open = await t.get(`/api/v1/rounds/${roundId}`, { bearer: t.network.userToken(bob) });
+ assert.strictEqual(open.json().round.blind, undefined, 'revealed to the person who voted');
+ assert.ok(open.json().round.entries.every((e) => e.model), 'both models are named');
+ });
+
+ await check('one vote per person per round: the second is 409', async () => {
+ const again = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
+ json: { outcome: 'right', pair },
+ });
+ assert.strictEqual(again.status, 409, again.text);
+ assert.strictEqual(again.json().code, 'bench.already_judged');
+ });
+
+ await check('another person votes on the same round', async () => {
+ const r = await t.get(`/api/v1/arena/rounds/${roundId}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': alice.subject },
+ json: { outcome: 'right', pair },
+ });
+ assert.strictEqual(r.status, 201, r.text);
+ });
+
+ await check('the task category has its own board, beside all and people', async () => {
+ const board = await t.get('/api/v1/boards/chat.arena?category=code');
+ assert.strictEqual(board.status, 200, board.text);
+ const out = board.json().board;
+ assert.strictEqual(out.category, 'code');
+ assert.strictEqual(out.rows.length, 2, 'only the models that played in this category');
+ for (const row of out.rows) {
+ assert.strictEqual(row.games, 2, `${row.subject} played both votes`);
+ assert.ok(row.elo > 900 && row.elo < 1100, `${row.subject} has an Elo`);
+ }
+ const all = await t.get('/api/v1/boards/chat.arena');
+ assert.strictEqual(all.json().board.category, 'all');
+ const people = await t.get('/api/v1/boards/chat.arena?voters=people');
+ assert.strictEqual(people.json().board.category, 'people');
+ });
+
+ await check('a vote names two entries of the round', async () => {
+ const make = await t.get('/api/v1/arena/rounds', {
+ bearer: exchange, json: { prompt: 'Write a haiku about rain', category: 'writing', entries: ENTRIES },
+ });
+ const id = make.json().round.id;
+ const bad = await t.get(`/api/v1/arena/rounds/${id}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
+ json: { outcome: 'left', pair: { left: pair.left, right: pair.right } },
+ });
+ assert.strictEqual(bad.status, 422, bad.text);
+ const good = await t.get(`/api/v1/arena/rounds/${id}/votes`, {
+ bearer: exchange, headers: { 'x-ov-subject': bob.subject },
+ json: { outcome: 'tie', pair: { left: make.json().entries[0].id, right: make.json().entries[1].id } },
+ });
+ assert.strictEqual(good.status, 201, good.text);
+ assert.strictEqual(good.json().judgement.outcome, 'tie');
+ });
+
+ await check('a round needs two different answers', async () => {
+ const one = await t.get('/api/v1/arena/rounds', {
+ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0]] },
+ });
+ assert.strictEqual(one.status, 422, one.text);
+ const blank = await t.get('/api/v1/arena/rounds', {
+ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [{ provider: 'x', model: 'y', answer: ' ' }, ENTRIES[1]] },
+ });
+ assert.strictEqual(blank.status, 422, blank.text);
+ const same = await t.get('/api/v1/arena/rounds', {
+ bearer: exchange, json: { prompt: 'hi', category: 'chat', entries: [ENTRIES[0], { ...ENTRIES[1], model: ENTRIES[0].model }] },
+ });
+ assert.strictEqual(same.status, 422, same.text);
+ assert.ok(same.json().detail.includes('two different models'));
+ });
+ } finally { await t.close(); }
+
+ done();
+})();
@@ -58,7 +58,7 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(param
@@ -58,7 +58,7 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(param
await check('bench.list_kinds lists every kind Bench runs', async () => {
await check('bench.list_kinds lists every kind Bench runs', async () => {
const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });
const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });
assert.strictEqual(r.json().result.isError, false);
assert.strictEqual(r.json().result.isError, false);
- assert.strictEqual(r.json().result.structuredContent.kinds.length, 25);
+ assert.strictEqual(r.json().result.structuredContent.kinds.length, require('../server/kinds').KINDS.length);
});
});
await check('bench.get_board renders a board without a person', async () => {
await check('bench.get_board renders a board without a person', async () => {
@@ -28,7 +28,7 @@ const HOSTILE = [
@@ -28,7 +28,7 @@ const HOSTILE = [
const r = await t.get('/api/v1/kinds');
const r = await t.get('/api/v1/kinds');
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
const byId = new Map(r.json().kinds.map((k) => [k.id, k]));
const byId = new Map(r.json().kinds.map((k) => [k.id, k]));
- assert.strictEqual(r.json().kinds.length, 25);
+ assert.strictEqual(r.json().kinds.length, require('../server/kinds').KINDS.length);
const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];
const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];
for (const id of want) {
for (const id of want) {
const k = byId.get(id);
const k = byId.get(id);