openvibers/openvibe.bench Commits

BENCH-7: Games match outcomes from service:games only, out of vote counts; BENCH-9: five text kinds (#9)

OP OpenVibers committed 15 days ago · deb93564bc · 5 files, +52 −8

* Accept Games match outcomes and define text benchmark kinds

* Keep automatic game outcomes out of vote counts

Changes

ddiff --git a/server/http/api.js b/server/http/api.js
iindex 11ee247..2544ee4 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -184,6 +184,11 @@ function createApi(ctx) {
const guard = isHuman ? requirePerson : principal.requireCapability('bench.judgement.cast');
return guard(req, res, async () => {
try {
++ const round = await bench.getRound(roundId, { viewerSubject: null });
++ if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
++ if (round.kind_id === 'games.match' && (judge !== 'game_outcome' || req.principal.requester !== 'service:games')) {
++ return problem(res, 403, 'bench.games_service_required', 'Only the Games service may record match outcomes.');
++ }
const subject = isHuman ? subjectOf(req) : req.principal.requester;
const out = await bench.castJudgement({
roundId, kindId: b.kind_id, judge,
ddiff --git a/server/kinds.js b/server/kinds.js
iindex 5248373..435504d 100644
---- a/server/kinds.js
++++ b/server/kinds.js
@@@ -19,14 +19,14 @@ const KINDS = Object.freeze([
{ id: 'meme.caption', room: 'meme', entry: 'image', judge: 'human_pairwise', name: 'Meme caption arena', metrics: ['elo', 'cost_usd'], schema: { entry: 'image', payload: { image: 'string', caption: 'string' } } },
{ id: 'vision.answer', room: 'vision', entry: 'model_answer', judge: 'automatic_grader', name: 'Vision answers', metrics: ['pass_rate', 'cost_usd'], schema: { entry: 'model_answer', payload: { answer: 'string' } } },
{ id: 'vision.pair', room: 'vision', entry: 'model_answer', judge: 'human_pairwise', name: 'Vision head-to-head', metrics: ['elo', 'cost_usd'], schema: { entry: 'model_answer', payload: { answer: 'string' } } },
-- { id: 'codes.patch', room: 'codes', entry: 'code_patch', judge: 'test_suite', name: 'Code patches', metrics: ['pass_rate', 'cost_usd'], schema: { entry: 'code_patch', payload: { patch: 'string' } } },
-- { id: 'codes.review', room: 'codes', entry: 'code_patch', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'code_patch', payload: { patch: 'string' } } },
++ { id: 'codes.patch', room: 'codes', entry: 'code_patch', judge: 'test_suite', name: 'Code patches', metrics: ['pass_rate', 'cost_usd'], schema: { entry: 'code_patch', payload: { patch: 'string' }, prompt: { repository: 'string', failing_test: 'string', constraints: 'string' }, rubric: ['Apply the patch in an isolated checkout', 'Run the pinned failing and regression tests', 'Pass only when all tests pass'], seeds: [{ repository: 'sample-js', failing_test: 'sum([]) should return 0', constraints: 'Change only sum.js' }] } },
++ { id: 'codes.review', room: 'codes', entry: 'review', judge: 'human_pairwise', name: 'Code diff review', metrics: ['elo', 'cost_usd'], schema: { entry: 'review', payload: { review: 'string' }, prompt: { diff: 'string', intent: 'string' }, rubric: ['Find specific correctness or security defects', 'Cite the affected lines and explain impact', 'Avoid speculative or duplicate findings'], seeds: [{ diff: 'A cache key changes from userId to a constant', intent: 'Review for cross-user data exposure' }] } },
{ id: 'surf.desk', room: 'surf', entry: 'agent_on_desk', judge: 'human_pairwise', name: 'Agents on desks', metrics: ['elo', 'cost_usd', 'ms'], schema: { entry: 'agent_on_desk', payload: { desk: 'object' } } },
-- { id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' } } },
++ { id: 'chat.reply', room: 'chat', entry: 'conversation_turn', judge: 'human_pairwise', name: 'Chat replies', metrics: ['elo', 'cost_usd'], schema: { entry: 'conversation_turn', payload: { reply: 'string' }, prompt: { conversation: 'array', latest_user_message: 'string' }, rubric: ['Answer the latest request accurately', 'Use prior turns when relevant', 'Be clear and avoid invented facts'], seeds: [{ conversation: [{ role: 'user', content: 'I have 20 minutes for dinner.' }], latest_user_message: 'Suggest a quick vegetarian meal.' }] } },
{ id: 'speech.tts', room: 'speech', entry: 'audio', judge: 'human_pairwise', name: 'Text to speech voices', metrics: ['elo', 'cost_usd'], schema: { entry: 'audio', payload: { audio_url: 'string' } } },
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
-- { id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' } } },
-- { id: 'search.answer', room: 'search', entry: 'cited_answer', judge: 'human_pairwise', name: 'Cited answers', metrics: ['elo', 'cost_usd'], schema: { entry: 'cited_answer', payload: { answer: 'string', citations: 'array' } } },
++ { id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
++ { id: 'search.answer', room: 'search', entry: 'cited_answer', judge: 'human_pairwise', name: 'Cited answers', metrics: ['elo', 'cost_usd'], schema: { entry: 'cited_answer', payload: { answer: 'string', citations: 'array' }, prompt: { question: 'string', source_urls: 'array' }, rubric: ['Answer from the supplied sources', 'Cite claims with supporting URLs', 'State when the sources do not establish an answer'], seeds: [{ question: 'What year was the first release announced?', source_urls: ['https://example.org/releases'] }] } },
]);
const byId = new Map(KINDS.map((k) => [k.id, k]));
ddiff --git a/server/store.js b/server/store.js
iindex d9a09bf..df03a61 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -259,7 +259,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const entryRows = await db.many('SELECT * FROM bench_entries WHERE round_id = $1 ORDER BY created_at, id', [r.id]);
const revealed = viewerSubject != null && (r.created_by === viewerSubject
|| await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, viewerSubject]) > 0);
-- const votes = num(await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1', [r.id]));
++ const votes = num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method <> 'game_outcome'", [r.id]));
const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }));
const out = { ...wireRound(r), entries, votes };
if (!revealed) out.blind = true;
@@@ -363,7 +363,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
let left = null;
let right = null;
if (judge === 'human_pairwise' || judge === 'game_outcome' || judge === 'audience_poll') {
-- if (judge !== 'audience_poll' && r.created_by === subject) throw new BenchError(403, 'bench.creator_cannot_vote', 'You cannot judge a round you started.');
++ if (judge === 'human_pairwise' && r.created_by === subject) throw new BenchError(403, 'bench.creator_cannot_vote', 'You cannot judge a round you started.');
const p = pair && typeof pair === 'object' ? pair : {};
if (!p.left || !p.right || p.left === p.right) throw new BenchError(422, 'bench.invalid_pair', 'A pairwise judgement needs two different entries.');
const prior = await db.maybe('SELECT 1 AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, subject]);
ddiff --git a/test/api.test.js b/test/api.test.js
iindex 5d1c3e8..be4e1a4 100644
---- a/test/api.test.js
++++ b/test/api.test.js
@@@ -22,6 +22,13 @@ const { boot, check, done } = require('./helpers/boot');
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
assert.ok(ids.includes(want), `missing kind ${want}`);
}
++ for (const id of ['chat.reply', 'translate.ja', 'search.answer', 'codes.patch', 'codes.review']) {
++ const kind = r.json().kinds.find((k) => k.id === id);
++ assert.ok(kind.schema.prompt && Object.keys(kind.schema.prompt).length, `${id} prompt shape`);
++ assert.ok(kind.schema.rubric?.length, `${id} judge rubric`);
++ assert.ok(kind.schema.seeds?.length, `${id} seed items`);
++ assert.ok(Object.keys(kind.schema.seeds[0]).every((key) => key in kind.schema.prompt), `${id} seed matches prompt`);
++ }
});
await check('a person creates a task and queues a round', async () => {
ddiff --git a/test/judges.test.js b/test/judges.test.js
iindex 0866aa8..baf55e1 100644
---- a/test/judges.test.js
++++ b/test/judges.test.js
@@@ -9,6 +9,7 @@ const { boot, check, done } = require('./helpers/boot');
const bob = t.network.addUser('bob');
const entrySvc = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
const judgeSvc = t.network.serviceToken({ sub: 'svc:judge', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
++ const gamesSvc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create', 'bench.judgement.cast'] });
let winRound;
let winPair;
@@@ -101,12 +102,43 @@ const { boot, check, done } = require('./helpers/boot');
await check('game outcome: a service casts a draw from a pair', async () => {
const { roundId, entryIds } = await makeRound('games.match', ['acme/alpha-1', 'beta/beta-2']);
-- const vote = await t.get(`/api/v1/rounds/${roundId}/judgements`, { bearer: judgeSvc, json: { judge: 'game_outcome', pair: { left: entryIds[0], right: entryIds[1] }, outcome: 'draw' } });
++ const refused = await t.get(`/api/v1/rounds/${roundId}/judgements`, { bearer: judgeSvc, json: { judge: 'game_outcome', pair: { left: entryIds[0], right: entryIds[1] }, outcome: 'left' } });
++ assert.strictEqual(refused.status, 403, refused.text);
++ const vote = await t.get(`/api/v1/rounds/${roundId}/judgements`, { bearer: gamesSvc, json: { judge: 'game_outcome', pair: { left: entryIds[0], right: entryIds[1] }, outcome: 'draw' } });
assert.strictEqual(vote.status, 201, vote.text);
assert.strictEqual(vote.json().judgement.judge, 'game_outcome');
assert.strictEqual(vote.json().judgement.outcome, 'draw');
});
++ await check('Games match reaches the JSON API with replay and cost per seat', async () => {
++ const task = await t.get('/api/v1/kinds/games.match/tasks', { bearer: gamesSvc, json: { prompt: 'Vibe Heroes draft duel', public: true } });
++ assert.strictEqual(task.status, 201, task.text);
++ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { bearer: gamesSvc, json: { seed: 17 } });
++ assert.strictEqual(rr.status, 201, rr.text);
++ const rid = rr.json().round.id;
++ const entries = [];
++ for (const [side, cost] of [[0, 0.012], [1, 0.019]]) {
++ const e = await t.get(`/api/v1/rounds/${rid}/entries`, { bearer: gamesSvc, json: {
++ provider: 'acme', model: `seat-${side}`, cost_usd: cost,
++ payload: { result: { game: 'vibe-heroes', side, replay_url: `https://openvibe.games/ai-matches/proof/games/1?side=${side}` } },
++ } });
++ assert.strictEqual(e.status, 201, e.text);
++ entries.push(e.json().entry.id);
++ }
++ const cast = await t.get(`/api/v1/rounds/${rid}/judgements`, { bearer: gamesSvc, json: { judge: 'game_outcome', pair: { left: entries[0], right: entries[1] }, outcome: 'left' } });
++ assert.strictEqual(cast.status, 201, cast.text);
++ assert.strictEqual(cast.json().judgement.outcome, 'left');
++ const roundApi = await t.get(`/api/v1/rounds/${rid}`);
++ assert.strictEqual(roundApi.json().round.votes, 0, 'an automatic result is not a person vote');
++ const api = await t.get('/api/v1/datasets/games.match.jsonl');
++ assert.strictEqual(api.status, 200, api.text);
++ const rows = api.text.trim().split('\n').map((line) => JSON.parse(line));
++ const matchEntries = rows.filter((row) => row.type === 'entry' && row.round_id === rid);
++ assert.deepStrictEqual(matchEntries.map((e) => e.cost_usd), [0.012, 0.019]);
++ assert.ok(matchEntries.every((e) => e.payload.result.replay_url.startsWith('https://openvibe.games/')));
++ assert.strictEqual(rows.find((row) => row.type === 'judgement' && row.round_id === rid).method, 'game_outcome');
++ });
++
await check('audience poll: counts are recorded but 100 viewers change Elo by one game', async () => {
const { roundId, entryIds } = await makeRound('chat.reply', ['poll/one', 'poll/two']);
const pair = { left: entryIds[0], right: entryIds[1] };