openvibers/openvibe.bench Commits

Use one fitted rating across boards and model reads (#189)

OP OpenVibers committed 3 days ago · 3d3eb33cd0 · 5 files, +110 −5

Changes

ddiff --git a/docs/shared-ratings.md b/docs/shared-ratings.md
nnew file mode 100644
iindex 0000000..e9eacb5
---- /dev/null
++++ b/docs/shared-ratings.md
@@@ -0,0 +1,26 @@
++# Shared ratings
++
++`GET /api/v1/ratings/:kindId?category=all` is the public rating read for other sites.
++It returns `{ board: { schema, kind_id, category, minimum_samples: 20, source,
++methodology, rows } }`, caches for sixty seconds and permits anonymous browser GETs.
++Each row contains a model subject, fitted Elo, games, wins/losses/ties, confidence
++and rank intervals. No prompts, answers, voters or site identifiers are returned.
++Models with fewer than twenty eligible games are omitted, including their counts.
++An empty board means insufficient evidence, never a baseline score of 1000.
++Private kinds and the separate adult willingness test are excluded.
++
++This reads `boardElo`, the same deterministic Bradley-Terry fit used by the board
++and JSONL export. `GET /api/v1/models/:model` now takes its rating for each kind
++from this same read. The incremental `bench_ratings` table remains for write-side
++accounting and events; it is no longer the model-summary rating source. Existing
++`/boards` reads retain their provisional views for compatibility.
++
++Design's `server/arena/bench.js` submits each vote to Bench judgements. Exchange's
++`server/bench.js` submits its arena votes to Bench's arena API. Bench kinds already
++write the same judgement stream. There is no second rating computation for these
++consumers. A model subject and kind identify the rating; clients must not average
++different subjects, kinds or provider revisions into a new rating.
++
++For Fable: rating, confidence bounds, eligible games, kind, and category come from
++this read. Empty rows need an insufficient-evidence state. Receipt-based latency
++and costs come from the separate AI metrics API, not these votes.
ddiff --git a/server/app.js b/server/app.js
iindex 911be0c..d096d62 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -139,6 +139,14 @@ async function createApp(opts = {}) {
}
if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;
}
++ paths['/api/v1/ratings/{kindId}'] = { get: {
++ operationId: 'bench.ratings.get', summary: 'Shared model ratings with at least twenty eligible games', security: [],
++ parameters: [
++ { name: 'kindId', in: 'path', required: true, schema: { type: 'string' } },
++ { name: 'category', in: 'query', schema: { type: 'string', default: 'all', maxLength: 80 } },
++ ],
++ responses: { '200': { description: 'The fitted rating board; sparse models are omitted' }, '404': { description: 'No public rating kind' }, '422': { description: 'Invalid category' } },
++ } };
res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });
});
// /.well-known/agent.json (openvibe-shared/agent-layer, SEO-2): what this site serves, for agents.
ddiff --git a/server/http/api.js b/server/http/api.js
iindex 0557c36..b134112 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -289,6 +289,22 @@ function createApi(ctx) {
});
// ── Boards, models, datasets ────────────────────────────
++ const ratingsCors = (_req, res, next) => {
++ res.set('Access-Control-Allow-Origin', '*').set('Access-Control-Allow-Methods', 'GET, OPTIONS');
++ res.set('Access-Control-Allow-Headers', 'Accept, Content-Type');
++ next();
++ };
++ r.options('/ratings/:kindId', ratingsCors, (_req, res) => res.status(204).end());
++ r.get('/ratings/:kindId', ratingsCors, async (req, res) => {
++ try {
++ if (req.query.category !== undefined && typeof req.query.category !== 'string') return invalid(res, 'Use one task category.');
++ const board = await bench.ratings(String(req.params.kindId || ''), { category: req.query.category || 'all' });
++ if (!board) return problem(res, 404, 'bench.kind_not_found', 'No public rating for this kind.');
++ res.set('Cache-Control', 'public, max-age=60');
++ return res.json({ board });
++ } catch (err) { return failed(res, err); }
++ });
++
r.get('/boards/:kindId', async (req, res) => {
try {
const kindId = String(req.params.kindId || '');
ddiff --git a/server/store.js b/server/store.js
iindex 58d2dda..699b370 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -1015,15 +1015,36 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
return rows;
}
++ // The public rating shared by model pages and other sites. It uses the same fit as the
++ // board and dataset; bench_ratings remains a write-side counter, never this read's score.
++ async function ratings(kindId, { category = CATEGORY_DEFAULT } = {}) {
++ const kind = await getKind(kindId);
++ if (!kind || isPrivateKind(kind.id) || kind.schema?.adult === true) return null;
++ const cat = String(category || CATEGORY_DEFAULT);
++ if (!/^[a-zA-Z0-9_.:-]{1,80}$/.test(cat)) throw new BenchError(422, 'bench.invalid_category', 'Use a task category of at most 80 characters.');
++ return {
++ schema: 'bench.ratings@1', kind_id: kindId, category: cat, type: 'elo', adult: false,
++ minimum_samples: 20, source: 'OpenVibe.Bench judgements',
++ methodology: 'Bradley-Terry fit of eligible pairwise games; one game has one vote. Ratings below twenty games are withheld.',
++ rows: (await boardElo(kindId, cat)).filter((row) => row.games >= 20),
++ };
++ }
++
async function modelSummary(model) {
const name = String(model || '');
-- const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);
++ const kinds = await db.many('SELECT DISTINCT r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY r.kind_id', [name]);
++ const modelRatings = [];
++ for (const { kind_id: kindId } of kinds) {
++ const result = await ratings(kindId);
++ const rating = result?.rows.find((row) => row.subject === name);
++ if (rating) modelRatings.push(rating);
++ }
const entries = (await db.many(
`SELECT e.*, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY e.created_at DESC LIMIT 100`,
[name],
)).map((e) => ({ ...wireEntry(e), kind_id: e.kind_id }));
const cost = num(await db.value('SELECT COALESCE(SUM(cost_usd), 0)::numeric AS n FROM bench_entries WHERE model = $1', [name]));
-- return { model: name, ratings, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };
++ return { model: name, ratings: modelRatings, ratings_minimum_samples: 20, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };
}
// ── Dataset (open, no voter ids) ─────────────────────────
@@@ -1174,7 +1195,7 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
createTask, getTask,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
-- board, valueRows, modelSummary, dataset, datasetJsonl,
++ board, ratings, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
};
ddiff --git a/test/boards.test.js b/test/boards.test.js
iindex 65ecd88..c33ae84 100644
---- a/test/boards.test.js
++++ b/test/boards.test.js
@@@ -74,8 +74,8 @@ const { boot, check, done } = require('./helpers/boot');
return { roundId: id, left: a.json().entry.id, right: b.json().entry.id };
}
-- async function seedGames(round, count) {
-- for (let i = 0; i < count; i++) {
++ async function seedGames(round, count, offset = 0) {
++ for (let i = offset; i < offset + count; i++) {
await t.ctx.s.db.exec(
`INSERT INTO bench_judgements (id, round_id, kind_id, judge, method, pair, outcome, subject, voter_kind)
VALUES ($1, $2, 'design.page', 'human_pairwise', 'human_pairwise', $3::jsonb, $4, $5, 'person')`,
@@@ -84,6 +84,40 @@ const { boot, check, done } = require('./helpers/boot');
}
}
++ await check('shared ratings withhold nineteen games and agree with the fitted board at twenty', async () => {
++ const before = await t.get('/api/v1/ratings/design.page');
++ assert.strictEqual(before.status, 200, before.text);
++ assert.deepStrictEqual(before.json().board.rows, []);
++ assert.strictEqual(before.json().board.minimum_samples, 20);
++ assert.strictEqual(before.headers.get('access-control-allow-origin'), '*');
++ assert.strictEqual(before.headers.get('cache-control'), 'public, max-age=60');
++ const round = await roundForCategory('ratings-floor');
++ await seedGames(round, 19);
++ assert.deepStrictEqual((await t.get('/api/v1/ratings/design.page?category=ratings-floor')).json().board.rows, []);
++ await seedGames(round, 1, 19);
++ const result = await t.get('/api/v1/ratings/design.page?category=ratings-floor');
++ const rows = result.json().board.rows;
++ assert.strictEqual(rows.length, 2);
++ assert.ok(rows.every((row) => row.games === 20));
++ const board = (await t.get('/api/v1/boards/design.page?category=ratings-floor')).json().board;
++ for (const row of rows) {
++ const fitted = board.rows.find((x) => x.subject === row.subject);
++ for (const field of ['elo', 'games', 'ci_lower', 'ci_upper', 'rank', 'rank_lo', 'rank_hi']) assert.strictEqual(row[field], fitted[field], field);
++ }
++ assert.ok(!result.text.includes('usr_') && !result.text.includes('prompt') && !result.text.includes('html'));
++ const all = (await t.get('/api/v1/ratings/design.page')).json().board.rows;
++ const model = (await t.get('/api/v1/models/alpha-1')).json();
++ assert.deepStrictEqual(model.ratings.find((row) => row.kind_id === 'design.page'), all.find((row) => row.subject === 'alpha-1'));
++ assert.strictEqual(model.ratings_minimum_samples, 20);
++ assert.strictEqual((await t.get('/api/v1/ratings/chat.willing')).status, 404);
++ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a&category=b')).status, 400);
++ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a%20b')).status, 422);
++ const options = await t.get('/api/v1/ratings/design.page', { method: 'OPTIONS' });
++ assert.strictEqual(options.status, 204);
++ const doc = (await t.get('/openapi.json')).json();
++ assert.ok(doc.paths['/api/v1/ratings/{kindId}'].get);
++ });
++
await check('a category board appears at 30 games and not at 29', async () => {
const arcade = await roundForCategory('arcade');
const writing = await roundForCategory('writing');