openvibers/openvibe.bench Commits
Use one fitted rating across boards and model reads (#189)
Changes
nnew file mode 100644@@@ -0,0 +1,26 @@++# Shared ratings++++`GET /api/v1/ratings/:kindId?category=all` is the public rating read for other sites.++It returns `{ board: { schema, kind_id, category, minimum_samples: 20, source,++methodology, rows } }`, caches for sixty seconds and permits anonymous browser GETs.++Each row contains a model subject, fitted Elo, games, wins/losses/ties, confidence++and rank intervals. No prompts, answers, voters or site identifiers are returned.++Models with fewer than twenty eligible games are omitted, including their counts.++An empty board means insufficient evidence, never a baseline score of 1000.++Private kinds and the separate adult willingness test are excluded.++++This reads `boardElo`, the same deterministic Bradley-Terry fit used by the board++and JSONL export. `GET /api/v1/models/:model` now takes its rating for each kind++from this same read. The incremental `bench_ratings` table remains for write-side++accounting and events; it is no longer the model-summary rating source. Existing++`/boards` reads retain their provisional views for compatibility.++++Design's `server/arena/bench.js` submits each vote to Bench judgements. Exchange's++`server/bench.js` submits its arena votes to Bench's arena API. Bench kinds already++write the same judgement stream. There is no second rating computation for these++consumers. A model subject and kind identify the rating; clients must not average++different subjects, kinds or provider revisions into a new rating.++++For Fable: rating, confidence bounds, eligible games, kind, and category come from++this read. Empty rows need an insufficient-evidence state. Receipt-based latency++and costs come from the separate AI metrics API, not these votes.@@@ -139,6 +139,14 @@ async function createApp(opts = {}) {}if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;}++ paths['/api/v1/ratings/{kindId}'] = { get: {++ operationId: 'bench.ratings.get', summary: 'Shared model ratings with at least twenty eligible games', security: [],++ parameters: [++ { name: 'kindId', in: 'path', required: true, schema: { type: 'string' } },++ { name: 'category', in: 'query', schema: { type: 'string', default: 'all', maxLength: 80 } },++ ],++ responses: { '200': { description: 'The fitted rating board; sparse models are omitted' }, '404': { description: 'No public rating kind' }, '422': { description: 'Invalid category' } },++ } };res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });});// /.well-known/agent.json (openvibe-shared/agent-layer, SEO-2): what this site serves, for agents.@@@ -289,6 +289,22 @@ function createApi(ctx) {});// ── Boards, models, datasets ────────────────────────────++ const ratingsCors = (_req, res, next) => {++ res.set('Access-Control-Allow-Origin', '*').set('Access-Control-Allow-Methods', 'GET, OPTIONS');++ res.set('Access-Control-Allow-Headers', 'Accept, Content-Type');++ next();++ };++ r.options('/ratings/:kindId', ratingsCors, (_req, res) => res.status(204).end());++ r.get('/ratings/:kindId', ratingsCors, async (req, res) => {++ try {++ if (req.query.category !== undefined && typeof req.query.category !== 'string') return invalid(res, 'Use one task category.');++ const board = await bench.ratings(String(req.params.kindId || ''), { category: req.query.category || 'all' });++ if (!board) return problem(res, 404, 'bench.kind_not_found', 'No public rating for this kind.');++ res.set('Cache-Control', 'public, max-age=60');++ return res.json({ board });++ } catch (err) { return failed(res, err); }++ });++r.get('/boards/:kindId', async (req, res) => {try {const kindId = String(req.params.kindId || '');@@@ -1015,15 +1015,36 @@ function createStore(s, { log = console, random = Math.random, config = null, aireturn rows;}++ // The public rating shared by model pages and other sites. It uses the same fit as the++ // board and dataset; bench_ratings remains a write-side counter, never this read's score.++ async function ratings(kindId, { category = CATEGORY_DEFAULT } = {}) {++ const kind = await getKind(kindId);++ if (!kind || isPrivateKind(kind.id) || kind.schema?.adult === true) return null;++ const cat = String(category || CATEGORY_DEFAULT);++ if (!/^[a-zA-Z0-9_.:-]{1,80}$/.test(cat)) throw new BenchError(422, 'bench.invalid_category', 'Use a task category of at most 80 characters.');++ return {++ schema: 'bench.ratings@1', kind_id: kindId, category: cat, type: 'elo', adult: false,++ minimum_samples: 20, source: 'OpenVibe.Bench judgements',++ methodology: 'Bradley-Terry fit of eligible pairwise games; one game has one vote. Ratings below twenty games are withheld.',++ rows: (await boardElo(kindId, cat)).filter((row) => row.games >= 20),++ };++ }++async function modelSummary(model) {const name = String(model || '');-- const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);++ const kinds = await db.many('SELECT DISTINCT r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY r.kind_id', [name]);++ const modelRatings = [];++ for (const { kind_id: kindId } of kinds) {++ const result = await ratings(kindId);++ const rating = result?.rows.find((row) => row.subject === name);++ if (rating) modelRatings.push(rating);++ }const entries = (await db.many(`SELECT e.*, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY e.created_at DESC LIMIT 100`,[name],)).map((e) => ({ ...wireEntry(e), kind_id: e.kind_id }));const cost = num(await db.value('SELECT COALESCE(SUM(cost_usd), 0)::numeric AS n FROM bench_entries WHERE model = $1', [name]));-- return { model: name, ratings, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };++ return { model: name, ratings: modelRatings, ratings_minimum_samples: 20, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };}// ── Dataset (open, no voter ids) ─────────────────────────@@@ -1174,7 +1195,7 @@ function createStore(s, { log = console, random = Math.random, config = null, aicreateTask, getTask,createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,nextJudgement, castJudgement, pairwiseGames,-- board, valueRows, modelSummary, dataset, datasetJsonl,++ board, ratings, valueRows, modelSummary, dataset, datasetJsonl,createEval, getEval, runEval, evalResults, evalDailyCount,KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,};@@@ -74,8 +74,8 @@ const { boot, check, done } = require('./helpers/boot');return { roundId: id, left: a.json().entry.id, right: b.json().entry.id };}-- async function seedGames(round, count) {-- for (let i = 0; i < count; i++) {++ async function seedGames(round, count, offset = 0) {++ for (let i = offset; i < offset + count; i++) {await t.ctx.s.db.exec(`INSERT INTO bench_judgements (id, round_id, kind_id, judge, method, pair, outcome, subject, voter_kind)VALUES ($1, $2, 'design.page', 'human_pairwise', 'human_pairwise', $3::jsonb, $4, $5, 'person')`,@@@ -84,6 +84,40 @@ const { boot, check, done } = require('./helpers/boot');}}++ await check('shared ratings withhold nineteen games and agree with the fitted board at twenty', async () => {++ const before = await t.get('/api/v1/ratings/design.page');++ assert.strictEqual(before.status, 200, before.text);++ assert.deepStrictEqual(before.json().board.rows, []);++ assert.strictEqual(before.json().board.minimum_samples, 20);++ assert.strictEqual(before.headers.get('access-control-allow-origin'), '*');++ assert.strictEqual(before.headers.get('cache-control'), 'public, max-age=60');++ const round = await roundForCategory('ratings-floor');++ await seedGames(round, 19);++ assert.deepStrictEqual((await t.get('/api/v1/ratings/design.page?category=ratings-floor')).json().board.rows, []);++ await seedGames(round, 1, 19);++ const result = await t.get('/api/v1/ratings/design.page?category=ratings-floor');++ const rows = result.json().board.rows;++ assert.strictEqual(rows.length, 2);++ assert.ok(rows.every((row) => row.games === 20));++ const board = (await t.get('/api/v1/boards/design.page?category=ratings-floor')).json().board;++ for (const row of rows) {++ const fitted = board.rows.find((x) => x.subject === row.subject);++ for (const field of ['elo', 'games', 'ci_lower', 'ci_upper', 'rank', 'rank_lo', 'rank_hi']) assert.strictEqual(row[field], fitted[field], field);++ }++ assert.ok(!result.text.includes('usr_') && !result.text.includes('prompt') && !result.text.includes('html'));++ const all = (await t.get('/api/v1/ratings/design.page')).json().board.rows;++ const model = (await t.get('/api/v1/models/alpha-1')).json();++ assert.deepStrictEqual(model.ratings.find((row) => row.kind_id === 'design.page'), all.find((row) => row.subject === 'alpha-1'));++ assert.strictEqual(model.ratings_minimum_samples, 20);++ assert.strictEqual((await t.get('/api/v1/ratings/chat.willing')).status, 404);++ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a&category=b')).status, 400);++ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a%20b')).status, 422);++ const options = await t.get('/api/v1/ratings/design.page', { method: 'OPTIONS' });++ assert.strictEqual(options.status, 204);++ const doc = (await t.get('/openapi.json')).json();++ assert.ok(doc.paths['/api/v1/ratings/{kindId}'].get);++ });++await check('a category board appears at 30 games and not at 29', async () => {const arcade = await roundForCategory('arcade');const writing = await roundForCategory('writing');
new file mode 100644
new file mode 100644
@@ -0,0 +1,26 @@
@@ -0,0 +1,26 @@
+# Shared ratings
+
+`GET /api/v1/ratings/:kindId?category=all` is the public rating read for other sites.
+It returns `{ board: { schema, kind_id, category, minimum_samples: 20, source,
+methodology, rows } }`, caches for sixty seconds and permits anonymous browser GETs.
+Each row contains a model subject, fitted Elo, games, wins/losses/ties, confidence
+and rank intervals. No prompts, answers, voters or site identifiers are returned.
+Models with fewer than twenty eligible games are omitted, including their counts.
+An empty board means insufficient evidence, never a baseline score of 1000.
+Private kinds and the separate adult willingness test are excluded.
+
+This reads `boardElo`, the same deterministic Bradley-Terry fit used by the board
+and JSONL export. `GET /api/v1/models/:model` now takes its rating for each kind
+from this same read. The incremental `bench_ratings` table remains for write-side
+accounting and events; it is no longer the model-summary rating source. Existing
+`/boards` reads retain their provisional views for compatibility.
+
+Design's `server/arena/bench.js` submits each vote to Bench judgements. Exchange's
+`server/bench.js` submits its arena votes to Bench's arena API. Bench kinds already
+write the same judgement stream. There is no second rating computation for these
+consumers. A model subject and kind identify the rating; clients must not average
+different subjects, kinds or provider revisions into a new rating.
+
+For Fable: rating, confidence bounds, eligible games, kind, and category come from
+this read. Empty rows need an insufficient-evidence state. Receipt-based latency
+and costs come from the separate AI metrics API, not these votes.
@@ -139,6 +139,14 @@ async function createApp(opts = {}) {
@@ -139,6 +139,14 @@ async function createApp(opts = {}) {
}
}
if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;
if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;
}
}
+ paths['/api/v1/ratings/{kindId}'] = { get: {
+ operationId: 'bench.ratings.get', summary: 'Shared model ratings with at least twenty eligible games', security: [],
+ parameters: [
+ { name: 'kindId', in: 'path', required: true, schema: { type: 'string' } },
+ { name: 'category', in: 'query', schema: { type: 'string', default: 'all', maxLength: 80 } },
+ ],
+ responses: { '200': { description: 'The fitted rating board; sparse models are omitted' }, '404': { description: 'No public rating kind' }, '422': { description: 'Invalid category' } },
+ } };
res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });
res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });
});
});
// /.well-known/agent.json (openvibe-shared/agent-layer, SEO-2): what this site serves, for agents.
// /.well-known/agent.json (openvibe-shared/agent-layer, SEO-2): what this site serves, for agents.
@@ -289,6 +289,22 @@ function createApi(ctx) {
@@ -289,6 +289,22 @@ function createApi(ctx) {
});
});
// ── Boards, models, datasets ────────────────────────────
// ── Boards, models, datasets ────────────────────────────
+ const ratingsCors = (_req, res, next) => {
+ res.set('Access-Control-Allow-Origin', '*').set('Access-Control-Allow-Methods', 'GET, OPTIONS');
+ res.set('Access-Control-Allow-Headers', 'Accept, Content-Type');
+ next();
+ };
+ r.options('/ratings/:kindId', ratingsCors, (_req, res) => res.status(204).end());
+ r.get('/ratings/:kindId', ratingsCors, async (req, res) => {
+ try {
+ if (req.query.category !== undefined && typeof req.query.category !== 'string') return invalid(res, 'Use one task category.');
+ const board = await bench.ratings(String(req.params.kindId || ''), { category: req.query.category || 'all' });
+ if (!board) return problem(res, 404, 'bench.kind_not_found', 'No public rating for this kind.');
+ res.set('Cache-Control', 'public, max-age=60');
+ return res.json({ board });
+ } catch (err) { return failed(res, err); }
+ });
+
r.get('/boards/:kindId', async (req, res) => {
r.get('/boards/:kindId', async (req, res) => {
try {
try {
const kindId = String(req.params.kindId || '');
const kindId = String(req.params.kindId || '');
@@ -1015,15 +1015,36 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
@@ -1015,15 +1015,36 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
return rows;
return rows;
}
}
+ // The public rating shared by model pages and other sites. It uses the same fit as the
+ // board and dataset; bench_ratings remains a write-side counter, never this read's score.
+ async function ratings(kindId, { category = CATEGORY_DEFAULT } = {}) {
+ const kind = await getKind(kindId);
+ if (!kind || isPrivateKind(kind.id) || kind.schema?.adult === true) return null;
+ const cat = String(category || CATEGORY_DEFAULT);
+ if (!/^[a-zA-Z0-9_.:-]{1,80}$/.test(cat)) throw new BenchError(422, 'bench.invalid_category', 'Use a task category of at most 80 characters.');
+ return {
+ schema: 'bench.ratings@1', kind_id: kindId, category: cat, type: 'elo', adult: false,
+ minimum_samples: 20, source: 'OpenVibe.Bench judgements',
+ methodology: 'Bradley-Terry fit of eligible pairwise games; one game has one vote. Ratings below twenty games are withheld.',
+ rows: (await boardElo(kindId, cat)).filter((row) => row.games >= 20),
+ };
+ }
+
async function modelSummary(model) {
async function modelSummary(model) {
const name = String(model || '');
const name = String(model || '');
- const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);
+ const kinds = await db.many('SELECT DISTINCT r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY r.kind_id', [name]);
+ const modelRatings = [];
+ for (const { kind_id: kindId } of kinds) {
+ const result = await ratings(kindId);
+ const rating = result?.rows.find((row) => row.subject === name);
+ if (rating) modelRatings.push(rating);
+ }
const entries = (await db.many(
const entries = (await db.many(
`SELECT e.*, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY e.created_at DESC LIMIT 100`,
`SELECT e.*, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.model = $1 ORDER BY e.created_at DESC LIMIT 100`,
[name],
[name],
)).map((e) => ({ ...wireEntry(e), kind_id: e.kind_id }));
)).map((e) => ({ ...wireEntry(e), kind_id: e.kind_id }));
const cost = num(await db.value('SELECT COALESCE(SUM(cost_usd), 0)::numeric AS n FROM bench_entries WHERE model = $1', [name]));
const cost = num(await db.value('SELECT COALESCE(SUM(cost_usd), 0)::numeric AS n FROM bench_entries WHERE model = $1', [name]));
- return { model: name, ratings, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };
+ return { model: name, ratings: modelRatings, ratings_minimum_samples: 20, entries, total_cost_usd: Math.round(cost * 1e6) / 1e6 };
}
}
// ── Dataset (open, no voter ids) ─────────────────────────
// ── Dataset (open, no voter ids) ─────────────────────────
@@ -1174,7 +1195,7 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
@@ -1174,7 +1195,7 @@ function createStore(s, { log = console, random = Math.random, config = null, ai
createTask, getTask,
createTask, getTask,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
createRound, createArenaRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
nextJudgement, castJudgement, pairwiseGames,
- board, valueRows, modelSummary, dataset, datasetJsonl,
+ board, ratings, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
createEval, getEval, runEval, evalResults, evalDailyCount,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
};
};
@@ -74,8 +74,8 @@ const { boot, check, done } = require('./helpers/boot');
@@ -74,8 +74,8 @@ const { boot, check, done } = require('./helpers/boot');
return { roundId: id, left: a.json().entry.id, right: b.json().entry.id };
return { roundId: id, left: a.json().entry.id, right: b.json().entry.id };
}
}
- async function seedGames(round, count) {
- for (let i = 0; i < count; i++) {
+ async function seedGames(round, count, offset = 0) {
+ for (let i = offset; i < offset + count; i++) {
await t.ctx.s.db.exec(
await t.ctx.s.db.exec(
`INSERT INTO bench_judgements (id, round_id, kind_id, judge, method, pair, outcome, subject, voter_kind)
`INSERT INTO bench_judgements (id, round_id, kind_id, judge, method, pair, outcome, subject, voter_kind)
VALUES ($1, $2, 'design.page', 'human_pairwise', 'human_pairwise', $3::jsonb, $4, $5, 'person')`,
VALUES ($1, $2, 'design.page', 'human_pairwise', 'human_pairwise', $3::jsonb, $4, $5, 'person')`,
@@ -84,6 +84,40 @@ const { boot, check, done } = require('./helpers/boot');
@@ -84,6 +84,40 @@ const { boot, check, done } = require('./helpers/boot');
}
}
}
}
+ await check('shared ratings withhold nineteen games and agree with the fitted board at twenty', async () => {
+ const before = await t.get('/api/v1/ratings/design.page');
+ assert.strictEqual(before.status, 200, before.text);
+ assert.deepStrictEqual(before.json().board.rows, []);
+ assert.strictEqual(before.json().board.minimum_samples, 20);
+ assert.strictEqual(before.headers.get('access-control-allow-origin'), '*');
+ assert.strictEqual(before.headers.get('cache-control'), 'public, max-age=60');
+ const round = await roundForCategory('ratings-floor');
+ await seedGames(round, 19);
+ assert.deepStrictEqual((await t.get('/api/v1/ratings/design.page?category=ratings-floor')).json().board.rows, []);
+ await seedGames(round, 1, 19);
+ const result = await t.get('/api/v1/ratings/design.page?category=ratings-floor');
+ const rows = result.json().board.rows;
+ assert.strictEqual(rows.length, 2);
+ assert.ok(rows.every((row) => row.games === 20));
+ const board = (await t.get('/api/v1/boards/design.page?category=ratings-floor')).json().board;
+ for (const row of rows) {
+ const fitted = board.rows.find((x) => x.subject === row.subject);
+ for (const field of ['elo', 'games', 'ci_lower', 'ci_upper', 'rank', 'rank_lo', 'rank_hi']) assert.strictEqual(row[field], fitted[field], field);
+ }
+ assert.ok(!result.text.includes('usr_') && !result.text.includes('prompt') && !result.text.includes('html'));
+ const all = (await t.get('/api/v1/ratings/design.page')).json().board.rows;
+ const model = (await t.get('/api/v1/models/alpha-1')).json();
+ assert.deepStrictEqual(model.ratings.find((row) => row.kind_id === 'design.page'), all.find((row) => row.subject === 'alpha-1'));
+ assert.strictEqual(model.ratings_minimum_samples, 20);
+ assert.strictEqual((await t.get('/api/v1/ratings/chat.willing')).status, 404);
+ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a&category=b')).status, 400);
+ assert.strictEqual((await t.get('/api/v1/ratings/design.page?category=a%20b')).status, 422);
+ const options = await t.get('/api/v1/ratings/design.page', { method: 'OPTIONS' });
+ assert.strictEqual(options.status, 204);
+ const doc = (await t.get('/openapi.json')).json();
+ assert.ok(doc.paths['/api/v1/ratings/{kindId}'].get);
+ });
+
await check('a category board appears at 30 games and not at 29', async () => {
await check('a category board appears at 30 games and not at 29', async () => {
const arcade = await roundForCategory('arcade');
const arcade = await roundForCategory('arcade');
const writing = await roundForCategory('writing');
const writing = await roundForCategory('writing');