openvibers/openvibe.report Commits
Read shared Bench ratings across Report AI boards (#177)
Changes
@@@ -9,12 +9,12 @@"port": "5080","domain": "openvibe.report","contracts": "openvibe-contracts v0.384.0",-- "sdk": "openvibe-sdk v0.125.0",++ "sdk": "openvibe-sdk v0.145.0","shared": "openvibe-shared v4.195.0","packages": ["openvibe-contracts v0.384.0","openvibe-publishing v1.7.0",-- "openvibe-sdk v0.125.0",++ "openvibe-sdk v0.145.0","openvibe-shared v4.195.0","pg v8.23.1","pglite v0.5.13 (dev)"nnew file mode 100644@@@ -0,0 +1,23 @@++# Shared ratings++++The `/ai` boards, comparison catalogue and model rating reads use Bench's fitted++ratings. Elo board reads use `openvibe-sdk/bench` `ratings.get(kind)`, cached for a++minute with the existing stale-source marker during an outage. Rows with fewer++than twenty eligible games are withheld. Exchange reads the same endpoint.++Model summaries are fitted by Bench too; Report has no local rating table.++++The older cost and Pareto board reads and benchmark registry remain in place;++they are separate data products, not an alternative Elo calculation. The++incremental Bench table remains write-side accounting only. Design and Exchange++already send their arena votes to Bench.++++Design's active `server/arena/store.js` reads Bench boards. Its legacy local++ratings table is retained for migration compatibility and is not a live rating++source. The optional legacy importer represents old votes as automatic-grader++evidence; those are not silently promoted to authenticated pairwise votes.++++For Fable: rating, lower/upper confidence bounds, games, kind and model subject++are the shared numbers. Empty ratings mean fewer than twenty games, while a++failed source retains the existing unavailable/stale state. Existing catalogue++model matching keeps the Bench subject visible when joining a dated API name to++an Exchange catalogue slug.@@@ -15,7 +15,7 @@"helmet": "^8.1.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0","pg": "^8.23.1"},@@@ -1091,9 +1091,9 @@}},"node_modules/openvibe-sdk": {-- "version": "0.125.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",-- "integrity": "sha512-hGYmiaYLk/QrZC7w0opFAlEQi9fgDct0qcwc4cVBNBzCPm8HSRZlibYBe5x+NVshM3kqgQ6LZihqvFj1LP51bg==",++ "version": "0.145.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",++ "integrity": "sha512-3kEEiEQSzgRHoCNdOQ/xmqNRNNf/QmJ269E6rPASDHpuciUTOwZkjIVMMTfGIf6VGBe73TdYdLjqMAMThSn93g==","license": "MIT","bin": {"openvibe-download": "bin/openvibe-download.js"@@@ -21,7 +21,7 @@"helmet": "^8.1.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0","pg": "^8.23.1"},@@@ -149,6 +149,11 @@ async function createApp(opts = {}) {}if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;}++ paths['/api/v1/ai/ratings/{kind}'] = { get: {++ operationId: 'report.ai.ratings', security: [], summary: 'The shared fitted Bench rating board',++ parameters: [{ name: 'kind', in: 'path', required: true, schema: { type: 'string' } }],++ responses: { '200': { description: 'Ratings with at least twenty eligible games and the source time' }, '422': { description: 'Invalid kind' }, '503': { description: 'Bench unavailable' } },++ } };res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });});app.disable('x-powered-by');@@@ -19,6 +19,8 @@* before any request is made.*/const { trim, requestJson, TIMEOUT_MS } = require('./source');++const { createClient } = require('openvibe-sdk/core');++const { createBenchClient: createSdkBench } = require('openvibe-sdk/bench');const TTL_MS = 60_000;const KIND_RE = /^[a-z0-9][a-z0-9._-]{0,79}$/;@@@ -59,6 +61,7 @@ function normaliseTab(value) {function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = console, now = () => Date.now(), timeoutMs = TIMEOUT_MS } = {}) {const baseUrl = trim(config.benchUrl || 'http://127.0.0.1:5230');const clock = typeof now === 'function' ? now : () => Date.now();++ const shared = createSdkBench(createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl });// The last good copy of every read, kept beyond its minute so a Bench outage never empties a page. Kinds and// boards are a fixed small set; models are capped, the oldest inserted out first.@@@ -73,12 +76,16 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co}/** One read: the fresh copy, the last good one (stale) while Bench is down, or null when there is none yet. */-- async function read(store, key, path, label, cap) {++ async function read(store, key, path, label, cap, loader = null) {const atMs = clock();const hit = store.get(key);if (hit && hit.atMs + TTL_MS > atMs) return { data: hit.data, at: hit.at, stale: false };if (hit && hit.retryAtMs > atMs) return { data: hit.data, at: hit.at, stale: true };-- const data = await requestJson(`${baseUrl}${path}`, { fetchImpl, log, timeoutMs, label });++ let data = null;++ if (loader) {++ try { data = await loader(); }++ catch (err) { log.warn(`[report] ${label} unavailable: ${String(err.message || err).slice(0, 160)}`); }++ } else data = await requestJson(`${baseUrl}${path}`, { fetchImpl, log, timeoutMs, label });if (data === null) {if (hit) hit.retryAtMs = atMs + TTL_MS;return hit ? { data: hit.data, at: hit.at, stale: true } : null;@@@ -102,6 +109,10 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co`/api/v1/boards/${encodeURIComponent(kindId)}?type=${type}`,`bench board (${kindId}, ${type})`,500,++ type === 'elo' ? async () => {++ const data = await shared.ratings.get(kindId, { signal: AbortSignal.timeout(timeoutMs) });++ return { ...data, board: { ...data.board, rows: (data.board?.rows || []).filter((row) => row.games >= 20) } };++ } : null,);}@@@ -48,6 +48,9 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analyticsr.use((req, res, next) => {res.set('Access-Control-Allow-Origin', '*');res.set('Cache-Control', CACHE);++ if (req.method === 'OPTIONS' && /^\/ai\/ratings\/[^/]+$/.test(req.path)) {++ return res.set('Access-Control-Allow-Methods', 'GET, OPTIONS').set('Access-Control-Allow-Headers', 'Accept, Content-Type, traceparent, X-OpenVibe-Request-Id').status(204).end();++ }if (req.method === 'OPTIONS' && req.path === '/me/stats/published') {res.set('Access-Control-Allow-Methods', 'GET, HEAD, PUT, OPTIONS');res.set('Access-Control-Allow-Headers', 'Authorization, Content-Type');@@@ -159,6 +162,13 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analyticsreturn rows;};++ r.get('/ai/ratings/:kind', async (req, res) => {++ if (!bench.isKindId(req.params.kind)) return problem(req, res, 422, 'report.invalid_kind', 'Use a Bench task kind.');++ const result = await bench.board(req.params.kind, 'elo');++ if (!result) return problem(req, res, 503, 'report.ai_source_unavailable', 'OpenVibe.Bench did not answer.');++ return res.set('Cache-Control', 'public, max-age=60').set('Access-Control-Allow-Origin', '*').json({ ...result.data, source_at: result.at, stale: result.stale });++ });++r.get('/ai/leaderboard', async (req, res) => {const all = await aiRows();if (!all) return problem(req, res, 502, 'report.ai_source_unavailable', 'OpenVibe.Bench did not answer.');@@@ -20,9 +20,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')}if (u.port !== '5230') return source(url, opts);if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.dataviz', room: 'design', name: 'Data viz' }] });-- if (u.pathname === '/api/v1/boards/design.dataviz') {-- if (u.searchParams.get('type') !== 'elo') return jsonResponse({ board: { kind_id: 'design.dataviz', rows: [] } });-- return jsonResponse({ board: { kind_id: 'design.dataviz', type: 'elo', rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, ci_lower: 960, ci_upper: 1040, games: 4 }, { subject: 'model-zero', elo: 1000, games: 0 }] } });++ if (u.pathname === '/api/v1/ratings/design.dataviz') {++ return jsonResponse({ board: { kind_id: 'design.dataviz', type: 'elo', rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, ci_lower: 960, ci_upper: 1040, games: 20 }, { subject: 'model-zero', elo: 1000, games: 0 }] } });}return jsonResponse({}, 404);};@@@ -45,8 +44,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')await check('a base rating with no games is not a rating', async () => {const r = await t.get('/api/v1/ai/leaderboard');const zero = r.json().rows.find((x) => x.id === 'model-zero');-- assert.ok(zero, 'listed');-- assert.strictEqual(zero.rating.value, null);++ assert.strictEqual(zero, undefined, 'a sparse Bench-only subject is withheld, including its count');const page = await t.get('/ai/models/model-zero');assert.ok(page.text.includes('Not rated on any task yet'), 'the model page says so');});@@@ -43,7 +43,7 @@ const catalogue = { models:[{ slug:'openai/gpt-5-nano', offers:[{provider:'openaif(u.pathname==='/api/v1/exchange/market')return jsonResponse({...market,next_cursor:null});if(u.pathname==='/api/v1/exchange/catalog')return jsonResponse(catalogue);if(u.pathname==='/api/v1/kinds')return jsonResponse({kinds:[{id:'chat.arena',name:'Chat'}]});-- if(u.pathname==='/api/v1/boards/chat.arena')return jsonResponse({board:{rows:[{subject:'gpt-5-nano-2025-08-07',elo:1000,games:4}]}});++ if(u.pathname==='/api/v1/ratings/chat.arena')return jsonResponse({board:{rows:[{subject:'gpt-5-nano-2025-08-07',elo:1000,games:20}]}});return source(url,opts);}});try {@@@ -14,12 +14,12 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')calls.push(u.pathname + u.search);if (down) throw new Error('Bench offline');if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page <design>', judge: 'human_pairwise' }] });-- if (u.pathname === '/api/v1/boards/design.page') {-- const type = u.searchParams.get('type');++ if (['/api/v1/boards/design.page', '/api/v1/ratings/design.page'].includes(u.pathname)) {++ const type = u.pathname.includes('/ratings/') ? 'elo' : u.searchParams.get('type');const rows = {elo: [-- { subject: 'openai/gpt-4o', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75, provisional: true },-- { subject: 'model-b', elo: 980, ci_lower: 940, ci_upper: 1020, games: 8, win_rate: 0.25, provisional: true },++ { subject: 'openai/gpt-4o', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75, provisional: false },++ { subject: 'model-b', elo: 980, ci_lower: 940, ci_upper: 1020, games: 20, win_rate: 0.25, provisional: false },],pareto: [{ subject: 'openai/gpt-4o', quality: 0.8 },@@@ -32,7 +32,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')}[type];return rows ? jsonResponse({ board: { kind_id: 'design.page', type, rows } }) : jsonResponse({}, 404);}-- if (u.pathname === '/api/v1/models/openai%2Fgpt-4o') return jsonResponse({ model: 'openai/gpt-4o', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75 }], entries: [{ kind_id: 'design.page' }], total_cost_usd: 0.8 });++ if (u.pathname === '/api/v1/models/openai%2Fgpt-4o') return jsonResponse({ model: 'openai/gpt-4o', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75 }], entries: [{ kind_id: 'design.page' }], total_cost_usd: 0.8 });return jsonResponse({}, 404);};const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });@@@ -44,10 +44,18 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')assert.ok(!r.text.includes('Page <design>'));});-- await check('Elo shows ranks, intervals, win rates and provisional state', async () => {++ await check('Shared Elo shows ranks, intervals and win rates', async () => {const r = await t.get('/ai/design.page');assert.strictEqual(r.status, 200);-- for (const s of ['<ol class="ovk-rank"', '<b>1,043</b><small>± 38</small>', '75% won', 'provisional', 'Ranking', '95% interval', 'dataset JSONL']) assert.ok(r.text.includes(s), s);++ for (const s of ['<ol class="ovk-rank"', '<b>1,043</b><small>± 38</small>', '75% won', 'Ranking', '95% interval', 'dataset JSONL']) assert.ok(r.text.includes(s), s);++ const data = await t.get('/api/v1/ai/ratings/design.page');++ assert.strictEqual(data.status, 200, data.text);++ assert.strictEqual(data.json().board.rows[0].elo, 1043);++ assert.strictEqual(data.json().board.rows[0].games, 20);++ assert.strictEqual(data.headers.get('access-control-allow-origin'), '*');++ const preflight = await t.get('/api/v1/ai/ratings/design.page', { method: 'OPTIONS' });++ assert.strictEqual(preflight.status, 204);++ assert.match(preflight.headers.get('access-control-allow-headers'), /traceparent/);});await check('Pareto has an inline SVG, frontier labels and an exact-value table', async () => {@@@ -98,7 +106,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')const u = new URL(String(url));if (u.port !== '5230') return source(url, opts);if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 2, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }, { id: 'speech.asr', room: 'speech', name: 'Speech', judge: 'automatic_grader' }] });-- if (u.pathname.startsWith('/api/v1/boards/')) return jsonResponse({ board: { rows: [] } });++ if ((/^\/api\/v1\/(boards|ratings)\//.test(u.pathname))) return jsonResponse({ board: { rows: [] } });return jsonResponse({}, 404);},env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },@@@ -121,7 +129,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')const u = new URL(String(url));if (u.port !== '5230') return source(url, opts);if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }] });-- if (u.pathname.startsWith('/api/v1/boards/')) return jsonResponse({ board: { rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, games: 0, provisional: true }] } });++ if ((/^\/api\/v1\/(boards|ratings)\//.test(u.pathname))) return jsonResponse({ board: { rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, games: 0, provisional: true }] } });return jsonResponse({}, 404);},env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },@@@ -149,7 +157,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')fetchImpl: async (url, opts) => {const u = new URL(String(url));if (u.port === '5230' && u.pathname === '/api/v1/kinds') return jsonResponse({ kinds: [{ id: 'code', name: 'Code' }] });-- if (u.port === '5230' && u.pathname === '/api/v1/boards/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });++ if (u.port === '5230' && u.pathname === '/api/v1/ratings/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });if (u.port === '5230' && u.pathname.includes('/summary')) return jsonResponse({ model: 'maker/model-a', runs: 4 });return source(url, opts);}, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },@@@ -55,7 +55,7 @@ const modelDetail = {],};--const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75 }], entries: [], total_cost_usd: 0.04 };++const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75 }], entries: [], total_cost_usd: 0.04 };(async () => {const source = stubFetch();@@@ -64,10 +64,10 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043if (u.port !== '5230') return source(url, opts);const p = u.pathname;if (p === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Design page arena', judge: 'human_pairwise' }] });-- if (p === '/api/v1/boards/design.page') {-- const type = u.searchParams.get('type');++ if (['/api/v1/boards/design.page', '/api/v1/ratings/design.page'].includes(p)) {++ const type = p.includes('/ratings/') ? 'elo' : u.searchParams.get('type');const rows = type === 'elo'-- ? [{ subject: 'model-a', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75, median_cost_usd: 0.02, median_ms: 1000, provisional: true }]++ ? [{ subject: 'model-a', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75, median_cost_usd: 0.02, median_ms: 1000, provisional: false }]: type === 'pareto' ? [{ subject: 'model-a', quality: 0.8 }] : [{ subject: 'model-a', entries: 2, solved: 2, total_cost_usd: 0.04, cost_per_solved: 0.02, median_ms: 1000 }];return jsonResponse({ board: { kind_id: 'design.page', type, rows } });}@@@ -54,5 +54,19 @@ const { check, done, jsonResponse } = require('./helpers/boot');assert.strictEqual(bench.normaliseLimit(1000), 100);});++ await check('rating boards use the anonymous SDK read and withhold sparse scores', async () => {++ const requests = [];++ const shared = createBenchClient({ config: { benchUrl: 'http://bench.test' }, fetchImpl: async (url, init) => {++ requests.push(String(url));++ assert.strictEqual(new Headers(init.headers).has('authorization'), false);++ return jsonResponse({ board: { minimum_samples: 20, rows: [{ subject: 'sparse', games: 19, elo: 1500 }, { subject: 'rated', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 }] } });++ } });++ const board = await shared.board('design.page');++ assert.strictEqual(requests[0], 'http://bench.test/api/v1/ratings/design.page');++ assert.deepStrictEqual(board.data.board.rows, [{ subject: 'rated', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 }]);++ assert.strictEqual((await shared.board('design.page')).data.board.rows[0].elo, 1100);++ assert.strictEqual(requests.length, 1);++ });++done();})().catch((e) => { console.error(e); process.exit(1); });
@@ -9,12 +9,12 @@
@@ -9,12 +9,12 @@
"port": "5080",
"port": "5080",
"domain": "openvibe.report",
"domain": "openvibe.report",
"contracts": "openvibe-contracts v0.384.0",
"contracts": "openvibe-contracts v0.384.0",
- "sdk": "openvibe-sdk v0.125.0",
+ "sdk": "openvibe-sdk v0.145.0",
"shared": "openvibe-shared v4.195.0",
"shared": "openvibe-shared v4.195.0",
"packages": [
"packages": [
"openvibe-contracts v0.384.0",
"openvibe-contracts v0.384.0",
"openvibe-publishing v1.7.0",
"openvibe-publishing v1.7.0",
- "openvibe-sdk v0.125.0",
+ "openvibe-sdk v0.145.0",
"openvibe-shared v4.195.0",
"openvibe-shared v4.195.0",
"pg v8.23.1",
"pg v8.23.1",
"pglite v0.5.13 (dev)"
"pglite v0.5.13 (dev)"
new file mode 100644
new file mode 100644
@@ -0,0 +1,23 @@
@@ -0,0 +1,23 @@
+# Shared ratings
+
+The `/ai` boards, comparison catalogue and model rating reads use Bench's fitted
+ratings. Elo board reads use `openvibe-sdk/bench` `ratings.get(kind)`, cached for a
+minute with the existing stale-source marker during an outage. Rows with fewer
+than twenty eligible games are withheld. Exchange reads the same endpoint.
+Model summaries are fitted by Bench too; Report has no local rating table.
+
+The older cost and Pareto board reads and benchmark registry remain in place;
+they are separate data products, not an alternative Elo calculation. The
+incremental Bench table remains write-side accounting only. Design and Exchange
+already send their arena votes to Bench.
+
+Design's active `server/arena/store.js` reads Bench boards. Its legacy local
+ratings table is retained for migration compatibility and is not a live rating
+source. The optional legacy importer represents old votes as automatic-grader
+evidence; those are not silently promoted to authenticated pairwise votes.
+
+For Fable: rating, lower/upper confidence bounds, games, kind and model subject
+are the shared numbers. Empty ratings mean fewer than twenty games, while a
+failed source retains the existing unavailable/stale state. Existing catalogue
+model matching keeps the Bench subject visible when joining a dated API name to
+an Exchange catalogue slug.
@@ -15,7 +15,7 @@
@@ -15,7 +15,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0",
"pg": "^8.23.1"
"pg": "^8.23.1"
},
},
@@ -1091,9 +1091,9 @@
@@ -1091,9 +1091,9 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.125.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",
- "integrity": "sha512-hGYmiaYLk/QrZC7w0opFAlEQi9fgDct0qcwc4cVBNBzCPm8HSRZlibYBe5x+NVshM3kqgQ6LZihqvFj1LP51bg==",
+ "version": "0.145.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
+ "integrity": "sha512-3kEEiEQSzgRHoCNdOQ/xmqNRNNf/QmJ269E6rPASDHpuciUTOwZkjIVMMTfGIf6VGBe73TdYdLjqMAMThSn93g==",
"license": "MIT",
"license": "MIT",
"bin": {
"bin": {
"openvibe-download": "bin/openvibe-download.js"
"openvibe-download": "bin/openvibe-download.js"
@@ -21,7 +21,7 @@
@@ -21,7 +21,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.125.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.195.0",
"pg": "^8.23.1"
"pg": "^8.23.1"
},
},
@@ -149,6 +149,11 @@ async function createApp(opts = {}) {
@@ -149,6 +149,11 @@ async function createApp(opts = {}) {
}
}
if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;
if (Object.keys(kept).some((k) => METHODS.has(k))) paths[p] = kept;
}
}
+ paths['/api/v1/ai/ratings/{kind}'] = { get: {
+ operationId: 'report.ai.ratings', security: [], summary: 'The shared fitted Bench rating board',
+ parameters: [{ name: 'kind', in: 'path', required: true, schema: { type: 'string' } }],
+ responses: { '200': { description: 'Ratings with at least twenty eligible games and the source time' }, '422': { description: 'Invalid kind' }, '503': { description: 'Bench unavailable' } },
+ } };
res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });
res.set('Cache-Control', 'public, max-age=3600').set('Access-Control-Allow-Origin', '*').json({ ...doc, paths });
});
});
app.disable('x-powered-by');
app.disable('x-powered-by');
@@ -19,6 +19,8 @@
@@ -19,6 +19,8 @@
* before any request is made.
* before any request is made.
*/
*/
const { trim, requestJson, TIMEOUT_MS } = require('./source');
const { trim, requestJson, TIMEOUT_MS } = require('./source');
+const { createClient } = require('openvibe-sdk/core');
+const { createBenchClient: createSdkBench } = require('openvibe-sdk/bench');
const TTL_MS = 60_000;
const TTL_MS = 60_000;
const KIND_RE = /^[a-z0-9][a-z0-9._-]{0,79}$/;
const KIND_RE = /^[a-z0-9][a-z0-9._-]{0,79}$/;
@@ -59,6 +61,7 @@ function normaliseTab(value) {
@@ -59,6 +61,7 @@ function normaliseTab(value) {
function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = console, now = () => Date.now(), timeoutMs = TIMEOUT_MS } = {}) {
function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = console, now = () => Date.now(), timeoutMs = TIMEOUT_MS } = {}) {
const baseUrl = trim(config.benchUrl || 'http://127.0.0.1:5230');
const baseUrl = trim(config.benchUrl || 'http://127.0.0.1:5230');
const clock = typeof now === 'function' ? now : () => Date.now();
const clock = typeof now === 'function' ? now : () => Date.now();
+ const shared = createSdkBench(createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl });
// The last good copy of every read, kept beyond its minute so a Bench outage never empties a page. Kinds and
// The last good copy of every read, kept beyond its minute so a Bench outage never empties a page. Kinds and
// boards are a fixed small set; models are capped, the oldest inserted out first.
// boards are a fixed small set; models are capped, the oldest inserted out first.
@@ -73,12 +76,16 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co
@@ -73,12 +76,16 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co
}
}
/** One read: the fresh copy, the last good one (stale) while Bench is down, or null when there is none yet. */
/** One read: the fresh copy, the last good one (stale) while Bench is down, or null when there is none yet. */
- async function read(store, key, path, label, cap) {
+ async function read(store, key, path, label, cap, loader = null) {
const atMs = clock();
const atMs = clock();
const hit = store.get(key);
const hit = store.get(key);
if (hit && hit.atMs + TTL_MS > atMs) return { data: hit.data, at: hit.at, stale: false };
if (hit && hit.atMs + TTL_MS > atMs) return { data: hit.data, at: hit.at, stale: false };
if (hit && hit.retryAtMs > atMs) return { data: hit.data, at: hit.at, stale: true };
if (hit && hit.retryAtMs > atMs) return { data: hit.data, at: hit.at, stale: true };
- const data = await requestJson(`${baseUrl}${path}`, { fetchImpl, log, timeoutMs, label });
+ let data = null;
+ if (loader) {
+ try { data = await loader(); }
+ catch (err) { log.warn(`[report] ${label} unavailable: ${String(err.message || err).slice(0, 160)}`); }
+ } else data = await requestJson(`${baseUrl}${path}`, { fetchImpl, log, timeoutMs, label });
if (data === null) {
if (data === null) {
if (hit) hit.retryAtMs = atMs + TTL_MS;
if (hit) hit.retryAtMs = atMs + TTL_MS;
return hit ? { data: hit.data, at: hit.at, stale: true } : null;
return hit ? { data: hit.data, at: hit.at, stale: true } : null;
@@ -102,6 +109,10 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co
@@ -102,6 +109,10 @@ function createBenchClient({ config = {}, fetchImpl = globalThis.fetch, log = co
`/api/v1/boards/${encodeURIComponent(kindId)}?type=${type}`,
`/api/v1/boards/${encodeURIComponent(kindId)}?type=${type}`,
`bench board (${kindId}, ${type})`,
`bench board (${kindId}, ${type})`,
500,
500,
+ type === 'elo' ? async () => {
+ const data = await shared.ratings.get(kindId, { signal: AbortSignal.timeout(timeoutMs) });
+ return { ...data, board: { ...data.board, rows: (data.board?.rows || []).filter((row) => row.games >= 20) } };
+ } : null,
);
);
}
}
@@ -48,6 +48,9 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
@@ -48,6 +48,9 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
r.use((req, res, next) => {
r.use((req, res, next) => {
res.set('Access-Control-Allow-Origin', '*');
res.set('Access-Control-Allow-Origin', '*');
res.set('Cache-Control', CACHE);
res.set('Cache-Control', CACHE);
+ if (req.method === 'OPTIONS' && /^\/ai\/ratings\/[^/]+$/.test(req.path)) {
+ return res.set('Access-Control-Allow-Methods', 'GET, OPTIONS').set('Access-Control-Allow-Headers', 'Accept, Content-Type, traceparent, X-OpenVibe-Request-Id').status(204).end();
+ }
if (req.method === 'OPTIONS' && req.path === '/me/stats/published') {
if (req.method === 'OPTIONS' && req.path === '/me/stats/published') {
res.set('Access-Control-Allow-Methods', 'GET, HEAD, PUT, OPTIONS');
res.set('Access-Control-Allow-Methods', 'GET, HEAD, PUT, OPTIONS');
res.set('Access-Control-Allow-Headers', 'Authorization, Content-Type');
res.set('Access-Control-Allow-Headers', 'Authorization, Content-Type');
@@ -159,6 +162,13 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
@@ -159,6 +162,13 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
return rows;
return rows;
};
};
+ r.get('/ai/ratings/:kind', async (req, res) => {
+ if (!bench.isKindId(req.params.kind)) return problem(req, res, 422, 'report.invalid_kind', 'Use a Bench task kind.');
+ const result = await bench.board(req.params.kind, 'elo');
+ if (!result) return problem(req, res, 503, 'report.ai_source_unavailable', 'OpenVibe.Bench did not answer.');
+ return res.set('Cache-Control', 'public, max-age=60').set('Access-Control-Allow-Origin', '*').json({ ...result.data, source_at: result.at, stale: result.stale });
+ });
+
r.get('/ai/leaderboard', async (req, res) => {
r.get('/ai/leaderboard', async (req, res) => {
const all = await aiRows();
const all = await aiRows();
if (!all) return problem(req, res, 502, 'report.ai_source_unavailable', 'OpenVibe.Bench did not answer.');
if (!all) return problem(req, res, 502, 'report.ai_source_unavailable', 'OpenVibe.Bench did not answer.');
@@ -20,9 +20,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -20,9 +20,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
}
}
if (u.port !== '5230') return source(url, opts);
if (u.port !== '5230') return source(url, opts);
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.dataviz', room: 'design', name: 'Data viz' }] });
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.dataviz', room: 'design', name: 'Data viz' }] });
- if (u.pathname === '/api/v1/boards/design.dataviz') {
- if (u.searchParams.get('type') !== 'elo') return jsonResponse({ board: { kind_id: 'design.dataviz', rows: [] } });
- return jsonResponse({ board: { kind_id: 'design.dataviz', type: 'elo', rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, ci_lower: 960, ci_upper: 1040, games: 4 }, { subject: 'model-zero', elo: 1000, games: 0 }] } });
+ if (u.pathname === '/api/v1/ratings/design.dataviz') {
+ return jsonResponse({ board: { kind_id: 'design.dataviz', type: 'elo', rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, ci_lower: 960, ci_upper: 1040, games: 20 }, { subject: 'model-zero', elo: 1000, games: 0 }] } });
}
}
return jsonResponse({}, 404);
return jsonResponse({}, 404);
};
};
@@ -45,8 +44,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -45,8 +44,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
await check('a base rating with no games is not a rating', async () => {
await check('a base rating with no games is not a rating', async () => {
const r = await t.get('/api/v1/ai/leaderboard');
const r = await t.get('/api/v1/ai/leaderboard');
const zero = r.json().rows.find((x) => x.id === 'model-zero');
const zero = r.json().rows.find((x) => x.id === 'model-zero');
- assert.ok(zero, 'listed');
- assert.strictEqual(zero.rating.value, null);
+ assert.strictEqual(zero, undefined, 'a sparse Bench-only subject is withheld, including its count');
const page = await t.get('/ai/models/model-zero');
const page = await t.get('/ai/models/model-zero');
assert.ok(page.text.includes('Not rated on any task yet'), 'the model page says so');
assert.ok(page.text.includes('Not rated on any task yet'), 'the model page says so');
});
});
@@ -43,7 +43,7 @@ const catalogue = { models:[{ slug:'openai/gpt-5-nano', offers:[{provider:'opena
@@ -43,7 +43,7 @@ const catalogue = { models:[{ slug:'openai/gpt-5-nano', offers:[{provider:'opena
if(u.pathname==='/api/v1/exchange/market')return jsonResponse({...market,next_cursor:null});
if(u.pathname==='/api/v1/exchange/market')return jsonResponse({...market,next_cursor:null});
if(u.pathname==='/api/v1/exchange/catalog')return jsonResponse(catalogue);
if(u.pathname==='/api/v1/exchange/catalog')return jsonResponse(catalogue);
if(u.pathname==='/api/v1/kinds')return jsonResponse({kinds:[{id:'chat.arena',name:'Chat'}]});
if(u.pathname==='/api/v1/kinds')return jsonResponse({kinds:[{id:'chat.arena',name:'Chat'}]});
- if(u.pathname==='/api/v1/boards/chat.arena')return jsonResponse({board:{rows:[{subject:'gpt-5-nano-2025-08-07',elo:1000,games:4}]}});
+ if(u.pathname==='/api/v1/ratings/chat.arena')return jsonResponse({board:{rows:[{subject:'gpt-5-nano-2025-08-07',elo:1000,games:20}]}});
return source(url,opts);
return source(url,opts);
}});
}});
try {
try {
@@ -14,12 +14,12 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -14,12 +14,12 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
calls.push(u.pathname + u.search);
calls.push(u.pathname + u.search);
if (down) throw new Error('Bench offline');
if (down) throw new Error('Bench offline');
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page <design>', judge: 'human_pairwise' }] });
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page <design>', judge: 'human_pairwise' }] });
- if (u.pathname === '/api/v1/boards/design.page') {
- const type = u.searchParams.get('type');
+ if (['/api/v1/boards/design.page', '/api/v1/ratings/design.page'].includes(u.pathname)) {
+ const type = u.pathname.includes('/ratings/') ? 'elo' : u.searchParams.get('type');
const rows = {
const rows = {
elo: [
elo: [
- { subject: 'openai/gpt-4o', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75, provisional: true },
- { subject: 'model-b', elo: 980, ci_lower: 940, ci_upper: 1020, games: 8, win_rate: 0.25, provisional: true },
+ { subject: 'openai/gpt-4o', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75, provisional: false },
+ { subject: 'model-b', elo: 980, ci_lower: 940, ci_upper: 1020, games: 20, win_rate: 0.25, provisional: false },
],
],
pareto: [
pareto: [
{ subject: 'openai/gpt-4o', quality: 0.8 },
{ subject: 'openai/gpt-4o', quality: 0.8 },
@@ -32,7 +32,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -32,7 +32,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
}[type];
}[type];
return rows ? jsonResponse({ board: { kind_id: 'design.page', type, rows } }) : jsonResponse({}, 404);
return rows ? jsonResponse({ board: { kind_id: 'design.page', type, rows } }) : jsonResponse({}, 404);
}
}
- if (u.pathname === '/api/v1/models/openai%2Fgpt-4o') return jsonResponse({ model: 'openai/gpt-4o', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75 }], entries: [{ kind_id: 'design.page' }], total_cost_usd: 0.8 });
+ if (u.pathname === '/api/v1/models/openai%2Fgpt-4o') return jsonResponse({ model: 'openai/gpt-4o', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75 }], entries: [{ kind_id: 'design.page' }], total_cost_usd: 0.8 });
return jsonResponse({}, 404);
return jsonResponse({}, 404);
};
};
const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
@@ -44,10 +44,18 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -44,10 +44,18 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
assert.ok(!r.text.includes('Page <design>'));
assert.ok(!r.text.includes('Page <design>'));
});
});
- await check('Elo shows ranks, intervals, win rates and provisional state', async () => {
+ await check('Shared Elo shows ranks, intervals and win rates', async () => {
const r = await t.get('/ai/design.page');
const r = await t.get('/ai/design.page');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
- for (const s of ['<ol class="ovk-rank"', '<b>1,043</b><small>± 38</small>', '75% won', 'provisional', 'Ranking', '95% interval', 'dataset JSONL']) assert.ok(r.text.includes(s), s);
+ for (const s of ['<ol class="ovk-rank"', '<b>1,043</b><small>± 38</small>', '75% won', 'Ranking', '95% interval', 'dataset JSONL']) assert.ok(r.text.includes(s), s);
+ const data = await t.get('/api/v1/ai/ratings/design.page');
+ assert.strictEqual(data.status, 200, data.text);
+ assert.strictEqual(data.json().board.rows[0].elo, 1043);
+ assert.strictEqual(data.json().board.rows[0].games, 20);
+ assert.strictEqual(data.headers.get('access-control-allow-origin'), '*');
+ const preflight = await t.get('/api/v1/ai/ratings/design.page', { method: 'OPTIONS' });
+ assert.strictEqual(preflight.status, 204);
+ assert.match(preflight.headers.get('access-control-allow-headers'), /traceparent/);
});
});
await check('Pareto has an inline SVG, frontier labels and an exact-value table', async () => {
await check('Pareto has an inline SVG, frontier labels and an exact-value table', async () => {
@@ -98,7 +106,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -98,7 +106,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
const u = new URL(String(url));
const u = new URL(String(url));
if (u.port !== '5230') return source(url, opts);
if (u.port !== '5230') return source(url, opts);
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 2, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }, { id: 'speech.asr', room: 'speech', name: 'Speech', judge: 'automatic_grader' }] });
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 2, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }, { id: 'speech.asr', room: 'speech', name: 'Speech', judge: 'automatic_grader' }] });
- if (u.pathname.startsWith('/api/v1/boards/')) return jsonResponse({ board: { rows: [] } });
+ if ((/^\/api\/v1\/(boards|ratings)\//.test(u.pathname))) return jsonResponse({ board: { rows: [] } });
return jsonResponse({}, 404);
return jsonResponse({}, 404);
},
},
env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
@@ -121,7 +129,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -121,7 +129,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
const u = new URL(String(url));
const u = new URL(String(url));
if (u.port !== '5230') return source(url, opts);
if (u.port !== '5230') return source(url, opts);
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }] });
if (u.pathname === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Page', judge: 'human_pairwise' }] });
- if (u.pathname.startsWith('/api/v1/boards/')) return jsonResponse({ board: { rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, games: 0, provisional: true }] } });
+ if ((/^\/api\/v1\/(boards|ratings)\//.test(u.pathname))) return jsonResponse({ board: { rows: [{ subject: 'gpt-5-nano-2025-08-07', elo: 1000, games: 0, provisional: true }] } });
return jsonResponse({}, 404);
return jsonResponse({}, 404);
},
},
env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
@@ -149,7 +157,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -149,7 +157,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
fetchImpl: async (url, opts) => {
fetchImpl: async (url, opts) => {
const u = new URL(String(url));
const u = new URL(String(url));
if (u.port === '5230' && u.pathname === '/api/v1/kinds') return jsonResponse({ kinds: [{ id: 'code', name: 'Code' }] });
if (u.port === '5230' && u.pathname === '/api/v1/kinds') return jsonResponse({ kinds: [{ id: 'code', name: 'Code' }] });
- if (u.port === '5230' && u.pathname === '/api/v1/boards/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });
+ if (u.port === '5230' && u.pathname === '/api/v1/ratings/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });
if (u.port === '5230' && u.pathname.includes('/summary')) return jsonResponse({ model: 'maker/model-a', runs: 4 });
if (u.port === '5230' && u.pathname.includes('/summary')) return jsonResponse({ model: 'maker/model-a', runs: 4 });
return source(url, opts);
return source(url, opts);
}, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
}, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
@@ -55,7 +55,7 @@ const modelDetail = {
@@ -55,7 +55,7 @@ const modelDetail = {
],
],
};
};
-const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75 }], entries: [], total_cost_usd: 0.04 };
+const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75 }], entries: [], total_cost_usd: 0.04 };
(async () => {
(async () => {
const source = stubFetch();
const source = stubFetch();
@@ -64,10 +64,10 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
@@ -64,10 +64,10 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
if (u.port !== '5230') return source(url, opts);
if (u.port !== '5230') return source(url, opts);
const p = u.pathname;
const p = u.pathname;
if (p === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Design page arena', judge: 'human_pairwise' }] });
if (p === '/api/v1/kinds') return jsonResponse({ count: 1, kinds: [{ id: 'design.page', room: 'design', name: 'Design page arena', judge: 'human_pairwise' }] });
- if (p === '/api/v1/boards/design.page') {
- const type = u.searchParams.get('type');
+ if (['/api/v1/boards/design.page', '/api/v1/ratings/design.page'].includes(p)) {
+ const type = p.includes('/ratings/') ? 'elo' : u.searchParams.get('type');
const rows = type === 'elo'
const rows = type === 'elo'
- ? [{ subject: 'model-a', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 8, win_rate: 0.75, median_cost_usd: 0.02, median_ms: 1000, provisional: true }]
+ ? [{ subject: 'model-a', elo: 1043, ci_lower: 1005, ci_upper: 1081, games: 20, win_rate: 0.75, median_cost_usd: 0.02, median_ms: 1000, provisional: false }]
: type === 'pareto' ? [{ subject: 'model-a', quality: 0.8 }] : [{ subject: 'model-a', entries: 2, solved: 2, total_cost_usd: 0.04, cost_per_solved: 0.02, median_ms: 1000 }];
: type === 'pareto' ? [{ subject: 'model-a', quality: 0.8 }] : [{ subject: 'model-a', entries: 2, solved: 2, total_cost_usd: 0.04, cost_per_solved: 0.02, median_ms: 1000 }];
return jsonResponse({ board: { kind_id: 'design.page', type, rows } });
return jsonResponse({ board: { kind_id: 'design.page', type, rows } });
}
}
@@ -54,5 +54,19 @@ const { check, done, jsonResponse } = require('./helpers/boot');
@@ -54,5 +54,19 @@ const { check, done, jsonResponse } = require('./helpers/boot');
assert.strictEqual(bench.normaliseLimit(1000), 100);
assert.strictEqual(bench.normaliseLimit(1000), 100);
});
});
+ await check('rating boards use the anonymous SDK read and withhold sparse scores', async () => {
+ const requests = [];
+ const shared = createBenchClient({ config: { benchUrl: 'http://bench.test' }, fetchImpl: async (url, init) => {
+ requests.push(String(url));
+ assert.strictEqual(new Headers(init.headers).has('authorization'), false);
+ return jsonResponse({ board: { minimum_samples: 20, rows: [{ subject: 'sparse', games: 19, elo: 1500 }, { subject: 'rated', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 }] } });
+ } });
+ const board = await shared.board('design.page');
+ assert.strictEqual(requests[0], 'http://bench.test/api/v1/ratings/design.page');
+ assert.deepStrictEqual(board.data.board.rows, [{ subject: 'rated', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 }]);
+ assert.strictEqual((await shared.board('design.page')).data.board.rows[0].elo, 1100);
+ assert.strictEqual(requests.length, 1);
+ });
+
done();
done();
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });