openvibers/openvibe.bench Commits
Battles, the gallery and ratings that admit the style (#33)
Changes
nnew file mode 100644@@@ -0,0 +1,33 @@++-- phase: expand++-- B9: battles, the gallery and the honest ratings.++--++-- A battle vote is a human_pairwise judgement with two extras: the kind's own question (games: which is more fun;++-- design: which looks more finished) and the answer to it. The network fingerprint is a hash of the voter's address,++-- never the address itself: it is only there to limit votes per network. The style of an output (its length and how++-- much markup it wears) is measured once, when the entry lands, so the ratings can control for it later.++++ALTER TABLE bench_judgements ADD COLUMN question text;++ALTER TABLE bench_judgements ADD COLUMN answer text;++ALTER TABLE bench_judgements ADD COLUMN net_hash text;++CREATE INDEX bench_judgements_net ON bench_judgements (net_hash, at DESC) WHERE net_hash IS NOT NULL;++++ALTER TABLE bench_entries ADD COLUMN style_length integer;++ALTER TABLE bench_entries ADD COLUMN style_markup integer;++++-- A pair a person has been shown. The unique pair keeps the promise that nobody sees the same pair twice.++CREATE TABLE bench_battle_seen (++ id text COLLATE "C" PRIMARY KEY, -- bbs_ + ULID++ subject text NOT NULL, -- the person++ pair_key text NOT NULL, -- the two entry ids, sorted and joined++ round_id text NOT NULL REFERENCES bench_rounds (id) ON DELETE CASCADE,++ kind_id text NOT NULL REFERENCES bench_kinds (id),++ a_entry text NOT NULL, -- the entry shown as pane A++ b_entry text NOT NULL, -- the entry shown as pane B++ net_hash text, -- the network fingerprint, for the per-network limit++ voted_at timestamptz, -- when the person cast the vote++ at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT bench_battle_seen_id CHECK (id ~ '^bbs_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_battle_seen_pair UNIQUE (subject, pair_key)++);++CREATE INDEX bench_battle_seen_subject ON bench_battle_seen (subject, at DESC);++CREATE INDEX bench_battle_seen_net ON bench_battle_seen (net_hash, at DESC) WHERE net_hash IS NOT NULL;@@@ -264,3 +264,76 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }.br-form + .br-fine { margin-top: 16px; }++++/* The gallery (B9): every round's outputs, live in sandboxed frames; six live at a time, the rest wait. */++.ga-head { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 0; }++.ga-gallery { max-width: 1180px; margin: 0 auto; padding: 0 16px 64px; }++.ga-gallery > h1, .ga-head > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }++.ga-filters { display: grid; gap: 10px; grid-template-columns: 1fr; margin: 18px 0 4px; }++.ga-filters label { display: grid; gap: 4px; min-width: 0; font-size: .74rem; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; color: var(--text-muted, #7386a3); }++.ga-filters input, .ga-filters select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; font-weight: 600; text-transform: none; letter-spacing: normal; max-width: 100%; }++.ga-filters input:focus-visible, .ga-filters select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }++.ga-grid { display: grid; gap: 16px; grid-template-columns: 1fr; margin-top: 18px; }++.ga-card { display: grid; gap: 12px; padding: 14px; border-radius: 20px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; align-content: start; }++.ga-prompt { margin: 0; font-size: .98rem; font-weight: 650; line-height: 1.45; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden; }++.ga-shots { display: grid; gap: 8px; grid-template-columns: repeat(2, minmax(0, 1fr)); }++.ga-shot { margin: 0; display: grid; gap: 4px; min-width: 0; }++.ga-shot figcaption { font-size: .74rem; font-weight: 700; color: var(--text-muted, #7386a3); overflow-wrap: anywhere; }++.ga-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }++.ga-still { display: grid; place-items: center; aspect-ratio: 4 / 3; padding: 8px; border: 1px dashed var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); color: var(--text-muted, #7386a3); font-size: .82rem; text-align: center; }++.ga-still img { width: 100%; height: 100%; object-fit: cover; border-radius: 12px; }++.ga-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 0; font-size: .84rem; color: var(--text-muted, #7386a3); }++.ga-votes { font-variant-numeric: tabular-nums; }++.ga-link { margin-left: auto; font-weight: 750; color: var(--room-strong) !important; text-decoration: none; }++.ga-link:hover { text-decoration: underline; }++@media (min-width: 640px) { .ga-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } }++@media (min-width: 1024px) {++ .ga-grid { grid-template-columns: repeat(3, minmax(0, 1fr)); }++ .ga-filters { grid-template-columns: repeat(4, minmax(0, 1fr)) auto; align-items: end; }++}++++/* The battles (B9): two outputs, one prompt, blind; the vote is a plain form post. */++.bt-battle { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }++.bt-battle > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.1; text-wrap: balance; }++.bt-prompt { font-size: clamp(1.3rem, 3.6vw, 2rem) !important; letter-spacing: -.02em !important; line-height: 1.25 !important; }++.bt-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 12px 0 0; font-size: .86rem; color: var(--text-muted, #7386a3); }++.bt-panes { display: grid; gap: 12px; grid-template-columns: 1fr; margin-top: 16px; }++.bt-pane { margin: 0; display: grid; gap: 6px; min-width: 0; }++.bt-pane figcaption { font-size: .76rem; font-weight: 800; letter-spacing: .05em; text-transform: uppercase; color: var(--text-muted, #7386a3); }++.bt-pane-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }++.bt-vote { margin-top: 18px; padding: 16px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); display: grid; gap: 12px; }++.bt-vote fieldset { margin: 0; padding: 0; border: 0; display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }++.bt-vote legend { width: 100%; margin-bottom: 8px; font-weight: 800; }++.bt-vote label { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0 16px; border-radius: 999px; border: 1px solid var(--border, #1f2d47); cursor: pointer; font-weight: 650; }++.bt-vote label:has(input:checked) { border-color: var(--room-accent); background: color-mix(in srgb, var(--room-accent) 14%, transparent); }++.bt-vote input { accent-color: var(--room-accent); }++.bt-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }++.bt-facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(90px, 1fr)); gap: 6px; margin: 0; font-size: .82rem; }++.bt-facts dt { color: var(--text-muted, #7386a3); font-size: .7rem; text-transform: uppercase; letter-spacing: .06em; font-weight: 800; }++.bt-facts dd { margin: 2px 0 0; font-weight: 650; overflow-wrap: anywhere; }++.bt-reveal-facts { display: grid; gap: 14px; grid-template-columns: 1fr; margin-top: 14px; }++.bt-signed-out { max-width: 720px; }++@media (min-width: 720px) {++ .bt-panes { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; }++ .bt-reveal-facts { grid-template-columns: repeat(2, minmax(0, 1fr)); }++}++++/* The ratings board (B9): raw and style controlled, side by side, with the intervals and the bands. */++.rt-board { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }++.rt-board > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }++.rt-list { list-style: none; margin: 20px 0 0; padding: 0; display: grid; gap: 10px; }++.rt-row { display: grid; gap: 10px; padding: 14px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }++.rt-cell { display: grid; gap: 2px; min-width: 0; align-content: start; }++.rt-label { font-size: .68rem; font-weight: 800; letter-spacing: .07em; text-transform: uppercase; color: var(--text-muted, #7386a3); }++.rt-model { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; overflow-wrap: anywhere; }++.rt-rating { font-size: 1.15rem; font-weight: 800; font-variant-numeric: tabular-nums; }++.rt-ci { color: var(--text-muted, #7386a3); font-size: .82rem; font-variant-numeric: tabular-nums; }++.rt-band { display: inline-block; margin-top: 4px; padding: 2px 10px; border-radius: 999px; background: color-mix(in srgb, var(--room-accent) 14%, transparent); color: var(--room-strong); font-size: .78rem; font-weight: 750; justify-self: start; }++.rt-votes { font-weight: 700; font-variant-numeric: tabular-nums; }++.rt-effects, .rt-data { margin-top: clamp(24px, 4vw, 36px); }++.rt-effects h2, .rt-data h2 { margin: 0 0 10px; font-size: 1.2rem; font-weight: 800; letter-spacing: -.01em; }++.rt-data code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; color: var(--room-strong); overflow-wrap: anywhere; }++@media (min-width: 900px) {++ .rt-row { grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1fr) auto; align-items: center; }++}++@@@ -1 +1,77 @@/* OpenVibe.Bench pages work without JavaScript; the main session designs this room later. */++/* The gallery (B9): at most six frames live at a time; the rest wait as a still until the reader asks. */++(function () {++ 'use strict';++++ var LIVE = 6;++++ function gallery() {++ var grid = document.querySelector('[data-gallery="1"]');++ if (!grid) return;++ var shots = Array.prototype.slice.call(grid.querySelectorAll('.ga-shot'));++ var live = shots.filter(function (s) { return s.getAttribute('data-live') === '1'; });++++ function release(shot) {++ var frame = shot.querySelector('iframe.ga-frame');++ if (!frame) return;++ if (shot.getAttribute('data-server') === '1') return; // the first frames stay live++ var still = document.createElement('button');++ still.type = 'button';++ still.className = 'ga-still';++ still.setAttribute('data-ga-still', '1');++ still.setAttribute('aria-label', 'Load the live frame');++ var note = document.createElement('span');++ note.className = 'ga-still-note';++ note.textContent = 'Live frame, on tap';++ still.appendChild(note);++ frame.parentNode.replaceChild(still, frame);++ shot.removeAttribute('data-live');++ live = live.filter(function (s) { return s !== shot; });++ }++++ function activate(shot) {++ if (!shot || shot.getAttribute('data-live') === '1') return;++ var src = shot.getAttribute('data-src');++ if (!src) return;++ while (live.length >= LIVE) {++ var victim = null;++ for (var i = 0; i < live.length; i++) {++ if (live[i].getAttribute('data-server') !== '1') { victim = live[i]; break; }++ }++ if (!victim) return;++ release(victim);++ }++ var still = shot.querySelector('[data-ga-still]');++ var frame = document.createElement('iframe');++ frame.className = 'ga-frame';++ frame.setAttribute('src', src);++ frame.setAttribute('sandbox', 'allow-scripts allow-same-origin');++ frame.setAttribute('referrerpolicy', 'no-referrer');++ frame.setAttribute('loading', 'lazy');++ frame.setAttribute('title', 'Output');++ if (still && still.parentNode) still.parentNode.replaceChild(frame, still);++ shot.setAttribute('data-live', '1');++ live.push(shot);++ }++++ grid.addEventListener('click', function (event) {++ var target = event.target;++ var shot = target && target.closest ? target.closest('.ga-shot') : null;++ if (shot) activate(shot);++ });++++ if ('IntersectionObserver' in window) {++ var observer = new IntersectionObserver(function (entries) {++ entries.forEach(function (entry) {++ if (entry.isIntersecting) activate(entry.target);++ });++ }, { rootMargin: '320px' });++ shots.forEach(function (shot) { observer.observe(shot); });++ } else {++ shots.forEach(activate);++ }++ }++++ if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', gallery);++ else gallery();++}());nnew file mode 100644@@@ -0,0 +1,99 @@++'use strict';++++/**++ * Re-derive the board from the open vote dump.++ *++ * node scripts/fit-ratings.js /tmp/votes.jsonl # the table++ * node scripts/fit-ratings.js /tmp/votes.jsonl --json # the same board as JSON++ * curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl | node scripts/fit-ratings.js++ *++ * The dump carries every vote without voter ids, plus the weight each vote carried in the live fit (so a voter++ * who disagreed with the consensus far too often counts less). The fit below is the same fit the board uses, with++ * the same seed, so the numbers here are the numbers on /ratings.++ */++++const fs = require('fs');++const path = require('path');++const style = require(path.join(__dirname, '..', 'server', 'rating', 'style'));++++const argv = process.argv.slice(2);++const asJson = argv.includes('--json');++const bootstrapIndex = argv.indexOf('--bootstrap');++const bootstrap = bootstrapIndex >= 0 ? Number(argv[bootstrapIndex + 1]) || style.BOOTSTRAP : style.BOOTSTRAP;++const file = argv.find((a) => !a.startsWith('--') && a !== String(bootstrap));++++function readInput() {++ if (!file) return fs.readFileSync(0, 'utf8');++ return fs.readFileSync(file, 'utf8');++}++++/** The dump's lines into the games the fit reads. */++function gamesOf(lines) {++ const games = [];++ const weights = new Map();++ for (const line of lines) {++ const text = String(line || '').trim();++ if (!text) continue;++ let v;++ try { v = JSON.parse(text); } catch { continue; }++ if (!v || !v.left || !v.right || !v.left.model || !v.right.model) continue;++ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';++ games.push({++ id: v.id == null ? null : String(v.id),++ a: v.left.model,++ b: v.right.model,++ outcome,++ fa: { length: Number(v.left.length) || 0, markup: Number(v.left.markup) || 0 },++ fb: { length: Number(v.right.length) || 0, markup: Number(v.right.markup) || 0 },++ });++ if (v.id != null && v.weight != null) weights.set(String(v.id), Number(v.weight));++ }++ return { games, weights };++}++++function main() {++ const text = readInput();++ const { games, weights } = gamesOf(text.split('\n'));++ if (!games.length) {++ process.stderr.write('No votes in the dump. Fetch one: curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl\n');++ process.exit(1);++ }++ const fit = style.fitRatings(games, { bootstrap, voteWeights: weights });++ if (asJson) {++ process.stdout.write(JSON.stringify({ models: fit.models, bands: fit.bands, effects: fit.effects, votes: fit.votes, bootstrap: fit.bootstrap, seed: fit.seed }, null, 2) + '\n');++ return;++ }++ const cell = (m, key) => {++ const c = m[key];++ const lo = Math.round(c.ci_lower);++ const hi = Math.round(c.ci_upper);++ return String(Math.round(c.rating)).padStart(5) + ' [' + String(lo).padStart(5) + ', ' + String(hi).padStart(5) + ']';++ };++ const lines = [];++ lines.push('OpenVibe.Bench — the board, re-derived from the open vote dump');++ lines.push(fit.votes + ' votes, ' + fit.bootstrap + ' bootstrap resamples, seed ' + fit.seed);++ lines.push('');++ lines.push('model'.padEnd(28) + 'votes raw rating (95% interval) band controlled rating (95% interval) band');++ lines.push('-'.repeat(120));++ for (const m of fit.models) {++ lines.push(++ String(m.model).padEnd(28)++ + String(m.votes).padStart(5) + ' '++ + cell(m, 'raw') + ' ' + String(m.raw.band || '').padEnd(8)++ + cell(m, 'controlled') + ' ' + String(m.controlled.band || ''),++ );++ }++ lines.push('');++ for (const [key, e] of Object.entries(fit.effects || {})) {++ lines.push('style effect ' + key + ':'++ + (e.elo_per_1000_chars != null ? ' ' + e.elo_per_1000_chars + ' Elo per 1000 characters' : '')++ + (e.elo_per_10_marks != null ? ' ' + e.elo_per_10_marks + ' Elo per 10 markup marks' : ''));++ }++ if (fit.dropped_covariates && fit.dropped_covariates.length) {++ lines.push('dropped (too close to another covariate): ' + fit.dropped_covariates.join(', '));++ }++ lines.push('down-weighted voters: ' + fit.voters.downweighted + ' of ' + fit.voters.total + ' (never named)');++ process.stdout.write(lines.join('\n') + '\n');++}++++main();@@@ -28,6 +28,9 @@ const { createApi } = require('./http/api');const { createRegistryApi } = require('./http/registry');const { createPageRoutes } = require('./http/pages');const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');++const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');++const { createBattlePages } = require('./http/battle');++const { createRatingsPages } = require('./http/ratings');const { createRenderRoutes } = require('./http/render');const { seedShowcaseTasks } = require('./bench/seed-tasks');const { createRegistry } = require('./bench/registry');@@@ -160,6 +163,7 @@ async function createApp(opts = {}) {app.use('/api/v1', createApi(ctx));app.use('/api/v1', createRegistryApi(ctx));app.use('/api/v1/showcase', createShowcaseRoutes(ctx));++ app.use('/api/v1/gallery', createGalleryRoutes(ctx));app.use(mcpRouter(ctx));app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));@@@ -176,6 +180,9 @@ async function createApp(opts = {}) {app.use('/render', createRenderRoutes(ctx));app.use('/vendor', require('./http/vendor').createVendorRoutes());app.use('/showcase', createShowcasePages(ctx));++ app.use('/gallery', createGalleryPages(ctx));++ app.use('/battle', createBattlePages(ctx));++ app.use('/ratings', createRatingsPages(ctx));app.use(createPageRoutes(ctx));app.use((req, res) => send(res, 404, {viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',nnew file mode 100644@@@ -0,0 +1,197 @@++'use strict';++++/**++ * Battles (spec 2): two outputs, one prompt, blind.++ *++ * A battle is a pair of entries from a finished round. The rules the research asks for are enforced here, not in++ * the page:++ *++ * - only a signed-in person votes (the page sends anonymous readers to sign in);++ * - a person never sees the same pair twice (bench_battle_seen holds the pair the moment it is shown);++ * - votes are limited per person and per network (the network is a hash of the address, never the address).++ *++ * The vote itself is a human_pairwise judgement, so the battle votes are the same votes the ratings are made of.++ * Two extras ride along: the kind's own question ("which is more fun" for games, "which looks more finished" for++ * design) and the network hash, which never leaves the database.++ */++++const crypto = require('crypto');++const { SHOWCASE_KIND_IDS } = require('../kinds');++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++++/** The question each kind asks about its own outputs. */++const QUESTIONS = Object.freeze({++ games: { id: 'more_fun', text: 'Which one is more fun?', options: ['A', 'B', 'tie'] },++ design: { id: 'more_finished', text: 'Which one looks more finished?', options: ['A', 'B', 'tie'] },++ other: { id: 'better', text: 'Which one is better?', options: ['A', 'B', 'tie'] },++});++++/** The question a kind asks. */++function questionFor(kindId) {++ const id = String(kindId || '');++ if (id.startsWith('games.')) return QUESTIONS.games;++ if (id.startsWith('design.')) return QUESTIONS.design;++ return QUESTIONS.other;++}++++/** The pair key: the two entry ids, sorted, so A and B in any order is the same pair. */++function pairKey(a, b) {++ return [String(a || ''), String(b || '')].sort().join('|');++}++++function createBattles(ctx) {++ const { s, config, bench } = ctx;++ const db = s.db;++++ /** The network fingerprint: a hash of the address, so the limit works without storing an address. */++ function netHashOf(req) {++ const ip = String((req && req.ip) || (req && req.socket && req.socket.remoteAddress) || '').trim();++ if (!ip) return null;++ return crypto.createHash('sha256').update('bench-battle|' + ip).digest('hex').slice(0, 40);++ }++++ const limits = () => (config && config.bench) || {};++++ /** How many votes this person and this network may still cast. */++ async function budget({ subject, netHash }) {++ const cfg = limits();++ const hourAgo = new Date(Date.now() - 3600_000).toISOString();++ const dayAgo = new Date(Date.now() - 86_400_000).toISOString();++ const personHour = num(await db.value(++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",++ [subject, hourAgo]));++ const personDay = num(await db.value(++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",++ [subject, dayAgo]));++ const netHour = netHash ? num(await db.value(++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",++ [netHash, hourAgo])) : 0;++ const netDay = netHash ? num(await db.value(++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",++ [netHash, dayAgo])) : 0;++ const perPersonHour = Math.max(1, num(cfg.battlePersonPerHour, 30));++ const perPersonDay = Math.max(1, num(cfg.battlePersonPerDay, 200));++ const perNetworkHour = Math.max(1, num(cfg.battleNetworkPerHour, 120));++ const perNetworkDay = Math.max(1, num(cfg.battleNetworkPerDay, 1000));++ const reasons = [];++ if (personHour >= perPersonHour) reasons.push('You have voted a lot in the last hour. Try again in a little while.');++ if (personDay >= perPersonDay) reasons.push('You have used your votes for today.');++ if (netHour >= perNetworkHour) reasons.push('This network has voted a lot in the last hour. Try again later.');++ if (netDay >= perNetworkDay) reasons.push('This network has used its votes for today.');++ return {++ allowed: reasons.length === 0,++ message: reasons.join(' '),++ person_hour: personHour, person_day: personDay, network_hour: netHour, network_day: netDay,++ limits: { person_hour: perPersonHour, person_day: perPersonDay, network_hour: perNetworkHour, network_day: perNetworkDay },++ };++ }++++ /**++ * A pair this person has not seen: a finished round, not their own, with two usable entries, and not a pair++ * they have been shown before. The pair is recorded as seen the moment it is shown, so nobody meets it twice.++ */++ async function pickPair({ subject, kind = 'all', random = Math.random, netHash = null }) {++ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;++ const rows = await db.many(++ `SELECT r.id, r.kind_id, t.prompt++ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND r.created_by <> $2++ AND (SELECT COUNT(*) FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) >= 2++ AND NOT EXISTS (SELECT 1 FROM bench_judgements j WHERE j.round_id = r.id AND j.subject = $2)++ AND NOT EXISTS (SELECT 1 FROM bench_battle_seen v WHERE v.subject = $2 AND v.round_id = r.id)++ ORDER BY r.created_at DESC, r.id LIMIT 60`,++ [kinds, subject]);++ if (!rows.length) return null;++ const row = rows[Math.floor(random() * rows.length) % rows.length];++ const entries = await db.many(++ `SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup++ FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id`,++ [row.id]);++ if (entries.length < 2) return null;++ // Prefer two different models, so a battle is always between two models.++ let pair = [entries[0], entries[1]];++ const other = entries.find((e) => e.model !== entries[0].model);++ if (other) pair = [entries[0], other];++ const [a, b] = pair;++ const key = pairKey(a.id, b.id);++ const id = s.newId('bbs');++ await db.exec(++ `INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, net_hash, at)++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::timestamptz)`,++ [id, subject, key, row.id, row.kind_id, a.id, b.id, netHash, s.iso()]);++ return {++ id, round_id: row.id, kind_id: row.kind_id, prompt: row.prompt,++ question: questionFor(row.kind_id),++ a: { id: a.id, provider: a.provider, model: a.model, cost_usd: num(a.cost_usd), ms: num(a.ms) },++ b: { id: b.id, provider: b.provider, model: b.model, cost_usd: num(b.cost_usd), ms: num(b.ms) },++ };++ }++++ /** The battle as it was shown, for the reveal and the share card. */++ async function battleOf(id, subject = null) {++ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(id || '')]);++ if (!seen) return null;++ if (subject && seen.subject !== subject) return null;++ const round = await db.maybe(++ 'SELECT r.id, r.kind_id, r.created_at, t.prompt FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.id = $1',++ [seen.round_id]);++ if (!round) return null;++ const entries = await db.many(++ 'SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup FROM bench_entries WHERE id = ANY($1::text[])',++ [[seen.a_entry, seen.b_entry]]);++ const a = entries.find((e) => e.id === seen.a_entry);++ const b = entries.find((e) => e.id === seen.b_entry);++ if (!a || !b) return null;++ const vote = await db.maybe(++ "SELECT outcome, question, answer FROM bench_judgements WHERE round_id = $1 AND subject = $2 AND judge = 'human_pairwise' ORDER BY at LIMIT 1",++ [seen.round_id, seen.subject]);++ return {++ id: seen.id, round_id: seen.round_id, kind_id: seen.kind_id, prompt: round.prompt,++ question: questionFor(seen.kind_id),++ voted_at: seen.voted_at ? new Date(seen.voted_at).toISOString() : null,++ a: { id: a.id, provider: a.provider, model: a.model, route: a.route, cost_usd: num(a.cost_usd), ms: num(a.ms), tokens: num(a.tokens), payload: a.payload },++ b: { id: b.id, provider: b.provider, model: b.model, route: b.route, cost_usd: num(b.cost_usd), ms: num(b.ms), tokens: num(b.tokens), payload: b.payload },++ vote: vote ? { outcome: vote.outcome, question: vote.question, answer: vote.answer } : null,++ };++ }++++ /** The vote. The rules are checked here; a caller that breaks them gets a refusal, not a quiet write. */++ async function recordVote({ subject, battleId, outcome, answer = null, netHash = null }) {++ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(battleId || '')]);++ if (!seen) return { ok: false, status: 404, code: 'bench.battle_not_found', message: 'That battle is gone.' };++ if (seen.subject !== subject) return { ok: false, status: 403, code: 'bench.battle_not_yours', message: 'That battle belongs to someone else.' };++ if (seen.voted_at) return { ok: false, status: 409, code: 'bench.already_voted', message: 'You have already voted on this pair.' };++ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };++ const won = map[String(outcome || '')];++ if (!won) return { ok: false, status: 422, code: 'bench.invalid_outcome', message: 'The outcome must be A, B, tie or both_bad.' };++ const room = await budget({ subject, netHash });++ if (!room.allowed) return { ok: false, status: 429, code: 'bench.limit_battle', message: room.message };++ const question = questionFor(seen.kind_id);++ const answerValue = answer == null || answer === '' ? null : String(answer).slice(0, 8);++ if (answerValue != null && !question.options.includes(answerValue)) {++ return { ok: false, status: 422, code: 'bench.invalid_answer', message: 'That answer does not fit the question.' };++ }++ try {++ await bench.castJudgement({++ roundId: seen.round_id,++ judge: 'human_pairwise',++ pair: { left: seen.a_entry, right: seen.b_entry },++ outcome: won,++ question: question.text,++ answer: answerValue,++ netHash,++ subject,++ });++ } catch (err) {++ if (err && err.status) return { ok: false, status: err.status, code: err.code || 'bench.invalid', message: err.detail || 'The vote was not accepted.' };++ throw err;++ }++ await db.exec('UPDATE bench_battle_seen SET voted_at = $2::timestamptz WHERE id = $1', [seen.id, s.iso()]);++ return { ok: true, battle_id: seen.id, outcome: won, question };++ }++++ return { pickPair, battleOf, recordVote, budget, netHashOf, questionFor, pairKey };++}++++module.exports = { createBattles, questionFor, pairKey, QUESTIONS };@@@ -84,6 +84,11 @@ function load(env = process.env) {maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),++ // Battles: how many votes one person and one network may cast.++ battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),++ battlePersonPerDay: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_DAY, 200)),++ battleNetworkPerHour: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_HOUR, 120)),++ battleNetworkPerDay: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_DAY, 1000)),},};}@@@ -29,6 +29,7 @@ const { asyncRouter } = require('./router');const { sameOrigin } = require('./same-origin');const { kindById } = require('../kinds');const { BenchError } = require('../store');++const boardLib = require('../rating/board');function createApi(ctx) {const { config, s, principal, bench } = ctx;@@@ -225,6 +226,28 @@ function createApi(ctx) {} catch (err) { return failed(res, err); }});++ // The open vote dump (stage B9): every human vote, no voter ids. It comes before the kind dataset route,++ // because '/datasets/votes.jsonl' would otherwise be read as the kind 'votes'.++ r.get('/datasets/votes.jsonl', async (req, res) => {++ try {++ const result = await boardLib.board(ctx);++ const text = boardLib.dumpJsonl(result);++ res.set('Cache-Control', 'public, max-age=60');++ res.set('Content-Disposition', 'attachment; filename="bench-votes.jsonl"');++ res.set('X-Bench-Votes', String(result.votes.length));++ return res.type('application/x-ndjson').send(text);++ } catch (err) { return failed(res, err); }++ });++++ // The board the votes make: raw and style controlled, with intervals and bands.++ r.get('/ratings', async (req, res) => {++ try {++ const result = await boardLib.board(ctx);++ res.set('Cache-Control', 'public, max-age=60');++ return res.json({ board: boardLib.boardView(result) });++ } catch (err) { return failed(res, err); }++ });++r.get('/datasets/:kindId.jsonl', async (req, res) => {try {const text = await bench.datasetJsonl(String(req.params.kindId || ''));nnew file mode 100644@@@ -0,0 +1,231 @@++'use strict';++++/**++ * The battle pages: /battle (blind), /battle/revealed/:id (after the vote) and /battle/card/:id.svg (the share card).++ *++ * The vote is a plain form post to the same origin, so it works with JavaScript switched off. The page itself is++ * server-rendered: the two panes are the same sandboxed render frames the showcase uses, and the model names stay++ * closed until the vote is in.++ */++++const express = require('express');++const { asyncRouter } = require('./router');++const { sameOrigin } = require('./same-origin');++const renderLib = require('../bench/render');++const { kindById } = require('../kinds');++const { esc, raw } = require('../render/html');++const { send } = require('../render/layout');++const { createBattles } = require('../bench/battles');++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++++/** The share card is a vector image: 1200 by 630, the size every social card expects. */++const CARD = { w: 1200, h: 630 };++++function paneOf(slot, letter) {++ const url = renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(slot.id);++ return '<figure class="bt-pane" data-slot="' + esc(letter) + '" data-entry-id="' + esc(slot.id) + '">'++ + '<figcaption><span class="bt-pane-label">Pane ' + esc(letter) + '</span></figcaption>'++ + '<iframe class="bt-pane-frame" src="' + esc(url) + '" title="Output ' + esc(letter) + '"'++ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'++ + '</figure>';++}++++function voteForm(battle) {++ const q = battle.question;++ return '<form class="bt-vote" method="post" action="/battle" data-battle-form="1">'++ + '<input type="hidden" name="battle" value="' + esc(battle.id) + '">'++ + '<fieldset><legend>Which is better?</legend>'++ + '<label><input type="radio" name="outcome" value="A" required> A</label>'++ + '<label><input type="radio" name="outcome" value="B"> B</label>'++ + '<label><input type="radio" name="outcome" value="tie"> Tie</label>'++ + '<label><input type="radio" name="outcome" value="both_bad"> Both bad</label>'++ + '</fieldset>'++ + '<fieldset><legend>' + esc(q.text) + '</legend>'++ + '<label><input type="radio" name="answer" value="A"> A</label>'++ + '<label><input type="radio" name="answer" value="B"> B</label>'++ + '<label><input type="radio" name="answer" value="tie"> Same</label>'++ + '</fieldset>'++ + '<button class="bm-btn" type="submit">Cast the vote</button>'++ + '</form>';++}++++function battleView(battle) {++ const kind = kindById(battle.kind_id);++ return '<section class="bt-battle" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'++ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'++ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'++ + '<span class="bm-fine">Blind: you see two outputs, not their names.</span></p>'++ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'++ + voteForm(battle)++ + '</section>';++}++++function revealView(battle) {++ const kind = kindById(battle.kind_id);++ const vote = battle.vote || {};++ const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';++ const fact = (slot, letter) => '<dl class="bt-facts" data-slot="' + esc(letter) + '">'++ + '<div><dt>Model</dt><dd>' + esc(slot.model) + '</dd></div>'++ + '<div><dt>Provider</dt><dd>' + esc(slot.provider) + '</dd></div>'++ + '<div><dt>Cost</dt><dd>$' + esc(slot.cost_usd) + '</dd></div>'++ + '<div><dt>Time</dt><dd>' + esc(slot.ms) + ' ms</dd></div>'++ + '</dl>';++ return '<section class="bt-battle bt-revealed" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'++ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'++ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'++ + '<span class="br-chip">' + esc('Your vote: ' + winner) + '</span>'++ + (vote.answer ? '<span class="br-chip">' + esc(battle.question.text + ': ' + vote.answer) + '</span>' : '')++ + '</p>'++ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'++ + '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'++ + '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'++ + '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'++ + '</section>';++}++++function signedOutView() {++ return '<section class="bt-battle bt-signed-out">'++ + '<h1>Battles: two outputs, one prompt, blind</h1>'++ + '<p class="bm-lede">Every battle shows two outputs of the same prompt, without their names. You say which one is '++ + 'better (or that both are bad), the names open, and your vote counts towards the board.</p>'++ + '<p class="bm-note">Battles are for signed-in people: one vote per person, one pair per person, and the votes '++ + 'are rate limited so the board stays honest.</p>'++ + '<p class="bm-actions"><a class="bm-btn" href="/auth/login?next=%2Fbattle">Sign in to fight</a>'++ + '<a class="bm-btn bm-btn-quiet" href="/gallery">Look at the gallery first</a></p>'++ + '</section>';++}++++/** The share card: both panes and the question. A vector card, so no rasteriser is needed. */++function cardSvg(battle, baseUrl) {++ const kind = kindById(battle.kind_id);++ const escXml = (s) => String(s == null ? '' : s)++ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');++ const thumb = (slot, x, letter) => {++ const payload = slot.payload || {};++ const svg = typeof payload.svg === 'string' ? payload.svg : '';++ if (svg && svg.trim()) {++ const uri = 'data:image/svg+xml;base64,' + Buffer.from(svg, 'utf8').toString('base64');++ return '<image href="' + uri + '" x="' + x + '" y="150" width="480" height="300" preserveAspectRatio="xMidYMid meet"/>';++ }++ return '<rect x="' + x + '" y="150" width="480" height="300" rx="18" fill="#16203a" stroke="#2b3a5c"/>'++ + '<text x="' + (x + 240) + '" y="292" fill="#7d8ea9" font-family="system-ui, sans-serif" font-size="26" text-anchor="middle">'++ + escXml(kind ? kind.name : 'output') + '</text>'++ + '<text x="' + (x + 240) + '" y="330" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="middle">live preview</text>';++ };++ const prompt = String(battle.prompt || '').replace(/\s+/g, ' ').trim();++ const short = prompt.length > 150 ? prompt.slice(0, 147) + '…' : prompt;++ return '<svg xmlns="http://www.w3.org/2000/svg" width="' + CARD.w + '" height="' + CARD.h + '" viewBox="0 0 ' + CARD.w + ' ' + CARD.h + '">'++ + '<rect width="' + CARD.w + '" height="' + CARD.h + '" fill="#0b1220"/>'++ + '<rect x="24" y="24" width="' + (CARD.w - 48) + '" height="' + (CARD.h - 48) + '" rx="28" fill="none" stroke="#24304a"/>'++ + '<text x="60" y="96" fill="#e6edf7" font-family="system-ui, sans-serif" font-size="52" font-weight="800">Vote: which is better?</text>'++ + '<text x="60" y="132" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="24">' + escXml(short) + '</text>'++ + thumb(battle.a, 60, 'A')++ + thumb(battle.b, 660, 'B')++ + '<text x="84" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">A</text>'++ + '<text x="684" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">B</text>'++ + '<text x="60" y="516" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="26">Two outputs, one prompt, blind.</text>'++ + '<text x="60" y="556" fill="#c9d6ea" font-family="system-ui, sans-serif" font-size="26" font-weight="700">'++ + escXml('OpenVibe.Bench') + '</text>'++ + '<text x="' + (CARD.w - 60) + '" y="556" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="end">'++ + escXml(baseUrl + '/battle') + '</text>'++ + '</svg>';++}++++function createBattlePages(ctx) {++ const r = express.Router();++ const { config, principal } = ctx;++ const battles = createBattles(ctx);++ ctx.battles = battles;++ r.use(express.urlencoded({ extended: false, limit: '16kb' }));++ r.use(principal.middleware);++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }++ : { kind: 'anonymous' });++ const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal.subject : null);++++ r.get('/', async (req, res) => {++ const subject = subjectOf(req);++ if (!subject) {++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Battles: two outputs, one prompt, blind',++ description: 'Pick the better output, blind. The names open after the vote.',++ body: raw(signedOutView()),++ });++ }++ const battle = await battles.pickPair({++ subject,++ kind: typeof req.query.kind === 'string' ? req.query.kind : 'all',++ random: ctx.random || Math.random,++ netHash: battles.netHashOf(req),++ });++ if (!battle) {++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Battles: nothing waiting',++ description: 'Pick the better output, blind. The names open after the vote.',++ body: raw('<section class="bt-battle"><h1>No battle is waiting right now</h1>'++ + '<p class="bm-lede">Every pair you have seen is voted on, and the next rounds are still running. '++ + 'Come back in a minute, or <a href="/gallery">look at the gallery</a>.</p></section>'),++ });++ }++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Battle: two outputs, one prompt',++ description: 'Pick the better output, blind. The names open after the vote.',++ headExtra: '<meta property="og:title" content="Vote: which is better?">'++ + '<meta property="og:description" content="Two outputs, one prompt, blind. Vote on OpenVibe.Bench.">'++ + '<meta property="og:image" content="' + esc(config.baseUrl + '/battle/card/' + battle.id + '.svg') + '">'++ + '<meta property="og:image:type" content="image/svg+xml">'++ + '<meta name="twitter:card" content="summary_large_image">',++ body: raw(battleView(battle)),++ });++ });++++ r.get('/card/:id.svg', async (req, res) => {++ const battle = await battles.battleOf(String(req.params.id || ''));++ if (!battle) return res.status(404).type('text/plain').send('No such card.');++ res.set('Cache-Control', 'public, max-age=60');++ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src data: 'self'");++ return res.type('image/svg+xml').send(cardSvg(battle, config.baseUrl));++ });++++ r.get('/revealed/:id', async (req, res) => {++ const subject = subjectOf(req);++ const battle = await battles.battleOf(String(req.params.id || ''), subject);++ if (!battle) {++ return send(res, 404, { config, viewer: viewerOf(req), path: req.originalUrl, title: 'No such battle', body: raw('<section><h1>No such battle</h1></section>') });++ }++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Battle: the names are open',++ description: 'The two outputs of one prompt, with their names.',++ body: raw(revealView(battle)),++ });++ });++++ r.post('/', async (req, res) => {++ const subject = subjectOf(req);++ const body = req.body && typeof req.body === 'object' ? req.body : {};++ if (!subject) return res.redirect('/auth/login?next=' + encodeURIComponent('/battle'));++ if (req.principal.viaSession && !sameOrigin(req, config)) {++ return res.status(403).type('text/plain').send('A signed-in vote must come from bench.openvibe.services itself.');++ }++ const out = await battles.recordVote({++ subject,++ battleId: String(body.battle || ''),++ outcome: body.outcome,++ answer: body.answer,++ netHash: battles.netHashOf(req),++ });++ if (!out.ok) {++ return res.status(out.status || 422).type('text/plain').send(out.message || 'The vote was not accepted.');++ }++ return res.redirect(303, '/battle/revealed/' + out.battle_id);++ });++++ return r;++}++++module.exports = { createBattlePages, cardSvg };nnew file mode 100644@@@ -0,0 +1,203 @@++'use strict';++++/**++ * The gallery (spec 1): every showcase round's outputs, live in the sandboxed render frames.++ *++ * The page is a filtered, ordered list of rounds: filter by kind, model and price; order by newest or by most++ * voted. The frames are lazy: at most six live frames run at a time, and the rest wait as a still (a screenshot++ * Bench's own job made, when there is one) until the reader asks for the live one.++ */++++const express = require('express');++const { asyncRouter } = require('./router');++const renderLib = require('../bench/render');++const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');++const { esc, raw } = require('../render/html');++const { send } = require('../render/layout');++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++/** How many frames may be live at once. */++const LIVE_FRAMES = 6;++++function parsePrice(value) {++ if (value == null || value === '') return null;++ const n = Number(value);++ if (!Number.isFinite(n) || n < 0) return null;++ return n;++}++++/** The screenshot Bench's own job made for an entry, when it made one. */++function screenshotOf(uiCheck) {++ if (!uiCheck || typeof uiCheck !== 'object') return null;++ for (const key of ['screenshot', 'screenshot_url', 'image', 'thumb', 'thumbnail']) {++ const v = uiCheck[key];++ if (typeof v === 'string' && /^https?:\/\//.test(v)) return v;++ }++ return null;++}++++async function loadGallery(ctx, { kind = 'all', model = '', price = null, order = 'new', limit = 60 } = {}) {++ const db = ctx.s.db;++ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;++ const rows = await db.many(++ `SELECT r.id, r.kind_id, r.created_at, t.prompt,++ (SELECT COUNT(*)::int FROM bench_judgements j WHERE j.round_id = r.id AND j.judge = 'human_pairwise') AS votes,++ (SELECT COUNT(*)::int FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) AS entries++ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[])++ AND ($2::text = '' OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.model = $2))++ AND ($3::numeric IS NULL OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL AND e.cost_usd <= $3))++ ORDER BY ${order === 'votes' ? 'votes DESC, r.created_at DESC' : 'r.created_at DESC, r.id DESC'}++ LIMIT $4`,++ [kinds, String(model || ''), price, Math.min(120, Math.max(1, num(limit, 60)))]);++ const out = [];++ for (const r of rows) {++ if (num(r.entries) < 2) continue;++ const entries = await db.many(++ 'SELECT id, provider, model, cost_usd, ms, ui_check FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id LIMIT 2',++ [r.id]);++ if (entries.length < 2) continue;++ const k = kindById(r.kind_id);++ out.push({++ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,++ prompt: r.prompt, votes: num(r.votes), created_at: new Date(r.created_at).toISOString(),++ limit_ms: k ? renderLib.limitMsOf(k) : renderLib.DEFAULT_LIMIT_MS,++ entries: entries.map((e) => ({++ id: e.id, model: e.model, provider: e.provider, cost_usd: num(e.cost_usd), ms: num(e.ms),++ screenshot: screenshotOf(e.ui_check),++ wrapper_url: renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(e.id),++ })),++ });++ }++ return out;++}++++/** The kinds and models the filter offers. */++async function facets(ctx) {++ const db = ctx.s.db;++ const kinds = await db.many(++ `SELECT r.kind_id AS id, COUNT(*)::int AS n FROM bench_rounds r++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) GROUP BY r.kind_id ORDER BY r.kind_id`,++ [SHOWCASE_KIND_IDS]);++ const models = await db.many(++ `SELECT e.model AS id, COUNT(*)::int AS n FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND e.model IS NOT NULL++ GROUP BY e.model ORDER BY n DESC, e.model LIMIT 40`,++ [SHOWCASE_KIND_IDS]);++ return {++ kinds: kinds.map((k) => ({ id: k.id, name: (kindById(k.id) || {}).name || k.id, count: num(k.n) })),++ models: models.map((m) => ({ id: m.id, count: num(m.n) })),++ };++}++++function frame(entry, live) {++ const src = live++ ? '<iframe class="ga-frame" data-live="1" data-server="1" src="' + esc(entry.wrapper_url) + '" title="Output of ' + esc(entry.model) + '"'++ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'++ : '<button type="button" class="ga-still" data-ga-still="1" aria-label="Load the live frame">'++ + (entry.screenshot++ ? '<img src="' + esc(entry.screenshot) + '" alt="A still of the output" loading="lazy">'++ : '<span class="ga-still-note">Live frame, on tap</span>')++ + '</button>';++ return '<figure class="ga-shot" data-entry-id="' + esc(entry.id) + '" data-src="' + esc(entry.wrapper_url) + '">'++ + '<figcaption>' + esc(entry.model) + '</figcaption>' + src + '</figure>';++}++++function card(round, live) {++ return '<article class="ga-card" data-round-id="' + esc(round.id) + '" data-kind="' + esc(round.kind_id) + '" data-votes="' + esc(round.votes) + '">'++ + '<h2 class="ga-prompt" data-prompt="1">' + esc(round.prompt) + '</h2>'++ + '<div class="ga-shots">' + round.entries.map((e) => frame(e, live)).join('') + '</div>'++ + '<p class="ga-meta"><span class="br-chip">' + esc(round.kind_name) + '</span>'++ + '<span class="ga-votes" data-votes="' + esc(round.votes) + '">' + esc(round.votes) + ' votes</span>'++ + '<span class="bm-fine">' + esc('$' + Math.max(...round.entries.map((e) => e.cost_usd)).toFixed(4)) + ' at most</span>'++ + '<a class="ga-link" href="/showcase/round/' + esc(round.id) + '">Open side by side</a></p>'++ + '</article>';++}++++function filterBar(ctx, { kind, model, price, order }) {++ const kinds = SHOWCASE_KIND_IDS.map((id) => {++ const k = kindById(id);++ return '<option value="' + esc(id) + '"' + (kind === id ? ' selected' : '') + '>' + esc(k ? k.name : id) + '</option>';++ }).join('');++ return '<form class="ga-filters" method="get" action="/gallery" data-ga-filters="1">'++ + '<label>Kind<select name="kind"><option value="all"' + (kind === 'all' ? ' selected' : '') + '>All kinds</option>' + kinds + '</select></label>'++ + '<label>Model<input type="text" name="model" value="' + esc(model || '') + '" placeholder="model name"></label>'++ + '<label>Price at most<input type="text" inputmode="decimal" name="price" value="' + esc(price == null ? '' : String(price)) + '" placeholder="$0.05"></label>'++ + '<label>Order<select name="order">'++ + '<option value="new"' + (order !== 'votes' ? ' selected' : '') + '>Newest</option>'++ + '<option value="votes"' + (order === 'votes' ? ' selected' : '') + '>Most voted</option>'++ + '</select></label>'++ + '<button class="bm-btn bm-btn-small" type="submit">Filter</button>'++ + '</form>';++}++++function galleryView(rounds, filters) {++ if (!rounds.length) {++ return '<section class="ga-gallery" data-gallery="1">'++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames.</p>'++ + '<div class="bm-empty"><h1>Nothing matches yet</h1><p>No round fits that filter. The first rounds are still generating.</p></div>'++ + '</section>';++ }++ return '<section class="ga-gallery" data-gallery="1">'++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. At most six frames run at '++ + 'once; the rest wait for you.</p>'++ + '<div class="ga-grid">' + rounds.map((r, i) => card(r, i * 2 < LIVE_FRAMES)).join('') + '</div>'++ + '</section>';++}++++function createGalleryRoutes(ctx) {++ const r = asyncRouter();++ r.get('/', async (req, res) => {++ const filters = {++ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',++ model: typeof req.query.model === 'string' ? req.query.model : '',++ price: parsePrice(req.query.price),++ order: req.query.order === 'votes' ? 'votes' : 'new',++ };++ const rounds = await loadGallery(ctx, filters);++ res.set('Cache-Control', 'public, max-age=30');++ return res.json({ count: rounds.length, filters, rounds });++ });++ r.get('/facets', async (req, res) => {++ res.set('Cache-Control', 'public, max-age=300');++ return res.json(await facets(ctx));++ });++ return r;++}++++function createGalleryPages(ctx) {++ const r = express.Router();++ const { config, principal } = ctx;++ r.use(principal.middleware);++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }++ : { kind: 'anonymous' });++++ r.get('/', async (req, res) => {++ const filters = {++ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',++ model: typeof req.query.model === 'string' ? req.query.model : '',++ price: parsePrice(req.query.price),++ order: req.query.order === 'votes' ? 'votes' : 'new',++ };++ const rounds = await loadGallery(ctx, filters);++ const f = await facets(ctx);++ const body = '<section class="ga-head">'++ + '<h1>Gallery</h1>'++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. Filter by kind, model '++ + 'and price; order by newest or by most voted.</p>'++ + filterBar(ctx, filters)++ + '<p class="bm-fine">' + esc(f.kinds.length + ' kinds, ' + f.models.length + ' models in the gallery') + '</p>'++ + '</section>' + galleryView(rounds, filters);++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Gallery: every showcase round',++ description: 'Every showcase round, live in sandboxed frames. Filter by kind, model and price.',++ body: raw(body),++ });++ });++++ return r;++}++++module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, LIVE_FRAMES };nnew file mode 100644@@@ -0,0 +1,98 @@++'use strict';++++/**++ * The ratings page: the board the votes make, raw and style controlled, side by side.++ *++ * The board is a Bradley-Terry fit over the human votes with the style of each output as covariates, and every++ * rating carries a bootstrap interval. Models whose intervals overlap share a band, so the page says "2nd-4th"++ * instead of pretending the third place is a fact. The open dump next to it carries every vote without voter ids,++ * and the fit can be re-derived from that file alone.++ */++++const express = require('express');++const { esc, raw } = require('../render/html');++const { send } = require('../render/layout');++const { board, boardView } = require('../rating/board');++++function interval(cell) {++ if (!cell) return '';++ const lo = cell.ci_lower == null ? '' : Math.round(cell.ci_lower);++ const hi = cell.ci_upper == null ? '' : Math.round(cell.ci_upper);++ return '<span class="rt-rating">' + esc(Math.round(cell.rating)) + '</span>'++ + '<span class="rt-ci">' + esc(lo + '–' + hi) + '</span>';++}++++function rows(view) {++ if (!view.models.length) {++ return '<div class="bm-empty"><h1>No votes yet</h1><p>The board fills up as people vote in the battles.</p></div>';++ }++ const body = view.models.map((m) => '<li class="rt-row" data-model="' + esc(m.model) + '">'++ + '<div class="rt-cell rt-cell-model"><span class="rt-label">Model</span>'++ + '<span class="rt-model">' + esc(m.model) + '</span></div>'++ + '<div class="rt-cell"><span class="rt-label">Raw</span>' + interval(m.raw)++ + '<span class="rt-band">' + esc(m.raw.band || '') + '</span></div>'++ + '<div class="rt-cell"><span class="rt-label">Style controlled</span>' + interval(m.controlled)++ + '<span class="rt-band">' + esc(m.controlled.band || '') + '</span></div>'++ + '<div class="rt-cell"><span class="rt-label">Votes</span><span class="rt-votes">' + esc(m.votes) + '</span></div>'++ + '</li>').join('');++ return '<ul class="rt-list">' + body + '</ul>';++}++++function effects(view) {++ const list = Object.entries(view.effects || {});++ if (!list.length) return '';++ const items = list.map(([key, e]) => '<div><dt>' + esc(key) + '</dt><dd>'++ + (e.elo_per_1000_chars != null ? esc(e.elo_per_1000_chars) + ' Elo per 1000 characters' : '')++ + (e.elo_per_1000_chars != null && e.elo_per_10_marks != null ? '<br>' : '')++ + (e.elo_per_10_marks != null ? esc(e.elo_per_10_marks) + ' Elo per 10 markup marks' : '')++ + '</dd></div>').join('');++ return '<section class="rt-effects"><h2>How much of the board is style</h2>'++ + '<dl class="br-facts">' + items + '</dl>'++ + '<p class="bm-fine">A positive number means the longer (or the more dressed up) output wins votes it did not '++ + 'earn. The controlled column removes that effect; the raw column is what people actually chose.</p>'++ + '</section>';++}++++function ratingsView(view) {++ return '<section class="rt-board" data-ratings="1">'++ + '<h1>Ratings</h1>'++ + '<p class="bm-lede">Every human vote in Bench is one game between two models. The board is fitted twice: on the '++ + 'votes alone (raw) and with the style of each output as covariates (style controlled), so you can see how much '++ + 'of a rank is length and formatting.</p>'++ + '<p class="bm-fine">' + esc(view.votes) + ' votes, '++ + esc(view.bootstrap) + ' bootstrap resamples, seed ' + esc(view.seed)++ + (view.voters && view.voters.downweighted ? ', ' + esc(view.voters.downweighted) + ' voters down-weighted (never named)' : '')++ + '</p>'++ + '<div class="rt-table-wrap">' + rows(view) + '</div>'++ + effects(view)++ + '<section class="rt-data"><h2>The open data</h2>'++ + '<p class="bm-lede">Every vote is published as JSONL, without voter ids: '++ + '<a href="/api/v1/datasets/votes.jsonl">/api/v1/datasets/votes.jsonl</a>. '++ + 'The fit is reproducible from it: <code>node scripts/fit-ratings.js /tmp/votes.jsonl</code> re-derives the '++ + 'same numbers (the dump carries the weight each vote carries in the fit).</p></section>'++ + '</section>';++}++++function createRatingsPages(ctx) {++ const r = express.Router();++ const { config, principal } = ctx;++ r.use(principal.middleware);++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }++ : { kind: 'anonymous' });++++ r.get('/', async (req, res) => {++ const result = await board(ctx);++ const view = boardView(result);++ return send(res, 200, {++ config, viewer: viewerOf(req), path: req.originalUrl,++ title: 'Ratings: raw and style controlled',++ description: 'The board the votes make: raw and style-controlled Bradley-Terry, with intervals and bands.',++ body: raw(ratingsView(view)),++ });++ });++++ return r;++}++++module.exports = { createRatingsPages };nnew file mode 100644@@@ -0,0 +1,123 @@++'use strict';++++/**++ * The board the votes make (spec 3).++ *++ * Every human pairwise vote in Bench is one game between two models. The board is fitted twice over those games:++ * once on the votes alone (raw) and once with the style of each output as covariates (controlled), so a reader can++ * see how much of the ranking is style. The intervals come from a bootstrap of the votes, and models whose++ * intervals overlap share a band.++ *++ * The board is a pure function of the votes, so it is cached on the shape of the vote set and recomputed when a++ * new vote lands. The open dump (/api/v1/datasets/votes.jsonl) carries the same weights, so the board can be++ * re-derived from the file alone.++ */++++const style = require('./style');++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++++/** The human votes, with the two models and the two styles of each vote. */++async function loadVotes(db) {++ const rows = await db.many(++ `SELECT j.id, j.at, j.kind_id, j.round_id, j.outcome, j.subject, j.question, j.answer,++ l.model AS left_model, l.style_length AS left_length, l.style_markup AS left_markup, l.payload AS left_payload,++ r.model AS right_model, r.style_length AS right_length, r.style_markup AS right_markup, r.payload AS right_payload++ FROM bench_judgements j++ JOIN bench_entries l ON l.id = (j.pair->>'left')++ JOIN bench_entries r ON r.id = (j.pair->>'right')++ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL++ ORDER BY j.at, j.id`);++ const out = [];++ for (const row of rows) {++ const left = styleOf(row.left_payload, row.left_length, row.left_markup);++ const right = styleOf(row.right_payload, row.right_length, row.right_markup);++ out.push({++ id: row.id,++ at: row.at instanceof Date ? row.at.toISOString() : String(row.at),++ kind_id: row.kind_id,++ round_id: row.round_id,++ outcome: row.outcome,++ question: row.question || null,++ answer: row.answer || null,++ voter: row.subject || null,++ a: { model: row.left_model, length: left.length, markup: left.markup },++ b: { model: row.right_model, length: right.length, markup: right.markup },++ });++ }++ return out;++}++++/** The style of an output: the columns when they are there, the payload when they are not (older entries). */++function styleOf(payload, length, markup) {++ if (length != null && markup != null) return { length: num(length), markup: num(markup) };++ let p = payload;++ if (typeof p === 'string') { try { p = JSON.parse(p); } catch { p = {}; } }++ const features = style.styleFeatures(p || {});++ return { length: length == null ? features.length : num(length), markup: markup == null ? features.markup : num(markup) };++}++++/** The games the fit reads. */++function gamesOf(votes) {++ const out = [];++ for (const v of votes) {++ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';++ out.push({ id: v.id, a: v.a.model, b: v.b.model, outcome, fa: v.a, fb: v.b, voter: v.voter });++ }++ return out;++}++++let cached = null;++++/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */++async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {++ const db = ctx.s.db;++ const votes = await loadVotes(db);++ const key = votes.length + ':' + (votes.length ? votes[votes.length - 1].id : '') + ':' + num(bootstrap);++ if (!force && cached && cached.key === key && Date.now() - cached.at < 5 * 60_000) return cached.value;++ const games = gamesOf(votes);++ const fit = style.fitRatings(games, { bootstrap });++ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));++ const enriched = votes.map((v) => ({ ...v, weight: weights.has(v.id) ? weights.get(v.id) : 1 }));++ const value = { fit, votes: enriched, generated_at: new Date().toISOString() };++ cached = { key, at: Date.now(), value };++ return value;++}++++/**++ * The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,++ * so the board can be re-derived from the file alone.++ */++function dumpJsonl(result) {++ const lines = result.votes.map((v) => JSON.stringify({++ id: v.id,++ at: v.at,++ kind_id: v.kind_id,++ round_id: v.round_id,++ left: { model: v.a.model, length: v.a.length, markup: v.a.markup },++ right: { model: v.b.model, length: v.b.length, markup: v.b.markup },++ outcome: v.outcome,++ question: v.question,++ answer: v.answer,++ weight: v.weight,++ }));++ return lines.join('\n') + (lines.length ? '\n' : '');++}++++/** The board as the page and the API show it. */++function boardView(result) {++ const fit = result.fit;++ return {++ models: fit.models,++ bands: fit.bands,++ effects: fit.effects,++ dropped_covariates: fit.dropped_covariates || [],++ voters: fit.voters,++ votes: fit.votes,++ bootstrap: fit.bootstrap,++ seed: fit.seed,++ generated_at: result.generated_at,++ };++}++++module.exports = { loadVotes, gamesOf, board, dumpJsonl, boardView, styleOf };@@@ -75,4 +75,12 @@ function costPerSolved(entries) {// Bradley–Terry with bootstrap intervals, voter weights, Wilson and percentiles: the tested library (lib.js, BENCH-2).const lib = require('./lib');--module.exports = { eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT };++// The style-controlled fit (stage B9): a conditional logit over the votes, with the length and the markup of++// each output as covariates, bootstrap intervals and rank bands.++const style = require('./style');++++module.exports = {++ eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT,++ style,++ fitRatings: style.fitRatings, rankBands: style.rankBands, styleFeatures: style.styleFeatures, markupCount: style.markupCount,++};nnew file mode 100644@@@ -0,0 +1,518 @@++'use strict';++++/**++ * Style-controlled Bradley-Terry for Bench's human votes (stage B9).++ *++ * The research (10-30) is blunt about two things: a board's rank is fragile once the whole board's uncertainty is++ * counted (an audit of 106,134 LMArena votes found only 1 of 54 neighbouring pairs to be a real ordering), and style++ * beats substance (length and formatting decide a large share of pairwise votes). So Bench does two things a plain++ * Elo ladder never does:++ *++ * 1. Style control. The same Bradley-Terry fit is made twice: once on the votes alone (the raw board, what people++ * actually chose) and once with the style of each output as covariates (the controlled board, what the model++ * would score if every output were written the same way). The covariates are the output's length and its++ * markup count, the two LMArena names as covariates; the effect sizes are reported so a reader can see how much++ * of the raw board is style.++ *++ * The model is a conditional logit (Bradley-Terry with per-output covariates):++ *++ * P(left wins) = sigmoid( theta_left - theta_right + SUM_k beta_k * (z_left,k - z_right,k) )++ *++ * where theta is the model's own strength (log-odds) and z the standardised covariates of the two outputs. A tie++ * counts as half a win. The fit is convex in theta for fixed beta, so a damped Newton direction with a++ * backtracking line search finds the maximum; theta is centred (its mean is zero) because only differences are++ * identified.++ *++ * 2. Bands, not places. Models whose bootstrap intervals overlap share a band, so the board says "2nd-4th" instead++ * of pretending the third place is a fact.++ *++ * Every number here is reproducible from the open vote dump (/api/v1/datasets/votes.jsonl): the dump carries each++ * vote's outcome, the two models, the two outputs' style features and the weight the fit gave that vote, so++ * scripts/fit-ratings.js re-derives the same board from the file alone.++ *++ * The maths is Bench's own (no copied code); the shape follows the published description of style-controlled++ * Bradley-Terry so the numbers mean what a reader expects.++ */++++const { percentile } = require('./lib');++++/** Elo points per unit of log-odds: the usual 400 / ln(10). */++const SCALE = 400 / Math.log(10);++/** The number a centred rating sits on, so the board reads like an Elo ladder. */++const ANCHOR = 1000;++/** The covariates, in the order they are reported. */++const COVARIATES = Object.freeze(['length', 'markup']);++/** The bootstrap resamples the board's intervals come from. */++const BOOTSTRAP = 1000;++/** The seed the bootstrap uses, so a rerun gives the same intervals. */++const SEED = 20261030;++++const sigmoid = (x) => (x >= 0 ? 1 / (1 + Math.exp(-x)) : (e => e / (1 + e))(Math.exp(x)));++++/** The text of an entry payload: whichever field its kind renders from. */++function contentOf(payload) {++ const p = payload && typeof payload === 'object' ? payload : {};++ for (const key of ['html', 'svg', 'glsl', 'text', 'code']) {++ if (typeof p[key] === 'string' && p[key].trim()) return p[key];++ }++ return '';++}++++/**++ * How much markup an output carries. Headings, bold, lists, code fences, links, table rows and HTML tags each count++ * once: the point is "how dressed up is this output", and the count has to be the same for every kind.++ */++function markupCount(text) {++ const s = String(text == null ? '' : text);++ let n = 0;++ n += (s.match(/^#{1,6}\s/gm) || []).length;++ n += (s.match(/\*\*|__/g) || []).length;++ n += (s.match(/^\s*[-*+]\s/gm) || []).length;++ n += (s.match(/^\s*\d+\.\s/gm) || []).length;++ n += (s.match(/```/g) || []).length;++ n += (s.match(/\[[^\]]*\]\([^)]*\)/g) || []).length;++ n += (s.match(/^\s*\|/gm) || []).length;++ n += (s.match(/<\/?[a-z][a-z0-9]*\b[^>]*>/gi) || []).length;++ return n;++}++++/** The style features of one output: how long it is, and how much markup it wears. */++function styleFeatures(payload) {++ const content = contentOf(payload);++ return { length: content.length, markup: markupCount(content) };++}++++/** A small seedable generator, so the same votes and seed always give the same intervals. */++function seededRandom(seed) {++ let state = (seed >>> 0) || 1;++ return () => {++ state = (state + 0x6D2B79F5) >>> 0;++ let value = state;++ value = Math.imul(value ^ (value >>> 15), value | 1);++ value ^= value + Math.imul(value ^ (value >>> 7), value | 61);++ return ((value ^ (value >>> 14)) >>> 0) / 4294967296;++ };++}++++function centre(values) {++ if (!values.length) return values;++ const mean = values.reduce((a, b) => a + b, 0) / values.length;++ for (let i = 0; i < values.length; i++) values[i] -= mean;++ return values;++}++++/**++ * The games a fit reads: { id, a, b, outcome, fa, fb, voter?, weight? }. The outcome is 'a', 'b' or 'tie';++ * 'both_bad' says nothing about the models and is skipped.++ */++function prepare(games, { covariates = COVARIATES } = {}) {++ const list = (Array.isArray(games) ? games : [])++ .filter((g) => g && g.a && g.b && g.a !== g.b && g.outcome !== 'both_bad');++ const subjects = [...new Set(list.flatMap((g) => [g.a, g.b]))].sort();++ const index = new Map(subjects.map((s, i) => [s, i]));++ const wanted = covariates.filter((k) => k === 'length' || k === 'markup');++++ // Standardise every covariate over the outputs the votes actually used.++ const stats = new Map();++ for (const key of wanted) {++ const values = [];++ for (const g of list) {++ values.push(Number((g.fa && g.fa[key]) || 0), Number((g.fb && g.fb[key]) || 0));++ }++ const mean = values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0;++ const variance = values.length ? values.reduce((a, b) => a + (b - mean) * (b - mean), 0) / values.length : 0;++ stats.set(key, { mean, sd: Math.sqrt(variance) || 1 });++ }++++ // The standardised difference of each covariate, per game.++ const diffs = wanted.map((key) => {++ const st = stats.get(key);++ const out = new Float64Array(list.length);++ for (let i = 0; i < list.length; i++) {++ const g = list[i];++ const za = (Number((g.fa && g.fa[key]) || 0) - st.mean) / st.sd;++ const zb = (Number((g.fb && g.fb[key]) || 0) - st.mean) / st.sd;++ out[i] = za - zb;++ }++ return out;++ });++++ // Two covariates that move together cannot be told apart, and the fit would trade them for ever: keep the first++ // and drop the one that repeats it. The board then says so instead of pretending both were measured.++ const keptKeys = [];++ const keptDiffs = [];++ const dropped = [];++ for (let j = 0; j < wanted.length; j++) {++ const clash = keptDiffs.some((other) => Math.abs(correlation(diffs[j], other)) > 0.95);++ if (clash) { dropped.push(wanted[j]); continue; }++ keptKeys.push(wanted[j]);++ keptDiffs.push(diffs[j]);++ }++ const kAll = keptKeys.length;++++ const n = list.length;++ const ai = new Int32Array(n);++ const bi = new Int32Array(n);++ const score = new Float64Array(n);++ const weight = new Float64Array(n);++ const dz = new Float64Array(n * kAll);++ const ids = new Array(n);++ const voters = new Array(n);++ let kept = 0;++ for (let gi = 0; gi < list.length; gi++) {++ const g = list[gi];++ const w = Number(g.weight == null ? 1 : g.weight);++ if (!(w > 0)) continue;++ const at = kept;++ ai[at] = index.get(g.a);++ bi[at] = index.get(g.b);++ score[at] = g.outcome === 'a' ? 1 : g.outcome === 'b' ? 0 : 0.5;++ weight[at] = w;++ ids[at] = g.id == null ? null : String(g.id);++ voters[at] = g.voter == null ? null : String(g.voter);++ for (let j = 0; j < kAll; j++) dz[at * kAll + j] = keptDiffs[j][gi];++ kept += 1;++ }++++ const keptStats = new Map();++ for (const key of keptKeys) keptStats.set(key, stats.get(key));++ return { ai, bi, score, weight, dz, ids, voters, count: kept, subjects, index, keys: keptKeys, stats: keptStats, dropped };++}++++/** The Pearson correlation of two equal-length series. */++function correlation(a, b) {++ const n = Math.min(a.length, b.length);++ if (!n) return 0;++ let ma = 0;++ let mb = 0;++ for (let i = 0; i < n; i++) { ma += a[i]; mb += b[i]; }++ ma /= n; mb /= n;++ let cov = 0;++ let va = 0;++ let vb = 0;++ for (let i = 0; i < n; i++) {++ cov += (a[i] - ma) * (b[i] - mb);++ va += (a[i] - ma) * (a[i] - ma);++ vb += (b[i] - mb) * (b[i] - mb);++ }++ if (!va || !vb) return 0;++ return cov / Math.sqrt(va * vb);++}++++/**++ * The conditional-logit fit over prepared games. `order` picks which games take part (the bootstrap resamples it);++ * `use` picks which of the prepared covariates enter the logit (the raw fit uses none of them). The hot loops work on++ * typed arrays, because the bootstrap runs this thousands of times.++ */++function fitPrepared(prepared, { order = null, use = null, ridge = 0.05, betaRidge = 0.002, maxIter = 120, tol = 1e-9, warm = null, lineSearch = true } = {}) {++ const { ai, bi, score, weight, dz, subjects, keys, stats } = prepared;++ const n = subjects.length;++ const kAll = keys.length;++ const cols = (use == null ? keys : use).map((key) => keys.indexOf(key)).filter((i) => i >= 0);++ const k = cols.length;++ const picks = order || null;++ const count = picks ? picks.length : prepared.count;++++ let theta = warm && warm.theta && warm.theta.length === n ? Float64Array.from(warm.theta) : new Float64Array(n);++ let beta = warm && warm.beta && warm.beta.length === k ? Float64Array.from(warm.beta) : new Float64Array(k);++++ const gTheta = new Float64Array(n);++ const hTheta = new Float64Array(n);++ const gBeta = new Float64Array(k);++ const hBeta = new Float64Array(k);++ const nextTheta = new Float64Array(n);++ const nextBeta = new Float64Array(k);++++ const logit = (p, th, be) => {++ let x = th[ai[p]] - th[bi[p]];++ for (let j = 0; j < k; j++) x += be[j] * dz[p * kAll + cols[j]];++ return x;++ };++++ const evaluate = (th, be) => {++ let value = 0;++ for (let p = 0; p < count; p++) {++ const idx = picks ? picks[p] : p;++ const x = logit(idx, th, be);++ const ph = sigmoid(x);++ const s = score[idx];++ value += -weight[idx] * (s * Math.log(ph) + (1 - s) * Math.log(1 - ph));++ }++ for (let i = 0; i < n; i++) value += ridge * th[i] * th[i];++ for (let j = 0; j < k; j++) value += betaRidge * be[j] * be[j];++ return value;++ };++++ const gradient = (th, be) => {++ gTheta.fill(0); hTheta.fill(0); gBeta.fill(0); hBeta.fill(0);++ for (let p = 0; p < count; p++) {++ const idx = picks ? picks[p] : p;++ const x = logit(idx, th, be);++ const ph = sigmoid(x);++ const s = score[idx];++ const w = weight[idx];++ const d = w * (s - ph);++ const a = ai[idx];++ const b = bi[idx];++ gTheta[a] += d;++ gTheta[b] -= d;++ const curv = w * ph * (1 - ph);++ hTheta[a] += curv;++ hTheta[b] += curv;++ for (let j = 0; j < k; j++) {++ const dzz = dz[idx * kAll + cols[j]];++ gBeta[j] += d * dzz;++ hBeta[j] += curv * dzz * dzz;++ }++ }++ for (let i = 0; i < n; i++) { gTheta[i] -= 2 * ridge * th[i]; hTheta[i] += 2 * ridge; }++ for (let j = 0; j < k; j++) { gBeta[j] -= 2 * betaRidge * be[j]; hBeta[j] += 2 * betaRidge; }++ };++++ for (let iter = 0; iter < maxIter; iter++) {++ gradient(theta, beta);++ let norm = 0;++ for (let i = 0; i < n; i++) norm += gTheta[i] * gTheta[i];++ for (let j = 0; j < k; j++) norm += gBeta[j] * gBeta[j];++ norm = Math.sqrt(norm);++ if (norm < tol) break;++ // A Newton direction with a diagonal curvature, damped by a backtracking line search.++ let step = 1;++ let moved = false;++ const base = lineSearch ? evaluate(theta, beta) : null;++ for (let attempt = 0; attempt < 12; attempt++) {++ let mean = 0;++ for (let i = 0; i < n; i++) {++ const v = theta[i] + step * (gTheta[i] / Math.max(hTheta[i], 1e-9));++ nextTheta[i] = v;++ mean += v;++ }++ mean /= n;++ for (let i = 0; i < n; i++) nextTheta[i] -= mean;++ for (let j = 0; j < k; j++) nextBeta[j] = beta[j] + step * (gBeta[j] / Math.max(hBeta[j], 1e-9));++ if (!lineSearch) { theta.set(nextTheta); beta.set(nextBeta); moved = true; break; }++ const value = evaluate(nextTheta, nextBeta);++ if (value <= base - 1e-12) {++ theta.set(nextTheta); beta.set(nextBeta); moved = true; break;++ }++ step *= 0.5;++ }++ if (!moved) break;++ }++++ return { theta: Array.from(theta), beta: Array.from(beta), subjects, keys, stats };++}++++/**++ * How often a voter disagrees with the consensus. A voter who disagrees far too often is down-weighted: their votes++ * count a quarter (nothing at all when they disagree with almost everything), and they are never named anywhere.++ */++function voterWeights(games, fit, { minVotes = 10, disagreeAt = 0.5, hopelessAt = 0.75 } = {}) {++ const rating = new Map();++ fit.subjects.forEach((subject, i) => rating.set(subject, fit.theta[i]));++ const voters = new Map();++ for (const g of games) {++ if (!g || g.voter == null) continue;++ if (g.outcome === 'tie' || g.outcome === 'both_bad') continue;++ if (g.a === g.b) continue;++ const ra = rating.get(g.a);++ const rb = rating.get(g.b);++ if (ra == null || rb == null) continue;++ const expected = ra > rb ? 1 : ra < rb ? -1 : 0;++ const observed = g.outcome === 'a' ? 1 : -1;++ const record = voters.get(g.voter) || { count: 0, disagreement: 0 };++ record.count += 1;++ record.disagreement += expected === observed ? 0 : 1;++ voters.set(g.voter, record);++ }++ const out = new Map();++ for (const [voter, record] of voters) {++ const rate = record.count ? record.disagreement / record.count : 0;++ let weight = 1;++ let downweighted = false;++ if (record.count >= minVotes && rate >= hopelessAt) { weight = 0; downweighted = true; }++ else if (record.count >= minVotes && rate > disagreeAt) { weight = 0.25; downweighted = true; }++ out.set(voter, { count: record.count, disagreement: rate, weight, downweighted });++ }++ return out;++}++++/**++ * The whole board: the raw fit, the style-controlled fit, both with bootstrap intervals, and the rank bands.++ *++ * The games are { id, a, b, outcome: 'a'|'b'|'tie', fa, fb, voter?, weight? }. The bootstrap resamples the votes and++ * gives the 2.5th and 97.5th percentile of each rating. `voteWeights` (when given) is the per-vote weight the caller++ * already computed, so the open dump and the live board agree.++ */++function fitRatings(games, {++ bootstrap = BOOTSTRAP,++ seed = SEED,++ ridge = 0.05,++ betaRidge = 0.002,++ covariates = COVARIATES,++ minVotes = 10,++ disagreeAt = 0.5,++ voteWeights = null,++} = {}) {++ const list = (Array.isArray(games) ? games : []).filter((g) => g && g.a && g.b);++ const empty = { models: [], bands: { raw: [], controlled: [] }, effects: {}, voters: { total: 0, downweighted: 0 }, votes: 0, bootstrap: 0, seed };++ if (!list.length) return empty;++++ // First pass: the consensus without voter weights, so the weights can be judged against it.++ const consensus = fitPrepared(prepare(list, { covariates: [] }), { use: [], ridge });++ const weights = voterWeights(list, consensus, { minVotes, disagreeAt });++ const weighted = list.map((g) => ({++ ...g,++ weight: voteWeights && g.id != null && voteWeights.get(g.id) != null++ ? voteWeights.get(g.id)++ : g.voter != null && weights.has(g.voter) ? weights.get(g.voter).weight++ : (g.weight == null ? 1 : g.weight),++ }));++++ const prepared = prepare(weighted, { covariates });++ const raw = fitPrepared(prepared, { use: [], ridge });++ const controlled = fitPrepared(prepared, { use: covariates, ridge, betaRidge });++++ const counts = new Map();++ for (const g of weighted) {++ counts.set(g.a, (counts.get(g.a) || 0) + 1);++ counts.set(g.b, (counts.get(g.b) || 0) + 1);++ }++++ const rawSamples = new Map();++ const ctrlSamples = new Map();++ const random = seededRandom(seed);++ const rounds = Math.max(0, Math.min(BOOTSTRAP, Number(bootstrap) || 0));++ const n = prepared.count;++ const rawIndex = new Map(raw.subjects.map((s, i) => [s, i]));++ const ctrlIndex = new Map(controlled.subjects.map((s, i) => [s, i]));++ const order = new Int32Array(n);++ for (let i = 0; i < rounds; i++) {++ for (let j = 0; j < n; j++) order[j] = Math.floor(random() * n);++ const r = fitPrepared(prepared, { order, use: [], ridge, maxIter: 60, warm: { theta: raw.theta, beta: [] } });++ const c = fitPrepared(prepared, { order, use: covariates, ridge, betaRidge, maxIter: 60, warm: { theta: controlled.theta, beta: controlled.beta } });++ for (const [subject, position] of rawIndex) {++ if (!rawSamples.has(subject)) { rawSamples.set(subject, []); ctrlSamples.set(subject, []); }++ rawSamples.get(subject).push(r.theta[position]);++ ctrlSamples.get(subject).push(c.theta[ctrlIndex.has(subject) ? ctrlIndex.get(subject) : 0]);++ }++ }++++ const toRating = (theta) => ANCHOR + SCALE * theta;++ const rawTheta = new Map(raw.subjects.map((s, i) => [s, raw.theta[i]]));++ const ctrlTheta = new Map(controlled.subjects.map((s, i) => [s, controlled.theta[i]]));++ const rows = raw.subjects.map((subject) => {++ const thetaRaw = rawTheta.get(subject);++ const thetaCtrl = ctrlTheta.get(subject);++ const rawP = rawSamples.get(subject) || [];++ const ctrlP = ctrlSamples.get(subject) || [];++ return {++ model: subject,++ votes: counts.get(subject) || 0,++ raw: {++ rating: Math.round(toRating(thetaRaw) * 100) / 100,++ ci_lower: rawP.length ? Math.round(toRating(percentile(rawP, 0.025)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,++ ci_upper: rawP.length ? Math.round(toRating(percentile(rawP, 0.975)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,++ },++ controlled: {++ rating: Math.round(toRating(thetaCtrl) * 100) / 100,++ ci_lower: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.025)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,++ ci_upper: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.975)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,++ },++ };++ }).sort((x, y) => y.raw.rating - x.raw.rating || x.model.localeCompare(y.model));++++ const rawBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.raw.rating, ci_lower: r.raw.ci_lower, ci_upper: r.raw.ci_upper })));++ const ctrlBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.controlled.rating, ci_lower: r.controlled.ci_lower, ci_upper: r.controlled.ci_upper })));++ const bandOf = (bands, model) => {++ const band = bands.find((b) => b.models.includes(model));++ return band ? band.label : null;++ };++ for (const row of rows) {++ row.raw.band = bandOf(rawBands, row.model);++ row.controlled.band = bandOf(ctrlBands, row.model);++ }++++ // The effect of each style covariate, in Elo points per 1000 characters or per 10 markup marks.++ const effects = {};++ const ctrlKeys = controlled.keys;++ for (let j = 0; j < ctrlKeys.length; j++) {++ const key = ctrlKeys[j];++ const beta = controlled.beta[j] == null ? 0 : controlled.beta[j];++ const st = controlled.stats.get(key);++ const perUnit = st ? beta * SCALE / st.sd : 0;++ effects[key] = {++ beta: Math.round(beta * 1e6) / 1e6,++ elo_per_1000_chars: key === 'length' ? Math.round(perUnit * 1000 * 100) / 100 : null,++ elo_per_10_marks: key === 'markup' ? Math.round(perUnit * 10 * 100) / 100 : null,++ mean: st ? Math.round(st.mean * 100) / 100 : null,++ sd: st ? Math.round(st.sd * 100) / 100 : null,++ };++ }++ const droppedCovariates = prepared.dropped || [];++++ const downweighted = [...weights.values()].filter((w) => w.downweighted).length;++ // The weight each vote carries, so the open dump can re-derive the same board from the file alone.++ const perVoteWeights = weighted.map((g) => ({ id: g.id == null ? null : String(g.id), weight: g.weight }));++ return {++ models: rows,++ vote_weights: perVoteWeights,++ bands: { raw: rawBands, controlled: ctrlBands },++ effects,++ voters: { total: weights.size, downweighted },++ dropped_covariates: droppedCovariates,++ votes: list.length,++ bootstrap: rounds,++ seed,++ };++}++++/**++ * Rank bands: models whose intervals overlap share a band. The rows come in rank order; a band grows while the next++ * model's interval still overlaps the band it would join.++ */++function rankBands(rows) {++ const sorted = [...(rows || [])].filter((r) => r && r.rating != null)++ .sort((a, b) => b.rating - a.rating || String(a.model).localeCompare(String(b.model)));++ const bands = [];++ let current = null;++ sorted.forEach((row, i) => {++ const lower = row.ci_lower == null ? row.rating : row.ci_lower;++ const upper = row.ci_upper == null ? row.rating : row.ci_upper;++ if (current && lower <= current.upper && upper >= current.lower) {++ current.models.push(row.model);++ current.lower = Math.min(current.lower, lower);++ current.upper = Math.max(current.upper, upper);++ current.last = i;++ } else {++ current = { models: [row.model], lower, upper, first: i, last: i };++ bands.push(current);++ }++ });++ return bands.map((band, i) => ({++ label: bandLabel(band.first + 1, band.last + 1),++ from: band.first + 1,++ to: band.last + 1,++ models: band.models,++ index: i,++ }));++}++++function ordinal(n) {++ const v = Math.round(Number(n));++ const t = v % 100;++ if (t >= 11 && t <= 13) return v + 'th';++ return v + ({ 1: 'st', 2: 'nd', 3: 'rd' }[v % 10] || 'th');++}++++/** "2nd-4th" for a band, "1st" for a single place. */++function bandLabel(from, to) {++ if (to <= from) return ordinal(from);++ return ordinal(from) + '\u2013' + ordinal(to);++}++++module.exports = {++ SCALE, ANCHOR, COVARIATES, BOOTSTRAP, SEED,++ styleFeatures, markupCount, contentOf,++ prepare, fitPrepared, voterWeights, fitRatings, rankBands, bandLabel, ordinal, seededRandom,++};@@@ -21,6 +21,9 @@ const NAV = [{ label: 'Home', href: '/' },{ label: 'Benchmarks', href: '/benchmarks' },{ label: 'Showcase', href: '/showcase' },++ { label: 'Gallery', href: '/gallery' },++ { label: 'Battles', href: '/battle' },++ { label: 'Ratings', href: '/ratings' },{ label: 'Docs', href: '/docs' },];@@@ -79,6 +82,7 @@ ${appIcon.headTags({ site: 'bench' })}<link rel="stylesheet" href="${ovServe.url('room.css')}"><link rel="stylesheet" href="${asset('css/bench.css')}"><script src="${asset('js/bench.js')}" defer></script>++${o.headExtra || ''}${render(scripts)}<meta name="ov-boost" content="bench@${esc(RELEASE)}"><script src="${require('openvibe-shared/serve').url('boost.js')}" data-main="#main" defer></script>`,@@@ -10,6 +10,7 @@ const crypto = require('crypto');const { ids } = require('openvibe-contracts');const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');const ratingLib = require('./rating');++const styleLib = require('./rating/style');const BASE_ELO = ratingLib.BASE_ELO;const K = 32;@@@ -277,12 +278,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');const id = s.newId('bte');++ // The style of the output (how long it is, how much markup it wears) is measured once, here, so the++ // ratings can control for style later without opening the payload again.++ const style = styleLib.styleFeatures(payload || {});const row = await db.maybe(-- `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at)-- VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz) RETURNING *`,++ `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at, style_length, style_markup)++ VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz, $14, $15) RETURNING *`,[id, r.id, String(provider || 'ai').slice(0, 80), String(model || 'unknown').slice(0, 120), JSON.stringify(route || {}),JSON.stringify(payload || {}), num(costUsd), Math.max(0, num(ms)), Math.max(0, num(tokens)), error || null,-- uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso()],++ uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso(), style.length, style.markup],);await maybeReady(r.id);return wireEntry(row);@@@ -354,7 +358,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =};}-- async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, subject }) {++ async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, question = null, answer = null, netHash = null, subject }) {if (!JUDGES.includes(judge)) throw new BenchError(422, 'bench.invalid_judge', 'Unknown judge.');try {return await s.tx(async () => {@@@ -392,10 +396,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =const id = s.newId('bju');const row = await db.maybe(-- `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts)-- VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb) RETURNING *`,++ `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts, question, answer, net_hash)++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb, $14, $15, $16) RETURNING *`,[id, r.id, r.kind_id, judge, judgeName, judgeVersion, judge, pair ? JSON.stringify(pair) : null,-- outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null],++ outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null,++ question == null ? null : String(question).slice(0, 200), answer == null ? null : String(answer).slice(0, 40),++ netHash == null ? null : String(netHash).slice(0, 64)],);// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {nnew file mode 100644@@@ -0,0 +1,115 @@++'use strict';++++/**++ * The battles (stage B9): blind pairs, the rules, and a vote that works without JavaScript.++ */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++const FORM = { 'content-type': 'application/x-www-form-urlencoded' };++++(async () => {++ // One vote per hour for this person: the rate limit is part of the rules under test.++ const t = await boot({ env: { BENCH_BATTLE_PERSON_PER_HOUR: '1' } });++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob');++ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });++++ // Two finished rounds, each with two entries: two battles to hand out.++ const rounds = [];++ for (const n of [1, 2]) {++ const task = await t.get('/api/v1/kinds/design.ui/tasks', { as: alice, json: { prompt: 'A dashboard, prompt ' + n + '.', public: true } });++ assert.strictEqual(task.status, 201, task.text);++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });++ assert.strictEqual(rr.status, 201, rr.text);++ const roundId = rr.json().round.id;++ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model: 'alpha-' + n, payload: { html: '<p>alpha ' + n + '</p>' }, cost_usd: 0.01 } });++ assert.strictEqual(e1.status, 201, e1.text);++ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: 'beta-' + n, payload: { html: '<p>beta ' + n + '</p>' }, cost_usd: 0.02 } });++ assert.strictEqual(e2.status, 201, e2.text);++ rounds.push({ id: roundId, a: e1.json().entry.id, b: e2.json().entry.id });++ }++++ await check('an anonymous reader is asked to sign in, and sees no pair', async () => {++ const r = await t.get('/battle');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(r.text.includes('Sign in'), 'the sign-in prompt');++ assert.ok(!r.text.includes('data-battle='), 'no pair for an anonymous reader');++ });++++ await check('a signed-in person gets a blind pair', async () => {++ const r = await t.get('/battle', { as: bob });++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(r.text.includes('data-battle="bbs_'), 'the battle id: ' + r.text.slice(0, 200));++ assert.ok(r.text.includes('data-slot="A"') && r.text.includes('data-slot="B"'), 'two panes');++ assert.ok(r.text.includes('Which one looks more finished?'), 'the kind\'s own question');++ assert.ok(!r.text.includes('alpha-1') && !r.text.includes('beta-1'), 'no model names before the vote');++ });++++ await check('the vote is a form post that works without JavaScript', async () => {++ const page = await t.get('/battle', { as: bob });++ const battleId = /data-battle="(bbs_[^"]+)"/.exec(page.text)[1];++ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(battleId) + '&outcome=A&answer=A' });++ assert.strictEqual(vote.status, 303, vote.text);++ assert.ok(String(vote.headers.get('location') || '').startsWith('/battle/revealed/'), 'the reveal page: ' + vote.headers.get('location'));++ const reveal = await t.get(vote.headers.get('location'), { as: bob });++ assert.strictEqual(reveal.status, 200, reveal.text);++ assert.ok(reveal.text.includes('alpha-') || reveal.text.includes('beta-'), 'the names are open after the vote');++ const row = await t.ctx.s.db.maybe("SELECT outcome, question, answer FROM bench_judgements WHERE judge = 'human_pairwise' ORDER BY at DESC LIMIT 1");++ assert.strictEqual(row.outcome, 'left');++ assert.strictEqual(row.question, 'Which one looks more finished?');++ assert.strictEqual(row.answer, 'A');++ });++++ await check('a person never sees the same pair twice', async () => {++ const seen = await t.ctx.s.db.many('SELECT round_id, pair_key, voted_at FROM bench_battle_seen WHERE subject = (SELECT subject FROM bench_judgements ORDER BY at DESC LIMIT 1)');++ const pairs = new Set(seen.map((s) => s.pair_key));++ assert.strictEqual(pairs.size, seen.length, 'no pair twice: ' + JSON.stringify(seen));++ // Both rounds are used up now: there is nothing left to show this person.++ const more = await t.get('/battle', { as: bob });++ assert.strictEqual(more.status, 200, more.text);++ assert.ok(more.text.includes('No battle is waiting'), 'no pair left: ' + more.text.slice(0, 300));++ });++++ await check('the rate limit stops the second vote', async () => {++ const before = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");++ const rows = await t.ctx.s.db.many('SELECT id, round_id FROM bench_battle_seen WHERE voted_at IS NULL');++ assert.ok(rows.length >= 1, 'a pair is waiting');++ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=B' });++ assert.strictEqual(vote.status, 429, vote.text);++ const after = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");++ assert.strictEqual(after, before, 'no vote was written');++ });++++ await check('an anonymous vote is refused, and a cross-site vote is refused', async () => {++ const rows = await t.ctx.s.db.many('SELECT id FROM bench_battle_seen WHERE voted_at IS NULL');++ const anon = await t.get('/battle', { method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });++ assert.ok(anon.status === 302 || anon.status === 303, 'the anonymous vote is sent to sign in: ' + anon.status);++ assert.ok(String(anon.headers.get('location') || '').startsWith('/auth/login'), 'to the sign-in page');++ const cross = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, origin: 'https://evil.example', body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });++ assert.strictEqual(cross.status, 403, cross.text);++ });++++ await check('the ratings board counts the battle vote, and the open data has no voter ids', async () => {++ const page = await t.get('/ratings', { as: bob });++ assert.strictEqual(page.status, 200, page.text);++ assert.ok(page.text.includes('Raw') && page.text.includes('Style controlled'), 'both columns');++ assert.ok(page.text.includes('alpha-'), 'a model on the board');++++ const dump = await t.get('/api/v1/datasets/votes.jsonl');++ assert.strictEqual(dump.status, 200, dump.text);++ const lines = dump.text.trim().split('\n').filter(Boolean);++ assert.ok(lines.length >= 1, 'the dump has the vote');++ const subject = await t.ctx.s.db.value("SELECT subject FROM bench_judgements WHERE judge = 'human_pairwise' LIMIT 1");++ for (const line of lines) {++ const v = JSON.parse(line);++ assert.ok(v.left && v.right && v.outcome, 'a vote line: ' + line);++ assert.ok(!('subject' in v) && !('voter' in v) && !('user' in v), 'no voter id in the line: ' + line);++ assert.ok(!line.includes(subject), 'the voter is nowhere in the dump');++ }++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,88 @@++'use strict';++++/**++ * The gallery (stage B9): every round's outputs, filtered and ordered, with at most six live frames.++ */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++(async () => {++ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '30', BENCH_ROUNDS_DAILY: '100' } });++ const alice = t.network.addUser('alice');++ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });++++ const make = async (kind, prompt, model, cost) => {++ const task = await t.get('/api/v1/kinds/' + kind + '/tasks', { as: alice, json: { prompt, public: true } });++ assert.strictEqual(task.status, 201, task.text);++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });++ const roundId = rr.json().round.id;++ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model, payload: { html: '<p>one</p>' }, cost_usd: cost } });++ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: model + '-b', payload: { html: '<p>two</p>' }, cost_usd: cost } });++ assert.strictEqual(e1.status, 201, e1.text);++ assert.strictEqual(e2.status, 201, e2.text);++ return roundId;++ };++++ const rounds = [];++ for (let i = 0; i < 5; i++) rounds.push(await make('design.ui', 'A page, number ' + i, 'model-' + i, 0.01 * (i + 1)));++ const gameRound = await make('games.play', 'A game, one', 'gamer-1', 0.5);++++ await check('the gallery lists the rounds', async () => {++ const r = await t.get('/gallery');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(r.text.includes('data-gallery="1"'), 'the gallery');++ assert.ok(r.text.includes('A page, number 0'), 'a prompt');++ assert.ok(r.text.includes('ga-card'), 'the cards');++ assert.ok(r.text.includes('model-0'), 'the model of an entry');++ });++++ await check('at most six frames are live, the rest wait for a tap', async () => {++ const r = await t.get('/gallery');++ const live = (r.text.match(/class="ga-frame"/g) || []).length;++ assert.ok(live <= 6, 'live frames: ' + live);++ assert.ok(live >= 4, 'the first cards are live: ' + live);++ assert.ok(r.text.includes('data-ga-still'), 'the stills');++ assert.ok(r.text.includes('Live frame, on tap'), 'the tap hint');++ });++++ await check('the filters narrow the gallery', async () => {++ const byKind = await t.get('/gallery?kind=games.play');++ assert.ok(byKind.text.includes('A game, one'), 'the game is there');++ assert.ok(!byKind.text.includes('A page, number 0'), 'the pages are not');++++ const byModel = await t.get('/gallery?model=model-2');++ assert.ok(byModel.text.includes('A page, number 2'), 'the model is there');++ assert.ok(!byModel.text.includes('A page, number 3'), 'the others are not');++++ const byPrice = await t.get('/gallery?price=0.02');++ assert.ok(byPrice.text.includes('A page, number 0'), 'the cheap one is there');++ assert.ok(!byPrice.text.includes('A page, number 4'), 'the expensive one is not');++ });++++ await check('the gallery orders by newest and by most voted', async () => {++ const newest = await t.get('/gallery?order=new');++ const votes = await t.get('/gallery?order=votes');++ assert.strictEqual(newest.status, 200);++ assert.strictEqual(votes.status, 200);++ const firstNewest = /data-round-id="([^"]+)"/.exec(newest.text)[1];++ const firstVotes = /data-round-id="([^"]+)"/.exec(votes.text)[1];++ assert.ok(firstNewest, 'a round in the newest order');++ assert.ok(firstVotes, 'a round in the most voted order');++ // The last round made is the newest; no votes yet, so the most-voted order falls back to the newest.++ assert.strictEqual(firstNewest, firstVotes, 'without votes both orders agree');++ });++++ await check('the gallery API answers the same thing', async () => {++ const r = await t.get('/api/v1/gallery?kind=design.ui');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(r.json().rounds.length >= 5, 'the rounds');++ assert.strictEqual(r.json().filters.kind, 'design.ui');++ const one = r.json().rounds[0];++ assert.ok(one.entries.length === 2, 'two entries');++ assert.ok(one.entries[0].wrapper_url.includes('/render/w/'), 'the sandboxed frame');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,119 @@++'use strict';++++/**++ * The style-controlled ratings (stage B9).++ *++ * These are the maths, on purpose synthetic: a fit that recovers known strengths, a fit that takes a planted++ * length bias back out, and bands that follow the intervals.++ */++++const assert = require('assert');++const style = require('../server/rating/style');++const { check, done } = require('./helpers/boot');++++/** A small generator with a seed, so the synthetic votes are the same every run. */++function rng(seed) {++ let st = seed >>> 0;++ return () => {++ st = (st * 1664525 + 1013904223) >>> 0;++ return st / 4294967296;++ };++}++++(async () => {++ await check('a fit recovers known strengths', async () => {++ const random = rng(11);++ const truth = [['alpha', 0.6], ['beta', 0.0], ['gamma', -0.5]];++ const games = [];++ for (let i = 0; i < 900; i++) {++ const x = truth[Math.floor(random() * truth.length)];++ const y = truth[Math.floor(random() * truth.length)];++ if (x === y) continue;++ const p = 1 / (1 + Math.exp(-(x[1] - y[1])));++ games.push({++ id: 'g' + i, a: x[0], b: y[0], outcome: random() < p ? 'a' : 'b',++ fa: { length: 100, markup: 2 }, fb: { length: 100, markup: 2 },++ });++ }++ const fit = style.fitRatings(games, { bootstrap: 120 });++ const byModel = new Map(fit.models.map((m) => [m.model, m]));++ const scale = 400 / Math.log(10);++ for (const [model, theta] of truth) {++ const expected = 1000 + scale * theta;++ const got = byModel.get(model).raw.rating;++ assert.ok(Math.abs(got - expected) < 40, model + ': expected ' + expected.toFixed(1) + ', got ' + got.toFixed(1));++ }++ });++++ await check('the style control takes a planted length bias back out', async () => {++ const random = rng(23);++ // Two equally good models, but voters prefer the longer output.++ const games = [];++ for (let i = 0; i < 400; i++) {++ const longA = random() < 0.6;++ const la = longA ? 3000 + Math.floor(random() * 2000) : 800 + Math.floor(random() * 600);++ const lb = longA ? 900 + Math.floor(random() * 700) : 2800 + Math.floor(random() * 2000);++ const p = 1 / (1 + Math.exp(-(la - lb) / 900));++ games.push({++ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',++ fa: { length: la, markup: Math.round(la / 120) }, fb: { length: lb, markup: Math.round(lb / 120) },++ });++ }++ const fit = style.fitRatings(games, { bootstrap: 120 });++ const raw = new Map(fit.models.map((m) => [m.model, m.raw.rating]));++ const controlled = new Map(fit.models.map((m) => [m.model, m.controlled.rating]));++ // The raw board says one model is clearly better; the controlled board does not.++ assert.ok(Math.abs(raw.get('alpha') - raw.get('beta')) > 30, 'the raw board is biased: ' + raw.get('alpha') + ' vs ' + raw.get('beta'));++ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < 30, 'the controlled board is even: ' + controlled.get('alpha') + ' vs ' + controlled.get('beta'));++ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < Math.abs(raw.get('alpha') - raw.get('beta')), 'the control shrinks the gap');++ assert.ok(fit.effects.length && fit.effects.length.beta > 0, 'the length effect is positive: ' + JSON.stringify(fit.effects.length));++ });++++ await check('the bands follow the intervals', async () => {++ const rows = [++ { model: 'a', rating: 1100, ci_lower: 1090, ci_upper: 1110 },++ { model: 'b', rating: 1080, ci_lower: 1040, ci_upper: 1105 }, // overlaps a++ { model: 'c', rating: 1000, ci_lower: 960, ci_upper: 1040 }, // overlaps b, not a++ { model: 'd', rating: 900, ci_lower: 880, ci_upper: 920 }, // alone++ ];++ const bands = style.rankBands(rows);++ assert.strictEqual(bands.length, 2, JSON.stringify(bands));++ assert.strictEqual(bands[0].label, '1st–3rd', bands[0].label);++ assert.deepStrictEqual(bands[0].models, ['a', 'b', 'c']);++ assert.strictEqual(bands[1].label, '4th', bands[1].label);++ assert.deepStrictEqual(bands[1].models, ['d']);++ assert.strictEqual(style.bandLabel(2, 4), '2nd–4th');++ assert.strictEqual(style.bandLabel(1, 1), '1st');++ });++++ await check('the style features count length and markup', async () => {++ const f = style.styleFeatures({ html: '<h1>Title</h1><p><strong>bold</strong></p><ul><li>one</li></ul>' });++ assert.ok(f.length > 20, 'length: ' + f.length);++ assert.ok(f.markup >= 4, 'markup: ' + f.markup);++ assert.strictEqual(style.markupCount('# Title\n- a\n- b'), 3);++ assert.strictEqual(style.markupCount('plain text'), 0);++ });++++ await check('a voter who disagrees with the consensus is down-weighted, never named', async () => {++ const random = rng(31);++ const games = [];++ for (let i = 0; i < 400; i++) {++ const p = 1 / (1 + Math.exp(-0.9));++ games.push({++ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',++ fa: { length: 100, markup: 1 }, fb: { length: 100, markup: 1 },++ voter: 'voter-' + (i % 12),++ });++ }++ // One voter always disagrees with the consensus.++ for (const g of games) if (g.voter === 'voter-3') g.outcome = g.outcome === 'a' ? 'b' : 'a';++ const fit = style.fitRatings(games, { bootstrap: 60 });++ assert.ok(fit.voters.downweighted >= 1, 'someone is down-weighted: ' + JSON.stringify(fit.voters));++ assert.ok(!JSON.stringify(fit).includes('voter-3'), 'the down-weighted voter is never named in the board');++ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));++ const odd = games.filter((g) => g.voter === 'voter-3');++ assert.ok(odd.every((g) => (weights.get(g.id) || 1) < 1), 'the odd voter\'s votes count less');++ });++++ done();++})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,33 @@
@@ -0,0 +1,33 @@
+-- phase: expand
+-- B9: battles, the gallery and the honest ratings.
+--
+-- A battle vote is a human_pairwise judgement with two extras: the kind's own question (games: which is more fun;
+-- design: which looks more finished) and the answer to it. The network fingerprint is a hash of the voter's address,
+-- never the address itself: it is only there to limit votes per network. The style of an output (its length and how
+-- much markup it wears) is measured once, when the entry lands, so the ratings can control for it later.
+
+ALTER TABLE bench_judgements ADD COLUMN question text;
+ALTER TABLE bench_judgements ADD COLUMN answer text;
+ALTER TABLE bench_judgements ADD COLUMN net_hash text;
+CREATE INDEX bench_judgements_net ON bench_judgements (net_hash, at DESC) WHERE net_hash IS NOT NULL;
+
+ALTER TABLE bench_entries ADD COLUMN style_length integer;
+ALTER TABLE bench_entries ADD COLUMN style_markup integer;
+
+-- A pair a person has been shown. The unique pair keeps the promise that nobody sees the same pair twice.
+CREATE TABLE bench_battle_seen (
+ id text COLLATE "C" PRIMARY KEY, -- bbs_ + ULID
+ subject text NOT NULL, -- the person
+ pair_key text NOT NULL, -- the two entry ids, sorted and joined
+ round_id text NOT NULL REFERENCES bench_rounds (id) ON DELETE CASCADE,
+ kind_id text NOT NULL REFERENCES bench_kinds (id),
+ a_entry text NOT NULL, -- the entry shown as pane A
+ b_entry text NOT NULL, -- the entry shown as pane B
+ net_hash text, -- the network fingerprint, for the per-network limit
+ voted_at timestamptz, -- when the person cast the vote
+ at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT bench_battle_seen_id CHECK (id ~ '^bbs_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_battle_seen_pair UNIQUE (subject, pair_key)
+);
+CREATE INDEX bench_battle_seen_subject ON bench_battle_seen (subject, at DESC);
+CREATE INDEX bench_battle_seen_net ON bench_battle_seen (net_hash, at DESC) WHERE net_hash IS NOT NULL;
@@ -264,3 +264,76 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -264,3 +264,76 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }
.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }
.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
.br-form + .br-fine { margin-top: 16px; }
.br-form + .br-fine { margin-top: 16px; }
+
+/* The gallery (B9): every round's outputs, live in sandboxed frames; six live at a time, the rest wait. */
+.ga-head { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 0; }
+.ga-gallery { max-width: 1180px; margin: 0 auto; padding: 0 16px 64px; }
+.ga-gallery > h1, .ga-head > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }
+.ga-filters { display: grid; gap: 10px; grid-template-columns: 1fr; margin: 18px 0 4px; }
+.ga-filters label { display: grid; gap: 4px; min-width: 0; font-size: .74rem; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
+.ga-filters input, .ga-filters select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; font-weight: 600; text-transform: none; letter-spacing: normal; max-width: 100%; }
+.ga-filters input:focus-visible, .ga-filters select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
+.ga-grid { display: grid; gap: 16px; grid-template-columns: 1fr; margin-top: 18px; }
+.ga-card { display: grid; gap: 12px; padding: 14px; border-radius: 20px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; align-content: start; }
+.ga-prompt { margin: 0; font-size: .98rem; font-weight: 650; line-height: 1.45; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden; }
+.ga-shots { display: grid; gap: 8px; grid-template-columns: repeat(2, minmax(0, 1fr)); }
+.ga-shot { margin: 0; display: grid; gap: 4px; min-width: 0; }
+.ga-shot figcaption { font-size: .74rem; font-weight: 700; color: var(--text-muted, #7386a3); overflow-wrap: anywhere; }
+.ga-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }
+.ga-still { display: grid; place-items: center; aspect-ratio: 4 / 3; padding: 8px; border: 1px dashed var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); color: var(--text-muted, #7386a3); font-size: .82rem; text-align: center; }
+.ga-still img { width: 100%; height: 100%; object-fit: cover; border-radius: 12px; }
+.ga-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 0; font-size: .84rem; color: var(--text-muted, #7386a3); }
+.ga-votes { font-variant-numeric: tabular-nums; }
+.ga-link { margin-left: auto; font-weight: 750; color: var(--room-strong) !important; text-decoration: none; }
+.ga-link:hover { text-decoration: underline; }
+@media (min-width: 640px) { .ga-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
+@media (min-width: 1024px) {
+ .ga-grid { grid-template-columns: repeat(3, minmax(0, 1fr)); }
+ .ga-filters { grid-template-columns: repeat(4, minmax(0, 1fr)) auto; align-items: end; }
+}
+
+/* The battles (B9): two outputs, one prompt, blind; the vote is a plain form post. */
+.bt-battle { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }
+.bt-battle > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.1; text-wrap: balance; }
+.bt-prompt { font-size: clamp(1.3rem, 3.6vw, 2rem) !important; letter-spacing: -.02em !important; line-height: 1.25 !important; }
+.bt-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 12px 0 0; font-size: .86rem; color: var(--text-muted, #7386a3); }
+.bt-panes { display: grid; gap: 12px; grid-template-columns: 1fr; margin-top: 16px; }
+.bt-pane { margin: 0; display: grid; gap: 6px; min-width: 0; }
+.bt-pane figcaption { font-size: .76rem; font-weight: 800; letter-spacing: .05em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
+.bt-pane-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }
+.bt-vote { margin-top: 18px; padding: 16px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); display: grid; gap: 12px; }
+.bt-vote fieldset { margin: 0; padding: 0; border: 0; display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
+.bt-vote legend { width: 100%; margin-bottom: 8px; font-weight: 800; }
+.bt-vote label { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0 16px; border-radius: 999px; border: 1px solid var(--border, #1f2d47); cursor: pointer; font-weight: 650; }
+.bt-vote label:has(input:checked) { border-color: var(--room-accent); background: color-mix(in srgb, var(--room-accent) 14%, transparent); }
+.bt-vote input { accent-color: var(--room-accent); }
+.bt-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }
+.bt-facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(90px, 1fr)); gap: 6px; margin: 0; font-size: .82rem; }
+.bt-facts dt { color: var(--text-muted, #7386a3); font-size: .7rem; text-transform: uppercase; letter-spacing: .06em; font-weight: 800; }
+.bt-facts dd { margin: 2px 0 0; font-weight: 650; overflow-wrap: anywhere; }
+.bt-reveal-facts { display: grid; gap: 14px; grid-template-columns: 1fr; margin-top: 14px; }
+.bt-signed-out { max-width: 720px; }
+@media (min-width: 720px) {
+ .bt-panes { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; }
+ .bt-reveal-facts { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+}
+
+/* The ratings board (B9): raw and style controlled, side by side, with the intervals and the bands. */
+.rt-board { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }
+.rt-board > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }
+.rt-list { list-style: none; margin: 20px 0 0; padding: 0; display: grid; gap: 10px; }
+.rt-row { display: grid; gap: 10px; padding: 14px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }
+.rt-cell { display: grid; gap: 2px; min-width: 0; align-content: start; }
+.rt-label { font-size: .68rem; font-weight: 800; letter-spacing: .07em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
+.rt-model { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; overflow-wrap: anywhere; }
+.rt-rating { font-size: 1.15rem; font-weight: 800; font-variant-numeric: tabular-nums; }
+.rt-ci { color: var(--text-muted, #7386a3); font-size: .82rem; font-variant-numeric: tabular-nums; }
+.rt-band { display: inline-block; margin-top: 4px; padding: 2px 10px; border-radius: 999px; background: color-mix(in srgb, var(--room-accent) 14%, transparent); color: var(--room-strong); font-size: .78rem; font-weight: 750; justify-self: start; }
+.rt-votes { font-weight: 700; font-variant-numeric: tabular-nums; }
+.rt-effects, .rt-data { margin-top: clamp(24px, 4vw, 36px); }
+.rt-effects h2, .rt-data h2 { margin: 0 0 10px; font-size: 1.2rem; font-weight: 800; letter-spacing: -.01em; }
+.rt-data code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; color: var(--room-strong); overflow-wrap: anywhere; }
+@media (min-width: 900px) {
+ .rt-row { grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1fr) auto; align-items: center; }
+}
+
@@ -1 +1,77 @@
@@ -1 +1,77 @@
/* OpenVibe.Bench pages work without JavaScript; the main session designs this room later. */
/* OpenVibe.Bench pages work without JavaScript; the main session designs this room later. */
+/* The gallery (B9): at most six frames live at a time; the rest wait as a still until the reader asks. */
+(function () {
+ 'use strict';
+
+ var LIVE = 6;
+
+ function gallery() {
+ var grid = document.querySelector('[data-gallery="1"]');
+ if (!grid) return;
+ var shots = Array.prototype.slice.call(grid.querySelectorAll('.ga-shot'));
+ var live = shots.filter(function (s) { return s.getAttribute('data-live') === '1'; });
+
+ function release(shot) {
+ var frame = shot.querySelector('iframe.ga-frame');
+ if (!frame) return;
+ if (shot.getAttribute('data-server') === '1') return; // the first frames stay live
+ var still = document.createElement('button');
+ still.type = 'button';
+ still.className = 'ga-still';
+ still.setAttribute('data-ga-still', '1');
+ still.setAttribute('aria-label', 'Load the live frame');
+ var note = document.createElement('span');
+ note.className = 'ga-still-note';
+ note.textContent = 'Live frame, on tap';
+ still.appendChild(note);
+ frame.parentNode.replaceChild(still, frame);
+ shot.removeAttribute('data-live');
+ live = live.filter(function (s) { return s !== shot; });
+ }
+
+ function activate(shot) {
+ if (!shot || shot.getAttribute('data-live') === '1') return;
+ var src = shot.getAttribute('data-src');
+ if (!src) return;
+ while (live.length >= LIVE) {
+ var victim = null;
+ for (var i = 0; i < live.length; i++) {
+ if (live[i].getAttribute('data-server') !== '1') { victim = live[i]; break; }
+ }
+ if (!victim) return;
+ release(victim);
+ }
+ var still = shot.querySelector('[data-ga-still]');
+ var frame = document.createElement('iframe');
+ frame.className = 'ga-frame';
+ frame.setAttribute('src', src);
+ frame.setAttribute('sandbox', 'allow-scripts allow-same-origin');
+ frame.setAttribute('referrerpolicy', 'no-referrer');
+ frame.setAttribute('loading', 'lazy');
+ frame.setAttribute('title', 'Output');
+ if (still && still.parentNode) still.parentNode.replaceChild(frame, still);
+ shot.setAttribute('data-live', '1');
+ live.push(shot);
+ }
+
+ grid.addEventListener('click', function (event) {
+ var target = event.target;
+ var shot = target && target.closest ? target.closest('.ga-shot') : null;
+ if (shot) activate(shot);
+ });
+
+ if ('IntersectionObserver' in window) {
+ var observer = new IntersectionObserver(function (entries) {
+ entries.forEach(function (entry) {
+ if (entry.isIntersecting) activate(entry.target);
+ });
+ }, { rootMargin: '320px' });
+ shots.forEach(function (shot) { observer.observe(shot); });
+ } else {
+ shots.forEach(activate);
+ }
+ }
+
+ if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', gallery);
+ else gallery();
+}());
new file mode 100644
new file mode 100644
@@ -0,0 +1,99 @@
@@ -0,0 +1,99 @@
+'use strict';
+
+/**
+ * Re-derive the board from the open vote dump.
+ *
+ * node scripts/fit-ratings.js /tmp/votes.jsonl # the table
+ * node scripts/fit-ratings.js /tmp/votes.jsonl --json # the same board as JSON
+ * curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl | node scripts/fit-ratings.js
+ *
+ * The dump carries every vote without voter ids, plus the weight each vote carried in the live fit (so a voter
+ * who disagreed with the consensus far too often counts less). The fit below is the same fit the board uses, with
+ * the same seed, so the numbers here are the numbers on /ratings.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const style = require(path.join(__dirname, '..', 'server', 'rating', 'style'));
+
+const argv = process.argv.slice(2);
+const asJson = argv.includes('--json');
+const bootstrapIndex = argv.indexOf('--bootstrap');
+const bootstrap = bootstrapIndex >= 0 ? Number(argv[bootstrapIndex + 1]) || style.BOOTSTRAP : style.BOOTSTRAP;
+const file = argv.find((a) => !a.startsWith('--') && a !== String(bootstrap));
+
+function readInput() {
+ if (!file) return fs.readFileSync(0, 'utf8');
+ return fs.readFileSync(file, 'utf8');
+}
+
+/** The dump's lines into the games the fit reads. */
+function gamesOf(lines) {
+ const games = [];
+ const weights = new Map();
+ for (const line of lines) {
+ const text = String(line || '').trim();
+ if (!text) continue;
+ let v;
+ try { v = JSON.parse(text); } catch { continue; }
+ if (!v || !v.left || !v.right || !v.left.model || !v.right.model) continue;
+ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';
+ games.push({
+ id: v.id == null ? null : String(v.id),
+ a: v.left.model,
+ b: v.right.model,
+ outcome,
+ fa: { length: Number(v.left.length) || 0, markup: Number(v.left.markup) || 0 },
+ fb: { length: Number(v.right.length) || 0, markup: Number(v.right.markup) || 0 },
+ });
+ if (v.id != null && v.weight != null) weights.set(String(v.id), Number(v.weight));
+ }
+ return { games, weights };
+}
+
+function main() {
+ const text = readInput();
+ const { games, weights } = gamesOf(text.split('\n'));
+ if (!games.length) {
+ process.stderr.write('No votes in the dump. Fetch one: curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl\n');
+ process.exit(1);
+ }
+ const fit = style.fitRatings(games, { bootstrap, voteWeights: weights });
+ if (asJson) {
+ process.stdout.write(JSON.stringify({ models: fit.models, bands: fit.bands, effects: fit.effects, votes: fit.votes, bootstrap: fit.bootstrap, seed: fit.seed }, null, 2) + '\n');
+ return;
+ }
+ const cell = (m, key) => {
+ const c = m[key];
+ const lo = Math.round(c.ci_lower);
+ const hi = Math.round(c.ci_upper);
+ return String(Math.round(c.rating)).padStart(5) + ' [' + String(lo).padStart(5) + ', ' + String(hi).padStart(5) + ']';
+ };
+ const lines = [];
+ lines.push('OpenVibe.Bench — the board, re-derived from the open vote dump');
+ lines.push(fit.votes + ' votes, ' + fit.bootstrap + ' bootstrap resamples, seed ' + fit.seed);
+ lines.push('');
+ lines.push('model'.padEnd(28) + 'votes raw rating (95% interval) band controlled rating (95% interval) band');
+ lines.push('-'.repeat(120));
+ for (const m of fit.models) {
+ lines.push(
+ String(m.model).padEnd(28)
+ + String(m.votes).padStart(5) + ' '
+ + cell(m, 'raw') + ' ' + String(m.raw.band || '').padEnd(8)
+ + cell(m, 'controlled') + ' ' + String(m.controlled.band || ''),
+ );
+ }
+ lines.push('');
+ for (const [key, e] of Object.entries(fit.effects || {})) {
+ lines.push('style effect ' + key + ':'
+ + (e.elo_per_1000_chars != null ? ' ' + e.elo_per_1000_chars + ' Elo per 1000 characters' : '')
+ + (e.elo_per_10_marks != null ? ' ' + e.elo_per_10_marks + ' Elo per 10 markup marks' : ''));
+ }
+ if (fit.dropped_covariates && fit.dropped_covariates.length) {
+ lines.push('dropped (too close to another covariate): ' + fit.dropped_covariates.join(', '));
+ }
+ lines.push('down-weighted voters: ' + fit.voters.downweighted + ' of ' + fit.voters.total + ' (never named)');
+ process.stdout.write(lines.join('\n') + '\n');
+}
+
+main();
@@ -28,6 +28,9 @@ const { createApi } = require('./http/api');
@@ -28,6 +28,9 @@ const { createApi } = require('./http/api');
const { createRegistryApi } = require('./http/registry');
const { createRegistryApi } = require('./http/registry');
const { createPageRoutes } = require('./http/pages');
const { createPageRoutes } = require('./http/pages');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
+const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
+const { createBattlePages } = require('./http/battle');
+const { createRatingsPages } = require('./http/ratings');
const { createRenderRoutes } = require('./http/render');
const { createRenderRoutes } = require('./http/render');
const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { createRegistry } = require('./bench/registry');
const { createRegistry } = require('./bench/registry');
@@ -160,6 +163,7 @@ async function createApp(opts = {}) {
@@ -160,6 +163,7 @@ async function createApp(opts = {}) {
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
+ app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use(mcpRouter(ctx));
app.use(mcpRouter(ctx));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
@@ -176,6 +180,9 @@ async function createApp(opts = {}) {
@@ -176,6 +180,9 @@ async function createApp(opts = {}) {
app.use('/render', createRenderRoutes(ctx));
app.use('/render', createRenderRoutes(ctx));
app.use('/vendor', require('./http/vendor').createVendorRoutes());
app.use('/vendor', require('./http/vendor').createVendorRoutes());
app.use('/showcase', createShowcasePages(ctx));
app.use('/showcase', createShowcasePages(ctx));
+ app.use('/gallery', createGalleryPages(ctx));
+ app.use('/battle', createBattlePages(ctx));
+ app.use('/ratings', createRatingsPages(ctx));
app.use(createPageRoutes(ctx));
app.use(createPageRoutes(ctx));
app.use((req, res) => send(res, 404, {
app.use((req, res) => send(res, 404, {
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
new file mode 100644
new file mode 100644
@@ -0,0 +1,197 @@
@@ -0,0 +1,197 @@
+'use strict';
+
+/**
+ * Battles (spec 2): two outputs, one prompt, blind.
+ *
+ * A battle is a pair of entries from a finished round. The rules the research asks for are enforced here, not in
+ * the page:
+ *
+ * - only a signed-in person votes (the page sends anonymous readers to sign in);
+ * - a person never sees the same pair twice (bench_battle_seen holds the pair the moment it is shown);
+ * - votes are limited per person and per network (the network is a hash of the address, never the address).
+ *
+ * The vote itself is a human_pairwise judgement, so the battle votes are the same votes the ratings are made of.
+ * Two extras ride along: the kind's own question ("which is more fun" for games, "which looks more finished" for
+ * design) and the network hash, which never leaves the database.
+ */
+
+const crypto = require('crypto');
+const { SHOWCASE_KIND_IDS } = require('../kinds');
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+
+/** The question each kind asks about its own outputs. */
+const QUESTIONS = Object.freeze({
+ games: { id: 'more_fun', text: 'Which one is more fun?', options: ['A', 'B', 'tie'] },
+ design: { id: 'more_finished', text: 'Which one looks more finished?', options: ['A', 'B', 'tie'] },
+ other: { id: 'better', text: 'Which one is better?', options: ['A', 'B', 'tie'] },
+});
+
+/** The question a kind asks. */
+function questionFor(kindId) {
+ const id = String(kindId || '');
+ if (id.startsWith('games.')) return QUESTIONS.games;
+ if (id.startsWith('design.')) return QUESTIONS.design;
+ return QUESTIONS.other;
+}
+
+/** The pair key: the two entry ids, sorted, so A and B in any order is the same pair. */
+function pairKey(a, b) {
+ return [String(a || ''), String(b || '')].sort().join('|');
+}
+
+function createBattles(ctx) {
+ const { s, config, bench } = ctx;
+ const db = s.db;
+
+ /** The network fingerprint: a hash of the address, so the limit works without storing an address. */
+ function netHashOf(req) {
+ const ip = String((req && req.ip) || (req && req.socket && req.socket.remoteAddress) || '').trim();
+ if (!ip) return null;
+ return crypto.createHash('sha256').update('bench-battle|' + ip).digest('hex').slice(0, 40);
+ }
+
+ const limits = () => (config && config.bench) || {};
+
+ /** How many votes this person and this network may still cast. */
+ async function budget({ subject, netHash }) {
+ const cfg = limits();
+ const hourAgo = new Date(Date.now() - 3600_000).toISOString();
+ const dayAgo = new Date(Date.now() - 86_400_000).toISOString();
+ const personHour = num(await db.value(
+ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
+ [subject, hourAgo]));
+ const personDay = num(await db.value(
+ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
+ [subject, dayAgo]));
+ const netHour = netHash ? num(await db.value(
+ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
+ [netHash, hourAgo])) : 0;
+ const netDay = netHash ? num(await db.value(
+ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
+ [netHash, dayAgo])) : 0;
+ const perPersonHour = Math.max(1, num(cfg.battlePersonPerHour, 30));
+ const perPersonDay = Math.max(1, num(cfg.battlePersonPerDay, 200));
+ const perNetworkHour = Math.max(1, num(cfg.battleNetworkPerHour, 120));
+ const perNetworkDay = Math.max(1, num(cfg.battleNetworkPerDay, 1000));
+ const reasons = [];
+ if (personHour >= perPersonHour) reasons.push('You have voted a lot in the last hour. Try again in a little while.');
+ if (personDay >= perPersonDay) reasons.push('You have used your votes for today.');
+ if (netHour >= perNetworkHour) reasons.push('This network has voted a lot in the last hour. Try again later.');
+ if (netDay >= perNetworkDay) reasons.push('This network has used its votes for today.');
+ return {
+ allowed: reasons.length === 0,
+ message: reasons.join(' '),
+ person_hour: personHour, person_day: personDay, network_hour: netHour, network_day: netDay,
+ limits: { person_hour: perPersonHour, person_day: perPersonDay, network_hour: perNetworkHour, network_day: perNetworkDay },
+ };
+ }
+
+ /**
+ * A pair this person has not seen: a finished round, not their own, with two usable entries, and not a pair
+ * they have been shown before. The pair is recorded as seen the moment it is shown, so nobody meets it twice.
+ */
+ async function pickPair({ subject, kind = 'all', random = Math.random, netHash = null }) {
+ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;
+ const rows = await db.many(
+ `SELECT r.id, r.kind_id, t.prompt
+ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id
+ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND r.created_by <> $2
+ AND (SELECT COUNT(*) FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) >= 2
+ AND NOT EXISTS (SELECT 1 FROM bench_judgements j WHERE j.round_id = r.id AND j.subject = $2)
+ AND NOT EXISTS (SELECT 1 FROM bench_battle_seen v WHERE v.subject = $2 AND v.round_id = r.id)
+ ORDER BY r.created_at DESC, r.id LIMIT 60`,
+ [kinds, subject]);
+ if (!rows.length) return null;
+ const row = rows[Math.floor(random() * rows.length) % rows.length];
+ const entries = await db.many(
+ `SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup
+ FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id`,
+ [row.id]);
+ if (entries.length < 2) return null;
+ // Prefer two different models, so a battle is always between two models.
+ let pair = [entries[0], entries[1]];
+ const other = entries.find((e) => e.model !== entries[0].model);
+ if (other) pair = [entries[0], other];
+ const [a, b] = pair;
+ const key = pairKey(a.id, b.id);
+ const id = s.newId('bbs');
+ await db.exec(
+ `INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, net_hash, at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::timestamptz)`,
+ [id, subject, key, row.id, row.kind_id, a.id, b.id, netHash, s.iso()]);
+ return {
+ id, round_id: row.id, kind_id: row.kind_id, prompt: row.prompt,
+ question: questionFor(row.kind_id),
+ a: { id: a.id, provider: a.provider, model: a.model, cost_usd: num(a.cost_usd), ms: num(a.ms) },
+ b: { id: b.id, provider: b.provider, model: b.model, cost_usd: num(b.cost_usd), ms: num(b.ms) },
+ };
+ }
+
+ /** The battle as it was shown, for the reveal and the share card. */
+ async function battleOf(id, subject = null) {
+ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(id || '')]);
+ if (!seen) return null;
+ if (subject && seen.subject !== subject) return null;
+ const round = await db.maybe(
+ 'SELECT r.id, r.kind_id, r.created_at, t.prompt FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.id = $1',
+ [seen.round_id]);
+ if (!round) return null;
+ const entries = await db.many(
+ 'SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup FROM bench_entries WHERE id = ANY($1::text[])',
+ [[seen.a_entry, seen.b_entry]]);
+ const a = entries.find((e) => e.id === seen.a_entry);
+ const b = entries.find((e) => e.id === seen.b_entry);
+ if (!a || !b) return null;
+ const vote = await db.maybe(
+ "SELECT outcome, question, answer FROM bench_judgements WHERE round_id = $1 AND subject = $2 AND judge = 'human_pairwise' ORDER BY at LIMIT 1",
+ [seen.round_id, seen.subject]);
+ return {
+ id: seen.id, round_id: seen.round_id, kind_id: seen.kind_id, prompt: round.prompt,
+ question: questionFor(seen.kind_id),
+ voted_at: seen.voted_at ? new Date(seen.voted_at).toISOString() : null,
+ a: { id: a.id, provider: a.provider, model: a.model, route: a.route, cost_usd: num(a.cost_usd), ms: num(a.ms), tokens: num(a.tokens), payload: a.payload },
+ b: { id: b.id, provider: b.provider, model: b.model, route: b.route, cost_usd: num(b.cost_usd), ms: num(b.ms), tokens: num(b.tokens), payload: b.payload },
+ vote: vote ? { outcome: vote.outcome, question: vote.question, answer: vote.answer } : null,
+ };
+ }
+
+ /** The vote. The rules are checked here; a caller that breaks them gets a refusal, not a quiet write. */
+ async function recordVote({ subject, battleId, outcome, answer = null, netHash = null }) {
+ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(battleId || '')]);
+ if (!seen) return { ok: false, status: 404, code: 'bench.battle_not_found', message: 'That battle is gone.' };
+ if (seen.subject !== subject) return { ok: false, status: 403, code: 'bench.battle_not_yours', message: 'That battle belongs to someone else.' };
+ if (seen.voted_at) return { ok: false, status: 409, code: 'bench.already_voted', message: 'You have already voted on this pair.' };
+ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };
+ const won = map[String(outcome || '')];
+ if (!won) return { ok: false, status: 422, code: 'bench.invalid_outcome', message: 'The outcome must be A, B, tie or both_bad.' };
+ const room = await budget({ subject, netHash });
+ if (!room.allowed) return { ok: false, status: 429, code: 'bench.limit_battle', message: room.message };
+ const question = questionFor(seen.kind_id);
+ const answerValue = answer == null || answer === '' ? null : String(answer).slice(0, 8);
+ if (answerValue != null && !question.options.includes(answerValue)) {
+ return { ok: false, status: 422, code: 'bench.invalid_answer', message: 'That answer does not fit the question.' };
+ }
+ try {
+ await bench.castJudgement({
+ roundId: seen.round_id,
+ judge: 'human_pairwise',
+ pair: { left: seen.a_entry, right: seen.b_entry },
+ outcome: won,
+ question: question.text,
+ answer: answerValue,
+ netHash,
+ subject,
+ });
+ } catch (err) {
+ if (err && err.status) return { ok: false, status: err.status, code: err.code || 'bench.invalid', message: err.detail || 'The vote was not accepted.' };
+ throw err;
+ }
+ await db.exec('UPDATE bench_battle_seen SET voted_at = $2::timestamptz WHERE id = $1', [seen.id, s.iso()]);
+ return { ok: true, battle_id: seen.id, outcome: won, question };
+ }
+
+ return { pickPair, battleOf, recordVote, budget, netHashOf, questionFor, pairKey };
+}
+
+module.exports = { createBattles, questionFor, pairKey, QUESTIONS };
@@ -84,6 +84,11 @@ function load(env = process.env) {
@@ -84,6 +84,11 @@ function load(env = process.env) {
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
+ // Battles: how many votes one person and one network may cast.
+ battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),
+ battlePersonPerDay: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_DAY, 200)),
+ battleNetworkPerHour: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_HOUR, 120)),
+ battleNetworkPerDay: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_DAY, 1000)),
},
},
};
};
}
}
@@ -29,6 +29,7 @@ const { asyncRouter } = require('./router');
@@ -29,6 +29,7 @@ const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const { sameOrigin } = require('./same-origin');
const { kindById } = require('../kinds');
const { kindById } = require('../kinds');
const { BenchError } = require('../store');
const { BenchError } = require('../store');
+const boardLib = require('../rating/board');
function createApi(ctx) {
function createApi(ctx) {
const { config, s, principal, bench } = ctx;
const { config, s, principal, bench } = ctx;
@@ -225,6 +226,28 @@ function createApi(ctx) {
@@ -225,6 +226,28 @@ function createApi(ctx) {
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
});
});
+ // The open vote dump (stage B9): every human vote, no voter ids. It comes before the kind dataset route,
+ // because '/datasets/votes.jsonl' would otherwise be read as the kind 'votes'.
+ r.get('/datasets/votes.jsonl', async (req, res) => {
+ try {
+ const result = await boardLib.board(ctx);
+ const text = boardLib.dumpJsonl(result);
+ res.set('Cache-Control', 'public, max-age=60');
+ res.set('Content-Disposition', 'attachment; filename="bench-votes.jsonl"');
+ res.set('X-Bench-Votes', String(result.votes.length));
+ return res.type('application/x-ndjson').send(text);
+ } catch (err) { return failed(res, err); }
+ });
+
+ // The board the votes make: raw and style controlled, with intervals and bands.
+ r.get('/ratings', async (req, res) => {
+ try {
+ const result = await boardLib.board(ctx);
+ res.set('Cache-Control', 'public, max-age=60');
+ return res.json({ board: boardLib.boardView(result) });
+ } catch (err) { return failed(res, err); }
+ });
+
r.get('/datasets/:kindId.jsonl', async (req, res) => {
r.get('/datasets/:kindId.jsonl', async (req, res) => {
try {
try {
const text = await bench.datasetJsonl(String(req.params.kindId || ''));
const text = await bench.datasetJsonl(String(req.params.kindId || ''));
new file mode 100644
new file mode 100644
@@ -0,0 +1,231 @@
@@ -0,0 +1,231 @@
+'use strict';
+
+/**
+ * The battle pages: /battle (blind), /battle/revealed/:id (after the vote) and /battle/card/:id.svg (the share card).
+ *
+ * The vote is a plain form post to the same origin, so it works with JavaScript switched off. The page itself is
+ * server-rendered: the two panes are the same sandboxed render frames the showcase uses, and the model names stay
+ * closed until the vote is in.
+ */
+
+const express = require('express');
+const { asyncRouter } = require('./router');
+const { sameOrigin } = require('./same-origin');
+const renderLib = require('../bench/render');
+const { kindById } = require('../kinds');
+const { esc, raw } = require('../render/html');
+const { send } = require('../render/layout');
+const { createBattles } = require('../bench/battles');
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+
+/** The share card is a vector image: 1200 by 630, the size every social card expects. */
+const CARD = { w: 1200, h: 630 };
+
+function paneOf(slot, letter) {
+ const url = renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(slot.id);
+ return '<figure class="bt-pane" data-slot="' + esc(letter) + '" data-entry-id="' + esc(slot.id) + '">'
+ + '<figcaption><span class="bt-pane-label">Pane ' + esc(letter) + '</span></figcaption>'
+ + '<iframe class="bt-pane-frame" src="' + esc(url) + '" title="Output ' + esc(letter) + '"'
+ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'
+ + '</figure>';
+}
+
+function voteForm(battle) {
+ const q = battle.question;
+ return '<form class="bt-vote" method="post" action="/battle" data-battle-form="1">'
+ + '<input type="hidden" name="battle" value="' + esc(battle.id) + '">'
+ + '<fieldset><legend>Which is better?</legend>'
+ + '<label><input type="radio" name="outcome" value="A" required> A</label>'
+ + '<label><input type="radio" name="outcome" value="B"> B</label>'
+ + '<label><input type="radio" name="outcome" value="tie"> Tie</label>'
+ + '<label><input type="radio" name="outcome" value="both_bad"> Both bad</label>'
+ + '</fieldset>'
+ + '<fieldset><legend>' + esc(q.text) + '</legend>'
+ + '<label><input type="radio" name="answer" value="A"> A</label>'
+ + '<label><input type="radio" name="answer" value="B"> B</label>'
+ + '<label><input type="radio" name="answer" value="tie"> Same</label>'
+ + '</fieldset>'
+ + '<button class="bm-btn" type="submit">Cast the vote</button>'
+ + '</form>';
+}
+
+function battleView(battle) {
+ const kind = kindById(battle.kind_id);
+ return '<section class="bt-battle" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'
+ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'
+ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'
+ + '<span class="bm-fine">Blind: you see two outputs, not their names.</span></p>'
+ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'
+ + voteForm(battle)
+ + '</section>';
+}
+
+function revealView(battle) {
+ const kind = kindById(battle.kind_id);
+ const vote = battle.vote || {};
+ const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';
+ const fact = (slot, letter) => '<dl class="bt-facts" data-slot="' + esc(letter) + '">'
+ + '<div><dt>Model</dt><dd>' + esc(slot.model) + '</dd></div>'
+ + '<div><dt>Provider</dt><dd>' + esc(slot.provider) + '</dd></div>'
+ + '<div><dt>Cost</dt><dd>$' + esc(slot.cost_usd) + '</dd></div>'
+ + '<div><dt>Time</dt><dd>' + esc(slot.ms) + ' ms</dd></div>'
+ + '</dl>';
+ return '<section class="bt-battle bt-revealed" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'
+ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'
+ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'
+ + '<span class="br-chip">' + esc('Your vote: ' + winner) + '</span>'
+ + (vote.answer ? '<span class="br-chip">' + esc(battle.question.text + ': ' + vote.answer) + '</span>' : '')
+ + '</p>'
+ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'
+ + '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'
+ + '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'
+ + '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'
+ + '</section>';
+}
+
+function signedOutView() {
+ return '<section class="bt-battle bt-signed-out">'
+ + '<h1>Battles: two outputs, one prompt, blind</h1>'
+ + '<p class="bm-lede">Every battle shows two outputs of the same prompt, without their names. You say which one is '
+ + 'better (or that both are bad), the names open, and your vote counts towards the board.</p>'
+ + '<p class="bm-note">Battles are for signed-in people: one vote per person, one pair per person, and the votes '
+ + 'are rate limited so the board stays honest.</p>'
+ + '<p class="bm-actions"><a class="bm-btn" href="/auth/login?next=%2Fbattle">Sign in to fight</a>'
+ + '<a class="bm-btn bm-btn-quiet" href="/gallery">Look at the gallery first</a></p>'
+ + '</section>';
+}
+
+/** The share card: both panes and the question. A vector card, so no rasteriser is needed. */
+function cardSvg(battle, baseUrl) {
+ const kind = kindById(battle.kind_id);
+ const escXml = (s) => String(s == null ? '' : s)
+ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
+ const thumb = (slot, x, letter) => {
+ const payload = slot.payload || {};
+ const svg = typeof payload.svg === 'string' ? payload.svg : '';
+ if (svg && svg.trim()) {
+ const uri = 'data:image/svg+xml;base64,' + Buffer.from(svg, 'utf8').toString('base64');
+ return '<image href="' + uri + '" x="' + x + '" y="150" width="480" height="300" preserveAspectRatio="xMidYMid meet"/>';
+ }
+ return '<rect x="' + x + '" y="150" width="480" height="300" rx="18" fill="#16203a" stroke="#2b3a5c"/>'
+ + '<text x="' + (x + 240) + '" y="292" fill="#7d8ea9" font-family="system-ui, sans-serif" font-size="26" text-anchor="middle">'
+ + escXml(kind ? kind.name : 'output') + '</text>'
+ + '<text x="' + (x + 240) + '" y="330" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="middle">live preview</text>';
+ };
+ const prompt = String(battle.prompt || '').replace(/\s+/g, ' ').trim();
+ const short = prompt.length > 150 ? prompt.slice(0, 147) + '…' : prompt;
+ return '<svg xmlns="http://www.w3.org/2000/svg" width="' + CARD.w + '" height="' + CARD.h + '" viewBox="0 0 ' + CARD.w + ' ' + CARD.h + '">'
+ + '<rect width="' + CARD.w + '" height="' + CARD.h + '" fill="#0b1220"/>'
+ + '<rect x="24" y="24" width="' + (CARD.w - 48) + '" height="' + (CARD.h - 48) + '" rx="28" fill="none" stroke="#24304a"/>'
+ + '<text x="60" y="96" fill="#e6edf7" font-family="system-ui, sans-serif" font-size="52" font-weight="800">Vote: which is better?</text>'
+ + '<text x="60" y="132" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="24">' + escXml(short) + '</text>'
+ + thumb(battle.a, 60, 'A')
+ + thumb(battle.b, 660, 'B')
+ + '<text x="84" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">A</text>'
+ + '<text x="684" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">B</text>'
+ + '<text x="60" y="516" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="26">Two outputs, one prompt, blind.</text>'
+ + '<text x="60" y="556" fill="#c9d6ea" font-family="system-ui, sans-serif" font-size="26" font-weight="700">'
+ + escXml('OpenVibe.Bench') + '</text>'
+ + '<text x="' + (CARD.w - 60) + '" y="556" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="end">'
+ + escXml(baseUrl + '/battle') + '</text>'
+ + '</svg>';
+}
+
+function createBattlePages(ctx) {
+ const r = express.Router();
+ const { config, principal } = ctx;
+ const battles = createBattles(ctx);
+ ctx.battles = battles;
+ r.use(express.urlencoded({ extended: false, limit: '16kb' }));
+ r.use(principal.middleware);
+ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
+ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
+ : { kind: 'anonymous' });
+ const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal.subject : null);
+
+ r.get('/', async (req, res) => {
+ const subject = subjectOf(req);
+ if (!subject) {
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Battles: two outputs, one prompt, blind',
+ description: 'Pick the better output, blind. The names open after the vote.',
+ body: raw(signedOutView()),
+ });
+ }
+ const battle = await battles.pickPair({
+ subject,
+ kind: typeof req.query.kind === 'string' ? req.query.kind : 'all',
+ random: ctx.random || Math.random,
+ netHash: battles.netHashOf(req),
+ });
+ if (!battle) {
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Battles: nothing waiting',
+ description: 'Pick the better output, blind. The names open after the vote.',
+ body: raw('<section class="bt-battle"><h1>No battle is waiting right now</h1>'
+ + '<p class="bm-lede">Every pair you have seen is voted on, and the next rounds are still running. '
+ + 'Come back in a minute, or <a href="/gallery">look at the gallery</a>.</p></section>'),
+ });
+ }
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Battle: two outputs, one prompt',
+ description: 'Pick the better output, blind. The names open after the vote.',
+ headExtra: '<meta property="og:title" content="Vote: which is better?">'
+ + '<meta property="og:description" content="Two outputs, one prompt, blind. Vote on OpenVibe.Bench.">'
+ + '<meta property="og:image" content="' + esc(config.baseUrl + '/battle/card/' + battle.id + '.svg') + '">'
+ + '<meta property="og:image:type" content="image/svg+xml">'
+ + '<meta name="twitter:card" content="summary_large_image">',
+ body: raw(battleView(battle)),
+ });
+ });
+
+ r.get('/card/:id.svg', async (req, res) => {
+ const battle = await battles.battleOf(String(req.params.id || ''));
+ if (!battle) return res.status(404).type('text/plain').send('No such card.');
+ res.set('Cache-Control', 'public, max-age=60');
+ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src data: 'self'");
+ return res.type('image/svg+xml').send(cardSvg(battle, config.baseUrl));
+ });
+
+ r.get('/revealed/:id', async (req, res) => {
+ const subject = subjectOf(req);
+ const battle = await battles.battleOf(String(req.params.id || ''), subject);
+ if (!battle) {
+ return send(res, 404, { config, viewer: viewerOf(req), path: req.originalUrl, title: 'No such battle', body: raw('<section><h1>No such battle</h1></section>') });
+ }
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Battle: the names are open',
+ description: 'The two outputs of one prompt, with their names.',
+ body: raw(revealView(battle)),
+ });
+ });
+
+ r.post('/', async (req, res) => {
+ const subject = subjectOf(req);
+ const body = req.body && typeof req.body === 'object' ? req.body : {};
+ if (!subject) return res.redirect('/auth/login?next=' + encodeURIComponent('/battle'));
+ if (req.principal.viaSession && !sameOrigin(req, config)) {
+ return res.status(403).type('text/plain').send('A signed-in vote must come from bench.openvibe.services itself.');
+ }
+ const out = await battles.recordVote({
+ subject,
+ battleId: String(body.battle || ''),
+ outcome: body.outcome,
+ answer: body.answer,
+ netHash: battles.netHashOf(req),
+ });
+ if (!out.ok) {
+ return res.status(out.status || 422).type('text/plain').send(out.message || 'The vote was not accepted.');
+ }
+ return res.redirect(303, '/battle/revealed/' + out.battle_id);
+ });
+
+ return r;
+}
+
+module.exports = { createBattlePages, cardSvg };
new file mode 100644
new file mode 100644
@@ -0,0 +1,203 @@
@@ -0,0 +1,203 @@
+'use strict';
+
+/**
+ * The gallery (spec 1): every showcase round's outputs, live in the sandboxed render frames.
+ *
+ * The page is a filtered, ordered list of rounds: filter by kind, model and price; order by newest or by most
+ * voted. The frames are lazy: at most six live frames run at a time, and the rest wait as a still (a screenshot
+ * Bench's own job made, when there is one) until the reader asks for the live one.
+ */
+
+const express = require('express');
+const { asyncRouter } = require('./router');
+const renderLib = require('../bench/render');
+const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');
+const { esc, raw } = require('../render/html');
+const { send } = require('../render/layout');
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+/** How many frames may be live at once. */
+const LIVE_FRAMES = 6;
+
+function parsePrice(value) {
+ if (value == null || value === '') return null;
+ const n = Number(value);
+ if (!Number.isFinite(n) || n < 0) return null;
+ return n;
+}
+
+/** The screenshot Bench's own job made for an entry, when it made one. */
+function screenshotOf(uiCheck) {
+ if (!uiCheck || typeof uiCheck !== 'object') return null;
+ for (const key of ['screenshot', 'screenshot_url', 'image', 'thumb', 'thumbnail']) {
+ const v = uiCheck[key];
+ if (typeof v === 'string' && /^https?:\/\//.test(v)) return v;
+ }
+ return null;
+}
+
+async function loadGallery(ctx, { kind = 'all', model = '', price = null, order = 'new', limit = 60 } = {}) {
+ const db = ctx.s.db;
+ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;
+ const rows = await db.many(
+ `SELECT r.id, r.kind_id, r.created_at, t.prompt,
+ (SELECT COUNT(*)::int FROM bench_judgements j WHERE j.round_id = r.id AND j.judge = 'human_pairwise') AS votes,
+ (SELECT COUNT(*)::int FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) AS entries
+ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id
+ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[])
+ AND ($2::text = '' OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.model = $2))
+ AND ($3::numeric IS NULL OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL AND e.cost_usd <= $3))
+ ORDER BY ${order === 'votes' ? 'votes DESC, r.created_at DESC' : 'r.created_at DESC, r.id DESC'}
+ LIMIT $4`,
+ [kinds, String(model || ''), price, Math.min(120, Math.max(1, num(limit, 60)))]);
+ const out = [];
+ for (const r of rows) {
+ if (num(r.entries) < 2) continue;
+ const entries = await db.many(
+ 'SELECT id, provider, model, cost_usd, ms, ui_check FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id LIMIT 2',
+ [r.id]);
+ if (entries.length < 2) continue;
+ const k = kindById(r.kind_id);
+ out.push({
+ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,
+ prompt: r.prompt, votes: num(r.votes), created_at: new Date(r.created_at).toISOString(),
+ limit_ms: k ? renderLib.limitMsOf(k) : renderLib.DEFAULT_LIMIT_MS,
+ entries: entries.map((e) => ({
+ id: e.id, model: e.model, provider: e.provider, cost_usd: num(e.cost_usd), ms: num(e.ms),
+ screenshot: screenshotOf(e.ui_check),
+ wrapper_url: renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(e.id),
+ })),
+ });
+ }
+ return out;
+}
+
+/** The kinds and models the filter offers. */
+async function facets(ctx) {
+ const db = ctx.s.db;
+ const kinds = await db.many(
+ `SELECT r.kind_id AS id, COUNT(*)::int AS n FROM bench_rounds r
+ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) GROUP BY r.kind_id ORDER BY r.kind_id`,
+ [SHOWCASE_KIND_IDS]);
+ const models = await db.many(
+ `SELECT e.model AS id, COUNT(*)::int AS n FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
+ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND e.model IS NOT NULL
+ GROUP BY e.model ORDER BY n DESC, e.model LIMIT 40`,
+ [SHOWCASE_KIND_IDS]);
+ return {
+ kinds: kinds.map((k) => ({ id: k.id, name: (kindById(k.id) || {}).name || k.id, count: num(k.n) })),
+ models: models.map((m) => ({ id: m.id, count: num(m.n) })),
+ };
+}
+
+function frame(entry, live) {
+ const src = live
+ ? '<iframe class="ga-frame" data-live="1" data-server="1" src="' + esc(entry.wrapper_url) + '" title="Output of ' + esc(entry.model) + '"'
+ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'
+ : '<button type="button" class="ga-still" data-ga-still="1" aria-label="Load the live frame">'
+ + (entry.screenshot
+ ? '<img src="' + esc(entry.screenshot) + '" alt="A still of the output" loading="lazy">'
+ : '<span class="ga-still-note">Live frame, on tap</span>')
+ + '</button>';
+ return '<figure class="ga-shot" data-entry-id="' + esc(entry.id) + '" data-src="' + esc(entry.wrapper_url) + '">'
+ + '<figcaption>' + esc(entry.model) + '</figcaption>' + src + '</figure>';
+}
+
+function card(round, live) {
+ return '<article class="ga-card" data-round-id="' + esc(round.id) + '" data-kind="' + esc(round.kind_id) + '" data-votes="' + esc(round.votes) + '">'
+ + '<h2 class="ga-prompt" data-prompt="1">' + esc(round.prompt) + '</h2>'
+ + '<div class="ga-shots">' + round.entries.map((e) => frame(e, live)).join('') + '</div>'
+ + '<p class="ga-meta"><span class="br-chip">' + esc(round.kind_name) + '</span>'
+ + '<span class="ga-votes" data-votes="' + esc(round.votes) + '">' + esc(round.votes) + ' votes</span>'
+ + '<span class="bm-fine">' + esc('$' + Math.max(...round.entries.map((e) => e.cost_usd)).toFixed(4)) + ' at most</span>'
+ + '<a class="ga-link" href="/showcase/round/' + esc(round.id) + '">Open side by side</a></p>'
+ + '</article>';
+}
+
+function filterBar(ctx, { kind, model, price, order }) {
+ const kinds = SHOWCASE_KIND_IDS.map((id) => {
+ const k = kindById(id);
+ return '<option value="' + esc(id) + '"' + (kind === id ? ' selected' : '') + '>' + esc(k ? k.name : id) + '</option>';
+ }).join('');
+ return '<form class="ga-filters" method="get" action="/gallery" data-ga-filters="1">'
+ + '<label>Kind<select name="kind"><option value="all"' + (kind === 'all' ? ' selected' : '') + '>All kinds</option>' + kinds + '</select></label>'
+ + '<label>Model<input type="text" name="model" value="' + esc(model || '') + '" placeholder="model name"></label>'
+ + '<label>Price at most<input type="text" inputmode="decimal" name="price" value="' + esc(price == null ? '' : String(price)) + '" placeholder="$0.05"></label>'
+ + '<label>Order<select name="order">'
+ + '<option value="new"' + (order !== 'votes' ? ' selected' : '') + '>Newest</option>'
+ + '<option value="votes"' + (order === 'votes' ? ' selected' : '') + '>Most voted</option>'
+ + '</select></label>'
+ + '<button class="bm-btn bm-btn-small" type="submit">Filter</button>'
+ + '</form>';
+}
+
+function galleryView(rounds, filters) {
+ if (!rounds.length) {
+ return '<section class="ga-gallery" data-gallery="1">'
+ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames.</p>'
+ + '<div class="bm-empty"><h1>Nothing matches yet</h1><p>No round fits that filter. The first rounds are still generating.</p></div>'
+ + '</section>';
+ }
+ return '<section class="ga-gallery" data-gallery="1">'
+ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. At most six frames run at '
+ + 'once; the rest wait for you.</p>'
+ + '<div class="ga-grid">' + rounds.map((r, i) => card(r, i * 2 < LIVE_FRAMES)).join('') + '</div>'
+ + '</section>';
+}
+
+function createGalleryRoutes(ctx) {
+ const r = asyncRouter();
+ r.get('/', async (req, res) => {
+ const filters = {
+ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',
+ model: typeof req.query.model === 'string' ? req.query.model : '',
+ price: parsePrice(req.query.price),
+ order: req.query.order === 'votes' ? 'votes' : 'new',
+ };
+ const rounds = await loadGallery(ctx, filters);
+ res.set('Cache-Control', 'public, max-age=30');
+ return res.json({ count: rounds.length, filters, rounds });
+ });
+ r.get('/facets', async (req, res) => {
+ res.set('Cache-Control', 'public, max-age=300');
+ return res.json(await facets(ctx));
+ });
+ return r;
+}
+
+function createGalleryPages(ctx) {
+ const r = express.Router();
+ const { config, principal } = ctx;
+ r.use(principal.middleware);
+ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
+ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
+ : { kind: 'anonymous' });
+
+ r.get('/', async (req, res) => {
+ const filters = {
+ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',
+ model: typeof req.query.model === 'string' ? req.query.model : '',
+ price: parsePrice(req.query.price),
+ order: req.query.order === 'votes' ? 'votes' : 'new',
+ };
+ const rounds = await loadGallery(ctx, filters);
+ const f = await facets(ctx);
+ const body = '<section class="ga-head">'
+ + '<h1>Gallery</h1>'
+ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. Filter by kind, model '
+ + 'and price; order by newest or by most voted.</p>'
+ + filterBar(ctx, filters)
+ + '<p class="bm-fine">' + esc(f.kinds.length + ' kinds, ' + f.models.length + ' models in the gallery') + '</p>'
+ + '</section>' + galleryView(rounds, filters);
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Gallery: every showcase round',
+ description: 'Every showcase round, live in sandboxed frames. Filter by kind, model and price.',
+ body: raw(body),
+ });
+ });
+
+ return r;
+}
+
+module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, LIVE_FRAMES };
new file mode 100644
new file mode 100644
@@ -0,0 +1,98 @@
@@ -0,0 +1,98 @@
+'use strict';
+
+/**
+ * The ratings page: the board the votes make, raw and style controlled, side by side.
+ *
+ * The board is a Bradley-Terry fit over the human votes with the style of each output as covariates, and every
+ * rating carries a bootstrap interval. Models whose intervals overlap share a band, so the page says "2nd-4th"
+ * instead of pretending the third place is a fact. The open dump next to it carries every vote without voter ids,
+ * and the fit can be re-derived from that file alone.
+ */
+
+const express = require('express');
+const { esc, raw } = require('../render/html');
+const { send } = require('../render/layout');
+const { board, boardView } = require('../rating/board');
+
+function interval(cell) {
+ if (!cell) return '';
+ const lo = cell.ci_lower == null ? '' : Math.round(cell.ci_lower);
+ const hi = cell.ci_upper == null ? '' : Math.round(cell.ci_upper);
+ return '<span class="rt-rating">' + esc(Math.round(cell.rating)) + '</span>'
+ + '<span class="rt-ci">' + esc(lo + '–' + hi) + '</span>';
+}
+
+function rows(view) {
+ if (!view.models.length) {
+ return '<div class="bm-empty"><h1>No votes yet</h1><p>The board fills up as people vote in the battles.</p></div>';
+ }
+ const body = view.models.map((m) => '<li class="rt-row" data-model="' + esc(m.model) + '">'
+ + '<div class="rt-cell rt-cell-model"><span class="rt-label">Model</span>'
+ + '<span class="rt-model">' + esc(m.model) + '</span></div>'
+ + '<div class="rt-cell"><span class="rt-label">Raw</span>' + interval(m.raw)
+ + '<span class="rt-band">' + esc(m.raw.band || '') + '</span></div>'
+ + '<div class="rt-cell"><span class="rt-label">Style controlled</span>' + interval(m.controlled)
+ + '<span class="rt-band">' + esc(m.controlled.band || '') + '</span></div>'
+ + '<div class="rt-cell"><span class="rt-label">Votes</span><span class="rt-votes">' + esc(m.votes) + '</span></div>'
+ + '</li>').join('');
+ return '<ul class="rt-list">' + body + '</ul>';
+}
+
+function effects(view) {
+ const list = Object.entries(view.effects || {});
+ if (!list.length) return '';
+ const items = list.map(([key, e]) => '<div><dt>' + esc(key) + '</dt><dd>'
+ + (e.elo_per_1000_chars != null ? esc(e.elo_per_1000_chars) + ' Elo per 1000 characters' : '')
+ + (e.elo_per_1000_chars != null && e.elo_per_10_marks != null ? '<br>' : '')
+ + (e.elo_per_10_marks != null ? esc(e.elo_per_10_marks) + ' Elo per 10 markup marks' : '')
+ + '</dd></div>').join('');
+ return '<section class="rt-effects"><h2>How much of the board is style</h2>'
+ + '<dl class="br-facts">' + items + '</dl>'
+ + '<p class="bm-fine">A positive number means the longer (or the more dressed up) output wins votes it did not '
+ + 'earn. The controlled column removes that effect; the raw column is what people actually chose.</p>'
+ + '</section>';
+}
+
+function ratingsView(view) {
+ return '<section class="rt-board" data-ratings="1">'
+ + '<h1>Ratings</h1>'
+ + '<p class="bm-lede">Every human vote in Bench is one game between two models. The board is fitted twice: on the '
+ + 'votes alone (raw) and with the style of each output as covariates (style controlled), so you can see how much '
+ + 'of a rank is length and formatting.</p>'
+ + '<p class="bm-fine">' + esc(view.votes) + ' votes, '
+ + esc(view.bootstrap) + ' bootstrap resamples, seed ' + esc(view.seed)
+ + (view.voters && view.voters.downweighted ? ', ' + esc(view.voters.downweighted) + ' voters down-weighted (never named)' : '')
+ + '</p>'
+ + '<div class="rt-table-wrap">' + rows(view) + '</div>'
+ + effects(view)
+ + '<section class="rt-data"><h2>The open data</h2>'
+ + '<p class="bm-lede">Every vote is published as JSONL, without voter ids: '
+ + '<a href="/api/v1/datasets/votes.jsonl">/api/v1/datasets/votes.jsonl</a>. '
+ + 'The fit is reproducible from it: <code>node scripts/fit-ratings.js /tmp/votes.jsonl</code> re-derives the '
+ + 'same numbers (the dump carries the weight each vote carries in the fit).</p></section>'
+ + '</section>';
+}
+
+function createRatingsPages(ctx) {
+ const r = express.Router();
+ const { config, principal } = ctx;
+ r.use(principal.middleware);
+ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
+ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
+ : { kind: 'anonymous' });
+
+ r.get('/', async (req, res) => {
+ const result = await board(ctx);
+ const view = boardView(result);
+ return send(res, 200, {
+ config, viewer: viewerOf(req), path: req.originalUrl,
+ title: 'Ratings: raw and style controlled',
+ description: 'The board the votes make: raw and style-controlled Bradley-Terry, with intervals and bands.',
+ body: raw(ratingsView(view)),
+ });
+ });
+
+ return r;
+}
+
+module.exports = { createRatingsPages };
new file mode 100644
new file mode 100644
@@ -0,0 +1,123 @@
@@ -0,0 +1,123 @@
+'use strict';
+
+/**
+ * The board the votes make (spec 3).
+ *
+ * Every human pairwise vote in Bench is one game between two models. The board is fitted twice over those games:
+ * once on the votes alone (raw) and once with the style of each output as covariates (controlled), so a reader can
+ * see how much of the ranking is style. The intervals come from a bootstrap of the votes, and models whose
+ * intervals overlap share a band.
+ *
+ * The board is a pure function of the votes, so it is cached on the shape of the vote set and recomputed when a
+ * new vote lands. The open dump (/api/v1/datasets/votes.jsonl) carries the same weights, so the board can be
+ * re-derived from the file alone.
+ */
+
+const style = require('./style');
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+
+/** The human votes, with the two models and the two styles of each vote. */
+async function loadVotes(db) {
+ const rows = await db.many(
+ `SELECT j.id, j.at, j.kind_id, j.round_id, j.outcome, j.subject, j.question, j.answer,
+ l.model AS left_model, l.style_length AS left_length, l.style_markup AS left_markup, l.payload AS left_payload,
+ r.model AS right_model, r.style_length AS right_length, r.style_markup AS right_markup, r.payload AS right_payload
+ FROM bench_judgements j
+ JOIN bench_entries l ON l.id = (j.pair->>'left')
+ JOIN bench_entries r ON r.id = (j.pair->>'right')
+ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL
+ ORDER BY j.at, j.id`);
+ const out = [];
+ for (const row of rows) {
+ const left = styleOf(row.left_payload, row.left_length, row.left_markup);
+ const right = styleOf(row.right_payload, row.right_length, row.right_markup);
+ out.push({
+ id: row.id,
+ at: row.at instanceof Date ? row.at.toISOString() : String(row.at),
+ kind_id: row.kind_id,
+ round_id: row.round_id,
+ outcome: row.outcome,
+ question: row.question || null,
+ answer: row.answer || null,
+ voter: row.subject || null,
+ a: { model: row.left_model, length: left.length, markup: left.markup },
+ b: { model: row.right_model, length: right.length, markup: right.markup },
+ });
+ }
+ return out;
+}
+
+/** The style of an output: the columns when they are there, the payload when they are not (older entries). */
+function styleOf(payload, length, markup) {
+ if (length != null && markup != null) return { length: num(length), markup: num(markup) };
+ let p = payload;
+ if (typeof p === 'string') { try { p = JSON.parse(p); } catch { p = {}; } }
+ const features = style.styleFeatures(p || {});
+ return { length: length == null ? features.length : num(length), markup: markup == null ? features.markup : num(markup) };
+}
+
+/** The games the fit reads. */
+function gamesOf(votes) {
+ const out = [];
+ for (const v of votes) {
+ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';
+ out.push({ id: v.id, a: v.a.model, b: v.b.model, outcome, fa: v.a, fb: v.b, voter: v.voter });
+ }
+ return out;
+}
+
+let cached = null;
+
+/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */
+async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
+ const db = ctx.s.db;
+ const votes = await loadVotes(db);
+ const key = votes.length + ':' + (votes.length ? votes[votes.length - 1].id : '') + ':' + num(bootstrap);
+ if (!force && cached && cached.key === key && Date.now() - cached.at < 5 * 60_000) return cached.value;
+ const games = gamesOf(votes);
+ const fit = style.fitRatings(games, { bootstrap });
+ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));
+ const enriched = votes.map((v) => ({ ...v, weight: weights.has(v.id) ? weights.get(v.id) : 1 }));
+ const value = { fit, votes: enriched, generated_at: new Date().toISOString() };
+ cached = { key, at: Date.now(), value };
+ return value;
+}
+
+/**
+ * The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,
+ * so the board can be re-derived from the file alone.
+ */
+function dumpJsonl(result) {
+ const lines = result.votes.map((v) => JSON.stringify({
+ id: v.id,
+ at: v.at,
+ kind_id: v.kind_id,
+ round_id: v.round_id,
+ left: { model: v.a.model, length: v.a.length, markup: v.a.markup },
+ right: { model: v.b.model, length: v.b.length, markup: v.b.markup },
+ outcome: v.outcome,
+ question: v.question,
+ answer: v.answer,
+ weight: v.weight,
+ }));
+ return lines.join('\n') + (lines.length ? '\n' : '');
+}
+
+/** The board as the page and the API show it. */
+function boardView(result) {
+ const fit = result.fit;
+ return {
+ models: fit.models,
+ bands: fit.bands,
+ effects: fit.effects,
+ dropped_covariates: fit.dropped_covariates || [],
+ voters: fit.voters,
+ votes: fit.votes,
+ bootstrap: fit.bootstrap,
+ seed: fit.seed,
+ generated_at: result.generated_at,
+ };
+}
+
+module.exports = { loadVotes, gamesOf, board, dumpJsonl, boardView, styleOf };
@@ -75,4 +75,12 @@ function costPerSolved(entries) {
@@ -75,4 +75,12 @@ function costPerSolved(entries) {
// Bradley–Terry with bootstrap intervals, voter weights, Wilson and percentiles: the tested library (lib.js, BENCH-2).
// Bradley–Terry with bootstrap intervals, voter weights, Wilson and percentiles: the tested library (lib.js, BENCH-2).
const lib = require('./lib');
const lib = require('./lib');
-module.exports = { eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT };
+// The style-controlled fit (stage B9): a conditional logit over the votes, with the length and the markup of
+// each output as covariates, bootstrap intervals and rank bands.
+const style = require('./style');
+
+module.exports = {
+ eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT,
+ style,
+ fitRatings: style.fitRatings, rankBands: style.rankBands, styleFeatures: style.styleFeatures, markupCount: style.markupCount,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,518 @@
@@ -0,0 +1,518 @@
+'use strict';
+
+/**
+ * Style-controlled Bradley-Terry for Bench's human votes (stage B9).
+ *
+ * The research (10-30) is blunt about two things: a board's rank is fragile once the whole board's uncertainty is
+ * counted (an audit of 106,134 LMArena votes found only 1 of 54 neighbouring pairs to be a real ordering), and style
+ * beats substance (length and formatting decide a large share of pairwise votes). So Bench does two things a plain
+ * Elo ladder never does:
+ *
+ * 1. Style control. The same Bradley-Terry fit is made twice: once on the votes alone (the raw board, what people
+ * actually chose) and once with the style of each output as covariates (the controlled board, what the model
+ * would score if every output were written the same way). The covariates are the output's length and its
+ * markup count, the two LMArena names as covariates; the effect sizes are reported so a reader can see how much
+ * of the raw board is style.
+ *
+ * The model is a conditional logit (Bradley-Terry with per-output covariates):
+ *
+ * P(left wins) = sigmoid( theta_left - theta_right + SUM_k beta_k * (z_left,k - z_right,k) )
+ *
+ * where theta is the model's own strength (log-odds) and z the standardised covariates of the two outputs. A tie
+ * counts as half a win. The fit is convex in theta for fixed beta, so a damped Newton direction with a
+ * backtracking line search finds the maximum; theta is centred (its mean is zero) because only differences are
+ * identified.
+ *
+ * 2. Bands, not places. Models whose bootstrap intervals overlap share a band, so the board says "2nd-4th" instead
+ * of pretending the third place is a fact.
+ *
+ * Every number here is reproducible from the open vote dump (/api/v1/datasets/votes.jsonl): the dump carries each
+ * vote's outcome, the two models, the two outputs' style features and the weight the fit gave that vote, so
+ * scripts/fit-ratings.js re-derives the same board from the file alone.
+ *
+ * The maths is Bench's own (no copied code); the shape follows the published description of style-controlled
+ * Bradley-Terry so the numbers mean what a reader expects.
+ */
+
+const { percentile } = require('./lib');
+
+/** Elo points per unit of log-odds: the usual 400 / ln(10). */
+const SCALE = 400 / Math.log(10);
+/** The number a centred rating sits on, so the board reads like an Elo ladder. */
+const ANCHOR = 1000;
+/** The covariates, in the order they are reported. */
+const COVARIATES = Object.freeze(['length', 'markup']);
+/** The bootstrap resamples the board's intervals come from. */
+const BOOTSTRAP = 1000;
+/** The seed the bootstrap uses, so a rerun gives the same intervals. */
+const SEED = 20261030;
+
+const sigmoid = (x) => (x >= 0 ? 1 / (1 + Math.exp(-x)) : (e => e / (1 + e))(Math.exp(x)));
+
+/** The text of an entry payload: whichever field its kind renders from. */
+function contentOf(payload) {
+ const p = payload && typeof payload === 'object' ? payload : {};
+ for (const key of ['html', 'svg', 'glsl', 'text', 'code']) {
+ if (typeof p[key] === 'string' && p[key].trim()) return p[key];
+ }
+ return '';
+}
+
+/**
+ * How much markup an output carries. Headings, bold, lists, code fences, links, table rows and HTML tags each count
+ * once: the point is "how dressed up is this output", and the count has to be the same for every kind.
+ */
+function markupCount(text) {
+ const s = String(text == null ? '' : text);
+ let n = 0;
+ n += (s.match(/^#{1,6}\s/gm) || []).length;
+ n += (s.match(/\*\*|__/g) || []).length;
+ n += (s.match(/^\s*[-*+]\s/gm) || []).length;
+ n += (s.match(/^\s*\d+\.\s/gm) || []).length;
+ n += (s.match(/```/g) || []).length;
+ n += (s.match(/\[[^\]]*\]\([^)]*\)/g) || []).length;
+ n += (s.match(/^\s*\|/gm) || []).length;
+ n += (s.match(/<\/?[a-z][a-z0-9]*\b[^>]*>/gi) || []).length;
+ return n;
+}
+
+/** The style features of one output: how long it is, and how much markup it wears. */
+function styleFeatures(payload) {
+ const content = contentOf(payload);
+ return { length: content.length, markup: markupCount(content) };
+}
+
+/** A small seedable generator, so the same votes and seed always give the same intervals. */
+function seededRandom(seed) {
+ let state = (seed >>> 0) || 1;
+ return () => {
+ state = (state + 0x6D2B79F5) >>> 0;
+ let value = state;
+ value = Math.imul(value ^ (value >>> 15), value | 1);
+ value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+ return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+function centre(values) {
+ if (!values.length) return values;
+ const mean = values.reduce((a, b) => a + b, 0) / values.length;
+ for (let i = 0; i < values.length; i++) values[i] -= mean;
+ return values;
+}
+
+/**
+ * The games a fit reads: { id, a, b, outcome, fa, fb, voter?, weight? }. The outcome is 'a', 'b' or 'tie';
+ * 'both_bad' says nothing about the models and is skipped.
+ */
+function prepare(games, { covariates = COVARIATES } = {}) {
+ const list = (Array.isArray(games) ? games : [])
+ .filter((g) => g && g.a && g.b && g.a !== g.b && g.outcome !== 'both_bad');
+ const subjects = [...new Set(list.flatMap((g) => [g.a, g.b]))].sort();
+ const index = new Map(subjects.map((s, i) => [s, i]));
+ const wanted = covariates.filter((k) => k === 'length' || k === 'markup');
+
+ // Standardise every covariate over the outputs the votes actually used.
+ const stats = new Map();
+ for (const key of wanted) {
+ const values = [];
+ for (const g of list) {
+ values.push(Number((g.fa && g.fa[key]) || 0), Number((g.fb && g.fb[key]) || 0));
+ }
+ const mean = values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0;
+ const variance = values.length ? values.reduce((a, b) => a + (b - mean) * (b - mean), 0) / values.length : 0;
+ stats.set(key, { mean, sd: Math.sqrt(variance) || 1 });
+ }
+
+ // The standardised difference of each covariate, per game.
+ const diffs = wanted.map((key) => {
+ const st = stats.get(key);
+ const out = new Float64Array(list.length);
+ for (let i = 0; i < list.length; i++) {
+ const g = list[i];
+ const za = (Number((g.fa && g.fa[key]) || 0) - st.mean) / st.sd;
+ const zb = (Number((g.fb && g.fb[key]) || 0) - st.mean) / st.sd;
+ out[i] = za - zb;
+ }
+ return out;
+ });
+
+ // Two covariates that move together cannot be told apart, and the fit would trade them for ever: keep the first
+ // and drop the one that repeats it. The board then says so instead of pretending both were measured.
+ const keptKeys = [];
+ const keptDiffs = [];
+ const dropped = [];
+ for (let j = 0; j < wanted.length; j++) {
+ const clash = keptDiffs.some((other) => Math.abs(correlation(diffs[j], other)) > 0.95);
+ if (clash) { dropped.push(wanted[j]); continue; }
+ keptKeys.push(wanted[j]);
+ keptDiffs.push(diffs[j]);
+ }
+ const kAll = keptKeys.length;
+
+ const n = list.length;
+ const ai = new Int32Array(n);
+ const bi = new Int32Array(n);
+ const score = new Float64Array(n);
+ const weight = new Float64Array(n);
+ const dz = new Float64Array(n * kAll);
+ const ids = new Array(n);
+ const voters = new Array(n);
+ let kept = 0;
+ for (let gi = 0; gi < list.length; gi++) {
+ const g = list[gi];
+ const w = Number(g.weight == null ? 1 : g.weight);
+ if (!(w > 0)) continue;
+ const at = kept;
+ ai[at] = index.get(g.a);
+ bi[at] = index.get(g.b);
+ score[at] = g.outcome === 'a' ? 1 : g.outcome === 'b' ? 0 : 0.5;
+ weight[at] = w;
+ ids[at] = g.id == null ? null : String(g.id);
+ voters[at] = g.voter == null ? null : String(g.voter);
+ for (let j = 0; j < kAll; j++) dz[at * kAll + j] = keptDiffs[j][gi];
+ kept += 1;
+ }
+
+ const keptStats = new Map();
+ for (const key of keptKeys) keptStats.set(key, stats.get(key));
+ return { ai, bi, score, weight, dz, ids, voters, count: kept, subjects, index, keys: keptKeys, stats: keptStats, dropped };
+}
+
+/** The Pearson correlation of two equal-length series. */
+function correlation(a, b) {
+ const n = Math.min(a.length, b.length);
+ if (!n) return 0;
+ let ma = 0;
+ let mb = 0;
+ for (let i = 0; i < n; i++) { ma += a[i]; mb += b[i]; }
+ ma /= n; mb /= n;
+ let cov = 0;
+ let va = 0;
+ let vb = 0;
+ for (let i = 0; i < n; i++) {
+ cov += (a[i] - ma) * (b[i] - mb);
+ va += (a[i] - ma) * (a[i] - ma);
+ vb += (b[i] - mb) * (b[i] - mb);
+ }
+ if (!va || !vb) return 0;
+ return cov / Math.sqrt(va * vb);
+}
+
+/**
+ * The conditional-logit fit over prepared games. `order` picks which games take part (the bootstrap resamples it);
+ * `use` picks which of the prepared covariates enter the logit (the raw fit uses none of them). The hot loops work on
+ * typed arrays, because the bootstrap runs this thousands of times.
+ */
+function fitPrepared(prepared, { order = null, use = null, ridge = 0.05, betaRidge = 0.002, maxIter = 120, tol = 1e-9, warm = null, lineSearch = true } = {}) {
+ const { ai, bi, score, weight, dz, subjects, keys, stats } = prepared;
+ const n = subjects.length;
+ const kAll = keys.length;
+ const cols = (use == null ? keys : use).map((key) => keys.indexOf(key)).filter((i) => i >= 0);
+ const k = cols.length;
+ const picks = order || null;
+ const count = picks ? picks.length : prepared.count;
+
+ let theta = warm && warm.theta && warm.theta.length === n ? Float64Array.from(warm.theta) : new Float64Array(n);
+ let beta = warm && warm.beta && warm.beta.length === k ? Float64Array.from(warm.beta) : new Float64Array(k);
+
+ const gTheta = new Float64Array(n);
+ const hTheta = new Float64Array(n);
+ const gBeta = new Float64Array(k);
+ const hBeta = new Float64Array(k);
+ const nextTheta = new Float64Array(n);
+ const nextBeta = new Float64Array(k);
+
+ const logit = (p, th, be) => {
+ let x = th[ai[p]] - th[bi[p]];
+ for (let j = 0; j < k; j++) x += be[j] * dz[p * kAll + cols[j]];
+ return x;
+ };
+
+ const evaluate = (th, be) => {
+ let value = 0;
+ for (let p = 0; p < count; p++) {
+ const idx = picks ? picks[p] : p;
+ const x = logit(idx, th, be);
+ const ph = sigmoid(x);
+ const s = score[idx];
+ value += -weight[idx] * (s * Math.log(ph) + (1 - s) * Math.log(1 - ph));
+ }
+ for (let i = 0; i < n; i++) value += ridge * th[i] * th[i];
+ for (let j = 0; j < k; j++) value += betaRidge * be[j] * be[j];
+ return value;
+ };
+
+ const gradient = (th, be) => {
+ gTheta.fill(0); hTheta.fill(0); gBeta.fill(0); hBeta.fill(0);
+ for (let p = 0; p < count; p++) {
+ const idx = picks ? picks[p] : p;
+ const x = logit(idx, th, be);
+ const ph = sigmoid(x);
+ const s = score[idx];
+ const w = weight[idx];
+ const d = w * (s - ph);
+ const a = ai[idx];
+ const b = bi[idx];
+ gTheta[a] += d;
+ gTheta[b] -= d;
+ const curv = w * ph * (1 - ph);
+ hTheta[a] += curv;
+ hTheta[b] += curv;
+ for (let j = 0; j < k; j++) {
+ const dzz = dz[idx * kAll + cols[j]];
+ gBeta[j] += d * dzz;
+ hBeta[j] += curv * dzz * dzz;
+ }
+ }
+ for (let i = 0; i < n; i++) { gTheta[i] -= 2 * ridge * th[i]; hTheta[i] += 2 * ridge; }
+ for (let j = 0; j < k; j++) { gBeta[j] -= 2 * betaRidge * be[j]; hBeta[j] += 2 * betaRidge; }
+ };
+
+ for (let iter = 0; iter < maxIter; iter++) {
+ gradient(theta, beta);
+ let norm = 0;
+ for (let i = 0; i < n; i++) norm += gTheta[i] * gTheta[i];
+ for (let j = 0; j < k; j++) norm += gBeta[j] * gBeta[j];
+ norm = Math.sqrt(norm);
+ if (norm < tol) break;
+ // A Newton direction with a diagonal curvature, damped by a backtracking line search.
+ let step = 1;
+ let moved = false;
+ const base = lineSearch ? evaluate(theta, beta) : null;
+ for (let attempt = 0; attempt < 12; attempt++) {
+ let mean = 0;
+ for (let i = 0; i < n; i++) {
+ const v = theta[i] + step * (gTheta[i] / Math.max(hTheta[i], 1e-9));
+ nextTheta[i] = v;
+ mean += v;
+ }
+ mean /= n;
+ for (let i = 0; i < n; i++) nextTheta[i] -= mean;
+ for (let j = 0; j < k; j++) nextBeta[j] = beta[j] + step * (gBeta[j] / Math.max(hBeta[j], 1e-9));
+ if (!lineSearch) { theta.set(nextTheta); beta.set(nextBeta); moved = true; break; }
+ const value = evaluate(nextTheta, nextBeta);
+ if (value <= base - 1e-12) {
+ theta.set(nextTheta); beta.set(nextBeta); moved = true; break;
+ }
+ step *= 0.5;
+ }
+ if (!moved) break;
+ }
+
+ return { theta: Array.from(theta), beta: Array.from(beta), subjects, keys, stats };
+}
+
+/**
+ * How often a voter disagrees with the consensus. A voter who disagrees far too often is down-weighted: their votes
+ * count a quarter (nothing at all when they disagree with almost everything), and they are never named anywhere.
+ */
+function voterWeights(games, fit, { minVotes = 10, disagreeAt = 0.5, hopelessAt = 0.75 } = {}) {
+ const rating = new Map();
+ fit.subjects.forEach((subject, i) => rating.set(subject, fit.theta[i]));
+ const voters = new Map();
+ for (const g of games) {
+ if (!g || g.voter == null) continue;
+ if (g.outcome === 'tie' || g.outcome === 'both_bad') continue;
+ if (g.a === g.b) continue;
+ const ra = rating.get(g.a);
+ const rb = rating.get(g.b);
+ if (ra == null || rb == null) continue;
+ const expected = ra > rb ? 1 : ra < rb ? -1 : 0;
+ const observed = g.outcome === 'a' ? 1 : -1;
+ const record = voters.get(g.voter) || { count: 0, disagreement: 0 };
+ record.count += 1;
+ record.disagreement += expected === observed ? 0 : 1;
+ voters.set(g.voter, record);
+ }
+ const out = new Map();
+ for (const [voter, record] of voters) {
+ const rate = record.count ? record.disagreement / record.count : 0;
+ let weight = 1;
+ let downweighted = false;
+ if (record.count >= minVotes && rate >= hopelessAt) { weight = 0; downweighted = true; }
+ else if (record.count >= minVotes && rate > disagreeAt) { weight = 0.25; downweighted = true; }
+ out.set(voter, { count: record.count, disagreement: rate, weight, downweighted });
+ }
+ return out;
+}
+
+/**
+ * The whole board: the raw fit, the style-controlled fit, both with bootstrap intervals, and the rank bands.
+ *
+ * The games are { id, a, b, outcome: 'a'|'b'|'tie', fa, fb, voter?, weight? }. The bootstrap resamples the votes and
+ * gives the 2.5th and 97.5th percentile of each rating. `voteWeights` (when given) is the per-vote weight the caller
+ * already computed, so the open dump and the live board agree.
+ */
+function fitRatings(games, {
+ bootstrap = BOOTSTRAP,
+ seed = SEED,
+ ridge = 0.05,
+ betaRidge = 0.002,
+ covariates = COVARIATES,
+ minVotes = 10,
+ disagreeAt = 0.5,
+ voteWeights = null,
+} = {}) {
+ const list = (Array.isArray(games) ? games : []).filter((g) => g && g.a && g.b);
+ const empty = { models: [], bands: { raw: [], controlled: [] }, effects: {}, voters: { total: 0, downweighted: 0 }, votes: 0, bootstrap: 0, seed };
+ if (!list.length) return empty;
+
+ // First pass: the consensus without voter weights, so the weights can be judged against it.
+ const consensus = fitPrepared(prepare(list, { covariates: [] }), { use: [], ridge });
+ const weights = voterWeights(list, consensus, { minVotes, disagreeAt });
+ const weighted = list.map((g) => ({
+ ...g,
+ weight: voteWeights && g.id != null && voteWeights.get(g.id) != null
+ ? voteWeights.get(g.id)
+ : g.voter != null && weights.has(g.voter) ? weights.get(g.voter).weight
+ : (g.weight == null ? 1 : g.weight),
+ }));
+
+ const prepared = prepare(weighted, { covariates });
+ const raw = fitPrepared(prepared, { use: [], ridge });
+ const controlled = fitPrepared(prepared, { use: covariates, ridge, betaRidge });
+
+ const counts = new Map();
+ for (const g of weighted) {
+ counts.set(g.a, (counts.get(g.a) || 0) + 1);
+ counts.set(g.b, (counts.get(g.b) || 0) + 1);
+ }
+
+ const rawSamples = new Map();
+ const ctrlSamples = new Map();
+ const random = seededRandom(seed);
+ const rounds = Math.max(0, Math.min(BOOTSTRAP, Number(bootstrap) || 0));
+ const n = prepared.count;
+ const rawIndex = new Map(raw.subjects.map((s, i) => [s, i]));
+ const ctrlIndex = new Map(controlled.subjects.map((s, i) => [s, i]));
+ const order = new Int32Array(n);
+ for (let i = 0; i < rounds; i++) {
+ for (let j = 0; j < n; j++) order[j] = Math.floor(random() * n);
+ const r = fitPrepared(prepared, { order, use: [], ridge, maxIter: 60, warm: { theta: raw.theta, beta: [] } });
+ const c = fitPrepared(prepared, { order, use: covariates, ridge, betaRidge, maxIter: 60, warm: { theta: controlled.theta, beta: controlled.beta } });
+ for (const [subject, position] of rawIndex) {
+ if (!rawSamples.has(subject)) { rawSamples.set(subject, []); ctrlSamples.set(subject, []); }
+ rawSamples.get(subject).push(r.theta[position]);
+ ctrlSamples.get(subject).push(c.theta[ctrlIndex.has(subject) ? ctrlIndex.get(subject) : 0]);
+ }
+ }
+
+ const toRating = (theta) => ANCHOR + SCALE * theta;
+ const rawTheta = new Map(raw.subjects.map((s, i) => [s, raw.theta[i]]));
+ const ctrlTheta = new Map(controlled.subjects.map((s, i) => [s, controlled.theta[i]]));
+ const rows = raw.subjects.map((subject) => {
+ const thetaRaw = rawTheta.get(subject);
+ const thetaCtrl = ctrlTheta.get(subject);
+ const rawP = rawSamples.get(subject) || [];
+ const ctrlP = ctrlSamples.get(subject) || [];
+ return {
+ model: subject,
+ votes: counts.get(subject) || 0,
+ raw: {
+ rating: Math.round(toRating(thetaRaw) * 100) / 100,
+ ci_lower: rawP.length ? Math.round(toRating(percentile(rawP, 0.025)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,
+ ci_upper: rawP.length ? Math.round(toRating(percentile(rawP, 0.975)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,
+ },
+ controlled: {
+ rating: Math.round(toRating(thetaCtrl) * 100) / 100,
+ ci_lower: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.025)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,
+ ci_upper: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.975)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,
+ },
+ };
+ }).sort((x, y) => y.raw.rating - x.raw.rating || x.model.localeCompare(y.model));
+
+ const rawBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.raw.rating, ci_lower: r.raw.ci_lower, ci_upper: r.raw.ci_upper })));
+ const ctrlBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.controlled.rating, ci_lower: r.controlled.ci_lower, ci_upper: r.controlled.ci_upper })));
+ const bandOf = (bands, model) => {
+ const band = bands.find((b) => b.models.includes(model));
+ return band ? band.label : null;
+ };
+ for (const row of rows) {
+ row.raw.band = bandOf(rawBands, row.model);
+ row.controlled.band = bandOf(ctrlBands, row.model);
+ }
+
+ // The effect of each style covariate, in Elo points per 1000 characters or per 10 markup marks.
+ const effects = {};
+ const ctrlKeys = controlled.keys;
+ for (let j = 0; j < ctrlKeys.length; j++) {
+ const key = ctrlKeys[j];
+ const beta = controlled.beta[j] == null ? 0 : controlled.beta[j];
+ const st = controlled.stats.get(key);
+ const perUnit = st ? beta * SCALE / st.sd : 0;
+ effects[key] = {
+ beta: Math.round(beta * 1e6) / 1e6,
+ elo_per_1000_chars: key === 'length' ? Math.round(perUnit * 1000 * 100) / 100 : null,
+ elo_per_10_marks: key === 'markup' ? Math.round(perUnit * 10 * 100) / 100 : null,
+ mean: st ? Math.round(st.mean * 100) / 100 : null,
+ sd: st ? Math.round(st.sd * 100) / 100 : null,
+ };
+ }
+ const droppedCovariates = prepared.dropped || [];
+
+ const downweighted = [...weights.values()].filter((w) => w.downweighted).length;
+ // The weight each vote carries, so the open dump can re-derive the same board from the file alone.
+ const perVoteWeights = weighted.map((g) => ({ id: g.id == null ? null : String(g.id), weight: g.weight }));
+ return {
+ models: rows,
+ vote_weights: perVoteWeights,
+ bands: { raw: rawBands, controlled: ctrlBands },
+ effects,
+ voters: { total: weights.size, downweighted },
+ dropped_covariates: droppedCovariates,
+ votes: list.length,
+ bootstrap: rounds,
+ seed,
+ };
+}
+
+/**
+ * Rank bands: models whose intervals overlap share a band. The rows come in rank order; a band grows while the next
+ * model's interval still overlaps the band it would join.
+ */
+function rankBands(rows) {
+ const sorted = [...(rows || [])].filter((r) => r && r.rating != null)
+ .sort((a, b) => b.rating - a.rating || String(a.model).localeCompare(String(b.model)));
+ const bands = [];
+ let current = null;
+ sorted.forEach((row, i) => {
+ const lower = row.ci_lower == null ? row.rating : row.ci_lower;
+ const upper = row.ci_upper == null ? row.rating : row.ci_upper;
+ if (current && lower <= current.upper && upper >= current.lower) {
+ current.models.push(row.model);
+ current.lower = Math.min(current.lower, lower);
+ current.upper = Math.max(current.upper, upper);
+ current.last = i;
+ } else {
+ current = { models: [row.model], lower, upper, first: i, last: i };
+ bands.push(current);
+ }
+ });
+ return bands.map((band, i) => ({
+ label: bandLabel(band.first + 1, band.last + 1),
+ from: band.first + 1,
+ to: band.last + 1,
+ models: band.models,
+ index: i,
+ }));
+}
+
+function ordinal(n) {
+ const v = Math.round(Number(n));
+ const t = v % 100;
+ if (t >= 11 && t <= 13) return v + 'th';
+ return v + ({ 1: 'st', 2: 'nd', 3: 'rd' }[v % 10] || 'th');
+}
+
+/** "2nd-4th" for a band, "1st" for a single place. */
+function bandLabel(from, to) {
+ if (to <= from) return ordinal(from);
+ return ordinal(from) + '\u2013' + ordinal(to);
+}
+
+module.exports = {
+ SCALE, ANCHOR, COVARIATES, BOOTSTRAP, SEED,
+ styleFeatures, markupCount, contentOf,
+ prepare, fitPrepared, voterWeights, fitRatings, rankBands, bandLabel, ordinal, seededRandom,
+};
@@ -21,6 +21,9 @@ const NAV = [
@@ -21,6 +21,9 @@ const NAV = [
{ label: 'Home', href: '/' },
{ label: 'Home', href: '/' },
{ label: 'Benchmarks', href: '/benchmarks' },
{ label: 'Benchmarks', href: '/benchmarks' },
{ label: 'Showcase', href: '/showcase' },
{ label: 'Showcase', href: '/showcase' },
+ { label: 'Gallery', href: '/gallery' },
+ { label: 'Battles', href: '/battle' },
+ { label: 'Ratings', href: '/ratings' },
{ label: 'Docs', href: '/docs' },
{ label: 'Docs', href: '/docs' },
];
];
@@ -79,6 +82,7 @@ ${appIcon.headTags({ site: 'bench' })}
@@ -79,6 +82,7 @@ ${appIcon.headTags({ site: 'bench' })}
<link rel="stylesheet" href="${ovServe.url('room.css')}">
<link rel="stylesheet" href="${ovServe.url('room.css')}">
<link rel="stylesheet" href="${asset('css/bench.css')}">
<link rel="stylesheet" href="${asset('css/bench.css')}">
<script src="${asset('js/bench.js')}" defer></script>
<script src="${asset('js/bench.js')}" defer></script>
+${o.headExtra || ''}
${render(scripts)}
${render(scripts)}
<meta name="ov-boost" content="bench@${esc(RELEASE)}">
<meta name="ov-boost" content="bench@${esc(RELEASE)}">
<script src="${require('openvibe-shared/serve').url('boost.js')}" data-main="#main" defer></script>`,
<script src="${require('openvibe-shared/serve').url('boost.js')}" data-main="#main" defer></script>`,
@@ -10,6 +10,7 @@ const crypto = require('crypto');
@@ -10,6 +10,7 @@ const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { ids } = require('openvibe-contracts');
const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');
const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');
const ratingLib = require('./rating');
const ratingLib = require('./rating');
+const styleLib = require('./rating/style');
const BASE_ELO = ratingLib.BASE_ELO;
const BASE_ELO = ratingLib.BASE_ELO;
const K = 32;
const K = 32;
@@ -277,12 +278,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -277,12 +278,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
const id = s.newId('bte');
const id = s.newId('bte');
+ // The style of the output (how long it is, how much markup it wears) is measured once, here, so the
+ // ratings can control for style later without opening the payload again.
+ const style = styleLib.styleFeatures(payload || {});
const row = await db.maybe(
const row = await db.maybe(
- `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at)
- VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz) RETURNING *`,
+ `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at, style_length, style_markup)
+ VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz, $14, $15) RETURNING *`,
[id, r.id, String(provider || 'ai').slice(0, 80), String(model || 'unknown').slice(0, 120), JSON.stringify(route || {}),
[id, r.id, String(provider || 'ai').slice(0, 80), String(model || 'unknown').slice(0, 120), JSON.stringify(route || {}),
JSON.stringify(payload || {}), num(costUsd), Math.max(0, num(ms)), Math.max(0, num(tokens)), error || null,
JSON.stringify(payload || {}), num(costUsd), Math.max(0, num(ms)), Math.max(0, num(tokens)), error || null,
- uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso()],
+ uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso(), style.length, style.markup],
);
);
await maybeReady(r.id);
await maybeReady(r.id);
return wireEntry(row);
return wireEntry(row);
@@ -354,7 +358,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -354,7 +358,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
};
};
}
}
- async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, subject }) {
+ async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, question = null, answer = null, netHash = null, subject }) {
if (!JUDGES.includes(judge)) throw new BenchError(422, 'bench.invalid_judge', 'Unknown judge.');
if (!JUDGES.includes(judge)) throw new BenchError(422, 'bench.invalid_judge', 'Unknown judge.');
try {
try {
return await s.tx(async () => {
return await s.tx(async () => {
@@ -392,10 +396,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -392,10 +396,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const id = s.newId('bju');
const id = s.newId('bju');
const row = await db.maybe(
const row = await db.maybe(
- `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb) RETURNING *`,
+ `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts, question, answer, net_hash)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb, $14, $15, $16) RETURNING *`,
[id, r.id, r.kind_id, judge, judgeName, judgeVersion, judge, pair ? JSON.stringify(pair) : null,
[id, r.id, r.kind_id, judge, judgeName, judgeVersion, judge, pair ? JSON.stringify(pair) : null,
- outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null],
+ outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null,
+ question == null ? null : String(question).slice(0, 200), answer == null ? null : String(answer).slice(0, 40),
+ netHash == null ? null : String(netHash).slice(0, 64)],
);
);
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+'use strict';
+
+/**
+ * The battles (stage B9): blind pairs, the rules, and a vote that works without JavaScript.
+ */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+const FORM = { 'content-type': 'application/x-www-form-urlencoded' };
+
+(async () => {
+ // One vote per hour for this person: the rate limit is part of the rules under test.
+ const t = await boot({ env: { BENCH_BATTLE_PERSON_PER_HOUR: '1' } });
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob');
+ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
+
+ // Two finished rounds, each with two entries: two battles to hand out.
+ const rounds = [];
+ for (const n of [1, 2]) {
+ const task = await t.get('/api/v1/kinds/design.ui/tasks', { as: alice, json: { prompt: 'A dashboard, prompt ' + n + '.', public: true } });
+ assert.strictEqual(task.status, 201, task.text);
+ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
+ assert.strictEqual(rr.status, 201, rr.text);
+ const roundId = rr.json().round.id;
+ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model: 'alpha-' + n, payload: { html: '<p>alpha ' + n + '</p>' }, cost_usd: 0.01 } });
+ assert.strictEqual(e1.status, 201, e1.text);
+ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: 'beta-' + n, payload: { html: '<p>beta ' + n + '</p>' }, cost_usd: 0.02 } });
+ assert.strictEqual(e2.status, 201, e2.text);
+ rounds.push({ id: roundId, a: e1.json().entry.id, b: e2.json().entry.id });
+ }
+
+ await check('an anonymous reader is asked to sign in, and sees no pair', async () => {
+ const r = await t.get('/battle');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(r.text.includes('Sign in'), 'the sign-in prompt');
+ assert.ok(!r.text.includes('data-battle='), 'no pair for an anonymous reader');
+ });
+
+ await check('a signed-in person gets a blind pair', async () => {
+ const r = await t.get('/battle', { as: bob });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(r.text.includes('data-battle="bbs_'), 'the battle id: ' + r.text.slice(0, 200));
+ assert.ok(r.text.includes('data-slot="A"') && r.text.includes('data-slot="B"'), 'two panes');
+ assert.ok(r.text.includes('Which one looks more finished?'), 'the kind\'s own question');
+ assert.ok(!r.text.includes('alpha-1') && !r.text.includes('beta-1'), 'no model names before the vote');
+ });
+
+ await check('the vote is a form post that works without JavaScript', async () => {
+ const page = await t.get('/battle', { as: bob });
+ const battleId = /data-battle="(bbs_[^"]+)"/.exec(page.text)[1];
+ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(battleId) + '&outcome=A&answer=A' });
+ assert.strictEqual(vote.status, 303, vote.text);
+ assert.ok(String(vote.headers.get('location') || '').startsWith('/battle/revealed/'), 'the reveal page: ' + vote.headers.get('location'));
+ const reveal = await t.get(vote.headers.get('location'), { as: bob });
+ assert.strictEqual(reveal.status, 200, reveal.text);
+ assert.ok(reveal.text.includes('alpha-') || reveal.text.includes('beta-'), 'the names are open after the vote');
+ const row = await t.ctx.s.db.maybe("SELECT outcome, question, answer FROM bench_judgements WHERE judge = 'human_pairwise' ORDER BY at DESC LIMIT 1");
+ assert.strictEqual(row.outcome, 'left');
+ assert.strictEqual(row.question, 'Which one looks more finished?');
+ assert.strictEqual(row.answer, 'A');
+ });
+
+ await check('a person never sees the same pair twice', async () => {
+ const seen = await t.ctx.s.db.many('SELECT round_id, pair_key, voted_at FROM bench_battle_seen WHERE subject = (SELECT subject FROM bench_judgements ORDER BY at DESC LIMIT 1)');
+ const pairs = new Set(seen.map((s) => s.pair_key));
+ assert.strictEqual(pairs.size, seen.length, 'no pair twice: ' + JSON.stringify(seen));
+ // Both rounds are used up now: there is nothing left to show this person.
+ const more = await t.get('/battle', { as: bob });
+ assert.strictEqual(more.status, 200, more.text);
+ assert.ok(more.text.includes('No battle is waiting'), 'no pair left: ' + more.text.slice(0, 300));
+ });
+
+ await check('the rate limit stops the second vote', async () => {
+ const before = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");
+ const rows = await t.ctx.s.db.many('SELECT id, round_id FROM bench_battle_seen WHERE voted_at IS NULL');
+ assert.ok(rows.length >= 1, 'a pair is waiting');
+ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=B' });
+ assert.strictEqual(vote.status, 429, vote.text);
+ const after = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");
+ assert.strictEqual(after, before, 'no vote was written');
+ });
+
+ await check('an anonymous vote is refused, and a cross-site vote is refused', async () => {
+ const rows = await t.ctx.s.db.many('SELECT id FROM bench_battle_seen WHERE voted_at IS NULL');
+ const anon = await t.get('/battle', { method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });
+ assert.ok(anon.status === 302 || anon.status === 303, 'the anonymous vote is sent to sign in: ' + anon.status);
+ assert.ok(String(anon.headers.get('location') || '').startsWith('/auth/login'), 'to the sign-in page');
+ const cross = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, origin: 'https://evil.example', body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });
+ assert.strictEqual(cross.status, 403, cross.text);
+ });
+
+ await check('the ratings board counts the battle vote, and the open data has no voter ids', async () => {
+ const page = await t.get('/ratings', { as: bob });
+ assert.strictEqual(page.status, 200, page.text);
+ assert.ok(page.text.includes('Raw') && page.text.includes('Style controlled'), 'both columns');
+ assert.ok(page.text.includes('alpha-'), 'a model on the board');
+
+ const dump = await t.get('/api/v1/datasets/votes.jsonl');
+ assert.strictEqual(dump.status, 200, dump.text);
+ const lines = dump.text.trim().split('\n').filter(Boolean);
+ assert.ok(lines.length >= 1, 'the dump has the vote');
+ const subject = await t.ctx.s.db.value("SELECT subject FROM bench_judgements WHERE judge = 'human_pairwise' LIMIT 1");
+ for (const line of lines) {
+ const v = JSON.parse(line);
+ assert.ok(v.left && v.right && v.outcome, 'a vote line: ' + line);
+ assert.ok(!('subject' in v) && !('voter' in v) && !('user' in v), 'no voter id in the line: ' + line);
+ assert.ok(!line.includes(subject), 'the voter is nowhere in the dump');
+ }
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,88 @@
@@ -0,0 +1,88 @@
+'use strict';
+
+/**
+ * The gallery (stage B9): every round's outputs, filtered and ordered, with at most six live frames.
+ */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+(async () => {
+ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '30', BENCH_ROUNDS_DAILY: '100' } });
+ const alice = t.network.addUser('alice');
+ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
+
+ const make = async (kind, prompt, model, cost) => {
+ const task = await t.get('/api/v1/kinds/' + kind + '/tasks', { as: alice, json: { prompt, public: true } });
+ assert.strictEqual(task.status, 201, task.text);
+ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
+ const roundId = rr.json().round.id;
+ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model, payload: { html: '<p>one</p>' }, cost_usd: cost } });
+ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: model + '-b', payload: { html: '<p>two</p>' }, cost_usd: cost } });
+ assert.strictEqual(e1.status, 201, e1.text);
+ assert.strictEqual(e2.status, 201, e2.text);
+ return roundId;
+ };
+
+ const rounds = [];
+ for (let i = 0; i < 5; i++) rounds.push(await make('design.ui', 'A page, number ' + i, 'model-' + i, 0.01 * (i + 1)));
+ const gameRound = await make('games.play', 'A game, one', 'gamer-1', 0.5);
+
+ await check('the gallery lists the rounds', async () => {
+ const r = await t.get('/gallery');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(r.text.includes('data-gallery="1"'), 'the gallery');
+ assert.ok(r.text.includes('A page, number 0'), 'a prompt');
+ assert.ok(r.text.includes('ga-card'), 'the cards');
+ assert.ok(r.text.includes('model-0'), 'the model of an entry');
+ });
+
+ await check('at most six frames are live, the rest wait for a tap', async () => {
+ const r = await t.get('/gallery');
+ const live = (r.text.match(/class="ga-frame"/g) || []).length;
+ assert.ok(live <= 6, 'live frames: ' + live);
+ assert.ok(live >= 4, 'the first cards are live: ' + live);
+ assert.ok(r.text.includes('data-ga-still'), 'the stills');
+ assert.ok(r.text.includes('Live frame, on tap'), 'the tap hint');
+ });
+
+ await check('the filters narrow the gallery', async () => {
+ const byKind = await t.get('/gallery?kind=games.play');
+ assert.ok(byKind.text.includes('A game, one'), 'the game is there');
+ assert.ok(!byKind.text.includes('A page, number 0'), 'the pages are not');
+
+ const byModel = await t.get('/gallery?model=model-2');
+ assert.ok(byModel.text.includes('A page, number 2'), 'the model is there');
+ assert.ok(!byModel.text.includes('A page, number 3'), 'the others are not');
+
+ const byPrice = await t.get('/gallery?price=0.02');
+ assert.ok(byPrice.text.includes('A page, number 0'), 'the cheap one is there');
+ assert.ok(!byPrice.text.includes('A page, number 4'), 'the expensive one is not');
+ });
+
+ await check('the gallery orders by newest and by most voted', async () => {
+ const newest = await t.get('/gallery?order=new');
+ const votes = await t.get('/gallery?order=votes');
+ assert.strictEqual(newest.status, 200);
+ assert.strictEqual(votes.status, 200);
+ const firstNewest = /data-round-id="([^"]+)"/.exec(newest.text)[1];
+ const firstVotes = /data-round-id="([^"]+)"/.exec(votes.text)[1];
+ assert.ok(firstNewest, 'a round in the newest order');
+ assert.ok(firstVotes, 'a round in the most voted order');
+ // The last round made is the newest; no votes yet, so the most-voted order falls back to the newest.
+ assert.strictEqual(firstNewest, firstVotes, 'without votes both orders agree');
+ });
+
+ await check('the gallery API answers the same thing', async () => {
+ const r = await t.get('/api/v1/gallery?kind=design.ui');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(r.json().rounds.length >= 5, 'the rounds');
+ assert.strictEqual(r.json().filters.kind, 'design.ui');
+ const one = r.json().rounds[0];
+ assert.ok(one.entries.length === 2, 'two entries');
+ assert.ok(one.entries[0].wrapper_url.includes('/render/w/'), 'the sandboxed frame');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,119 @@
@@ -0,0 +1,119 @@
+'use strict';
+
+/**
+ * The style-controlled ratings (stage B9).
+ *
+ * These are the maths, on purpose synthetic: a fit that recovers known strengths, a fit that takes a planted
+ * length bias back out, and bands that follow the intervals.
+ */
+
+const assert = require('assert');
+const style = require('../server/rating/style');
+const { check, done } = require('./helpers/boot');
+
+/** A small generator with a seed, so the synthetic votes are the same every run. */
+function rng(seed) {
+ let st = seed >>> 0;
+ return () => {
+ st = (st * 1664525 + 1013904223) >>> 0;
+ return st / 4294967296;
+ };
+}
+
+(async () => {
+ await check('a fit recovers known strengths', async () => {
+ const random = rng(11);
+ const truth = [['alpha', 0.6], ['beta', 0.0], ['gamma', -0.5]];
+ const games = [];
+ for (let i = 0; i < 900; i++) {
+ const x = truth[Math.floor(random() * truth.length)];
+ const y = truth[Math.floor(random() * truth.length)];
+ if (x === y) continue;
+ const p = 1 / (1 + Math.exp(-(x[1] - y[1])));
+ games.push({
+ id: 'g' + i, a: x[0], b: y[0], outcome: random() < p ? 'a' : 'b',
+ fa: { length: 100, markup: 2 }, fb: { length: 100, markup: 2 },
+ });
+ }
+ const fit = style.fitRatings(games, { bootstrap: 120 });
+ const byModel = new Map(fit.models.map((m) => [m.model, m]));
+ const scale = 400 / Math.log(10);
+ for (const [model, theta] of truth) {
+ const expected = 1000 + scale * theta;
+ const got = byModel.get(model).raw.rating;
+ assert.ok(Math.abs(got - expected) < 40, model + ': expected ' + expected.toFixed(1) + ', got ' + got.toFixed(1));
+ }
+ });
+
+ await check('the style control takes a planted length bias back out', async () => {
+ const random = rng(23);
+ // Two equally good models, but voters prefer the longer output.
+ const games = [];
+ for (let i = 0; i < 400; i++) {
+ const longA = random() < 0.6;
+ const la = longA ? 3000 + Math.floor(random() * 2000) : 800 + Math.floor(random() * 600);
+ const lb = longA ? 900 + Math.floor(random() * 700) : 2800 + Math.floor(random() * 2000);
+ const p = 1 / (1 + Math.exp(-(la - lb) / 900));
+ games.push({
+ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',
+ fa: { length: la, markup: Math.round(la / 120) }, fb: { length: lb, markup: Math.round(lb / 120) },
+ });
+ }
+ const fit = style.fitRatings(games, { bootstrap: 120 });
+ const raw = new Map(fit.models.map((m) => [m.model, m.raw.rating]));
+ const controlled = new Map(fit.models.map((m) => [m.model, m.controlled.rating]));
+ // The raw board says one model is clearly better; the controlled board does not.
+ assert.ok(Math.abs(raw.get('alpha') - raw.get('beta')) > 30, 'the raw board is biased: ' + raw.get('alpha') + ' vs ' + raw.get('beta'));
+ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < 30, 'the controlled board is even: ' + controlled.get('alpha') + ' vs ' + controlled.get('beta'));
+ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < Math.abs(raw.get('alpha') - raw.get('beta')), 'the control shrinks the gap');
+ assert.ok(fit.effects.length && fit.effects.length.beta > 0, 'the length effect is positive: ' + JSON.stringify(fit.effects.length));
+ });
+
+ await check('the bands follow the intervals', async () => {
+ const rows = [
+ { model: 'a', rating: 1100, ci_lower: 1090, ci_upper: 1110 },
+ { model: 'b', rating: 1080, ci_lower: 1040, ci_upper: 1105 }, // overlaps a
+ { model: 'c', rating: 1000, ci_lower: 960, ci_upper: 1040 }, // overlaps b, not a
+ { model: 'd', rating: 900, ci_lower: 880, ci_upper: 920 }, // alone
+ ];
+ const bands = style.rankBands(rows);
+ assert.strictEqual(bands.length, 2, JSON.stringify(bands));
+ assert.strictEqual(bands[0].label, '1st–3rd', bands[0].label);
+ assert.deepStrictEqual(bands[0].models, ['a', 'b', 'c']);
+ assert.strictEqual(bands[1].label, '4th', bands[1].label);
+ assert.deepStrictEqual(bands[1].models, ['d']);
+ assert.strictEqual(style.bandLabel(2, 4), '2nd–4th');
+ assert.strictEqual(style.bandLabel(1, 1), '1st');
+ });
+
+ await check('the style features count length and markup', async () => {
+ const f = style.styleFeatures({ html: '<h1>Title</h1><p><strong>bold</strong></p><ul><li>one</li></ul>' });
+ assert.ok(f.length > 20, 'length: ' + f.length);
+ assert.ok(f.markup >= 4, 'markup: ' + f.markup);
+ assert.strictEqual(style.markupCount('# Title\n- a\n- b'), 3);
+ assert.strictEqual(style.markupCount('plain text'), 0);
+ });
+
+ await check('a voter who disagrees with the consensus is down-weighted, never named', async () => {
+ const random = rng(31);
+ const games = [];
+ for (let i = 0; i < 400; i++) {
+ const p = 1 / (1 + Math.exp(-0.9));
+ games.push({
+ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',
+ fa: { length: 100, markup: 1 }, fb: { length: 100, markup: 1 },
+ voter: 'voter-' + (i % 12),
+ });
+ }
+ // One voter always disagrees with the consensus.
+ for (const g of games) if (g.voter === 'voter-3') g.outcome = g.outcome === 'a' ? 'b' : 'a';
+ const fit = style.fitRatings(games, { bootstrap: 60 });
+ assert.ok(fit.voters.downweighted >= 1, 'someone is down-weighted: ' + JSON.stringify(fit.voters));
+ assert.ok(!JSON.stringify(fit).includes('voter-3'), 'the down-weighted voter is never named in the board');
+ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));
+ const odd = games.filter((g) => g.voter === 'voter-3');
+ assert.ok(odd.every((g) => (weights.get(g.id) || 1) < 1), 'the odd voter\'s votes count less');
+ });
+
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });