openvibers/openvibe.bench Commits

Battles, the gallery and ratings that admit the style (#33)

OP OpenVibers committed 14 days ago · a52f76b76c · 19 files, +2034 −8

The gallery (/gallery) lists every showcase round with its outputs live in
the sandboxed render frames: filter by kind, model and price, order by
newest or by most voted, and keep at most six frames live at a time (the
rest wait as a still, or as the frame on tap).

The battles (/battle) put two outputs of one prompt side by side, blind.
The vote is a plain form post to the same origin, so it works without
JavaScript: A, B, tie or both bad, plus the kind's own question (games:
which is more fun; design: which looks more finished). The names open
after the vote. The rules are enforced in the server, not the page: a
signed-in person only, one pair per person, and votes limited per person
and per network (the network is a hash of the address, never the
address). A voter who disagrees with the consensus far too often is
down-weighted, and never named. The share card is a vector image with
both panes and the question.

The ratings (/ratings) are a Bradley-Terry fit over the human votes,
fitted twice: on the votes alone (raw) and with the length and the markup
of each output as covariates (style controlled), so a reader can see how
much of a rank is style. Every rating carries a bootstrap interval over
1000 resamples, and models whose intervals overlap share a band
(2nd-4th). The maths is Bench's own. Every vote is published as
JSONL without voter ids at /api/v1/datasets/votes.jsonl, and
scripts/fit-ratings.js re-derives the same board from that file alone.

Changes

ddiff --git a/migrations/0005_battles.sql b/migrations/0005_battles.sql
nnew file mode 100644
iindex 0000000..a9db606
---- /dev/null
++++ b/migrations/0005_battles.sql
@@@ -0,0 +1,33 @@
++-- phase: expand
++-- B9: battles, the gallery and the honest ratings.
++--
++-- A battle vote is a human_pairwise judgement with two extras: the kind's own question (games: which is more fun;
++-- design: which looks more finished) and the answer to it. The network fingerprint is a hash of the voter's address,
++-- never the address itself: it is only there to limit votes per network. The style of an output (its length and how
++-- much markup it wears) is measured once, when the entry lands, so the ratings can control for it later.
++
++ALTER TABLE bench_judgements ADD COLUMN question text;
++ALTER TABLE bench_judgements ADD COLUMN answer text;
++ALTER TABLE bench_judgements ADD COLUMN net_hash text;
++CREATE INDEX bench_judgements_net ON bench_judgements (net_hash, at DESC) WHERE net_hash IS NOT NULL;
++
++ALTER TABLE bench_entries ADD COLUMN style_length integer;
++ALTER TABLE bench_entries ADD COLUMN style_markup integer;
++
++-- A pair a person has been shown. The unique pair keeps the promise that nobody sees the same pair twice.
++CREATE TABLE bench_battle_seen (
++ id text COLLATE "C" PRIMARY KEY, -- bbs_ + ULID
++ subject text NOT NULL, -- the person
++ pair_key text NOT NULL, -- the two entry ids, sorted and joined
++ round_id text NOT NULL REFERENCES bench_rounds (id) ON DELETE CASCADE,
++ kind_id text NOT NULL REFERENCES bench_kinds (id),
++ a_entry text NOT NULL, -- the entry shown as pane A
++ b_entry text NOT NULL, -- the entry shown as pane B
++ net_hash text, -- the network fingerprint, for the per-network limit
++ voted_at timestamptz, -- when the person cast the vote
++ at timestamptz NOT NULL DEFAULT now(),
++ CONSTRAINT bench_battle_seen_id CHECK (id ~ '^bbs_[0-9A-HJKMNP-TV-Z]{26}$'),
++ CONSTRAINT bench_battle_seen_pair UNIQUE (subject, pair_key)
++);
++CREATE INDEX bench_battle_seen_subject ON bench_battle_seen (subject, at DESC);
++CREATE INDEX bench_battle_seen_net ON bench_battle_seen (net_hash, at DESC) WHERE net_hash IS NOT NULL;
ddiff --git a/public/css/bench.css b/public/css/bench.css
iindex 13097eb..bcf2a3b 100644
---- a/public/css/bench.css
++++ b/public/css/bench.css
@@@ -264,3 +264,76 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.br-field input, .br-field select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; max-width: 100%; }
.br-field input:focus-visible, .br-field select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
.br-form + .br-fine { margin-top: 16px; }
++
++/* The gallery (B9): every round's outputs, live in sandboxed frames; six live at a time, the rest wait. */
++.ga-head { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 0; }
++.ga-gallery { max-width: 1180px; margin: 0 auto; padding: 0 16px 64px; }
++.ga-gallery > h1, .ga-head > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }
++.ga-filters { display: grid; gap: 10px; grid-template-columns: 1fr; margin: 18px 0 4px; }
++.ga-filters label { display: grid; gap: 4px; min-width: 0; font-size: .74rem; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
++.ga-filters input, .ga-filters select { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border, #1f2d47); background: var(--bg-input, var(--bg-primary, #0a0f1c)); color: var(--text-primary, #e6edf7); font: inherit; font-weight: 600; text-transform: none; letter-spacing: normal; max-width: 100%; }
++.ga-filters input:focus-visible, .ga-filters select:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 1px; }
++.ga-grid { display: grid; gap: 16px; grid-template-columns: 1fr; margin-top: 18px; }
++.ga-card { display: grid; gap: 12px; padding: 14px; border-radius: 20px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; align-content: start; }
++.ga-prompt { margin: 0; font-size: .98rem; font-weight: 650; line-height: 1.45; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden; }
++.ga-shots { display: grid; gap: 8px; grid-template-columns: repeat(2, minmax(0, 1fr)); }
++.ga-shot { margin: 0; display: grid; gap: 4px; min-width: 0; }
++.ga-shot figcaption { font-size: .74rem; font-weight: 700; color: var(--text-muted, #7386a3); overflow-wrap: anywhere; }
++.ga-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }
++.ga-still { display: grid; place-items: center; aspect-ratio: 4 / 3; padding: 8px; border: 1px dashed var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); color: var(--text-muted, #7386a3); font-size: .82rem; text-align: center; }
++.ga-still img { width: 100%; height: 100%; object-fit: cover; border-radius: 12px; }
++.ga-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 0; font-size: .84rem; color: var(--text-muted, #7386a3); }
++.ga-votes { font-variant-numeric: tabular-nums; }
++.ga-link { margin-left: auto; font-weight: 750; color: var(--room-strong) !important; text-decoration: none; }
++.ga-link:hover { text-decoration: underline; }
++@media (min-width: 640px) { .ga-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
++@media (min-width: 1024px) {
++ .ga-grid { grid-template-columns: repeat(3, minmax(0, 1fr)); }
++ .ga-filters { grid-template-columns: repeat(4, minmax(0, 1fr)) auto; align-items: end; }
++}
++
++/* The battles (B9): two outputs, one prompt, blind; the vote is a plain form post. */
++.bt-battle { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }
++.bt-battle > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.1; text-wrap: balance; }
++.bt-prompt { font-size: clamp(1.3rem, 3.6vw, 2rem) !important; letter-spacing: -.02em !important; line-height: 1.25 !important; }
++.bt-meta { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; margin: 12px 0 0; font-size: .86rem; color: var(--text-muted, #7386a3); }
++.bt-panes { display: grid; gap: 12px; grid-template-columns: 1fr; margin-top: 16px; }
++.bt-pane { margin: 0; display: grid; gap: 6px; min-width: 0; }
++.bt-pane figcaption { font-size: .76rem; font-weight: 800; letter-spacing: .05em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
++.bt-pane-frame { width: 100%; max-width: 100%; aspect-ratio: 4 / 3; border: 1px solid var(--border, #1f2d47); border-radius: 12px; background: var(--bg-primary, #0a0f1c); display: block; }
++.bt-vote { margin-top: 18px; padding: 16px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); display: grid; gap: 12px; }
++.bt-vote fieldset { margin: 0; padding: 0; border: 0; display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
++.bt-vote legend { width: 100%; margin-bottom: 8px; font-weight: 800; }
++.bt-vote label { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0 16px; border-radius: 999px; border: 1px solid var(--border, #1f2d47); cursor: pointer; font-weight: 650; }
++.bt-vote label:has(input:checked) { border-color: var(--room-accent); background: color-mix(in srgb, var(--room-accent) 14%, transparent); }
++.bt-vote input { accent-color: var(--room-accent); }
++.bt-vote button { justify-self: start; min-height: 44px; padding: 0 20px; border: 0; border-radius: 999px; background: var(--room-fill); color: var(--room-on); font: inherit; font-weight: 750; cursor: pointer; }
++.bt-facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(90px, 1fr)); gap: 6px; margin: 0; font-size: .82rem; }
++.bt-facts dt { color: var(--text-muted, #7386a3); font-size: .7rem; text-transform: uppercase; letter-spacing: .06em; font-weight: 800; }
++.bt-facts dd { margin: 2px 0 0; font-weight: 650; overflow-wrap: anywhere; }
++.bt-reveal-facts { display: grid; gap: 14px; grid-template-columns: 1fr; margin-top: 14px; }
++.bt-signed-out { max-width: 720px; }
++@media (min-width: 720px) {
++ .bt-panes { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; }
++ .bt-reveal-facts { grid-template-columns: repeat(2, minmax(0, 1fr)); }
++}
++
++/* The ratings board (B9): raw and style controlled, side by side, with the intervals and the bands. */
++.rt-board { max-width: 1180px; margin: 0 auto; padding: clamp(20px, 4vw, 40px) 16px 64px; }
++.rt-board > h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.8rem); font-weight: 800; letter-spacing: -.03em; line-height: 1.08; }
++.rt-list { list-style: none; margin: 20px 0 0; padding: 0; display: grid; gap: 10px; }
++.rt-row { display: grid; gap: 10px; padding: 14px; border-radius: 18px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); min-width: 0; }
++.rt-cell { display: grid; gap: 2px; min-width: 0; align-content: start; }
++.rt-label { font-size: .68rem; font-weight: 800; letter-spacing: .07em; text-transform: uppercase; color: var(--text-muted, #7386a3); }
++.rt-model { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; overflow-wrap: anywhere; }
++.rt-rating { font-size: 1.15rem; font-weight: 800; font-variant-numeric: tabular-nums; }
++.rt-ci { color: var(--text-muted, #7386a3); font-size: .82rem; font-variant-numeric: tabular-nums; }
++.rt-band { display: inline-block; margin-top: 4px; padding: 2px 10px; border-radius: 999px; background: color-mix(in srgb, var(--room-accent) 14%, transparent); color: var(--room-strong); font-size: .78rem; font-weight: 750; justify-self: start; }
++.rt-votes { font-weight: 700; font-variant-numeric: tabular-nums; }
++.rt-effects, .rt-data { margin-top: clamp(24px, 4vw, 36px); }
++.rt-effects h2, .rt-data h2 { margin: 0 0 10px; font-size: 1.2rem; font-weight: 800; letter-spacing: -.01em; }
++.rt-data code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-weight: 700; color: var(--room-strong); overflow-wrap: anywhere; }
++@media (min-width: 900px) {
++ .rt-row { grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1fr) auto; align-items: center; }
++}
++
ddiff --git a/public/js/bench.js b/public/js/bench.js
iindex 4656b49..71dbea1 100644
---- a/public/js/bench.js
++++ b/public/js/bench.js
@@@ -1 +1,77 @@
/* OpenVibe.Bench pages work without JavaScript; the main session designs this room later. */
++/* The gallery (B9): at most six frames live at a time; the rest wait as a still until the reader asks. */
++(function () {
++ 'use strict';
++
++ var LIVE = 6;
++
++ function gallery() {
++ var grid = document.querySelector('[data-gallery="1"]');
++ if (!grid) return;
++ var shots = Array.prototype.slice.call(grid.querySelectorAll('.ga-shot'));
++ var live = shots.filter(function (s) { return s.getAttribute('data-live') === '1'; });
++
++ function release(shot) {
++ var frame = shot.querySelector('iframe.ga-frame');
++ if (!frame) return;
++ if (shot.getAttribute('data-server') === '1') return; // the first frames stay live
++ var still = document.createElement('button');
++ still.type = 'button';
++ still.className = 'ga-still';
++ still.setAttribute('data-ga-still', '1');
++ still.setAttribute('aria-label', 'Load the live frame');
++ var note = document.createElement('span');
++ note.className = 'ga-still-note';
++ note.textContent = 'Live frame, on tap';
++ still.appendChild(note);
++ frame.parentNode.replaceChild(still, frame);
++ shot.removeAttribute('data-live');
++ live = live.filter(function (s) { return s !== shot; });
++ }
++
++ function activate(shot) {
++ if (!shot || shot.getAttribute('data-live') === '1') return;
++ var src = shot.getAttribute('data-src');
++ if (!src) return;
++ while (live.length >= LIVE) {
++ var victim = null;
++ for (var i = 0; i < live.length; i++) {
++ if (live[i].getAttribute('data-server') !== '1') { victim = live[i]; break; }
++ }
++ if (!victim) return;
++ release(victim);
++ }
++ var still = shot.querySelector('[data-ga-still]');
++ var frame = document.createElement('iframe');
++ frame.className = 'ga-frame';
++ frame.setAttribute('src', src);
++ frame.setAttribute('sandbox', 'allow-scripts allow-same-origin');
++ frame.setAttribute('referrerpolicy', 'no-referrer');
++ frame.setAttribute('loading', 'lazy');
++ frame.setAttribute('title', 'Output');
++ if (still && still.parentNode) still.parentNode.replaceChild(frame, still);
++ shot.setAttribute('data-live', '1');
++ live.push(shot);
++ }
++
++ grid.addEventListener('click', function (event) {
++ var target = event.target;
++ var shot = target && target.closest ? target.closest('.ga-shot') : null;
++ if (shot) activate(shot);
++ });
++
++ if ('IntersectionObserver' in window) {
++ var observer = new IntersectionObserver(function (entries) {
++ entries.forEach(function (entry) {
++ if (entry.isIntersecting) activate(entry.target);
++ });
++ }, { rootMargin: '320px' });
++ shots.forEach(function (shot) { observer.observe(shot); });
++ } else {
++ shots.forEach(activate);
++ }
++ }
++
++ if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', gallery);
++ else gallery();
++}());
ddiff --git a/scripts/fit-ratings.js b/scripts/fit-ratings.js
nnew file mode 100644
iindex 0000000..f1bd9f2
---- /dev/null
++++ b/scripts/fit-ratings.js
@@@ -0,0 +1,99 @@
++'use strict';
++
++/**
++ * Re-derive the board from the open vote dump.
++ *
++ * node scripts/fit-ratings.js /tmp/votes.jsonl # the table
++ * node scripts/fit-ratings.js /tmp/votes.jsonl --json # the same board as JSON
++ * curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl | node scripts/fit-ratings.js
++ *
++ * The dump carries every vote without voter ids, plus the weight each vote carried in the live fit (so a voter
++ * who disagreed with the consensus far too often counts less). The fit below is the same fit the board uses, with
++ * the same seed, so the numbers here are the numbers on /ratings.
++ */
++
++const fs = require('fs');
++const path = require('path');
++const style = require(path.join(__dirname, '..', 'server', 'rating', 'style'));
++
++const argv = process.argv.slice(2);
++const asJson = argv.includes('--json');
++const bootstrapIndex = argv.indexOf('--bootstrap');
++const bootstrap = bootstrapIndex >= 0 ? Number(argv[bootstrapIndex + 1]) || style.BOOTSTRAP : style.BOOTSTRAP;
++const file = argv.find((a) => !a.startsWith('--') && a !== String(bootstrap));
++
++function readInput() {
++ if (!file) return fs.readFileSync(0, 'utf8');
++ return fs.readFileSync(file, 'utf8');
++}
++
++/** The dump's lines into the games the fit reads. */
++function gamesOf(lines) {
++ const games = [];
++ const weights = new Map();
++ for (const line of lines) {
++ const text = String(line || '').trim();
++ if (!text) continue;
++ let v;
++ try { v = JSON.parse(text); } catch { continue; }
++ if (!v || !v.left || !v.right || !v.left.model || !v.right.model) continue;
++ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';
++ games.push({
++ id: v.id == null ? null : String(v.id),
++ a: v.left.model,
++ b: v.right.model,
++ outcome,
++ fa: { length: Number(v.left.length) || 0, markup: Number(v.left.markup) || 0 },
++ fb: { length: Number(v.right.length) || 0, markup: Number(v.right.markup) || 0 },
++ });
++ if (v.id != null && v.weight != null) weights.set(String(v.id), Number(v.weight));
++ }
++ return { games, weights };
++}
++
++function main() {
++ const text = readInput();
++ const { games, weights } = gamesOf(text.split('\n'));
++ if (!games.length) {
++ process.stderr.write('No votes in the dump. Fetch one: curl -s https://bench.openvibe.services/api/v1/datasets/votes.jsonl\n');
++ process.exit(1);
++ }
++ const fit = style.fitRatings(games, { bootstrap, voteWeights: weights });
++ if (asJson) {
++ process.stdout.write(JSON.stringify({ models: fit.models, bands: fit.bands, effects: fit.effects, votes: fit.votes, bootstrap: fit.bootstrap, seed: fit.seed }, null, 2) + '\n');
++ return;
++ }
++ const cell = (m, key) => {
++ const c = m[key];
++ const lo = Math.round(c.ci_lower);
++ const hi = Math.round(c.ci_upper);
++ return String(Math.round(c.rating)).padStart(5) + ' [' + String(lo).padStart(5) + ', ' + String(hi).padStart(5) + ']';
++ };
++ const lines = [];
++ lines.push('OpenVibe.Bench — the board, re-derived from the open vote dump');
++ lines.push(fit.votes + ' votes, ' + fit.bootstrap + ' bootstrap resamples, seed ' + fit.seed);
++ lines.push('');
++ lines.push('model'.padEnd(28) + 'votes raw rating (95% interval) band controlled rating (95% interval) band');
++ lines.push('-'.repeat(120));
++ for (const m of fit.models) {
++ lines.push(
++ String(m.model).padEnd(28)
++ + String(m.votes).padStart(5) + ' '
++ + cell(m, 'raw') + ' ' + String(m.raw.band || '').padEnd(8)
++ + cell(m, 'controlled') + ' ' + String(m.controlled.band || ''),
++ );
++ }
++ lines.push('');
++ for (const [key, e] of Object.entries(fit.effects || {})) {
++ lines.push('style effect ' + key + ':'
++ + (e.elo_per_1000_chars != null ? ' ' + e.elo_per_1000_chars + ' Elo per 1000 characters' : '')
++ + (e.elo_per_10_marks != null ? ' ' + e.elo_per_10_marks + ' Elo per 10 markup marks' : ''));
++ }
++ if (fit.dropped_covariates && fit.dropped_covariates.length) {
++ lines.push('dropped (too close to another covariate): ' + fit.dropped_covariates.join(', '));
++ }
++ lines.push('down-weighted voters: ' + fit.voters.downweighted + ' of ' + fit.voters.total + ' (never named)');
++ process.stdout.write(lines.join('\n') + '\n');
++}
++
++main();
ddiff --git a/server/app.js b/server/app.js
iindex 101a87b..d4ede81 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -28,6 +28,9 @@ const { createApi } = require('./http/api');
const { createRegistryApi } = require('./http/registry');
const { createPageRoutes } = require('./http/pages');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
++const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
++const { createBattlePages } = require('./http/battle');
++const { createRatingsPages } = require('./http/ratings');
const { createRenderRoutes } = require('./http/render');
const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { createRegistry } = require('./bench/registry');
@@@ -160,6 +163,7 @@ async function createApp(opts = {}) {
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
++ app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use(mcpRouter(ctx));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
@@@ -176,6 +180,9 @@ async function createApp(opts = {}) {
app.use('/render', createRenderRoutes(ctx));
app.use('/vendor', require('./http/vendor').createVendorRoutes());
app.use('/showcase', createShowcasePages(ctx));
++ app.use('/gallery', createGalleryPages(ctx));
++ app.use('/battle', createBattlePages(ctx));
++ app.use('/ratings', createRatingsPages(ctx));
app.use(createPageRoutes(ctx));
app.use((req, res) => send(res, 404, {
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
ddiff --git a/server/bench/battles.js b/server/bench/battles.js
nnew file mode 100644
iindex 0000000..5c13996
---- /dev/null
++++ b/server/bench/battles.js
@@@ -0,0 +1,197 @@
++'use strict';
++
++/**
++ * Battles (spec 2): two outputs, one prompt, blind.
++ *
++ * A battle is a pair of entries from a finished round. The rules the research asks for are enforced here, not in
++ * the page:
++ *
++ * - only a signed-in person votes (the page sends anonymous readers to sign in);
++ * - a person never sees the same pair twice (bench_battle_seen holds the pair the moment it is shown);
++ * - votes are limited per person and per network (the network is a hash of the address, never the address).
++ *
++ * The vote itself is a human_pairwise judgement, so the battle votes are the same votes the ratings are made of.
++ * Two extras ride along: the kind's own question ("which is more fun" for games, "which looks more finished" for
++ * design) and the network hash, which never leaves the database.
++ */
++
++const crypto = require('crypto');
++const { SHOWCASE_KIND_IDS } = require('../kinds');
++
++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
++
++/** The question each kind asks about its own outputs. */
++const QUESTIONS = Object.freeze({
++ games: { id: 'more_fun', text: 'Which one is more fun?', options: ['A', 'B', 'tie'] },
++ design: { id: 'more_finished', text: 'Which one looks more finished?', options: ['A', 'B', 'tie'] },
++ other: { id: 'better', text: 'Which one is better?', options: ['A', 'B', 'tie'] },
++});
++
++/** The question a kind asks. */
++function questionFor(kindId) {
++ const id = String(kindId || '');
++ if (id.startsWith('games.')) return QUESTIONS.games;
++ if (id.startsWith('design.')) return QUESTIONS.design;
++ return QUESTIONS.other;
++}
++
++/** The pair key: the two entry ids, sorted, so A and B in any order is the same pair. */
++function pairKey(a, b) {
++ return [String(a || ''), String(b || '')].sort().join('|');
++}
++
++function createBattles(ctx) {
++ const { s, config, bench } = ctx;
++ const db = s.db;
++
++ /** The network fingerprint: a hash of the address, so the limit works without storing an address. */
++ function netHashOf(req) {
++ const ip = String((req && req.ip) || (req && req.socket && req.socket.remoteAddress) || '').trim();
++ if (!ip) return null;
++ return crypto.createHash('sha256').update('bench-battle|' + ip).digest('hex').slice(0, 40);
++ }
++
++ const limits = () => (config && config.bench) || {};
++
++ /** How many votes this person and this network may still cast. */
++ async function budget({ subject, netHash }) {
++ const cfg = limits();
++ const hourAgo = new Date(Date.now() - 3600_000).toISOString();
++ const dayAgo = new Date(Date.now() - 86_400_000).toISOString();
++ const personHour = num(await db.value(
++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
++ [subject, hourAgo]));
++ const personDay = num(await db.value(
++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE subject = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
++ [subject, dayAgo]));
++ const netHour = netHash ? num(await db.value(
++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
++ [netHash, hourAgo])) : 0;
++ const netDay = netHash ? num(await db.value(
++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE net_hash = $1 AND judge = 'human_pairwise' AND at >= $2::timestamptz",
++ [netHash, dayAgo])) : 0;
++ const perPersonHour = Math.max(1, num(cfg.battlePersonPerHour, 30));
++ const perPersonDay = Math.max(1, num(cfg.battlePersonPerDay, 200));
++ const perNetworkHour = Math.max(1, num(cfg.battleNetworkPerHour, 120));
++ const perNetworkDay = Math.max(1, num(cfg.battleNetworkPerDay, 1000));
++ const reasons = [];
++ if (personHour >= perPersonHour) reasons.push('You have voted a lot in the last hour. Try again in a little while.');
++ if (personDay >= perPersonDay) reasons.push('You have used your votes for today.');
++ if (netHour >= perNetworkHour) reasons.push('This network has voted a lot in the last hour. Try again later.');
++ if (netDay >= perNetworkDay) reasons.push('This network has used its votes for today.');
++ return {
++ allowed: reasons.length === 0,
++ message: reasons.join(' '),
++ person_hour: personHour, person_day: personDay, network_hour: netHour, network_day: netDay,
++ limits: { person_hour: perPersonHour, person_day: perPersonDay, network_hour: perNetworkHour, network_day: perNetworkDay },
++ };
++ }
++
++ /**
++ * A pair this person has not seen: a finished round, not their own, with two usable entries, and not a pair
++ * they have been shown before. The pair is recorded as seen the moment it is shown, so nobody meets it twice.
++ */
++ async function pickPair({ subject, kind = 'all', random = Math.random, netHash = null }) {
++ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;
++ const rows = await db.many(
++ `SELECT r.id, r.kind_id, t.prompt
++ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id
++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND r.created_by <> $2
++ AND (SELECT COUNT(*) FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) >= 2
++ AND NOT EXISTS (SELECT 1 FROM bench_judgements j WHERE j.round_id = r.id AND j.subject = $2)
++ AND NOT EXISTS (SELECT 1 FROM bench_battle_seen v WHERE v.subject = $2 AND v.round_id = r.id)
++ ORDER BY r.created_at DESC, r.id LIMIT 60`,
++ [kinds, subject]);
++ if (!rows.length) return null;
++ const row = rows[Math.floor(random() * rows.length) % rows.length];
++ const entries = await db.many(
++ `SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup
++ FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id`,
++ [row.id]);
++ if (entries.length < 2) return null;
++ // Prefer two different models, so a battle is always between two models.
++ let pair = [entries[0], entries[1]];
++ const other = entries.find((e) => e.model !== entries[0].model);
++ if (other) pair = [entries[0], other];
++ const [a, b] = pair;
++ const key = pairKey(a.id, b.id);
++ const id = s.newId('bbs');
++ await db.exec(
++ `INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, net_hash, at)
++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::timestamptz)`,
++ [id, subject, key, row.id, row.kind_id, a.id, b.id, netHash, s.iso()]);
++ return {
++ id, round_id: row.id, kind_id: row.kind_id, prompt: row.prompt,
++ question: questionFor(row.kind_id),
++ a: { id: a.id, provider: a.provider, model: a.model, cost_usd: num(a.cost_usd), ms: num(a.ms) },
++ b: { id: b.id, provider: b.provider, model: b.model, cost_usd: num(b.cost_usd), ms: num(b.ms) },
++ };
++ }
++
++ /** The battle as it was shown, for the reveal and the share card. */
++ async function battleOf(id, subject = null) {
++ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(id || '')]);
++ if (!seen) return null;
++ if (subject && seen.subject !== subject) return null;
++ const round = await db.maybe(
++ 'SELECT r.id, r.kind_id, r.created_at, t.prompt FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.id = $1',
++ [seen.round_id]);
++ if (!round) return null;
++ const entries = await db.many(
++ 'SELECT id, provider, model, route, cost_usd, ms, tokens, payload, style_length, style_markup FROM bench_entries WHERE id = ANY($1::text[])',
++ [[seen.a_entry, seen.b_entry]]);
++ const a = entries.find((e) => e.id === seen.a_entry);
++ const b = entries.find((e) => e.id === seen.b_entry);
++ if (!a || !b) return null;
++ const vote = await db.maybe(
++ "SELECT outcome, question, answer FROM bench_judgements WHERE round_id = $1 AND subject = $2 AND judge = 'human_pairwise' ORDER BY at LIMIT 1",
++ [seen.round_id, seen.subject]);
++ return {
++ id: seen.id, round_id: seen.round_id, kind_id: seen.kind_id, prompt: round.prompt,
++ question: questionFor(seen.kind_id),
++ voted_at: seen.voted_at ? new Date(seen.voted_at).toISOString() : null,
++ a: { id: a.id, provider: a.provider, model: a.model, route: a.route, cost_usd: num(a.cost_usd), ms: num(a.ms), tokens: num(a.tokens), payload: a.payload },
++ b: { id: b.id, provider: b.provider, model: b.model, route: b.route, cost_usd: num(b.cost_usd), ms: num(b.ms), tokens: num(b.tokens), payload: b.payload },
++ vote: vote ? { outcome: vote.outcome, question: vote.question, answer: vote.answer } : null,
++ };
++ }
++
++ /** The vote. The rules are checked here; a caller that breaks them gets a refusal, not a quiet write. */
++ async function recordVote({ subject, battleId, outcome, answer = null, netHash = null }) {
++ const seen = await db.maybe('SELECT * FROM bench_battle_seen WHERE id = $1', [String(battleId || '')]);
++ if (!seen) return { ok: false, status: 404, code: 'bench.battle_not_found', message: 'That battle is gone.' };
++ if (seen.subject !== subject) return { ok: false, status: 403, code: 'bench.battle_not_yours', message: 'That battle belongs to someone else.' };
++ if (seen.voted_at) return { ok: false, status: 409, code: 'bench.already_voted', message: 'You have already voted on this pair.' };
++ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };
++ const won = map[String(outcome || '')];
++ if (!won) return { ok: false, status: 422, code: 'bench.invalid_outcome', message: 'The outcome must be A, B, tie or both_bad.' };
++ const room = await budget({ subject, netHash });
++ if (!room.allowed) return { ok: false, status: 429, code: 'bench.limit_battle', message: room.message };
++ const question = questionFor(seen.kind_id);
++ const answerValue = answer == null || answer === '' ? null : String(answer).slice(0, 8);
++ if (answerValue != null && !question.options.includes(answerValue)) {
++ return { ok: false, status: 422, code: 'bench.invalid_answer', message: 'That answer does not fit the question.' };
++ }
++ try {
++ await bench.castJudgement({
++ roundId: seen.round_id,
++ judge: 'human_pairwise',
++ pair: { left: seen.a_entry, right: seen.b_entry },
++ outcome: won,
++ question: question.text,
++ answer: answerValue,
++ netHash,
++ subject,
++ });
++ } catch (err) {
++ if (err && err.status) return { ok: false, status: err.status, code: err.code || 'bench.invalid', message: err.detail || 'The vote was not accepted.' };
++ throw err;
++ }
++ await db.exec('UPDATE bench_battle_seen SET voted_at = $2::timestamptz WHERE id = $1', [seen.id, s.iso()]);
++ return { ok: true, battle_id: seen.id, outcome: won, question };
++ }
++
++ return { pickPair, battleOf, recordVote, budget, netHashOf, questionFor, pairKey };
++}
++
++module.exports = { createBattles, questionFor, pairKey, QUESTIONS };
ddiff --git a/server/config.js b/server/config.js
iindex edf78d5..a78adb7 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -84,6 +84,11 @@ function load(env = process.env) {
maxPrompt: Math.max(20, int(env.BENCH_MAX_PROMPT, 4000)),
evalsPerDay: Math.max(1, int(env.BENCH_EVALS_PER_DAY, 20)),
provisionalGames: Math.max(1, int(env.BENCH_PROVISIONAL_GAMES, 10)),
++ // Battles: how many votes one person and one network may cast.
++ battlePersonPerHour: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_HOUR, 30)),
++ battlePersonPerDay: Math.max(1, int(env.BENCH_BATTLE_PERSON_PER_DAY, 200)),
++ battleNetworkPerHour: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_HOUR, 120)),
++ battleNetworkPerDay: Math.max(1, int(env.BENCH_BATTLE_NETWORK_PER_DAY, 1000)),
},
};
}
ddiff --git a/server/http/api.js b/server/http/api.js
iindex 2544ee4..c5ec752 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -29,6 +29,7 @@ const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const { kindById } = require('../kinds');
const { BenchError } = require('../store');
++const boardLib = require('../rating/board');
function createApi(ctx) {
const { config, s, principal, bench } = ctx;
@@@ -225,6 +226,28 @@ function createApi(ctx) {
} catch (err) { return failed(res, err); }
});
++ // The open vote dump (stage B9): every human vote, no voter ids. It comes before the kind dataset route,
++ // because '/datasets/votes.jsonl' would otherwise be read as the kind 'votes'.
++ r.get('/datasets/votes.jsonl', async (req, res) => {
++ try {
++ const result = await boardLib.board(ctx);
++ const text = boardLib.dumpJsonl(result);
++ res.set('Cache-Control', 'public, max-age=60');
++ res.set('Content-Disposition', 'attachment; filename="bench-votes.jsonl"');
++ res.set('X-Bench-Votes', String(result.votes.length));
++ return res.type('application/x-ndjson').send(text);
++ } catch (err) { return failed(res, err); }
++ });
++
++ // The board the votes make: raw and style controlled, with intervals and bands.
++ r.get('/ratings', async (req, res) => {
++ try {
++ const result = await boardLib.board(ctx);
++ res.set('Cache-Control', 'public, max-age=60');
++ return res.json({ board: boardLib.boardView(result) });
++ } catch (err) { return failed(res, err); }
++ });
++
r.get('/datasets/:kindId.jsonl', async (req, res) => {
try {
const text = await bench.datasetJsonl(String(req.params.kindId || ''));
ddiff --git a/server/http/battle.js b/server/http/battle.js
nnew file mode 100644
iindex 0000000..278510e
---- /dev/null
++++ b/server/http/battle.js
@@@ -0,0 +1,231 @@
++'use strict';
++
++/**
++ * The battle pages: /battle (blind), /battle/revealed/:id (after the vote) and /battle/card/:id.svg (the share card).
++ *
++ * The vote is a plain form post to the same origin, so it works with JavaScript switched off. The page itself is
++ * server-rendered: the two panes are the same sandboxed render frames the showcase uses, and the model names stay
++ * closed until the vote is in.
++ */
++
++const express = require('express');
++const { asyncRouter } = require('./router');
++const { sameOrigin } = require('./same-origin');
++const renderLib = require('../bench/render');
++const { kindById } = require('../kinds');
++const { esc, raw } = require('../render/html');
++const { send } = require('../render/layout');
++const { createBattles } = require('../bench/battles');
++
++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
++
++/** The share card is a vector image: 1200 by 630, the size every social card expects. */
++const CARD = { w: 1200, h: 630 };
++
++function paneOf(slot, letter) {
++ const url = renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(slot.id);
++ return '<figure class="bt-pane" data-slot="' + esc(letter) + '" data-entry-id="' + esc(slot.id) + '">'
++ + '<figcaption><span class="bt-pane-label">Pane ' + esc(letter) + '</span></figcaption>'
++ + '<iframe class="bt-pane-frame" src="' + esc(url) + '" title="Output ' + esc(letter) + '"'
++ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'
++ + '</figure>';
++}
++
++function voteForm(battle) {
++ const q = battle.question;
++ return '<form class="bt-vote" method="post" action="/battle" data-battle-form="1">'
++ + '<input type="hidden" name="battle" value="' + esc(battle.id) + '">'
++ + '<fieldset><legend>Which is better?</legend>'
++ + '<label><input type="radio" name="outcome" value="A" required> A</label>'
++ + '<label><input type="radio" name="outcome" value="B"> B</label>'
++ + '<label><input type="radio" name="outcome" value="tie"> Tie</label>'
++ + '<label><input type="radio" name="outcome" value="both_bad"> Both bad</label>'
++ + '</fieldset>'
++ + '<fieldset><legend>' + esc(q.text) + '</legend>'
++ + '<label><input type="radio" name="answer" value="A"> A</label>'
++ + '<label><input type="radio" name="answer" value="B"> B</label>'
++ + '<label><input type="radio" name="answer" value="tie"> Same</label>'
++ + '</fieldset>'
++ + '<button class="bm-btn" type="submit">Cast the vote</button>'
++ + '</form>';
++}
++
++function battleView(battle) {
++ const kind = kindById(battle.kind_id);
++ return '<section class="bt-battle" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'
++ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'
++ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'
++ + '<span class="bm-fine">Blind: you see two outputs, not their names.</span></p>'
++ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'
++ + voteForm(battle)
++ + '</section>';
++}
++
++function revealView(battle) {
++ const kind = kindById(battle.kind_id);
++ const vote = battle.vote || {};
++ const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';
++ const fact = (slot, letter) => '<dl class="bt-facts" data-slot="' + esc(letter) + '">'
++ + '<div><dt>Model</dt><dd>' + esc(slot.model) + '</dd></div>'
++ + '<div><dt>Provider</dt><dd>' + esc(slot.provider) + '</dd></div>'
++ + '<div><dt>Cost</dt><dd>$' + esc(slot.cost_usd) + '</dd></div>'
++ + '<div><dt>Time</dt><dd>' + esc(slot.ms) + ' ms</dd></div>'
++ + '</dl>';
++ return '<section class="bt-battle bt-revealed" data-battle="' + esc(battle.id) + '" data-kind="' + esc(battle.kind_id) + '">'
++ + '<h1 class="bt-prompt" data-prompt="1">' + esc(battle.prompt) + '</h1>'
++ + '<p class="bt-meta"><span class="br-chip">' + esc(kind ? kind.name : battle.kind_id) + '</span>'
++ + '<span class="br-chip">' + esc('Your vote: ' + winner) + '</span>'
++ + (vote.answer ? '<span class="br-chip">' + esc(battle.question.text + ': ' + vote.answer) + '</span>' : '')
++ + '</p>'
++ + '<div class="bt-panes">' + paneOf(battle.a, 'A') + paneOf(battle.b, 'B') + '</div>'
++ + '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'
++ + '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'
++ + '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'
++ + '</section>';
++}
++
++function signedOutView() {
++ return '<section class="bt-battle bt-signed-out">'
++ + '<h1>Battles: two outputs, one prompt, blind</h1>'
++ + '<p class="bm-lede">Every battle shows two outputs of the same prompt, without their names. You say which one is '
++ + 'better (or that both are bad), the names open, and your vote counts towards the board.</p>'
++ + '<p class="bm-note">Battles are for signed-in people: one vote per person, one pair per person, and the votes '
++ + 'are rate limited so the board stays honest.</p>'
++ + '<p class="bm-actions"><a class="bm-btn" href="/auth/login?next=%2Fbattle">Sign in to fight</a>'
++ + '<a class="bm-btn bm-btn-quiet" href="/gallery">Look at the gallery first</a></p>'
++ + '</section>';
++}
++
++/** The share card: both panes and the question. A vector card, so no rasteriser is needed. */
++function cardSvg(battle, baseUrl) {
++ const kind = kindById(battle.kind_id);
++ const escXml = (s) => String(s == null ? '' : s)
++ .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
++ const thumb = (slot, x, letter) => {
++ const payload = slot.payload || {};
++ const svg = typeof payload.svg === 'string' ? payload.svg : '';
++ if (svg && svg.trim()) {
++ const uri = 'data:image/svg+xml;base64,' + Buffer.from(svg, 'utf8').toString('base64');
++ return '<image href="' + uri + '" x="' + x + '" y="150" width="480" height="300" preserveAspectRatio="xMidYMid meet"/>';
++ }
++ return '<rect x="' + x + '" y="150" width="480" height="300" rx="18" fill="#16203a" stroke="#2b3a5c"/>'
++ + '<text x="' + (x + 240) + '" y="292" fill="#7d8ea9" font-family="system-ui, sans-serif" font-size="26" text-anchor="middle">'
++ + escXml(kind ? kind.name : 'output') + '</text>'
++ + '<text x="' + (x + 240) + '" y="330" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="middle">live preview</text>';
++ };
++ const prompt = String(battle.prompt || '').replace(/\s+/g, ' ').trim();
++ const short = prompt.length > 150 ? prompt.slice(0, 147) + '…' : prompt;
++ return '<svg xmlns="http://www.w3.org/2000/svg" width="' + CARD.w + '" height="' + CARD.h + '" viewBox="0 0 ' + CARD.w + ' ' + CARD.h + '">'
++ + '<rect width="' + CARD.w + '" height="' + CARD.h + '" fill="#0b1220"/>'
++ + '<rect x="24" y="24" width="' + (CARD.w - 48) + '" height="' + (CARD.h - 48) + '" rx="28" fill="none" stroke="#24304a"/>'
++ + '<text x="60" y="96" fill="#e6edf7" font-family="system-ui, sans-serif" font-size="52" font-weight="800">Vote: which is better?</text>'
++ + '<text x="60" y="132" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="24">' + escXml(short) + '</text>'
++ + thumb(battle.a, 60, 'A')
++ + thumb(battle.b, 660, 'B')
++ + '<text x="84" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">A</text>'
++ + '<text x="684" y="196" fill="#7fd1a8" font-family="system-ui, sans-serif" font-size="30" font-weight="800">B</text>'
++ + '<text x="60" y="516" fill="#8fa3c2" font-family="system-ui, sans-serif" font-size="26">Two outputs, one prompt, blind.</text>'
++ + '<text x="60" y="556" fill="#c9d6ea" font-family="system-ui, sans-serif" font-size="26" font-weight="700">'
++ + escXml('OpenVibe.Bench') + '</text>'
++ + '<text x="' + (CARD.w - 60) + '" y="556" fill="#55658a" font-family="system-ui, sans-serif" font-size="20" text-anchor="end">'
++ + escXml(baseUrl + '/battle') + '</text>'
++ + '</svg>';
++}
++
++function createBattlePages(ctx) {
++ const r = express.Router();
++ const { config, principal } = ctx;
++ const battles = createBattles(ctx);
++ ctx.battles = battles;
++ r.use(express.urlencoded({ extended: false, limit: '16kb' }));
++ r.use(principal.middleware);
++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
++ : { kind: 'anonymous' });
++ const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal.subject : null);
++
++ r.get('/', async (req, res) => {
++ const subject = subjectOf(req);
++ if (!subject) {
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Battles: two outputs, one prompt, blind',
++ description: 'Pick the better output, blind. The names open after the vote.',
++ body: raw(signedOutView()),
++ });
++ }
++ const battle = await battles.pickPair({
++ subject,
++ kind: typeof req.query.kind === 'string' ? req.query.kind : 'all',
++ random: ctx.random || Math.random,
++ netHash: battles.netHashOf(req),
++ });
++ if (!battle) {
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Battles: nothing waiting',
++ description: 'Pick the better output, blind. The names open after the vote.',
++ body: raw('<section class="bt-battle"><h1>No battle is waiting right now</h1>'
++ + '<p class="bm-lede">Every pair you have seen is voted on, and the next rounds are still running. '
++ + 'Come back in a minute, or <a href="/gallery">look at the gallery</a>.</p></section>'),
++ });
++ }
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Battle: two outputs, one prompt',
++ description: 'Pick the better output, blind. The names open after the vote.',
++ headExtra: '<meta property="og:title" content="Vote: which is better?">'
++ + '<meta property="og:description" content="Two outputs, one prompt, blind. Vote on OpenVibe.Bench.">'
++ + '<meta property="og:image" content="' + esc(config.baseUrl + '/battle/card/' + battle.id + '.svg') + '">'
++ + '<meta property="og:image:type" content="image/svg+xml">'
++ + '<meta name="twitter:card" content="summary_large_image">',
++ body: raw(battleView(battle)),
++ });
++ });
++
++ r.get('/card/:id.svg', async (req, res) => {
++ const battle = await battles.battleOf(String(req.params.id || ''));
++ if (!battle) return res.status(404).type('text/plain').send('No such card.');
++ res.set('Cache-Control', 'public, max-age=60');
++ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src data: 'self'");
++ return res.type('image/svg+xml').send(cardSvg(battle, config.baseUrl));
++ });
++
++ r.get('/revealed/:id', async (req, res) => {
++ const subject = subjectOf(req);
++ const battle = await battles.battleOf(String(req.params.id || ''), subject);
++ if (!battle) {
++ return send(res, 404, { config, viewer: viewerOf(req), path: req.originalUrl, title: 'No such battle', body: raw('<section><h1>No such battle</h1></section>') });
++ }
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Battle: the names are open',
++ description: 'The two outputs of one prompt, with their names.',
++ body: raw(revealView(battle)),
++ });
++ });
++
++ r.post('/', async (req, res) => {
++ const subject = subjectOf(req);
++ const body = req.body && typeof req.body === 'object' ? req.body : {};
++ if (!subject) return res.redirect('/auth/login?next=' + encodeURIComponent('/battle'));
++ if (req.principal.viaSession && !sameOrigin(req, config)) {
++ return res.status(403).type('text/plain').send('A signed-in vote must come from bench.openvibe.services itself.');
++ }
++ const out = await battles.recordVote({
++ subject,
++ battleId: String(body.battle || ''),
++ outcome: body.outcome,
++ answer: body.answer,
++ netHash: battles.netHashOf(req),
++ });
++ if (!out.ok) {
++ return res.status(out.status || 422).type('text/plain').send(out.message || 'The vote was not accepted.');
++ }
++ return res.redirect(303, '/battle/revealed/' + out.battle_id);
++ });
++
++ return r;
++}
++
++module.exports = { createBattlePages, cardSvg };
ddiff --git a/server/http/gallery.js b/server/http/gallery.js
nnew file mode 100644
iindex 0000000..9fa6b2e
---- /dev/null
++++ b/server/http/gallery.js
@@@ -0,0 +1,203 @@
++'use strict';
++
++/**
++ * The gallery (spec 1): every showcase round's outputs, live in the sandboxed render frames.
++ *
++ * The page is a filtered, ordered list of rounds: filter by kind, model and price; order by newest or by most
++ * voted. The frames are lazy: at most six live frames run at a time, and the rest wait as a still (a screenshot
++ * Bench's own job made, when there is one) until the reader asks for the live one.
++ */
++
++const express = require('express');
++const { asyncRouter } = require('./router');
++const renderLib = require('../bench/render');
++const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');
++const { esc, raw } = require('../render/html');
++const { send } = require('../render/layout');
++
++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
++/** How many frames may be live at once. */
++const LIVE_FRAMES = 6;
++
++function parsePrice(value) {
++ if (value == null || value === '') return null;
++ const n = Number(value);
++ if (!Number.isFinite(n) || n < 0) return null;
++ return n;
++}
++
++/** The screenshot Bench's own job made for an entry, when it made one. */
++function screenshotOf(uiCheck) {
++ if (!uiCheck || typeof uiCheck !== 'object') return null;
++ for (const key of ['screenshot', 'screenshot_url', 'image', 'thumb', 'thumbnail']) {
++ const v = uiCheck[key];
++ if (typeof v === 'string' && /^https?:\/\//.test(v)) return v;
++ }
++ return null;
++}
++
++async function loadGallery(ctx, { kind = 'all', model = '', price = null, order = 'new', limit = 60 } = {}) {
++ const db = ctx.s.db;
++ const kinds = kind && kind !== 'all' ? [String(kind)] : SHOWCASE_KIND_IDS;
++ const rows = await db.many(
++ `SELECT r.id, r.kind_id, r.created_at, t.prompt,
++ (SELECT COUNT(*)::int FROM bench_judgements j WHERE j.round_id = r.id AND j.judge = 'human_pairwise') AS votes,
++ (SELECT COUNT(*)::int FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL) AS entries
++ FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id
++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[])
++ AND ($2::text = '' OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.model = $2))
++ AND ($3::numeric IS NULL OR EXISTS (SELECT 1 FROM bench_entries e WHERE e.round_id = r.id AND e.error IS NULL AND e.cost_usd <= $3))
++ ORDER BY ${order === 'votes' ? 'votes DESC, r.created_at DESC' : 'r.created_at DESC, r.id DESC'}
++ LIMIT $4`,
++ [kinds, String(model || ''), price, Math.min(120, Math.max(1, num(limit, 60)))]);
++ const out = [];
++ for (const r of rows) {
++ if (num(r.entries) < 2) continue;
++ const entries = await db.many(
++ 'SELECT id, provider, model, cost_usd, ms, ui_check FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id LIMIT 2',
++ [r.id]);
++ if (entries.length < 2) continue;
++ const k = kindById(r.kind_id);
++ out.push({
++ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,
++ prompt: r.prompt, votes: num(r.votes), created_at: new Date(r.created_at).toISOString(),
++ limit_ms: k ? renderLib.limitMsOf(k) : renderLib.DEFAULT_LIMIT_MS,
++ entries: entries.map((e) => ({
++ id: e.id, model: e.model, provider: e.provider, cost_usd: num(e.cost_usd), ms: num(e.ms),
++ screenshot: screenshotOf(e.ui_check),
++ wrapper_url: renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(e.id),
++ })),
++ });
++ }
++ return out;
++}
++
++/** The kinds and models the filter offers. */
++async function facets(ctx) {
++ const db = ctx.s.db;
++ const kinds = await db.many(
++ `SELECT r.kind_id AS id, COUNT(*)::int AS n FROM bench_rounds r
++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) GROUP BY r.kind_id ORDER BY r.kind_id`,
++ [SHOWCASE_KIND_IDS]);
++ const models = await db.many(
++ `SELECT e.model AS id, COUNT(*)::int AS n FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
++ WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) AND e.model IS NOT NULL
++ GROUP BY e.model ORDER BY n DESC, e.model LIMIT 40`,
++ [SHOWCASE_KIND_IDS]);
++ return {
++ kinds: kinds.map((k) => ({ id: k.id, name: (kindById(k.id) || {}).name || k.id, count: num(k.n) })),
++ models: models.map((m) => ({ id: m.id, count: num(m.n) })),
++ };
++}
++
++function frame(entry, live) {
++ const src = live
++ ? '<iframe class="ga-frame" data-live="1" data-server="1" src="' + esc(entry.wrapper_url) + '" title="Output of ' + esc(entry.model) + '"'
++ + ' sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy"></iframe>'
++ : '<button type="button" class="ga-still" data-ga-still="1" aria-label="Load the live frame">'
++ + (entry.screenshot
++ ? '<img src="' + esc(entry.screenshot) + '" alt="A still of the output" loading="lazy">'
++ : '<span class="ga-still-note">Live frame, on tap</span>')
++ + '</button>';
++ return '<figure class="ga-shot" data-entry-id="' + esc(entry.id) + '" data-src="' + esc(entry.wrapper_url) + '">'
++ + '<figcaption>' + esc(entry.model) + '</figcaption>' + src + '</figure>';
++}
++
++function card(round, live) {
++ return '<article class="ga-card" data-round-id="' + esc(round.id) + '" data-kind="' + esc(round.kind_id) + '" data-votes="' + esc(round.votes) + '">'
++ + '<h2 class="ga-prompt" data-prompt="1">' + esc(round.prompt) + '</h2>'
++ + '<div class="ga-shots">' + round.entries.map((e) => frame(e, live)).join('') + '</div>'
++ + '<p class="ga-meta"><span class="br-chip">' + esc(round.kind_name) + '</span>'
++ + '<span class="ga-votes" data-votes="' + esc(round.votes) + '">' + esc(round.votes) + ' votes</span>'
++ + '<span class="bm-fine">' + esc('$' + Math.max(...round.entries.map((e) => e.cost_usd)).toFixed(4)) + ' at most</span>'
++ + '<a class="ga-link" href="/showcase/round/' + esc(round.id) + '">Open side by side</a></p>'
++ + '</article>';
++}
++
++function filterBar(ctx, { kind, model, price, order }) {
++ const kinds = SHOWCASE_KIND_IDS.map((id) => {
++ const k = kindById(id);
++ return '<option value="' + esc(id) + '"' + (kind === id ? ' selected' : '') + '>' + esc(k ? k.name : id) + '</option>';
++ }).join('');
++ return '<form class="ga-filters" method="get" action="/gallery" data-ga-filters="1">'
++ + '<label>Kind<select name="kind"><option value="all"' + (kind === 'all' ? ' selected' : '') + '>All kinds</option>' + kinds + '</select></label>'
++ + '<label>Model<input type="text" name="model" value="' + esc(model || '') + '" placeholder="model name"></label>'
++ + '<label>Price at most<input type="text" inputmode="decimal" name="price" value="' + esc(price == null ? '' : String(price)) + '" placeholder="$0.05"></label>'
++ + '<label>Order<select name="order">'
++ + '<option value="new"' + (order !== 'votes' ? ' selected' : '') + '>Newest</option>'
++ + '<option value="votes"' + (order === 'votes' ? ' selected' : '') + '>Most voted</option>'
++ + '</select></label>'
++ + '<button class="bm-btn bm-btn-small" type="submit">Filter</button>'
++ + '</form>';
++}
++
++function galleryView(rounds, filters) {
++ if (!rounds.length) {
++ return '<section class="ga-gallery" data-gallery="1">'
++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames.</p>'
++ + '<div class="bm-empty"><h1>Nothing matches yet</h1><p>No round fits that filter. The first rounds are still generating.</p></div>'
++ + '</section>';
++ }
++ return '<section class="ga-gallery" data-gallery="1">'
++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. At most six frames run at '
++ + 'once; the rest wait for you.</p>'
++ + '<div class="ga-grid">' + rounds.map((r, i) => card(r, i * 2 < LIVE_FRAMES)).join('') + '</div>'
++ + '</section>';
++}
++
++function createGalleryRoutes(ctx) {
++ const r = asyncRouter();
++ r.get('/', async (req, res) => {
++ const filters = {
++ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',
++ model: typeof req.query.model === 'string' ? req.query.model : '',
++ price: parsePrice(req.query.price),
++ order: req.query.order === 'votes' ? 'votes' : 'new',
++ };
++ const rounds = await loadGallery(ctx, filters);
++ res.set('Cache-Control', 'public, max-age=30');
++ return res.json({ count: rounds.length, filters, rounds });
++ });
++ r.get('/facets', async (req, res) => {
++ res.set('Cache-Control', 'public, max-age=300');
++ return res.json(await facets(ctx));
++ });
++ return r;
++}
++
++function createGalleryPages(ctx) {
++ const r = express.Router();
++ const { config, principal } = ctx;
++ r.use(principal.middleware);
++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
++ : { kind: 'anonymous' });
++
++ r.get('/', async (req, res) => {
++ const filters = {
++ kind: typeof req.query.kind === 'string' && req.query.kind ? req.query.kind : 'all',
++ model: typeof req.query.model === 'string' ? req.query.model : '',
++ price: parsePrice(req.query.price),
++ order: req.query.order === 'votes' ? 'votes' : 'new',
++ };
++ const rounds = await loadGallery(ctx, filters);
++ const f = await facets(ctx);
++ const body = '<section class="ga-head">'
++ + '<h1>Gallery</h1>'
++ + '<p class="bm-lede">Every showcase round, with its outputs live in sandboxed frames. Filter by kind, model '
++ + 'and price; order by newest or by most voted.</p>'
++ + filterBar(ctx, filters)
++ + '<p class="bm-fine">' + esc(f.kinds.length + ' kinds, ' + f.models.length + ' models in the gallery') + '</p>'
++ + '</section>' + galleryView(rounds, filters);
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Gallery: every showcase round',
++ description: 'Every showcase round, live in sandboxed frames. Filter by kind, model and price.',
++ body: raw(body),
++ });
++ });
++
++ return r;
++}
++
++module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, LIVE_FRAMES };
ddiff --git a/server/http/ratings.js b/server/http/ratings.js
nnew file mode 100644
iindex 0000000..ef32e82
---- /dev/null
++++ b/server/http/ratings.js
@@@ -0,0 +1,98 @@
++'use strict';
++
++/**
++ * The ratings page: the board the votes make, raw and style controlled, side by side.
++ *
++ * The board is a Bradley-Terry fit over the human votes with the style of each output as covariates, and every
++ * rating carries a bootstrap interval. Models whose intervals overlap share a band, so the page says "2nd-4th"
++ * instead of pretending the third place is a fact. The open dump next to it carries every vote without voter ids,
++ * and the fit can be re-derived from that file alone.
++ */
++
++const express = require('express');
++const { esc, raw } = require('../render/html');
++const { send } = require('../render/layout');
++const { board, boardView } = require('../rating/board');
++
++function interval(cell) {
++ if (!cell) return '';
++ const lo = cell.ci_lower == null ? '' : Math.round(cell.ci_lower);
++ const hi = cell.ci_upper == null ? '' : Math.round(cell.ci_upper);
++ return '<span class="rt-rating">' + esc(Math.round(cell.rating)) + '</span>'
++ + '<span class="rt-ci">' + esc(lo + '–' + hi) + '</span>';
++}
++
++function rows(view) {
++ if (!view.models.length) {
++ return '<div class="bm-empty"><h1>No votes yet</h1><p>The board fills up as people vote in the battles.</p></div>';
++ }
++ const body = view.models.map((m) => '<li class="rt-row" data-model="' + esc(m.model) + '">'
++ + '<div class="rt-cell rt-cell-model"><span class="rt-label">Model</span>'
++ + '<span class="rt-model">' + esc(m.model) + '</span></div>'
++ + '<div class="rt-cell"><span class="rt-label">Raw</span>' + interval(m.raw)
++ + '<span class="rt-band">' + esc(m.raw.band || '') + '</span></div>'
++ + '<div class="rt-cell"><span class="rt-label">Style controlled</span>' + interval(m.controlled)
++ + '<span class="rt-band">' + esc(m.controlled.band || '') + '</span></div>'
++ + '<div class="rt-cell"><span class="rt-label">Votes</span><span class="rt-votes">' + esc(m.votes) + '</span></div>'
++ + '</li>').join('');
++ return '<ul class="rt-list">' + body + '</ul>';
++}
++
++function effects(view) {
++ const list = Object.entries(view.effects || {});
++ if (!list.length) return '';
++ const items = list.map(([key, e]) => '<div><dt>' + esc(key) + '</dt><dd>'
++ + (e.elo_per_1000_chars != null ? esc(e.elo_per_1000_chars) + ' Elo per 1000 characters' : '')
++ + (e.elo_per_1000_chars != null && e.elo_per_10_marks != null ? '<br>' : '')
++ + (e.elo_per_10_marks != null ? esc(e.elo_per_10_marks) + ' Elo per 10 markup marks' : '')
++ + '</dd></div>').join('');
++ return '<section class="rt-effects"><h2>How much of the board is style</h2>'
++ + '<dl class="br-facts">' + items + '</dl>'
++ + '<p class="bm-fine">A positive number means the longer (or the more dressed up) output wins votes it did not '
++ + 'earn. The controlled column removes that effect; the raw column is what people actually chose.</p>'
++ + '</section>';
++}
++
++function ratingsView(view) {
++ return '<section class="rt-board" data-ratings="1">'
++ + '<h1>Ratings</h1>'
++ + '<p class="bm-lede">Every human vote in Bench is one game between two models. The board is fitted twice: on the '
++ + 'votes alone (raw) and with the style of each output as covariates (style controlled), so you can see how much '
++ + 'of a rank is length and formatting.</p>'
++ + '<p class="bm-fine">' + esc(view.votes) + ' votes, '
++ + esc(view.bootstrap) + ' bootstrap resamples, seed ' + esc(view.seed)
++ + (view.voters && view.voters.downweighted ? ', ' + esc(view.voters.downweighted) + ' voters down-weighted (never named)' : '')
++ + '</p>'
++ + '<div class="rt-table-wrap">' + rows(view) + '</div>'
++ + effects(view)
++ + '<section class="rt-data"><h2>The open data</h2>'
++ + '<p class="bm-lede">Every vote is published as JSONL, without voter ids: '
++ + '<a href="/api/v1/datasets/votes.jsonl">/api/v1/datasets/votes.jsonl</a>. '
++ + 'The fit is reproducible from it: <code>node scripts/fit-ratings.js /tmp/votes.jsonl</code> re-derives the '
++ + 'same numbers (the dump carries the weight each vote carries in the fit).</p></section>'
++ + '</section>';
++}
++
++function createRatingsPages(ctx) {
++ const r = express.Router();
++ const { config, principal } = ctx;
++ r.use(principal.middleware);
++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
++ : { kind: 'anonymous' });
++
++ r.get('/', async (req, res) => {
++ const result = await board(ctx);
++ const view = boardView(result);
++ return send(res, 200, {
++ config, viewer: viewerOf(req), path: req.originalUrl,
++ title: 'Ratings: raw and style controlled',
++ description: 'The board the votes make: raw and style-controlled Bradley-Terry, with intervals and bands.',
++ body: raw(ratingsView(view)),
++ });
++ });
++
++ return r;
++}
++
++module.exports = { createRatingsPages };
ddiff --git a/server/rating/board.js b/server/rating/board.js
nnew file mode 100644
iindex 0000000..3234377
---- /dev/null
++++ b/server/rating/board.js
@@@ -0,0 +1,123 @@
++'use strict';
++
++/**
++ * The board the votes make (spec 3).
++ *
++ * Every human pairwise vote in Bench is one game between two models. The board is fitted twice over those games:
++ * once on the votes alone (raw) and once with the style of each output as covariates (controlled), so a reader can
++ * see how much of the ranking is style. The intervals come from a bootstrap of the votes, and models whose
++ * intervals overlap share a band.
++ *
++ * The board is a pure function of the votes, so it is cached on the shape of the vote set and recomputed when a
++ * new vote lands. The open dump (/api/v1/datasets/votes.jsonl) carries the same weights, so the board can be
++ * re-derived from the file alone.
++ */
++
++const style = require('./style');
++
++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
++
++/** The human votes, with the two models and the two styles of each vote. */
++async function loadVotes(db) {
++ const rows = await db.many(
++ `SELECT j.id, j.at, j.kind_id, j.round_id, j.outcome, j.subject, j.question, j.answer,
++ l.model AS left_model, l.style_length AS left_length, l.style_markup AS left_markup, l.payload AS left_payload,
++ r.model AS right_model, r.style_length AS right_length, r.style_markup AS right_markup, r.payload AS right_payload
++ FROM bench_judgements j
++ JOIN bench_entries l ON l.id = (j.pair->>'left')
++ JOIN bench_entries r ON r.id = (j.pair->>'right')
++ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL
++ ORDER BY j.at, j.id`);
++ const out = [];
++ for (const row of rows) {
++ const left = styleOf(row.left_payload, row.left_length, row.left_markup);
++ const right = styleOf(row.right_payload, row.right_length, row.right_markup);
++ out.push({
++ id: row.id,
++ at: row.at instanceof Date ? row.at.toISOString() : String(row.at),
++ kind_id: row.kind_id,
++ round_id: row.round_id,
++ outcome: row.outcome,
++ question: row.question || null,
++ answer: row.answer || null,
++ voter: row.subject || null,
++ a: { model: row.left_model, length: left.length, markup: left.markup },
++ b: { model: row.right_model, length: right.length, markup: right.markup },
++ });
++ }
++ return out;
++}
++
++/** The style of an output: the columns when they are there, the payload when they are not (older entries). */
++function styleOf(payload, length, markup) {
++ if (length != null && markup != null) return { length: num(length), markup: num(markup) };
++ let p = payload;
++ if (typeof p === 'string') { try { p = JSON.parse(p); } catch { p = {}; } }
++ const features = style.styleFeatures(p || {});
++ return { length: length == null ? features.length : num(length), markup: markup == null ? features.markup : num(markup) };
++}
++
++/** The games the fit reads. */
++function gamesOf(votes) {
++ const out = [];
++ for (const v of votes) {
++ const outcome = v.outcome === 'left' ? 'a' : v.outcome === 'right' ? 'b' : v.outcome === 'tie' ? 'tie' : 'both_bad';
++ out.push({ id: v.id, a: v.a.model, b: v.b.model, outcome, fa: v.a, fb: v.b, voter: v.voter });
++ }
++ return out;
++}
++
++let cached = null;
++
++/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */
++async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
++ const db = ctx.s.db;
++ const votes = await loadVotes(db);
++ const key = votes.length + ':' + (votes.length ? votes[votes.length - 1].id : '') + ':' + num(bootstrap);
++ if (!force && cached && cached.key === key && Date.now() - cached.at < 5 * 60_000) return cached.value;
++ const games = gamesOf(votes);
++ const fit = style.fitRatings(games, { bootstrap });
++ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));
++ const enriched = votes.map((v) => ({ ...v, weight: weights.has(v.id) ? weights.get(v.id) : 1 }));
++ const value = { fit, votes: enriched, generated_at: new Date().toISOString() };
++ cached = { key, at: Date.now(), value };
++ return value;
++}
++
++/**
++ * The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,
++ * so the board can be re-derived from the file alone.
++ */
++function dumpJsonl(result) {
++ const lines = result.votes.map((v) => JSON.stringify({
++ id: v.id,
++ at: v.at,
++ kind_id: v.kind_id,
++ round_id: v.round_id,
++ left: { model: v.a.model, length: v.a.length, markup: v.a.markup },
++ right: { model: v.b.model, length: v.b.length, markup: v.b.markup },
++ outcome: v.outcome,
++ question: v.question,
++ answer: v.answer,
++ weight: v.weight,
++ }));
++ return lines.join('\n') + (lines.length ? '\n' : '');
++}
++
++/** The board as the page and the API show it. */
++function boardView(result) {
++ const fit = result.fit;
++ return {
++ models: fit.models,
++ bands: fit.bands,
++ effects: fit.effects,
++ dropped_covariates: fit.dropped_covariates || [],
++ voters: fit.voters,
++ votes: fit.votes,
++ bootstrap: fit.bootstrap,
++ seed: fit.seed,
++ generated_at: result.generated_at,
++ };
++}
++
++module.exports = { loadVotes, gamesOf, board, dumpJsonl, boardView, styleOf };
ddiff --git a/server/rating/index.js b/server/rating/index.js
iindex 696c487..4e5e939 100644
---- a/server/rating/index.js
++++ b/server/rating/index.js
@@@ -75,4 +75,12 @@ function costPerSolved(entries) {
// Bradley–Terry with bootstrap intervals, voter weights, Wilson and percentiles: the tested library (lib.js, BENCH-2).
const lib = require('./lib');
--module.exports = { eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT };
++// The style-controlled fit (stage B9): a conditional logit over the votes, with the length and the markup of
++// each output as covariates, bootstrap intervals and rank bands.
++const style = require('./style');
++
++module.exports = {
++ eloUpdate, bradleyTerry: lib.bradleyTerry, voterWeights: lib.voterWeights, wilson: lib.wilson, percentile: lib.percentile, pareto, costPerSolved, BASE_ELO, K_DEFAULT,
++ style,
++ fitRatings: style.fitRatings, rankBands: style.rankBands, styleFeatures: style.styleFeatures, markupCount: style.markupCount,
++};
ddiff --git a/server/rating/style.js b/server/rating/style.js
nnew file mode 100644
iindex 0000000..b69d84f
---- /dev/null
++++ b/server/rating/style.js
@@@ -0,0 +1,518 @@
++'use strict';
++
++/**
++ * Style-controlled Bradley-Terry for Bench's human votes (stage B9).
++ *
++ * The research (10-30) is blunt about two things: a board's rank is fragile once the whole board's uncertainty is
++ * counted (an audit of 106,134 LMArena votes found only 1 of 54 neighbouring pairs to be a real ordering), and style
++ * beats substance (length and formatting decide a large share of pairwise votes). So Bench does two things a plain
++ * Elo ladder never does:
++ *
++ * 1. Style control. The same Bradley-Terry fit is made twice: once on the votes alone (the raw board, what people
++ * actually chose) and once with the style of each output as covariates (the controlled board, what the model
++ * would score if every output were written the same way). The covariates are the output's length and its
++ * markup count, the two LMArena names as covariates; the effect sizes are reported so a reader can see how much
++ * of the raw board is style.
++ *
++ * The model is a conditional logit (Bradley-Terry with per-output covariates):
++ *
++ * P(left wins) = sigmoid( theta_left - theta_right + SUM_k beta_k * (z_left,k - z_right,k) )
++ *
++ * where theta is the model's own strength (log-odds) and z the standardised covariates of the two outputs. A tie
++ * counts as half a win. The fit is convex in theta for fixed beta, so a damped Newton direction with a
++ * backtracking line search finds the maximum; theta is centred (its mean is zero) because only differences are
++ * identified.
++ *
++ * 2. Bands, not places. Models whose bootstrap intervals overlap share a band, so the board says "2nd-4th" instead
++ * of pretending the third place is a fact.
++ *
++ * Every number here is reproducible from the open vote dump (/api/v1/datasets/votes.jsonl): the dump carries each
++ * vote's outcome, the two models, the two outputs' style features and the weight the fit gave that vote, so
++ * scripts/fit-ratings.js re-derives the same board from the file alone.
++ *
++ * The maths is Bench's own (no copied code); the shape follows the published description of style-controlled
++ * Bradley-Terry so the numbers mean what a reader expects.
++ */
++
++const { percentile } = require('./lib');
++
++/** Elo points per unit of log-odds: the usual 400 / ln(10). */
++const SCALE = 400 / Math.log(10);
++/** The number a centred rating sits on, so the board reads like an Elo ladder. */
++const ANCHOR = 1000;
++/** The covariates, in the order they are reported. */
++const COVARIATES = Object.freeze(['length', 'markup']);
++/** The bootstrap resamples the board's intervals come from. */
++const BOOTSTRAP = 1000;
++/** The seed the bootstrap uses, so a rerun gives the same intervals. */
++const SEED = 20261030;
++
++const sigmoid = (x) => (x >= 0 ? 1 / (1 + Math.exp(-x)) : (e => e / (1 + e))(Math.exp(x)));
++
++/** The text of an entry payload: whichever field its kind renders from. */
++function contentOf(payload) {
++ const p = payload && typeof payload === 'object' ? payload : {};
++ for (const key of ['html', 'svg', 'glsl', 'text', 'code']) {
++ if (typeof p[key] === 'string' && p[key].trim()) return p[key];
++ }
++ return '';
++}
++
++/**
++ * How much markup an output carries. Headings, bold, lists, code fences, links, table rows and HTML tags each count
++ * once: the point is "how dressed up is this output", and the count has to be the same for every kind.
++ */
++function markupCount(text) {
++ const s = String(text == null ? '' : text);
++ let n = 0;
++ n += (s.match(/^#{1,6}\s/gm) || []).length;
++ n += (s.match(/\*\*|__/g) || []).length;
++ n += (s.match(/^\s*[-*+]\s/gm) || []).length;
++ n += (s.match(/^\s*\d+\.\s/gm) || []).length;
++ n += (s.match(/```/g) || []).length;
++ n += (s.match(/\[[^\]]*\]\([^)]*\)/g) || []).length;
++ n += (s.match(/^\s*\|/gm) || []).length;
++ n += (s.match(/<\/?[a-z][a-z0-9]*\b[^>]*>/gi) || []).length;
++ return n;
++}
++
++/** The style features of one output: how long it is, and how much markup it wears. */
++function styleFeatures(payload) {
++ const content = contentOf(payload);
++ return { length: content.length, markup: markupCount(content) };
++}
++
++/** A small seedable generator, so the same votes and seed always give the same intervals. */
++function seededRandom(seed) {
++ let state = (seed >>> 0) || 1;
++ return () => {
++ state = (state + 0x6D2B79F5) >>> 0;
++ let value = state;
++ value = Math.imul(value ^ (value >>> 15), value | 1);
++ value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
++ return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
++ };
++}
++
++function centre(values) {
++ if (!values.length) return values;
++ const mean = values.reduce((a, b) => a + b, 0) / values.length;
++ for (let i = 0; i < values.length; i++) values[i] -= mean;
++ return values;
++}
++
++/**
++ * The games a fit reads: { id, a, b, outcome, fa, fb, voter?, weight? }. The outcome is 'a', 'b' or 'tie';
++ * 'both_bad' says nothing about the models and is skipped.
++ */
++function prepare(games, { covariates = COVARIATES } = {}) {
++ const list = (Array.isArray(games) ? games : [])
++ .filter((g) => g && g.a && g.b && g.a !== g.b && g.outcome !== 'both_bad');
++ const subjects = [...new Set(list.flatMap((g) => [g.a, g.b]))].sort();
++ const index = new Map(subjects.map((s, i) => [s, i]));
++ const wanted = covariates.filter((k) => k === 'length' || k === 'markup');
++
++ // Standardise every covariate over the outputs the votes actually used.
++ const stats = new Map();
++ for (const key of wanted) {
++ const values = [];
++ for (const g of list) {
++ values.push(Number((g.fa && g.fa[key]) || 0), Number((g.fb && g.fb[key]) || 0));
++ }
++ const mean = values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0;
++ const variance = values.length ? values.reduce((a, b) => a + (b - mean) * (b - mean), 0) / values.length : 0;
++ stats.set(key, { mean, sd: Math.sqrt(variance) || 1 });
++ }
++
++ // The standardised difference of each covariate, per game.
++ const diffs = wanted.map((key) => {
++ const st = stats.get(key);
++ const out = new Float64Array(list.length);
++ for (let i = 0; i < list.length; i++) {
++ const g = list[i];
++ const za = (Number((g.fa && g.fa[key]) || 0) - st.mean) / st.sd;
++ const zb = (Number((g.fb && g.fb[key]) || 0) - st.mean) / st.sd;
++ out[i] = za - zb;
++ }
++ return out;
++ });
++
++ // Two covariates that move together cannot be told apart, and the fit would trade them for ever: keep the first
++ // and drop the one that repeats it. The board then says so instead of pretending both were measured.
++ const keptKeys = [];
++ const keptDiffs = [];
++ const dropped = [];
++ for (let j = 0; j < wanted.length; j++) {
++ const clash = keptDiffs.some((other) => Math.abs(correlation(diffs[j], other)) > 0.95);
++ if (clash) { dropped.push(wanted[j]); continue; }
++ keptKeys.push(wanted[j]);
++ keptDiffs.push(diffs[j]);
++ }
++ const kAll = keptKeys.length;
++
++ const n = list.length;
++ const ai = new Int32Array(n);
++ const bi = new Int32Array(n);
++ const score = new Float64Array(n);
++ const weight = new Float64Array(n);
++ const dz = new Float64Array(n * kAll);
++ const ids = new Array(n);
++ const voters = new Array(n);
++ let kept = 0;
++ for (let gi = 0; gi < list.length; gi++) {
++ const g = list[gi];
++ const w = Number(g.weight == null ? 1 : g.weight);
++ if (!(w > 0)) continue;
++ const at = kept;
++ ai[at] = index.get(g.a);
++ bi[at] = index.get(g.b);
++ score[at] = g.outcome === 'a' ? 1 : g.outcome === 'b' ? 0 : 0.5;
++ weight[at] = w;
++ ids[at] = g.id == null ? null : String(g.id);
++ voters[at] = g.voter == null ? null : String(g.voter);
++ for (let j = 0; j < kAll; j++) dz[at * kAll + j] = keptDiffs[j][gi];
++ kept += 1;
++ }
++
++ const keptStats = new Map();
++ for (const key of keptKeys) keptStats.set(key, stats.get(key));
++ return { ai, bi, score, weight, dz, ids, voters, count: kept, subjects, index, keys: keptKeys, stats: keptStats, dropped };
++}
++
++/** The Pearson correlation of two equal-length series. */
++function correlation(a, b) {
++ const n = Math.min(a.length, b.length);
++ if (!n) return 0;
++ let ma = 0;
++ let mb = 0;
++ for (let i = 0; i < n; i++) { ma += a[i]; mb += b[i]; }
++ ma /= n; mb /= n;
++ let cov = 0;
++ let va = 0;
++ let vb = 0;
++ for (let i = 0; i < n; i++) {
++ cov += (a[i] - ma) * (b[i] - mb);
++ va += (a[i] - ma) * (a[i] - ma);
++ vb += (b[i] - mb) * (b[i] - mb);
++ }
++ if (!va || !vb) return 0;
++ return cov / Math.sqrt(va * vb);
++}
++
++/**
++ * The conditional-logit fit over prepared games. `order` picks which games take part (the bootstrap resamples it);
++ * `use` picks which of the prepared covariates enter the logit (the raw fit uses none of them). The hot loops work on
++ * typed arrays, because the bootstrap runs this thousands of times.
++ */
++function fitPrepared(prepared, { order = null, use = null, ridge = 0.05, betaRidge = 0.002, maxIter = 120, tol = 1e-9, warm = null, lineSearch = true } = {}) {
++ const { ai, bi, score, weight, dz, subjects, keys, stats } = prepared;
++ const n = subjects.length;
++ const kAll = keys.length;
++ const cols = (use == null ? keys : use).map((key) => keys.indexOf(key)).filter((i) => i >= 0);
++ const k = cols.length;
++ const picks = order || null;
++ const count = picks ? picks.length : prepared.count;
++
++ let theta = warm && warm.theta && warm.theta.length === n ? Float64Array.from(warm.theta) : new Float64Array(n);
++ let beta = warm && warm.beta && warm.beta.length === k ? Float64Array.from(warm.beta) : new Float64Array(k);
++
++ const gTheta = new Float64Array(n);
++ const hTheta = new Float64Array(n);
++ const gBeta = new Float64Array(k);
++ const hBeta = new Float64Array(k);
++ const nextTheta = new Float64Array(n);
++ const nextBeta = new Float64Array(k);
++
++ const logit = (p, th, be) => {
++ let x = th[ai[p]] - th[bi[p]];
++ for (let j = 0; j < k; j++) x += be[j] * dz[p * kAll + cols[j]];
++ return x;
++ };
++
++ const evaluate = (th, be) => {
++ let value = 0;
++ for (let p = 0; p < count; p++) {
++ const idx = picks ? picks[p] : p;
++ const x = logit(idx, th, be);
++ const ph = sigmoid(x);
++ const s = score[idx];
++ value += -weight[idx] * (s * Math.log(ph) + (1 - s) * Math.log(1 - ph));
++ }
++ for (let i = 0; i < n; i++) value += ridge * th[i] * th[i];
++ for (let j = 0; j < k; j++) value += betaRidge * be[j] * be[j];
++ return value;
++ };
++
++ const gradient = (th, be) => {
++ gTheta.fill(0); hTheta.fill(0); gBeta.fill(0); hBeta.fill(0);
++ for (let p = 0; p < count; p++) {
++ const idx = picks ? picks[p] : p;
++ const x = logit(idx, th, be);
++ const ph = sigmoid(x);
++ const s = score[idx];
++ const w = weight[idx];
++ const d = w * (s - ph);
++ const a = ai[idx];
++ const b = bi[idx];
++ gTheta[a] += d;
++ gTheta[b] -= d;
++ const curv = w * ph * (1 - ph);
++ hTheta[a] += curv;
++ hTheta[b] += curv;
++ for (let j = 0; j < k; j++) {
++ const dzz = dz[idx * kAll + cols[j]];
++ gBeta[j] += d * dzz;
++ hBeta[j] += curv * dzz * dzz;
++ }
++ }
++ for (let i = 0; i < n; i++) { gTheta[i] -= 2 * ridge * th[i]; hTheta[i] += 2 * ridge; }
++ for (let j = 0; j < k; j++) { gBeta[j] -= 2 * betaRidge * be[j]; hBeta[j] += 2 * betaRidge; }
++ };
++
++ for (let iter = 0; iter < maxIter; iter++) {
++ gradient(theta, beta);
++ let norm = 0;
++ for (let i = 0; i < n; i++) norm += gTheta[i] * gTheta[i];
++ for (let j = 0; j < k; j++) norm += gBeta[j] * gBeta[j];
++ norm = Math.sqrt(norm);
++ if (norm < tol) break;
++ // A Newton direction with a diagonal curvature, damped by a backtracking line search.
++ let step = 1;
++ let moved = false;
++ const base = lineSearch ? evaluate(theta, beta) : null;
++ for (let attempt = 0; attempt < 12; attempt++) {
++ let mean = 0;
++ for (let i = 0; i < n; i++) {
++ const v = theta[i] + step * (gTheta[i] / Math.max(hTheta[i], 1e-9));
++ nextTheta[i] = v;
++ mean += v;
++ }
++ mean /= n;
++ for (let i = 0; i < n; i++) nextTheta[i] -= mean;
++ for (let j = 0; j < k; j++) nextBeta[j] = beta[j] + step * (gBeta[j] / Math.max(hBeta[j], 1e-9));
++ if (!lineSearch) { theta.set(nextTheta); beta.set(nextBeta); moved = true; break; }
++ const value = evaluate(nextTheta, nextBeta);
++ if (value <= base - 1e-12) {
++ theta.set(nextTheta); beta.set(nextBeta); moved = true; break;
++ }
++ step *= 0.5;
++ }
++ if (!moved) break;
++ }
++
++ return { theta: Array.from(theta), beta: Array.from(beta), subjects, keys, stats };
++}
++
++/**
++ * How often a voter disagrees with the consensus. A voter who disagrees far too often is down-weighted: their votes
++ * count a quarter (nothing at all when they disagree with almost everything), and they are never named anywhere.
++ */
++function voterWeights(games, fit, { minVotes = 10, disagreeAt = 0.5, hopelessAt = 0.75 } = {}) {
++ const rating = new Map();
++ fit.subjects.forEach((subject, i) => rating.set(subject, fit.theta[i]));
++ const voters = new Map();
++ for (const g of games) {
++ if (!g || g.voter == null) continue;
++ if (g.outcome === 'tie' || g.outcome === 'both_bad') continue;
++ if (g.a === g.b) continue;
++ const ra = rating.get(g.a);
++ const rb = rating.get(g.b);
++ if (ra == null || rb == null) continue;
++ const expected = ra > rb ? 1 : ra < rb ? -1 : 0;
++ const observed = g.outcome === 'a' ? 1 : -1;
++ const record = voters.get(g.voter) || { count: 0, disagreement: 0 };
++ record.count += 1;
++ record.disagreement += expected === observed ? 0 : 1;
++ voters.set(g.voter, record);
++ }
++ const out = new Map();
++ for (const [voter, record] of voters) {
++ const rate = record.count ? record.disagreement / record.count : 0;
++ let weight = 1;
++ let downweighted = false;
++ if (record.count >= minVotes && rate >= hopelessAt) { weight = 0; downweighted = true; }
++ else if (record.count >= minVotes && rate > disagreeAt) { weight = 0.25; downweighted = true; }
++ out.set(voter, { count: record.count, disagreement: rate, weight, downweighted });
++ }
++ return out;
++}
++
++/**
++ * The whole board: the raw fit, the style-controlled fit, both with bootstrap intervals, and the rank bands.
++ *
++ * The games are { id, a, b, outcome: 'a'|'b'|'tie', fa, fb, voter?, weight? }. The bootstrap resamples the votes and
++ * gives the 2.5th and 97.5th percentile of each rating. `voteWeights` (when given) is the per-vote weight the caller
++ * already computed, so the open dump and the live board agree.
++ */
++function fitRatings(games, {
++ bootstrap = BOOTSTRAP,
++ seed = SEED,
++ ridge = 0.05,
++ betaRidge = 0.002,
++ covariates = COVARIATES,
++ minVotes = 10,
++ disagreeAt = 0.5,
++ voteWeights = null,
++} = {}) {
++ const list = (Array.isArray(games) ? games : []).filter((g) => g && g.a && g.b);
++ const empty = { models: [], bands: { raw: [], controlled: [] }, effects: {}, voters: { total: 0, downweighted: 0 }, votes: 0, bootstrap: 0, seed };
++ if (!list.length) return empty;
++
++ // First pass: the consensus without voter weights, so the weights can be judged against it.
++ const consensus = fitPrepared(prepare(list, { covariates: [] }), { use: [], ridge });
++ const weights = voterWeights(list, consensus, { minVotes, disagreeAt });
++ const weighted = list.map((g) => ({
++ ...g,
++ weight: voteWeights && g.id != null && voteWeights.get(g.id) != null
++ ? voteWeights.get(g.id)
++ : g.voter != null && weights.has(g.voter) ? weights.get(g.voter).weight
++ : (g.weight == null ? 1 : g.weight),
++ }));
++
++ const prepared = prepare(weighted, { covariates });
++ const raw = fitPrepared(prepared, { use: [], ridge });
++ const controlled = fitPrepared(prepared, { use: covariates, ridge, betaRidge });
++
++ const counts = new Map();
++ for (const g of weighted) {
++ counts.set(g.a, (counts.get(g.a) || 0) + 1);
++ counts.set(g.b, (counts.get(g.b) || 0) + 1);
++ }
++
++ const rawSamples = new Map();
++ const ctrlSamples = new Map();
++ const random = seededRandom(seed);
++ const rounds = Math.max(0, Math.min(BOOTSTRAP, Number(bootstrap) || 0));
++ const n = prepared.count;
++ const rawIndex = new Map(raw.subjects.map((s, i) => [s, i]));
++ const ctrlIndex = new Map(controlled.subjects.map((s, i) => [s, i]));
++ const order = new Int32Array(n);
++ for (let i = 0; i < rounds; i++) {
++ for (let j = 0; j < n; j++) order[j] = Math.floor(random() * n);
++ const r = fitPrepared(prepared, { order, use: [], ridge, maxIter: 60, warm: { theta: raw.theta, beta: [] } });
++ const c = fitPrepared(prepared, { order, use: covariates, ridge, betaRidge, maxIter: 60, warm: { theta: controlled.theta, beta: controlled.beta } });
++ for (const [subject, position] of rawIndex) {
++ if (!rawSamples.has(subject)) { rawSamples.set(subject, []); ctrlSamples.set(subject, []); }
++ rawSamples.get(subject).push(r.theta[position]);
++ ctrlSamples.get(subject).push(c.theta[ctrlIndex.has(subject) ? ctrlIndex.get(subject) : 0]);
++ }
++ }
++
++ const toRating = (theta) => ANCHOR + SCALE * theta;
++ const rawTheta = new Map(raw.subjects.map((s, i) => [s, raw.theta[i]]));
++ const ctrlTheta = new Map(controlled.subjects.map((s, i) => [s, controlled.theta[i]]));
++ const rows = raw.subjects.map((subject) => {
++ const thetaRaw = rawTheta.get(subject);
++ const thetaCtrl = ctrlTheta.get(subject);
++ const rawP = rawSamples.get(subject) || [];
++ const ctrlP = ctrlSamples.get(subject) || [];
++ return {
++ model: subject,
++ votes: counts.get(subject) || 0,
++ raw: {
++ rating: Math.round(toRating(thetaRaw) * 100) / 100,
++ ci_lower: rawP.length ? Math.round(toRating(percentile(rawP, 0.025)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,
++ ci_upper: rawP.length ? Math.round(toRating(percentile(rawP, 0.975)) * 100) / 100 : Math.round(toRating(thetaRaw) * 100) / 100,
++ },
++ controlled: {
++ rating: Math.round(toRating(thetaCtrl) * 100) / 100,
++ ci_lower: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.025)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,
++ ci_upper: ctrlP.length ? Math.round(toRating(percentile(ctrlP, 0.975)) * 100) / 100 : Math.round(toRating(thetaCtrl) * 100) / 100,
++ },
++ };
++ }).sort((x, y) => y.raw.rating - x.raw.rating || x.model.localeCompare(y.model));
++
++ const rawBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.raw.rating, ci_lower: r.raw.ci_lower, ci_upper: r.raw.ci_upper })));
++ const ctrlBands = rankBands(rows.map((r) => ({ model: r.model, rating: r.controlled.rating, ci_lower: r.controlled.ci_lower, ci_upper: r.controlled.ci_upper })));
++ const bandOf = (bands, model) => {
++ const band = bands.find((b) => b.models.includes(model));
++ return band ? band.label : null;
++ };
++ for (const row of rows) {
++ row.raw.band = bandOf(rawBands, row.model);
++ row.controlled.band = bandOf(ctrlBands, row.model);
++ }
++
++ // The effect of each style covariate, in Elo points per 1000 characters or per 10 markup marks.
++ const effects = {};
++ const ctrlKeys = controlled.keys;
++ for (let j = 0; j < ctrlKeys.length; j++) {
++ const key = ctrlKeys[j];
++ const beta = controlled.beta[j] == null ? 0 : controlled.beta[j];
++ const st = controlled.stats.get(key);
++ const perUnit = st ? beta * SCALE / st.sd : 0;
++ effects[key] = {
++ beta: Math.round(beta * 1e6) / 1e6,
++ elo_per_1000_chars: key === 'length' ? Math.round(perUnit * 1000 * 100) / 100 : null,
++ elo_per_10_marks: key === 'markup' ? Math.round(perUnit * 10 * 100) / 100 : null,
++ mean: st ? Math.round(st.mean * 100) / 100 : null,
++ sd: st ? Math.round(st.sd * 100) / 100 : null,
++ };
++ }
++ const droppedCovariates = prepared.dropped || [];
++
++ const downweighted = [...weights.values()].filter((w) => w.downweighted).length;
++ // The weight each vote carries, so the open dump can re-derive the same board from the file alone.
++ const perVoteWeights = weighted.map((g) => ({ id: g.id == null ? null : String(g.id), weight: g.weight }));
++ return {
++ models: rows,
++ vote_weights: perVoteWeights,
++ bands: { raw: rawBands, controlled: ctrlBands },
++ effects,
++ voters: { total: weights.size, downweighted },
++ dropped_covariates: droppedCovariates,
++ votes: list.length,
++ bootstrap: rounds,
++ seed,
++ };
++}
++
++/**
++ * Rank bands: models whose intervals overlap share a band. The rows come in rank order; a band grows while the next
++ * model's interval still overlaps the band it would join.
++ */
++function rankBands(rows) {
++ const sorted = [...(rows || [])].filter((r) => r && r.rating != null)
++ .sort((a, b) => b.rating - a.rating || String(a.model).localeCompare(String(b.model)));
++ const bands = [];
++ let current = null;
++ sorted.forEach((row, i) => {
++ const lower = row.ci_lower == null ? row.rating : row.ci_lower;
++ const upper = row.ci_upper == null ? row.rating : row.ci_upper;
++ if (current && lower <= current.upper && upper >= current.lower) {
++ current.models.push(row.model);
++ current.lower = Math.min(current.lower, lower);
++ current.upper = Math.max(current.upper, upper);
++ current.last = i;
++ } else {
++ current = { models: [row.model], lower, upper, first: i, last: i };
++ bands.push(current);
++ }
++ });
++ return bands.map((band, i) => ({
++ label: bandLabel(band.first + 1, band.last + 1),
++ from: band.first + 1,
++ to: band.last + 1,
++ models: band.models,
++ index: i,
++ }));
++}
++
++function ordinal(n) {
++ const v = Math.round(Number(n));
++ const t = v % 100;
++ if (t >= 11 && t <= 13) return v + 'th';
++ return v + ({ 1: 'st', 2: 'nd', 3: 'rd' }[v % 10] || 'th');
++}
++
++/** "2nd-4th" for a band, "1st" for a single place. */
++function bandLabel(from, to) {
++ if (to <= from) return ordinal(from);
++ return ordinal(from) + '\u2013' + ordinal(to);
++}
++
++module.exports = {
++ SCALE, ANCHOR, COVARIATES, BOOTSTRAP, SEED,
++ styleFeatures, markupCount, contentOf,
++ prepare, fitPrepared, voterWeights, fitRatings, rankBands, bandLabel, ordinal, seededRandom,
++};
ddiff --git a/server/render/layout.js b/server/render/layout.js
iindex 5518179..469a6ee 100644
---- a/server/render/layout.js
++++ b/server/render/layout.js
@@@ -21,6 +21,9 @@ const NAV = [
{ label: 'Home', href: '/' },
{ label: 'Benchmarks', href: '/benchmarks' },
{ label: 'Showcase', href: '/showcase' },
++ { label: 'Gallery', href: '/gallery' },
++ { label: 'Battles', href: '/battle' },
++ { label: 'Ratings', href: '/ratings' },
{ label: 'Docs', href: '/docs' },
];
@@@ -79,6 +82,7 @@ ${appIcon.headTags({ site: 'bench' })}
<link rel="stylesheet" href="${ovServe.url('room.css')}">
<link rel="stylesheet" href="${asset('css/bench.css')}">
<script src="${asset('js/bench.js')}" defer></script>
++${o.headExtra || ''}
${render(scripts)}
<meta name="ov-boost" content="bench@${esc(RELEASE)}">
<script src="${require('openvibe-shared/serve').url('boost.js')}" data-main="#main" defer></script>`,
ddiff --git a/server/store.js b/server/store.js
iindex d438ded..74a1f01 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -10,6 +10,7 @@ const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');
const ratingLib = require('./rating');
++const styleLib = require('./rating/style');
const BASE_ELO = ratingLib.BASE_ELO;
const K = 32;
@@@ -277,12 +278,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
const id = s.newId('bte');
++ // The style of the output (how long it is, how much markup it wears) is measured once, here, so the
++ // ratings can control for style later without opening the payload again.
++ const style = styleLib.styleFeatures(payload || {});
const row = await db.maybe(
-- `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at)
-- VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz) RETURNING *`,
++ `INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, error, ui_check, run_id, created_at, style_length, style_markup)
++ VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11::jsonb, $12, $13::timestamptz, $14, $15) RETURNING *`,
[id, r.id, String(provider || 'ai').slice(0, 80), String(model || 'unknown').slice(0, 120), JSON.stringify(route || {}),
JSON.stringify(payload || {}), num(costUsd), Math.max(0, num(ms)), Math.max(0, num(tokens)), error || null,
-- uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso()],
++ uiCheck == null ? null : JSON.stringify(uiCheck), runId || null, s.iso(), style.length, style.markup],
);
await maybeReady(r.id);
return wireEntry(row);
@@@ -354,7 +358,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
};
}
-- async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, subject }) {
++ async function castJudgement({ roundId, kindId, judge, judgeName = null, judgeVersion = null, pair = null, outcome = null, score = null, voteCounts = null, question = null, answer = null, netHash = null, subject }) {
if (!JUDGES.includes(judge)) throw new BenchError(422, 'bench.invalid_judge', 'Unknown judge.');
try {
return await s.tx(async () => {
@@@ -392,10 +396,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const id = s.newId('bju');
const row = await db.maybe(
-- `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts)
-- VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb) RETURNING *`,
++ `INSERT INTO bench_judgements (id, round_id, kind_id, judge, judge_name, judge_version, method, pair, outcome, score, subject, at, vote_counts, question, answer, net_hash)
++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12::timestamptz, $13::jsonb, $14, $15, $16) RETURNING *`,
[id, r.id, r.kind_id, judge, judgeName, judgeVersion, judge, pair ? JSON.stringify(pair) : null,
-- outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null],
++ outcome || null, score == null ? null : num(score), subject || null, s.iso(), voteCounts ? JSON.stringify(voteCounts) : null,
++ question == null ? null : String(question).slice(0, 200), answer == null ? null : String(answer).slice(0, 40),
++ netHash == null ? null : String(netHash).slice(0, 64)],
);
// One stream poll is one game, regardless of audience size. The counts are evidence, not Elo weight.
if (judge === 'human_pairwise' || judge === 'game_outcome' || (judge === 'audience_poll' && voteCounts.voters > 0)) {
ddiff --git a/test/battle.test.js b/test/battle.test.js
nnew file mode 100644
iindex 0000000..29f6f5e
---- /dev/null
++++ b/test/battle.test.js
@@@ -0,0 +1,115 @@
++'use strict';
++
++/**
++ * The battles (stage B9): blind pairs, the rules, and a vote that works without JavaScript.
++ */
++
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++
++const FORM = { 'content-type': 'application/x-www-form-urlencoded' };
++
++(async () => {
++ // One vote per hour for this person: the rate limit is part of the rules under test.
++ const t = await boot({ env: { BENCH_BATTLE_PERSON_PER_HOUR: '1' } });
++ const alice = t.network.addUser('alice');
++ const bob = t.network.addUser('bob');
++ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
++
++ // Two finished rounds, each with two entries: two battles to hand out.
++ const rounds = [];
++ for (const n of [1, 2]) {
++ const task = await t.get('/api/v1/kinds/design.ui/tasks', { as: alice, json: { prompt: 'A dashboard, prompt ' + n + '.', public: true } });
++ assert.strictEqual(task.status, 201, task.text);
++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
++ assert.strictEqual(rr.status, 201, rr.text);
++ const roundId = rr.json().round.id;
++ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model: 'alpha-' + n, payload: { html: '<p>alpha ' + n + '</p>' }, cost_usd: 0.01 } });
++ assert.strictEqual(e1.status, 201, e1.text);
++ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: 'beta-' + n, payload: { html: '<p>beta ' + n + '</p>' }, cost_usd: 0.02 } });
++ assert.strictEqual(e2.status, 201, e2.text);
++ rounds.push({ id: roundId, a: e1.json().entry.id, b: e2.json().entry.id });
++ }
++
++ await check('an anonymous reader is asked to sign in, and sees no pair', async () => {
++ const r = await t.get('/battle');
++ assert.strictEqual(r.status, 200, r.text);
++ assert.ok(r.text.includes('Sign in'), 'the sign-in prompt');
++ assert.ok(!r.text.includes('data-battle='), 'no pair for an anonymous reader');
++ });
++
++ await check('a signed-in person gets a blind pair', async () => {
++ const r = await t.get('/battle', { as: bob });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.ok(r.text.includes('data-battle="bbs_'), 'the battle id: ' + r.text.slice(0, 200));
++ assert.ok(r.text.includes('data-slot="A"') && r.text.includes('data-slot="B"'), 'two panes');
++ assert.ok(r.text.includes('Which one looks more finished?'), 'the kind\'s own question');
++ assert.ok(!r.text.includes('alpha-1') && !r.text.includes('beta-1'), 'no model names before the vote');
++ });
++
++ await check('the vote is a form post that works without JavaScript', async () => {
++ const page = await t.get('/battle', { as: bob });
++ const battleId = /data-battle="(bbs_[^"]+)"/.exec(page.text)[1];
++ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(battleId) + '&outcome=A&answer=A' });
++ assert.strictEqual(vote.status, 303, vote.text);
++ assert.ok(String(vote.headers.get('location') || '').startsWith('/battle/revealed/'), 'the reveal page: ' + vote.headers.get('location'));
++ const reveal = await t.get(vote.headers.get('location'), { as: bob });
++ assert.strictEqual(reveal.status, 200, reveal.text);
++ assert.ok(reveal.text.includes('alpha-') || reveal.text.includes('beta-'), 'the names are open after the vote');
++ const row = await t.ctx.s.db.maybe("SELECT outcome, question, answer FROM bench_judgements WHERE judge = 'human_pairwise' ORDER BY at DESC LIMIT 1");
++ assert.strictEqual(row.outcome, 'left');
++ assert.strictEqual(row.question, 'Which one looks more finished?');
++ assert.strictEqual(row.answer, 'A');
++ });
++
++ await check('a person never sees the same pair twice', async () => {
++ const seen = await t.ctx.s.db.many('SELECT round_id, pair_key, voted_at FROM bench_battle_seen WHERE subject = (SELECT subject FROM bench_judgements ORDER BY at DESC LIMIT 1)');
++ const pairs = new Set(seen.map((s) => s.pair_key));
++ assert.strictEqual(pairs.size, seen.length, 'no pair twice: ' + JSON.stringify(seen));
++ // Both rounds are used up now: there is nothing left to show this person.
++ const more = await t.get('/battle', { as: bob });
++ assert.strictEqual(more.status, 200, more.text);
++ assert.ok(more.text.includes('No battle is waiting'), 'no pair left: ' + more.text.slice(0, 300));
++ });
++
++ await check('the rate limit stops the second vote', async () => {
++ const before = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");
++ const rows = await t.ctx.s.db.many('SELECT id, round_id FROM bench_battle_seen WHERE voted_at IS NULL');
++ assert.ok(rows.length >= 1, 'a pair is waiting');
++ const vote = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=B' });
++ assert.strictEqual(vote.status, 429, vote.text);
++ const after = await t.ctx.s.db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE judge = 'human_pairwise'");
++ assert.strictEqual(after, before, 'no vote was written');
++ });
++
++ await check('an anonymous vote is refused, and a cross-site vote is refused', async () => {
++ const rows = await t.ctx.s.db.many('SELECT id FROM bench_battle_seen WHERE voted_at IS NULL');
++ const anon = await t.get('/battle', { method: 'POST', headers: FORM, body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });
++ assert.ok(anon.status === 302 || anon.status === 303, 'the anonymous vote is sent to sign in: ' + anon.status);
++ assert.ok(String(anon.headers.get('location') || '').startsWith('/auth/login'), 'to the sign-in page');
++ const cross = await t.get('/battle', { as: bob, method: 'POST', headers: FORM, origin: 'https://evil.example', body: 'battle=' + encodeURIComponent(rows[0].id) + '&outcome=A' });
++ assert.strictEqual(cross.status, 403, cross.text);
++ });
++
++ await check('the ratings board counts the battle vote, and the open data has no voter ids', async () => {
++ const page = await t.get('/ratings', { as: bob });
++ assert.strictEqual(page.status, 200, page.text);
++ assert.ok(page.text.includes('Raw') && page.text.includes('Style controlled'), 'both columns');
++ assert.ok(page.text.includes('alpha-'), 'a model on the board');
++
++ const dump = await t.get('/api/v1/datasets/votes.jsonl');
++ assert.strictEqual(dump.status, 200, dump.text);
++ const lines = dump.text.trim().split('\n').filter(Boolean);
++ assert.ok(lines.length >= 1, 'the dump has the vote');
++ const subject = await t.ctx.s.db.value("SELECT subject FROM bench_judgements WHERE judge = 'human_pairwise' LIMIT 1");
++ for (const line of lines) {
++ const v = JSON.parse(line);
++ assert.ok(v.left && v.right && v.outcome, 'a vote line: ' + line);
++ assert.ok(!('subject' in v) && !('voter' in v) && !('user' in v), 'no voter id in the line: ' + line);
++ assert.ok(!line.includes(subject), 'the voter is nowhere in the dump');
++ }
++ });
++
++ await t.close();
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/gallery.test.js b/test/gallery.test.js
nnew file mode 100644
iindex 0000000..b03b862
---- /dev/null
++++ b/test/gallery.test.js
@@@ -0,0 +1,88 @@
++'use strict';
++
++/**
++ * The gallery (stage B9): every round's outputs, filtered and ordered, with at most six live frames.
++ */
++
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++
++(async () => {
++ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '30', BENCH_ROUNDS_DAILY: '100' } });
++ const alice = t.network.addUser('alice');
++ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
++
++ const make = async (kind, prompt, model, cost) => {
++ const task = await t.get('/api/v1/kinds/' + kind + '/tasks', { as: alice, json: { prompt, public: true } });
++ assert.strictEqual(task.status, 201, task.text);
++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
++ const roundId = rr.json().round.id;
++ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model, payload: { html: '<p>one</p>' }, cost_usd: cost } });
++ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: model + '-b', payload: { html: '<p>two</p>' }, cost_usd: cost } });
++ assert.strictEqual(e1.status, 201, e1.text);
++ assert.strictEqual(e2.status, 201, e2.text);
++ return roundId;
++ };
++
++ const rounds = [];
++ for (let i = 0; i < 5; i++) rounds.push(await make('design.ui', 'A page, number ' + i, 'model-' + i, 0.01 * (i + 1)));
++ const gameRound = await make('games.play', 'A game, one', 'gamer-1', 0.5);
++
++ await check('the gallery lists the rounds', async () => {
++ const r = await t.get('/gallery');
++ assert.strictEqual(r.status, 200, r.text);
++ assert.ok(r.text.includes('data-gallery="1"'), 'the gallery');
++ assert.ok(r.text.includes('A page, number 0'), 'a prompt');
++ assert.ok(r.text.includes('ga-card'), 'the cards');
++ assert.ok(r.text.includes('model-0'), 'the model of an entry');
++ });
++
++ await check('at most six frames are live, the rest wait for a tap', async () => {
++ const r = await t.get('/gallery');
++ const live = (r.text.match(/class="ga-frame"/g) || []).length;
++ assert.ok(live <= 6, 'live frames: ' + live);
++ assert.ok(live >= 4, 'the first cards are live: ' + live);
++ assert.ok(r.text.includes('data-ga-still'), 'the stills');
++ assert.ok(r.text.includes('Live frame, on tap'), 'the tap hint');
++ });
++
++ await check('the filters narrow the gallery', async () => {
++ const byKind = await t.get('/gallery?kind=games.play');
++ assert.ok(byKind.text.includes('A game, one'), 'the game is there');
++ assert.ok(!byKind.text.includes('A page, number 0'), 'the pages are not');
++
++ const byModel = await t.get('/gallery?model=model-2');
++ assert.ok(byModel.text.includes('A page, number 2'), 'the model is there');
++ assert.ok(!byModel.text.includes('A page, number 3'), 'the others are not');
++
++ const byPrice = await t.get('/gallery?price=0.02');
++ assert.ok(byPrice.text.includes('A page, number 0'), 'the cheap one is there');
++ assert.ok(!byPrice.text.includes('A page, number 4'), 'the expensive one is not');
++ });
++
++ await check('the gallery orders by newest and by most voted', async () => {
++ const newest = await t.get('/gallery?order=new');
++ const votes = await t.get('/gallery?order=votes');
++ assert.strictEqual(newest.status, 200);
++ assert.strictEqual(votes.status, 200);
++ const firstNewest = /data-round-id="([^"]+)"/.exec(newest.text)[1];
++ const firstVotes = /data-round-id="([^"]+)"/.exec(votes.text)[1];
++ assert.ok(firstNewest, 'a round in the newest order');
++ assert.ok(firstVotes, 'a round in the most voted order');
++ // The last round made is the newest; no votes yet, so the most-voted order falls back to the newest.
++ assert.strictEqual(firstNewest, firstVotes, 'without votes both orders agree');
++ });
++
++ await check('the gallery API answers the same thing', async () => {
++ const r = await t.get('/api/v1/gallery?kind=design.ui');
++ assert.strictEqual(r.status, 200, r.text);
++ assert.ok(r.json().rounds.length >= 5, 'the rounds');
++ assert.strictEqual(r.json().filters.kind, 'design.ui');
++ const one = r.json().rounds[0];
++ assert.ok(one.entries.length === 2, 'two entries');
++ assert.ok(one.entries[0].wrapper_url.includes('/render/w/'), 'the sandboxed frame');
++ });
++
++ await t.close();
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/rating-style.test.js b/test/rating-style.test.js
nnew file mode 100644
iindex 0000000..5b12edc
---- /dev/null
++++ b/test/rating-style.test.js
@@@ -0,0 +1,119 @@
++'use strict';
++
++/**
++ * The style-controlled ratings (stage B9).
++ *
++ * These are the maths, on purpose synthetic: a fit that recovers known strengths, a fit that takes a planted
++ * length bias back out, and bands that follow the intervals.
++ */
++
++const assert = require('assert');
++const style = require('../server/rating/style');
++const { check, done } = require('./helpers/boot');
++
++/** A small generator with a seed, so the synthetic votes are the same every run. */
++function rng(seed) {
++ let st = seed >>> 0;
++ return () => {
++ st = (st * 1664525 + 1013904223) >>> 0;
++ return st / 4294967296;
++ };
++}
++
++(async () => {
++ await check('a fit recovers known strengths', async () => {
++ const random = rng(11);
++ const truth = [['alpha', 0.6], ['beta', 0.0], ['gamma', -0.5]];
++ const games = [];
++ for (let i = 0; i < 900; i++) {
++ const x = truth[Math.floor(random() * truth.length)];
++ const y = truth[Math.floor(random() * truth.length)];
++ if (x === y) continue;
++ const p = 1 / (1 + Math.exp(-(x[1] - y[1])));
++ games.push({
++ id: 'g' + i, a: x[0], b: y[0], outcome: random() < p ? 'a' : 'b',
++ fa: { length: 100, markup: 2 }, fb: { length: 100, markup: 2 },
++ });
++ }
++ const fit = style.fitRatings(games, { bootstrap: 120 });
++ const byModel = new Map(fit.models.map((m) => [m.model, m]));
++ const scale = 400 / Math.log(10);
++ for (const [model, theta] of truth) {
++ const expected = 1000 + scale * theta;
++ const got = byModel.get(model).raw.rating;
++ assert.ok(Math.abs(got - expected) < 40, model + ': expected ' + expected.toFixed(1) + ', got ' + got.toFixed(1));
++ }
++ });
++
++ await check('the style control takes a planted length bias back out', async () => {
++ const random = rng(23);
++ // Two equally good models, but voters prefer the longer output.
++ const games = [];
++ for (let i = 0; i < 400; i++) {
++ const longA = random() < 0.6;
++ const la = longA ? 3000 + Math.floor(random() * 2000) : 800 + Math.floor(random() * 600);
++ const lb = longA ? 900 + Math.floor(random() * 700) : 2800 + Math.floor(random() * 2000);
++ const p = 1 / (1 + Math.exp(-(la - lb) / 900));
++ games.push({
++ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',
++ fa: { length: la, markup: Math.round(la / 120) }, fb: { length: lb, markup: Math.round(lb / 120) },
++ });
++ }
++ const fit = style.fitRatings(games, { bootstrap: 120 });
++ const raw = new Map(fit.models.map((m) => [m.model, m.raw.rating]));
++ const controlled = new Map(fit.models.map((m) => [m.model, m.controlled.rating]));
++ // The raw board says one model is clearly better; the controlled board does not.
++ assert.ok(Math.abs(raw.get('alpha') - raw.get('beta')) > 30, 'the raw board is biased: ' + raw.get('alpha') + ' vs ' + raw.get('beta'));
++ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < 30, 'the controlled board is even: ' + controlled.get('alpha') + ' vs ' + controlled.get('beta'));
++ assert.ok(Math.abs(controlled.get('alpha') - controlled.get('beta')) < Math.abs(raw.get('alpha') - raw.get('beta')), 'the control shrinks the gap');
++ assert.ok(fit.effects.length && fit.effects.length.beta > 0, 'the length effect is positive: ' + JSON.stringify(fit.effects.length));
++ });
++
++ await check('the bands follow the intervals', async () => {
++ const rows = [
++ { model: 'a', rating: 1100, ci_lower: 1090, ci_upper: 1110 },
++ { model: 'b', rating: 1080, ci_lower: 1040, ci_upper: 1105 }, // overlaps a
++ { model: 'c', rating: 1000, ci_lower: 960, ci_upper: 1040 }, // overlaps b, not a
++ { model: 'd', rating: 900, ci_lower: 880, ci_upper: 920 }, // alone
++ ];
++ const bands = style.rankBands(rows);
++ assert.strictEqual(bands.length, 2, JSON.stringify(bands));
++ assert.strictEqual(bands[0].label, '1st–3rd', bands[0].label);
++ assert.deepStrictEqual(bands[0].models, ['a', 'b', 'c']);
++ assert.strictEqual(bands[1].label, '4th', bands[1].label);
++ assert.deepStrictEqual(bands[1].models, ['d']);
++ assert.strictEqual(style.bandLabel(2, 4), '2nd–4th');
++ assert.strictEqual(style.bandLabel(1, 1), '1st');
++ });
++
++ await check('the style features count length and markup', async () => {
++ const f = style.styleFeatures({ html: '<h1>Title</h1><p><strong>bold</strong></p><ul><li>one</li></ul>' });
++ assert.ok(f.length > 20, 'length: ' + f.length);
++ assert.ok(f.markup >= 4, 'markup: ' + f.markup);
++ assert.strictEqual(style.markupCount('# Title\n- a\n- b'), 3);
++ assert.strictEqual(style.markupCount('plain text'), 0);
++ });
++
++ await check('a voter who disagrees with the consensus is down-weighted, never named', async () => {
++ const random = rng(31);
++ const games = [];
++ for (let i = 0; i < 400; i++) {
++ const p = 1 / (1 + Math.exp(-0.9));
++ games.push({
++ id: 'g' + i, a: 'alpha', b: 'beta', outcome: random() < p ? 'a' : 'b',
++ fa: { length: 100, markup: 1 }, fb: { length: 100, markup: 1 },
++ voter: 'voter-' + (i % 12),
++ });
++ }
++ // One voter always disagrees with the consensus.
++ for (const g of games) if (g.voter === 'voter-3') g.outcome = g.outcome === 'a' ? 'b' : 'a';
++ const fit = style.fitRatings(games, { bootstrap: 60 });
++ assert.ok(fit.voters.downweighted >= 1, 'someone is down-weighted: ' + JSON.stringify(fit.voters));
++ assert.ok(!JSON.stringify(fit).includes('voter-3'), 'the down-weighted voter is never named in the board');
++ const weights = new Map((fit.vote_weights || []).map((w) => [w.id, w.weight]));
++ const odd = games.filter((g) => g.voter === 'voter-3');
++ assert.ok(odd.every((g) => (weights.get(g.id) || 1) < 1), 'the odd voter\'s votes count less');
++ });
++
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });