openvibers/openvibe.bench Commits
Pick for me, model watch, badges and embeds (B10) (#37)
Changes
nnew file mode 100644@@@ -0,0 +1,26 @@++-- phase: expand++-- The watches (spec 6): a person follows a benchmark, or a model on a benchmark. The rows go into the account export++-- and are deleted with the person.++CREATE TABLE IF NOT EXISTS bench_watches (++ id TEXT PRIMARY KEY,++ subject TEXT NOT NULL,++ benchmark_id TEXT NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,++ model TEXT,++ user_id BIGINT,++ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),++ last_notified_at TIMESTAMPTZ++);++CREATE UNIQUE INDEX IF NOT EXISTS bench_watches_once ON bench_watches (subject, benchmark_id, COALESCE(model, ''));++CREATE INDEX IF NOT EXISTS bench_watches_benchmark ON bench_watches (benchmark_id);++++-- The board as it was last seen, so the next fit can be compared with it. The model is a model, not a person.++CREATE TABLE IF NOT EXISTS bench_watch_ratings (++ kind_id TEXT NOT NULL,++ model TEXT NOT NULL,++ rating DOUBLE PRECISION,++ ci_lower DOUBLE PRECISION,++ ci_upper DOUBLE PRECISION,++ band TEXT,++ updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),++ PRIMARY KEY (kind_id, model)++);@@@ -360,3 +360,43 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex.rt-bar { height: 6px; margin-top: 4px; }.rt-ci { display: block; font-size: .78rem; color: var(--text-secondary, #96a7c2); }}++++/* ---- Pick for me (B10): plain controls, then the frontier ---- */++.pk-form { display: grid; gap: 12px; margin: 18px 0 8px; padding: 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }++.pk-form label { display: grid; gap: 6px; font-size: .8rem; font-weight: 800; color: var(--text-secondary); }++.pk-form select, .pk-form input[type="text"] { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border); background: var(--bg-primary); color: var(--text-primary); font: inherit; font-size: .95rem; font-weight: 500; width: 100%; }++.pk-form select:focus-visible, .pk-form input:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 2px; }++.pk-check { display: flex; flex-direction: row; align-items: center; gap: 10px; min-height: 44px; }++.pk-check input { width: 20px; height: 20px; accent-color: var(--room-accent); }++.pk-words input { font-weight: 500; }++.pk-notes { margin: 4px 0 0; }++.pk-why { margin: 10px 0 14px; padding: 12px 14px; border-radius: 14px; background: color-mix(in srgb, var(--room-accent) 12%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 30%, transparent); font-weight: 600; line-height: 1.55; }++.pk-table td, .pk-table th { white-space: nowrap; }++.pk-score { font-weight: 800; }++.pk-band { font-size: .74rem; color: var(--text-muted); white-space: nowrap; }++.pk-thin { opacity: .55; }++.pk-thin-note { font-size: .72rem; color: var(--text-muted); }++.pk-table .pk-thin-note { white-space: nowrap; }++.pk-samples { display: grid; gap: 12px; grid-template-columns: repeat(auto-fill, minmax(min(100%, 280px), 1fr)); margin-top: 12px; }++.pk-card { border: 1px solid var(--border); border-radius: 16px; overflow: hidden; background: var(--bg-card); min-width: 0; }++.pk-card-h { margin: 0; padding: 10px 12px; font-size: .85rem; font-weight: 800; overflow-wrap: anywhere; }++.pk-shot { aspect-ratio: 16 / 10; background: var(--bg-primary); overflow: hidden; }++.pk-shot .ga-shot { margin: 0; height: 100%; min-width: 0; }++.pk-shot .ga-shot figcaption { display: none; }++.pk-shot iframe, .pk-shot img { display: block; width: 100%; height: 100%; border: 0; object-fit: cover; }++.pk-shot .ga-frame { aspect-ratio: auto; }++.pk-prompt { padding: 0 12px 12px; margin: 0; }++++/* ---- Watching (B10) ---- */++.wk-list { list-style: none; margin: 12px 0 0; padding: 0; display: grid; gap: 10px; }++.wk-item { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; justify-content: space-between; padding: 14px 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }++.wk-what { display: grid; gap: 6px; min-width: 0; overflow-wrap: anywhere; }++.wk-form { margin: 0; }++++/* ---- Use this (B10): the copy boxes ---- */++.br-usethis { display: grid; gap: 12px; }++.br-usethis-box { display: grid; gap: 10px; padding: 14px 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }++.br-usethis-h { margin: 0; font-size: .95rem; overflow-wrap: anywhere; }++.br-usethis-row { display: grid; gap: 6px; min-width: 0; }++.br-usethis-row textarea { width: 100%; min-height: 44px; padding: 10px 12px; border-radius: 12px; border: 1px solid var(--border); background: var(--bg-primary); color: var(--text-secondary); font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: .8rem; line-height: 1.45; resize: vertical; }++.br-follow { margin: 0; }@@@ -29,6 +29,12 @@ const { createRegistryApi } = require('./http/registry');const { createPageRoutes } = require('./http/pages');const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');++const { createPickPages, createPickRoutes } = require('./http/pick');++const { createBadgeRoutes } = require('./http/badge');++const { createEmbedRoutes } = require('./http/embed');++const { createWatchPages } = require('./http/watch');++const { createWatches } = require('./watch/watch');++const { createBattles } = require('./bench/battles');const { createBattlePages } = require('./http/battle');const { createRatingsPages } = require('./http/ratings');const { createRenderRoutes } = require('./http/render');@@@ -98,6 +104,15 @@ async function createApp(opts = {}) {ctx.telemetry = createHttpTelemetry({ service: 'bench', sink: opts.telemetrySink || createPgTelemetrySink({ db: s.db, log }), skipPrefixes: ['/shared'], intervalMs: 60_000 });app.use(ctx.telemetry.middleware());app.locals.ctx = ctx;++ ctx.watches = createWatches(ctx);++ ctx.battles = createBattles(ctx);++ // Where the ratings are refitted, the watches are checked: a score that moved by more than its own interval, or++ // a model that overtook another one, is worth one notice.++ s.onRatingsChanged = async (kindId, models) => {++ try { await ctx.watches.check(kindId, { models }); } catch (err) {++ log.warn('[OpenVibe.Bench] watch check failed: ' + (err && err.message ? err.message : err));++ }++ };const valkey = opts.valkey !== undefined ? opts.valkey : (config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null);ctx.valkey = valkey;@@@ -124,6 +139,9 @@ async function createApp(opts = {}) {crossOriginResourcePolicy: { policy: 'same-site' },referrerPolicy: { policy: 'strict-origin-when-cross-origin' },}));++ // ── Badges and embeds: no session, no cookies, before the cookie parser ──++ app.use('/badge', createBadgeRoutes(ctx));++ app.use('/embed', createEmbedRoutes(ctx));app.use(cookieParser());// ── Machine endpoints ───────────────────────────────────@@@ -164,6 +182,7 @@ async function createApp(opts = {}) {app.use('/api/v1', createRegistryApi(ctx));app.use('/api/v1/showcase', createShowcaseRoutes(ctx));app.use('/api/v1/gallery', createGalleryRoutes(ctx));++ app.use('/api/v1', createPickRoutes(ctx));app.use(mcpRouter(ctx));app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));@@@ -181,6 +200,8 @@ async function createApp(opts = {}) {app.use('/vendor', require('./http/vendor').createVendorRoutes());app.use('/showcase', createShowcasePages(ctx));app.use('/gallery', createGalleryPages(ctx));++ app.use('/pick', createPickPages(ctx));++ app.use(createWatchPages(ctx));app.use('/battle', createBattlePages(ctx));app.use('/ratings', createRatingsPages(ctx));app.use(createPageRoutes(ctx));nnew file mode 100644@@@ -0,0 +1,107 @@++'use strict';++++/**++ * The README badge (spec 3): /badge/:benchmark/:model.svg++ *++ * The badge carries the benchmark, its version and the model's score with its spread, plus the band the model sits in++ * when the board knows it — never a bare rank. The text is escaped and the width comes from the text, so a long++ * title still fits. An unknown benchmark or a model with no data gets a grey "no data" badge and a 404.++ */++++const express = require('express');++const { esc } = require('../render/html');++++/** The height of a badge and the width one character takes at this size. */++const HEIGHT = 20;++const CHAR_W = 6.9;++const PAD = 10;++++/** The XML escape for text inside the SVG. */++function xml(s) {++ return String(s == null ? '' : s)++ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')++ .replace(/"/g, '"').replace(/'/g, ''');++}++++/** The width of a run of text, from the text itself. */++function widthOf(text) {++ return Math.max(20, Math.ceil(String(text).length * CHAR_W) + PAD * 2);++}++++/**++ * A shields-shaped badge. `label` on the left, `message` on the right; the width comes from both.++ * The fills are the standard badge greys: a badge is a picture of its own, so it cannot borrow the site's theme.++ */++function badgeSvg({ label, message, tone = 'value', title = null }) {++ const lw = widthOf(label);++ const mw = widthOf(message);++ const width = lw + mw;++ const labelFill = tone === 'nodata' ? '#9f9f9f' : '#555';++ const valueFill = tone === 'nodata' ? '#c8c8c8' : '#007ec6';++ const labelText = tone === 'nodata' ? '#ffffff' : '#ffffff';++ const name = title || (label + ' · ' + message);++ return '<svg xmlns="http://www.w3.org/2000/svg" width="' + width + '" height="' + HEIGHT + '" role="img" aria-label="' + xml(name) + '">'++ + '<title>' + xml(name) + '</title>'++ + '<rect width="' + width + '" height="' + HEIGHT + '" rx="3" fill="' + valueFill + '"/>'++ + '<rect width="' + lw + '" height="' + HEIGHT + '" rx="3" fill="' + labelFill + '"/>'++ + '<rect x="' + (lw - 3) + '" width="3" height="' + HEIGHT + '" fill="' + valueFill + '"/>'++ + '<g fill="' + labelText + '" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">'++ + '<text x="' + (PAD - 0.5) + '" y="14">' + xml(label) + '</text>'++ + '<text x="' + (lw + PAD - 0.5) + '" y="14">' + xml(message) + '</text>'++ + '</g></svg>';++}++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++const fmtScore = (v) => (v == null ? '—' : String(Math.round(Number(v) * 10) / 10));++++/** The no-data badge: grey, with the reason, and a 404 beside it. */++function noData(reason) {++ const svg = badgeSvg({ label: 'OpenVibe Bench', message: 'no data' + (reason ? ' (' + reason + ')' : ''), tone: 'nodata' });++ return { svg, status: 404 };++}++++/**++ * The badge for one benchmark and one model. The score is the median of the model's verified runs on that benchmark;++ * the spread is how much those runs differ from each other, and the band is where the model sits on that benchmark's++ * board.++ */++async function badgeFor(ctx, benchmarkId, model) {++ const registry = ctx.registry;++ const benchmark = await registry.getBenchmark(String(benchmarkId || ''));++ if (!benchmark) return noData('unknown benchmark');++ const rankings = await registry.rankings({ benchmark: benchmark.id, metric: 'score' });++ const entries = (rankings && rankings.entries) || [];++ const row = entries.find((e) => String(e.model) === String(model || ''));++ if (!row || row.score == null) return noData('no runs');++ const version = num(benchmark.latest_version, 1);++ const title = String(benchmark.title || benchmark.id);++ const spread = row.consistency != null ? Math.round(num(row.consistency) * 10) / 10 : null;++ let band = null;++ try {++ const board = await require('../rating/board').boardForKind(ctx, benchmark.kind);++ for (const b of ((board.fit && board.fit.bands && board.fit.bands.controlled) || [])) {++ if ((b.models || []).includes(String(model))) { band = b.label; break; }++ }++ } catch { /* the board is a nicety here: a badge without a band still carries the score */ }++ const message = spread != null ? fmtScore(row.score) + ' ±' + spread : fmtScore(row.score);++ const label = title + ' v' + version + (band ? ' · band ' + band : '');++ return { svg: badgeSvg({ label, message, title: label + ': ' + message }), status: 200 };++}++++function createBadgeRoutes(ctx) {++ const r = express.Router();++ r.get('/:benchmark/:model.svg', async (req, res) => {++ const model = String(req.params.model || '').replace(/\.svg$/i, '');++ const out = await badgeFor(ctx, String(req.params.benchmark || ''), model);++ res.status(out.status);++ res.set('Content-Type', 'image/svg+xml; charset=utf-8');++ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'");++ res.set('X-Content-Type-Options', 'nosniff');++ res.set('Cross-Origin-Resource-Policy', 'cross-origin');++ res.set('Cache-Control', 'public, max-age=3600');++ return res.send(out.svg);++ });++ return r;++}++++module.exports = { createBadgeRoutes, badgeFor, badgeSvg, xml, widthOf, noData };@@@ -15,6 +15,7 @@ const renderLib = require('../bench/render');const { kindById } = require('../kinds');const { esc, raw } = require('../render/html');const { send } = require('../render/layout');++const { battleUseThis } = require('../views/usethis');const { createBattles } = require('../bench/battles');const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));@@@ -61,7 +62,7 @@ function battleView(battle) {+ '</section>';}--function revealView(battle) {++function revealView(battle, { baseUrl = '' } = {}) {const kind = kindById(battle.kind_id);const vote = battle.vote || {};const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';@@@ -81,6 +82,7 @@ function revealView(battle) {+ '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'+ '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'+ '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'++ + String(battleUseThis({ baseUrl, battle, title: kind ? kind.name : battle.kind_id }))+ '</section>';}@@@ -201,7 +203,7 @@ function createBattlePages(ctx) {config, viewer: viewerOf(req), path: req.originalUrl,title: 'Battle: the names are open',description: 'The two outputs of one prompt, with their names.',-- body: raw(revealView(battle)),++ body: raw(revealView(battle, { baseUrl: ctx.config.baseUrl })),});});nnew file mode 100644@@@ -0,0 +1,94 @@++'use strict';++++/**++ * The battle embed (spec 3): /embed/battle/:id++ *++ * A page of one battle that another site may frame. Only this route may be framed: every other route keeps refusing++ * frames. The page reads and writes no cookie, the outputs stay in their sandboxed frames, and voting opens Bench in++ * a new tab: the widget itself never posts anything.++ */++++const express = require('express');++const { esc } = require('../render/html');++const renderLib = require('../bench/render');++const { kindById } = require('../kinds');++++/**++ * The style of the embed page. It stands alone on someone else's site, so it takes no theme of ours: the visitor's++ * own light or dark setting paints it through the browser's system colours, which adapt by themselves.++ */++const STYLE = [++ ':root{color-scheme:light dark;--em-bg:Canvas;--em-text:CanvasText;--em-muted:GrayText;--em-line:color-mix(in srgb,CanvasText 22%,transparent);--em-panel:color-mix(in srgb,CanvasText 6%,Canvas);--em-fill:Highlight;--em-on:HighlightText}',++ 'body{margin:0;background:var(--em-bg);color:var(--em-text);font-family:Inter,-apple-system,BlinkMacSystemFont,Segoe UI,sans-serif;padding:12px}',++ '.em-wrap{display:grid;gap:10px;max-width:900px;margin:0 auto}',++ '.em-h{font-size:.95rem;font-weight:700;margin:0;overflow-wrap:anywhere}',++ '.em-p{font-size:.86rem;margin:0;color:var(--em-muted);line-height:1.5}',++ '.em-panes{display:grid;gap:10px;grid-template-columns:repeat(auto-fit,minmax(min(100%,260px),1fr))}',++ '.em-pane{margin:0;border:1px solid var(--em-line);border-radius:12px;overflow:hidden;background:var(--em-panel)}',++ '.em-pane iframe{display:block;width:100%;aspect-ratio:16/10;height:auto;border:0}',++ '.em-cap{font-size:.78rem;padding:6px 8px;margin:0;color:var(--em-muted);font-weight:600;overflow-wrap:anywhere}',++ '.em-links{display:flex;flex-wrap:wrap;gap:10px;align-items:center}',++ '.em-link{display:inline-flex;align-items:center;min-height:36px;padding:0 12px;border-radius:999px;background:var(--em-fill);color:var(--em-on);font-weight:700;font-size:.86rem;text-decoration:none}',++ '.em-back{font-size:.78rem;color:var(--em-muted);text-decoration:none}',++ '.em-link:focus-visible,.em-back:focus-visible{outline:2px solid var(--em-fill);outline-offset:2px}',++].join('');++++/** The question a battle asks, which is an object on the battle and a string on an old one. */++function questionText(question, kind) {++ if (typeof question === 'string' && question) return question;++ if (question && typeof question.text === 'string' && question.text) return question.text;++ return kind ? 'Two ' + kind.name + ' outputs, side by side.' : 'Two outputs, side by side.';++}++++const pane = (entry, label) => '<figure class="em-pane">'++ + '<iframe src="' + esc(renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(entry.id)) + '"'++ + ' title="' + esc(label + ' output') + '" sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '"'++ + ' referrerpolicy="no-referrer" loading="lazy"></iframe>'++ + '<figcaption class="em-cap">' + esc(label) + '</figcaption></figure>';++++/** The embed page for one battle. */++function embedHtml(battle, { baseUrl }) {++ const kind = kindById(battle.kind_id);++ const label = (e) => String(e.model || 'model');++ return '<!doctype html><html lang="en"><head><meta charset="utf-8">'++ + '<meta name="viewport" content="width=device-width, initial-scale=1">'++ + '<meta name="robots" content="noindex, nofollow">'++ + '<title>' + esc((kind ? kind.name : battle.kind_id) + ' — OpenVibe Bench') + '</title>'++ + '<style>' + STYLE + '</style></head><body><div class="em-wrap">'++ + '<h1 class="em-h">' + esc(battle.prompt || (kind ? kind.name : battle.kind_id)) + '</h1>'++ + '<p class="em-p">' + esc(questionText(battle.question, kind)) + '</p>'++ + '<div class="em-panes">' + pane(battle.a, label(battle.a)) + pane(battle.b, label(battle.b)) + '</div>'++ + '<div class="em-links">'++ + '<a class="em-link" href="' + esc(baseUrl + '/battle') + '" target="_blank" rel="noopener">Vote on OpenVibe Bench</a>'++ + '<a class="em-back" href="' + esc(baseUrl + '/battle/revealed/' + battle.id) + '" target="_blank" rel="noopener">OpenVibe Bench</a>'++ + '</div></div></body></html>';++}++++function createEmbedRoutes(ctx) {++ const r = express.Router();++ r.get('/battle/:id', async (req, res) => {++ const battle = await ctx.battles.battleOf(String(req.params.id || ''));++ if (!battle) {++ res.status(404);++ res.set('Content-Type', 'text/html; charset=utf-8');++ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors *");++ res.set('X-Content-Type-Options', 'nosniff');++ res.removeHeader('X-Frame-Options');++ return res.send('<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="robots" content="noindex, nofollow">'++ + '<title>No such battle</title><style>' + STYLE + '</style></head><body><div class="em-wrap">'++ + '<h1 class="em-h">No such battle</h1><p class="em-p">This battle is not here.</p></div></body></html>');++ }++ const baseUrl = ctx.config.baseUrl || 'https://bench.openvibe.services';++ res.status(200);++ res.set('Content-Type', 'text/html; charset=utf-8');++ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; frame-src " + renderLib.RENDER_ORIGIN++ + "; img-src data:; form-action 'none'; base-uri 'none'; frame-ancestors *");++ res.set('X-Content-Type-Options', 'nosniff');++ res.removeHeader('X-Frame-Options');++ res.set('Cache-Control', 'public, max-age=60');++ return res.send(embedHtml(battle, { baseUrl }));++ });++ return r;++}++++module.exports = { createEmbedRoutes, embedHtml };@@@ -200,4 +200,4 @@ function createGalleryPages(ctx) {return r;}--module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, LIVE_FRAMES };++module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, frame, card, screenshotOf, LIVE_FRAMES };@@@ -105,7 +105,7 @@ function createPageRoutes(ctx) {jsonLd: [seo.jsonLd.webPage({ name: out.benchmark.title, url: `${base()}/benchmarks/${encodeURIComponent(id)}`, description: out.benchmark.description || out.benchmark.title, siteUrl: base() })],body: raw(String(results.benchmarkPage({benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,-- rankings, metric, runs, codesRunUrl: config.codesRunUrl || null,++ rankings, metric, runs, codesRunUrl: config.codesRunUrl || null, viewer: req.viewer, baseUrl: base(),}))),});});@@@ -197,7 +197,7 @@ function createPageRoutes(ctx) {index: summary.runs > 0, cache: PUBLIC_CACHE,title: model,description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,-- body: raw(String(results.modelPage({ model, summary, runs, titles }))),++ body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base() }))),}, summary.runs > 0 || runs.length ? 200 : 404);});nnew file mode 100644@@@ -0,0 +1,61 @@++'use strict';++++/**++ * The pick routes (spec 3): the page at /pick and the same answer as JSON at /api/v1/pick.++ * Both read the same three controls — kind, top price per solved task and fast — and the free text that sets them.++ */++++const express = require('express');++const { asyncRouter } = require('./router');++const { send } = require('../render/layout');++const { raw } = require('../render/html');++const pickLib = require('../pick/pick');++const { pickPage } = require('../views/pick');++++const str = (v) => (typeof v === 'string' ? v : '');++++function readControls(query) {++ const maxCost = str(query.max_cost);++ return {++ kind: str(query.kind).slice(0, 80),++ maxCost: maxCost === '' ? null : Number(maxCost),++ fast: str(query.fast) === '1' || str(query.fast) === 'true',++ q: str(query.q).slice(0, 200),++ };++}++++const viewerOf = (req) => (req.principal && req.principal.kind === 'user'++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }++ : { kind: 'anonymous' });++++function createPickPages(ctx) {++ const r = express.Router();++ r.use(ctx.principal.middleware);++ r.get('/', async (req, res) => {++ const controls = readControls(req.query);++ const answer = await pickLib.answer(ctx, controls);++ const body = pickPage(answer);++ return send(res, 200, {++ config: ctx.config,++ viewer: viewerOf(req),++ path: req.originalUrl,++ title: 'Pick for me: the frontier of a kind',++ description: 'Plain controls, then the models no other model beats on both score and cost per solved task.',++ body: raw(body),++ });++ });++ return r;++}++++function createPickRoutes(ctx) {++ const r = asyncRouter();++ r.get('/pick', async (req, res) => {++ const controls = readControls(req.query);++ const answer = await pickLib.answer(ctx, controls);++ res.set('Cache-Control', 'public, max-age=30');++ return res.json(answer);++ });++ return r;++}++++module.exports = { createPickPages, createPickRoutes, readControls };nnew file mode 100644@@@ -0,0 +1,81 @@++'use strict';++++/**++ * The watch routes (spec 6): follow and unfollow, and the Watching page.++ * Only a signed-in person may follow: the rows belong to them and are deleted with them.++ */++++const express = require('express');++const { asyncRouter } = require('./router');++const { sameOrigin } = require('./same-origin');++const { send } = require('../render/layout');++const { raw } = require('../render/html');++const { watchingPage } = require('../views/watch');++++const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal : null);++/** The claims of a token, read without verifying it: the id is only used to address the notice. */++function decodeClaims(token) {++ const parts = String(token || '').split('.');++ if (parts.length !== 3) return null;++ try { return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); } catch { return null; }++}++++/** The Network user id of the person, when their token carries it: the notice needs it. */++const networkUserId = (req) => {++ const principal = req.principal || {};++ const claims = principal.claims || decodeClaims(principal.token);++ const id = claims && (claims.id != null ? claims.id : claims.sub);++ const n = Number(id);++ return Number.isFinite(n) && n > 0 ? n : null;++};++++function createWatchPages(ctx) {++ const r = express.Router();++ r.use(express.urlencoded({ extended: false, limit: '16kb' }));++ r.use(express.json({ limit: '64kb' }));++ r.use(ctx.principal.middleware);++++ r.get('/watching', async (req, res) => {++ const principal = subjectOf(req);++ const watches = principal ? await ctx.watches.listFor(principal.subject) : [];++ const body = watchingPage(watches, { viewer: principal });++ return send(res, 200, {++ config: ctx.config,++ viewer: principal ? { kind: 'user', subject: principal.subject, displayName: principal.displayName || principal.username } : { kind: 'anonymous' },++ path: req.originalUrl,++ title: 'Watching: the benchmarks and models you follow',++ description: 'The benchmarks and models you follow, and what Bench tells you when they move.',++ body: raw(body),++ });++ });++++ r.post('/watch/follow', async (req, res) => {++ if (!sameOrigin(req, ctx.config)) return res.status(403).type('text/plain').send('That request did not come from this site.');++ const principal = subjectOf(req);++ if (!principal) return res.status(401).type('text/plain').send('Sign in to follow a benchmark.');++ const body = req.body || {};++ const benchmarkId = String(body.benchmark_id || '').trim();++ const model = body.model ? String(body.model).trim() : null;++ const out = await ctx.watches.follow({++ subject: principal.subject,++ benchmarkId,++ model: model || null,++ userId: networkUserId(req),++ });++ if (!out.ok) return res.status(out.status || 400).type('text/plain').send(out.error || 'Could not follow that.');++ return res.redirect(303, String(body.back || '/watching'));++ });++++ r.post('/watch/unfollow', async (req, res) => {++ if (!sameOrigin(req, ctx.config)) return res.status(403).type('text/plain').send('That request did not come from this site.');++ const principal = subjectOf(req);++ if (!principal) return res.status(401).type('text/plain').send('Sign in to stop watching.');++ const body = req.body || {};++ await ctx.watches.unfollow({ subject: principal.subject, id: String(body.id || '') });++ return res.redirect(303, String(body.back || '/watching'));++ });++++ return r;++}++++module.exports = { createWatchPages, networkUserId };nnew file mode 100644@@@ -0,0 +1,42 @@++'use strict';++++/**++ * The frontier (spec 3, "Pick for me"): the models no other model beats on both score and cost per solved task.++ *++ * The rule is deliberately narrow, because the whole point of the pick is that it never hides a model:++ *++ * - a model is dropped when some other model is at least as good on score AND at least as cheap per solved task,++ * with at least one of the two strictly better (that other model dominates it);++ * - two models that are exactly equal on both axes are a tie, and a tie keeps both (neither dominates the other);++ * - a model with no score or no cost cannot be compared at all, so it is never on the frontier and never++ * dominates anyone.++ *++ * Pure maths, no I/O: the tests are the proof.++ */++++const num = (v) => (v == null || Number.isNaN(Number(v)) ? null : Number(v));++++/** True when `other` dominates `row`: at least as good on both axes, and strictly better on one. */++function dominates(other, row) {++ const os = num(other && other.score);++ const oc = num(other && other.cost);++ const rs = num(row && row.score);++ const rc = num(row && row.cost);++ if (os == null || oc == null || rs == null || rc == null) return false;++ if (!(os >= rs && oc <= rc)) return false;++ // Exactly equal on both axes is a tie, not domination: both models stay.++ return !(os === rs && oc === rc);++}++++/**++ * The frontier of a list of rows. Each row needs a `score` and a `cost`; rows missing either are left out.++ * The rows that come back keep the order they came in.++ */++function frontier(rows) {++ const list = (Array.isArray(rows) ? rows : [])++ .filter((r) => r && num(r.score) != null && num(r.cost) != null)++ .map((r) => ({ ...r, score: num(r.score), cost: num(r.cost) }));++ return list.filter((row) => !list.some((other) => other !== row && dominates(other, row)));++}++++module.exports = { frontier, dominates };nnew file mode 100644@@@ -0,0 +1,115 @@++'use strict';++++/**++ * The plain keyword parser behind the free-text box on /pick (spec 3).++ *++ * There is no model here and no network call: the words in the box are read with rules, so the pick stays free and++ * the same words always give the same answer. The parser only fills in the three controls the pick has — the kind,++ * the top price per solved task and "fast" — and says in its notes what it read, so the reader can see the reading++ * and change it.++ */++++/** The numbers people write as words. */++const WORD_NUMBERS = Object.freeze({++ a: 1, an: 1, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10,++ eleven: 11, twelve: 12, fifteen: 15, twenty: 20, fifty: 50, hundred: 100,++});++++/** The words that mean "fast": a time cap per task. */++const FAST_WORDS = Object.freeze(['fast', 'quicker', 'quick', 'quickly', 'snappy', 'speedy', 'speed', 'instant', 'instantly', 'low latency']);++++/**++ * What people write for the kinds Bench has. The kind's own id and name are added to these at parse time, so a new++ * kind is readable without touching this table.++ */++const KIND_ALIASES = Object.freeze({++ 'games.play': ['playable games', 'playable game', 'browser game', 'game', 'games'],++ 'games.three': ['three.js', 'three js', '3d scene', 'three'],++ 'design.svg': ['svg', 'svg art', 'illustration', 'drawing'],++ 'design.ui': ['ui', 'interface screens', 'screen', 'screens'],++ 'design.dataviz': ['dataviz', 'data viz', 'data visualisation', 'chart', 'graph'],++ 'design.motion': ['motion', 'animation', 'effects'],++ 'design.interface': ['component system', 'component', 'design system'],++ 'design.page': ['landing page', 'web page', 'website', 'page'],++ 'design.shader': ['shader', 'glsl'],++ 'design.model3d': ['3d model', 'gltf', 'model3d'],++ 'design.component': ['component arena'],++});++++const clean = (text) => String(text == null ? '' : text).toLowerCase().replace(/[,;]/g, ' ').replace(/\s+/g, ' ').trim();++++/** The kind the words name, or null. The longest alias wins, so "landing page" beats "page". */++function parseKind(text, kinds) {++ const hay = ' ' + clean(text) + ' ';++ const list = (Array.isArray(kinds) ? kinds : []).map((k) => ({ id: k.id, name: k.name }));++ const candidates = [];++ for (const k of list) {++ const aliases = new Set(KIND_ALIASES[k.id] || []);++ if (k.name) aliases.add(String(k.name).toLowerCase());++ aliases.add(String(k.id).toLowerCase());++ aliases.add(String(k.id).toLowerCase().replace(/\./g, ' '));++ for (const alias of aliases) {++ const a = clean(alias);++ if (!a) continue;++ if (hay.includes(' ' + a + ' ')) candidates.push({ id: k.id, alias: a });++ }++ }++ if (!candidates.length) return null;++ candidates.sort((x, y) => y.alias.length - x.alias.length || x.id.localeCompare(y.id));++ return candidates[0].id;++}++++/** The top price per solved task the words name, in dollars, or null. */++function parseCost(text) {++ const t = ' ' + clean(text) + ' ';++ // Dollars: $0.02++ const dollar = /\$\s*(\d+(?:\.\d+)?)/.exec(t);++ // Cents: 2 cents, 2¢, two cents, half a cent++ const cent = /\b(\d+(?:\.\d+)?|a|an|one|two|three|four|five|six|seven|eight|nine|ten|twelve|fifteen|twenty)\s*(?:cents?|¢)/.exec(t);++ const half = /\bhalf\s+(?:a\s+)?cent/.test(t);++ if (half) return 0.005;++ if (cent) {++ const raw = cent[1];++ const n = WORD_NUMBERS[raw] != null ? WORD_NUMBERS[raw] : Number(raw);++ if (Number.isFinite(n)) return Math.round(n / 100 * 1e6) / 1e6;++ }++ if (dollar) {++ const n = Number(dollar[1]);++ if (Number.isFinite(n)) return Math.round(n * 1e6) / 1e6;++ }++ return null;++}++++/** True when the words ask for fast. */++function parseFast(text) {++ const t = clean(text);++ return FAST_WORDS.some((w) => new RegExp('(^|[^a-z])' + w + '([^a-z]|$)').test(t));++}++++/**++ * Read the free text into the three controls. Returns { kind, max_cost, fast, notes }.++ * Nothing is guessed: an unknown word changes nothing.++ */++function parsePick(text, { kinds = [] } = {}) {++ const raw = clean(text);++ const notes = [];++ if (!raw) return { kind: null, max_cost: null, fast: false, notes, text: '' };++++ const kind = parseKind(raw, kinds);++ if (kind) notes.push('read the kind as ' + kind);++ else notes.push('no kind in the words: the kind stays as it was');++++ const maxCost = parseCost(raw);++ if (maxCost != null) {++ const cents = Math.round(maxCost * 10000) / 100;++ notes.push('read the price as at most ' + (cents + ' cents') + ' a solved task');++ } else notes.push('no price in the words: any price');++++ const fast = parseFast(raw);++ if (fast) notes.push('read "fast" as a time cap from the run times');++ else notes.push('no "fast" in the words: no time cap');++++ return { kind, max_cost: maxCost, fast, notes, text: raw };++}++++module.exports = { parsePick, parseKind, parseCost, parseFast, KIND_ALIASES, FAST_WORDS, WORD_NUMBERS };nnew file mode 100644@@@ -0,0 +1,177 @@++'use strict';++++/**++ * The pick (spec 3): the answer to "which model should I use for this job", read straight from the run data.++ *++ * Nothing here calls a model or a network service: the answer is the frontier of the kind — the models no other++ * model beats on both score and cost per solved task — with the numbers that produced it. A model with fewer than++ * three runs is greyed and says so; no number is ever invented.++ */++++const { frontier } = require('./frontier');++const { parsePick } = require('./parser');++const { KINDS, kindById, SHOWCASE_KIND_IDS } = require('../kinds');++const renderLib = require('../bench/render');++const { screenshotOf } = require('../http/gallery');++++/** The price steps the form offers, in dollars a solved task. */++const PRICE_STEPS = Object.freeze([++ { value: 0.005, label: '0.5¢ a solved task' },++ { value: 0.02, label: '2¢ a solved task' },++ { value: 0.1, label: '10¢ a solved task' },++]);++++/** Fewer runs than this and the numbers are too thin to stand on. */++const MIN_RUNS = 3;++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++++function median(list) {++ const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);++ if (!xs.length) return null;++ const mid = Math.floor(xs.length / 2);++ return xs.length % 2 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2;++}++++/** The band each model sits in, from the style-controlled board. */++function bandMap(fit) {++ const out = new Map();++ const bands = (fit && fit.bands && fit.bands.controlled) || [];++ for (const band of bands) for (const model of band.models || []) out.set(model, band.label);++ return out;++}++++/** The band a model sits in, from the board of one kind. */++async function boardForKind(ctx, kindId) {++ const board = require('../rating/board');++ return board.boardForKind(ctx, kindId);++}++++/** The kind with the most runs, when the reader did not name one. */++async function busiestKind(ctx) {++ const rows = await ctx.s.db.many(++ `SELECT r.kind_id AS id, COUNT(e.id)::int AS n++ FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id++ WHERE e.error IS NULL GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`);++ if (rows && rows[0] && rows[0].id) return rows[0].id;++ return SHOWCASE_KIND_IDS[0] || 'games.play';++}++++/** One model's newest output in the kind, the way the gallery shows it. */++async function sampleFor(ctx, kindId, model) {++ const rows = await ctx.s.db.many(++ `SELECT e.id, e.model, e.provider, e.cost_usd, e.ms, e.ui_check, r.id AS round_id, t.prompt++ FROM bench_entries e++ JOIN bench_rounds r ON r.id = e.round_id++ JOIN bench_tasks t ON t.id = r.task_id++ WHERE r.kind_id = $1 AND e.model = $2 AND e.error IS NULL AND r.status = 'finished'++ ORDER BY r.created_at DESC, e.id LIMIT 1`,++ [String(kindId), String(model)]);++ const row = rows && rows[0];++ if (!row) return null;++ return {++ id: row.id,++ round_id: row.round_id,++ prompt: row.prompt,++ model: row.model,++ provider: row.provider,++ cost_usd: num(row.cost_usd),++ ms: num(row.ms),++ screenshot: screenshotOf(row.ui_check),++ wrapper_url: renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(row.id),++ };++}++++/** Why the top pick is first, in one sentence, from the numbers. */++function whySentence(rows, kindName) {++ const top = rows[0];++ if (!top) return '';++ const rivals = rows.slice(1);++ const cents = (v) => (v == null ? 'an unknown price' : (Math.round(v * 10000) / 100) + '¢');++ const head = top.model + ' is first on ' + kindName + ' because no other model beats it on both score and cost per solved task: '++ + 'it scores ' + top.score + ' at ' + cents(top.cost_per_solved) + ' a solved task';++ if (!rivals.length) return head + ', and it is the only model with enough data to stand on.';++ const rival = rivals[0];++ const lower = rival.score < top.score++ ? 'scores lower (' + rival.score + ')'++ : 'costs more (' + cents(rival.cost_per_solved) + ')';++ return head + '. The closest, ' + rival.model + ', ' + lower + '.';++}++++/**++ * The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier++ * with every number that made it.++ */++async function answer(ctx, { kind = '', maxCost = null, fast = false, q = '', kinds = KINDS } = {}) {++ const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };++ const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);++ const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));++ const wantFast = parsed.fast || !!fast;++++ const all = await ctx.bench.valueRows(kindId);++ const fit = await boardForKind(ctx, kindId);++ const bands = bandMap(fit && fit.fit);++++ // The price comes first: a model over it is out whatever else it does.++ const priced = price == null ? all : all.filter((r) => r.cost_per_solved != null && r.cost_per_solved <= price + 1e-9);++++ // "Fast" is a time cap taken from the run data: the middle of the models' median times.++ const fastCap = wantFast ? median(priced.map((r) => r.median_ms).filter((v) => v > 0)) : null;++ const timed = fastCap == null ? priced : priced.filter((r) => !r.median_ms || r.median_ms <= fastCap + 1e-9);++++ // The frontier maths works on { model, score, cost }: the cost is the cost per solved task.++ const comparable = timed.filter((r) => r.score != null && r.cost_per_solved != null)++ .map((r) => ({ ...r, model: r.subject, cost: r.cost_per_solved }));++ const front = frontier(comparable);++ const order = new Map(comparable.map((r, i) => [r.subject, i]));++ front.sort((a, b) => (b.score - a.score)++ || ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))++ || a.subject.localeCompare(b.subject));++++ const rows = [];++ for (const r of front) {++ rows.push({++ model: r.subject,++ score: r.score,++ band: bands.get(r.subject) || null,++ votes: r.votes,++ cost_per_solved: r.cost_per_solved,++ median_ms: r.median_ms,++ runs: r.runs,++ thin: r.runs < MIN_RUNS,++ sample: await sampleFor(ctx, kindId, r.subject),++ });++ }++++ const kindInfo = kindById(kindId) || { id: kindId, name: kindId };++ const onFront = new Set(front.map((r) => r.subject));++ const alsoRan = timed.filter((r) => !onFront.has(r.subject)).map((r) => ({++ model: r.subject,++ score: r.score,++ cost_per_solved: r.cost_per_solved,++ median_ms: r.median_ms,++ runs: r.runs,++ thin: r.runs < MIN_RUNS,++ }));++ const others = alsoRan.length;++ const thinElsewhere = alsoRan.filter((r) => r.thin).length;++++ return {++ kind: { id: kindInfo.id, name: kindInfo.name, room: kindInfo.room || null },++ controls: {++ kind: kindId,++ max_cost: price == null ? null : price,++ fast: wantFast,++ q: String(q || ''),++ notes: parsed.notes || [],++ },++ filters: { max_cost: price == null ? null : price, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },++ rows,++ also_ran: alsoRan,++ why: whySentence(rows, kindInfo.name),++ counts: { models: all.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },++ generated_at: new Date().toISOString(),++ };++}++++module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence };@@@ -68,6 +68,7 @@ function gamesOf(votes) {}let cached = null;++const kindCache = new Map();/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {@@@ -84,6 +85,22 @@ async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {return value;}++/**++ * The board of one kind: the votes of that kind alone, fitted the same way as the whole board. A pick and a watch both++ * ask about one kind at a time, and the band a model sits in there is the band that kind gives it.++ */++async function boardForKind(ctx, kindId, { bootstrap = style.BOOTSTRAP, force = false } = {}) {++ const db = ctx.s.db;++ const votes = (await loadVotes(db)).filter((v) => String(v.kind_id) === String(kindId));++ const key = String(kindId) + ':' + votes.length + ':' + (votes.length ? votes[votes.length - 1].id : '') + ':' + num(bootstrap);++ const hit = kindCache.get(String(kindId));++ if (!force && hit && hit.key === key && Date.now() - hit.at < 5 * 60_000) return hit.value;++ const fit = style.fitRatings(gamesOf(votes), { bootstrap });++ const value = { fit, votes, generated_at: new Date().toISOString() };++ kindCache.set(String(kindId), { key, at: Date.now(), value });++ return value;++}++/*** The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,* so the board can be re-derived from the file alone.@@@ -120,4 +137,4 @@ function boardView(result) {};}--module.exports = { loadVotes, gamesOf, board, dumpJsonl, boardView, styleOf };++module.exports = { loadVotes, gamesOf, board, boardForKind, dumpJsonl, boardView, styleOf };@@@ -23,7 +23,9 @@ const NAV = [{ label: 'Showcase', href: '/showcase' },{ label: 'Gallery', href: '/gallery' },{ label: 'Battles', href: '/battle' },++ { label: 'Pick for me', href: '/pick' },{ label: 'Ratings', href: '/ratings' },++ { label: 'Watching', href: '/watching' },{ label: 'Docs', href: '/docs' },];@@@ -456,6 +456,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =);}await insertOutbox('bench.rating.changed', { kind_id: kindId, category: CATEGORY_DEFAULT, subjects: models });++ // The ratings have been refitted: this is where the watches are checked.++ if (typeof s.onRatingsChanged === 'function') {++ try { await s.onRatingsChanged(kindId, models); } catch (err) {++ if (log && typeof log.warn === 'function') log.warn('[OpenVibe.Bench] watch check failed: ' + (err && err.message ? err.message : err));++ }++ }}// ── Boards ──────────────────────────────────────────────@@@ -558,6 +564,44 @@ function createStore(s, { log = console, random = Math.random, config = null } =return { kind_id: kindId, category, period, type, rows };}++ /**++ * Every model that ran a kind, with the numbers the pick reads: the score (how often its output passed), the cost++ * per solved task, the median time a task took and how many runs it has. Every row is kept, including the ones++ * another model beats on both score and cost: the pick works out the frontier itself.++ */++ async function valueRows(kindId) {++ const cost = await costStats(kindId);++ const pass = await passRate(kindId);++ const eloRows = await boardElo(kindId);++ const rows = [];++ for (const [model, c] of cost) {++ const p = pass.get(model);++ const e = eloRows.find((x) => x.subject === model) || {};++ const trials = num(p && p.trials);++ const passed = num(p && p.passed);++ const quality = trials ? passed / trials : (num(e.win_rate) || 0);++ const totalCost = num(c.total_cost);++ const solved = passed || num(c.usable);++ rows.push({++ subject: model,++ score: Math.round(quality * 1000) / 10,++ quality: Math.round(quality * 1e6) / 1e6,++ votes: num(e.games),++ win_rate: num(e.win_rate) || 0,++ runs: num(c.usable),++ entries: num(c.entries),++ solved,++ cost_per_solved: solved ? Math.round((totalCost / solved) * 1e6) / 1e6 : null,++ median_cost_usd: num(c.median_cost),++ median_ms: num(c.median_ms),++ });++ }++ rows.sort((a, b) => (b.score - a.score)++ || ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))++ || a.subject.localeCompare(b.subject));++ return rows;++ }++async function modelSummary(model) {const name = String(model || '');const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);@@@ -660,7 +704,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =createTask, getTask,createRound, getRound, addEntry, maybeReady, finishRound,nextJudgement, castJudgement, pairwiseGames,-- board, modelSummary, dataset, datasetJsonl,++ board, valueRows, modelSummary, dataset, datasetJsonl,createEval, getEval, runEval, evalResults, evalDailyCount,KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,};nnew file mode 100644@@@ -0,0 +1,92 @@++'use strict';++++/**++ * The pick page (spec 3): plain controls, then the frontier as a table, then one sample output per model.++ * The page works without JavaScript: the form is a GET, and the words in the free-text box are read on the server.++ */++++const { html, raw } = require('../render/html');++const { frame } = require('../http/gallery');++const { PRICE_STEPS } = require('../pick/pick');++const { KINDS } = require('../kinds');++++const cents = (v) => (v == null ? '—' : (Math.round(v * 10000) / 100) + '¢');++const msText = (v) => (v == null ? '—' : v >= 1000 ? (Math.round(v / 100) / 10) + 's' : Math.round(v) + 'ms');++++function pickForm(controls) {++ const kinds = KINDS.map((k) => html`<option value="${k.id}"${k.id === controls.kind ? ' selected' : ''}>${k.name}</option>`);++ const prices = [html`<option value=""${controls.max_cost == null ? ' selected' : ''}>Any price</option>`].concat(++ PRICE_STEPS.map((p) => html`<option value="${p.value}"${controls.max_cost != null && Math.abs(controls.max_cost - p.value) < 1e-9 ? ' selected' : ''}>${p.label}</option>`));++ return html`<form class="pk-form" method="get" action="/pick" data-pk-form="1">++<label>Kind<select name="kind">${kinds}</select></label>++<label>Top price per solved task<select name="max_cost">${prices}</select></label>++<label class="pk-check"><input type="checkbox" name="fast" value="1"${controls.fast ? ' checked' : ''}> Fast</label>++<label class="pk-words">Or say it in words<input type="text" name="q" value="${controls.q || ''}" placeholder="landing page under 2 cents, fast"></label>++<button class="bm-btn" type="submit">Pick</button>++</form>`;++}++++function pickTable(a) {++ if (!a.rows.length) {++ return html`<div class="bm-empty"><h1>Nothing to pick yet</h1><p>${a.counts.models++ ? 'No model on this frontier has both a score and a cost per solved task yet, so there is no answer to give. Nothing is guessed here.'++ : 'No model has run this kind yet, so there is nothing to compare. The answer appears as soon as runs are published.'}</p></div>`;++ }++ const rows = a.rows.map((r) => html`<tr class="${r.thin ? 'pk-thin' : ''}">++<td><a class="br-model" href="/models/${r.model}">${r.model}</a></td>++<td><span class="pk-score">${r.score}</span> <span class="pk-band">${r.band ? 'band ' + r.band : 'no band yet'}</span></td>++<td class="br-num">${cents(r.cost_per_solved)}</td>++<td class="br-num">${msText(r.median_ms)}</td>++<td class="br-num">${r.runs}${r.thin ? html` <span class="pk-thin-note">too few runs</span>` : ''}</td>++</tr>`);++ return html`<div class="br-table-wrap"><table class="br-table pk-table"><thead><tr>++<th scope="col">Model</th><th scope="col">Score</th><th scope="col" class="br-num">Cost per solved task</th>++<th scope="col" class="br-num">Median time</th><th scope="col" class="br-num">Runs</th>++</tr></thead><tbody>${rows}</tbody></table></div>`;++}++++function pickSamples(a) {++ const withSample = a.rows.filter((r) => r.sample);++ if (!withSample.length) return '';++ return html`<section class="bm-section" aria-labelledby="pk-samples-h"><h2 id="pk-samples-h">One output each</h2>++<p class="bm-fine">The newest published output of every model on the frontier, live in its sandboxed frame.</p>++<div class="pk-samples">${withSample.map((r) => html`<article class="pk-card">++<h3 class="pk-card-h">${r.model}</h3>++<div class="pk-shot">${raw(frame(r.sample, !r.sample.screenshot))}</div>++<p class="bm-fine pk-prompt">${r.sample.prompt}</p>++</article>`)}</div></section>`;++}++++function pickPage(a) {++ const notes = (a.controls.notes || []).length++ ? html`<p class="pk-notes bm-fine">Read from the words: ${a.controls.notes.join('; ')}.</p>`++ : '';++ const thin = a.rows.filter((r) => r.thin).length;++ const thinNote = thin++ ? html`<p class="pk-thin-note bm-fine">${thin} model${thin === 1 ? '' : 's'} on this frontier have fewer than ${a.filters.min_runs} runs. Their numbers are shown greyed, because that is all the data there is.</p>`++ : '';++ const fastNote = a.filters.fast && a.filters.fast_cap_ms != null++ ? html`<p class="bm-fine">Fast: a median time of ${msText(a.filters.fast_cap_ms)} a task or under, the middle of the run times.</p>`++ : '';++ const also = (a.also_ran || []).length ? html`<section class="bm-section" aria-labelledby="pk-also-h"><h2 id="pk-also-h">Also ran this kind</h2>++<ul class="pk-also">${(a.also_ran || []).map((r) => html`<li class="${r.thin ? 'pk-thin' : ''}"><b>${r.model}</b> · score ${r.score} · ${cents(r.cost_per_solved)} a solved task · ${r.runs} run${r.runs === 1 ? '' : 's'}${r.thin ? ' — too few runs to stand on, so these numbers are greyed' : ' — another model is at least as good on score and at least as cheap'}</li>`)}</ul>++</section>` : '';++ return html`<div class="bm-wrap pk" id="pk-pick">++<header class="bm-page-h"><p class="bm-kicker">OpenVibe.Bench</p><h1>Pick for me</h1>++<p class="bm-lede">Say the job with plain controls. No model is called and nothing is charged: the answer comes from the runs Bench has published.</p></header>++${pickForm(a.controls)}++<p class="bm-fine">The free-text box is read by a plain parser, never by a model. It only sets the three controls above.</p>++${notes}++<section class="bm-section pk-answer" aria-labelledby="pk-answer-h">++<h2 id="pk-answer-h">The frontier for ${a.kind.name}</h2>++<p class="pk-why">${a.why}</p>++${fastNote}++${pickTable(a)}++${thinNote}++</section>++${also}++${pickSamples(a)}++</div>`;++}++++module.exports = { pickPage, pickForm, pickTable, pickSamples };@@@ -7,6 +7,7 @@* Ids and classes start with `br-`.*/const { html, raw, render } = require('../render/html');++const { benchmarkUseThis, modelUseThis } = require('./usethis');const renderLib = require('../bench/render');const MEDIA_ORIGIN = 'https://openvibe.media';@@@ -201,7 +202,7 @@ function benchmarkCard(b) {return html`<li class="br-card"><a class="br-card-link" href="${benchmarkHref(b.id)}"><span class="br-card-top">${kindChip(b.kind)}<span class="br-card-v">v${b.latest_version || 0}</span></span><span class="br-card-title">${b.title}</span><span class="br-card-desc">${b.description}</span><span class="br-card-foot">${plural(tasks, 'task')}${b.runs != null ? html` · ${plural(b.runs, 'run')}` : ''}${b.parent ? ' · a fork' : ''}</span></a></li>`;}--function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null }) {++function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '' }) {const d = definition || {};const tasks = Array.isArray(d.tasks) ? d.tasks : [];const shown = tasks.filter((t) => !t.holdout && t.prompt);@@@ -227,6 +228,7 @@ ${entries.length ? rankingsTable(entries, metric) : empty('No verified runs yet'<ol class="br-tasks">${shown.map((t) => html`<li class="br-task"><span class="br-task-id br-mono">${t.id}</span><p>${t.prompt}</p></li>`)}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section><section class="bm-section" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section>++${benchmarkUseThis({ baseUrl, benchmark, entries, viewer })}<p class="br-fine br-api">The same data as JSON: <a href="/api/v1/benchmarks/${encodeURIComponent(benchmark.id)}">definition</a> · <a href="/api/v1/rankings?benchmark=${encodeURIComponent(benchmark.id)}&metric=${metric}">rankings</a></p></div>`;}@@@ -322,7 +324,7 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta</div>`;}--function modelPage({ model, summary, runs = [], titles = {} }) {++function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '' }) {const rows = (summary && summary.benchmarks) || [];return html`<div class="bm-wrap br" id="br-model"><nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>@@@ -330,6 +332,7 @@ function modelPage({ model, summary, runs = [], titles = {} }) {<section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}++${modelUseThis({ baseUrl, model, rows, viewer })}</div>`;}nnew file mode 100644@@@ -0,0 +1,88 @@++'use strict';++++/**++ * "Use this" (spec 3): the copy boxes. A benchmark and a model page carry the badge's Markdown and HTML; the battle++ * page carries the embed's iframe snippet. The boxes are plain textareas, so copying works without JavaScript.++ */++++const { html } = require('../render/html');++++/** The badge's URL for a benchmark and a model. */++function badgeUrl(baseUrl, benchmarkId, model) {++ return String(baseUrl || '').replace(/\/+$/, '') + '/badge/' + encodeURIComponent(benchmarkId) + '/' + encodeURIComponent(model) + '.svg';++}++++/** The Markdown and the HTML of the badge. */++function badgeSnippets(baseUrl, benchmarkId, model, title) {++ const url = badgeUrl(baseUrl, benchmarkId, model);++ const alt = (title ? title + ' on OpenVibe Bench' : 'OpenVibe Bench') + ' badge';++ return { markdown: '', html: '<img src="' + url + '" alt="' + alt + '">' };++}++++/** The embed's snippet: an iframe that may be framed by other sites. */++function embedSnippets(baseUrl, battleId, title) {++ const url = String(baseUrl || '').replace(/\/+$/, '') + '/embed/battle/' + encodeURIComponent(battleId);++ const name = title || 'OpenVibe Bench battle';++ return {++ markdown: '[' + name + '](' + url + ')',++ html: '<iframe src="' + url + '" width="100%" height="560" style="border:0" loading="lazy" title="' + name + '"></iframe>',++ };++}++++/** A copy box: one textarea per snippet. */++function copyBox({ heading, pairs }) {++ return html`<div class="br-usethis-box"><h3 class="br-usethis-h">${heading}</h3>++${pairs.map((p) => html`<label class="br-usethis-row"><span class="br-fine">${p.label}</span><textarea readonly rows="2" aria-label="${p.label}">${p.value}</textarea></label>`)}++</div>`;++}++++/** The follow form: a plain POST, so it works without JavaScript. */++function followForm({ benchmarkId, model = null, back = '/watching', signedIn = false }) {++ if (!signedIn) return html`<p class="br-fine">Sign in to follow this, and Bench will tell you when it moves.</p>`;++ return html`<form class="br-follow" method="post" action="/watch/follow">++<input type="hidden" name="benchmark_id" value="${benchmarkId}">++${model ? html`<input type="hidden" name="model" value="${model}">` : ''}++<input type="hidden" name="back" value="${back}">++<button class="bm-btn bm-btn-small" type="submit">${model ? 'Follow this model' : 'Follow this benchmark'}</button>++</form>`;++}++++/** The benchmark page: a badge for each model it ranks, and the follow button. */++function benchmarkUseThis({ baseUrl, benchmark, entries = [], viewer = null, back = '' }) {++ const boxes = entries.slice(0, 4).map((e) => {++ const s = badgeSnippets(baseUrl, benchmark.id, e.model, benchmark.title);++ return copyBox({ heading: e.model, pairs: [{ label: 'Markdown', value: s.markdown }, { label: 'HTML', value: s.html }] });++ });++ return html`<section class="bm-section" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>++<p class="br-fine">A badge for your README, and a watch that tells you when this benchmark moves.</p>++${boxes}++${followForm({ benchmarkId: benchmark.id, back: back || ('/benchmarks/' + benchmark.id), signedIn: Boolean(viewer && viewer.kind === 'user') })}++</section>`;++}++++/** The model page: a badge for each benchmark the model ran, and a watch for each. */++function modelUseThis({ baseUrl, model, rows = [], viewer = null, back = '' }) {++ const boxes = rows.slice(0, 4).map((b) => {++ const s = badgeSnippets(baseUrl, b.benchmark_id, model, b.title);++ return html`<div class="br-usethis-box"><h3 class="br-usethis-h">${b.title}</h3>++<label class="br-usethis-row"><span class="br-fine">Markdown</span><textarea readonly rows="2" aria-label="Markdown">${s.markdown}</textarea></label>++<label class="br-usethis-row"><span class="br-fine">HTML</span><textarea readonly rows="2" aria-label="HTML">${s.html}</textarea></label>++${followForm({ benchmarkId: b.benchmark_id, model, back: back || ('/models/' + encodeURIComponent(model)), signedIn: Boolean(viewer && viewer.kind === 'user') })}++</div>`;++ });++ return html`<section class="bm-section" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>++<p class="br-fine">A badge for your README, and a watch for each benchmark this model ran.</p>++${boxes}++</section>`;++}++++/** The battle page: the embed's iframe snippet. */++function battleUseThis({ baseUrl, battle, title }) {++ const s = embedSnippets(baseUrl, battle.id, title);++ return html`<section class="bm-section br-usethis" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>++<p class="br-fine">An embed of this battle for another site. It may be framed; voting opens Bench in a new tab.</p>++<label class="br-usethis-row"><span class="br-fine">HTML</span><textarea readonly rows="3" aria-label="HTML">${s.html}</textarea></label>++</section>`;++}++++module.exports = { badgeUrl, badgeSnippets, embedSnippets, copyBox, followForm, benchmarkUseThis, modelUseThis, battleUseThis };nnew file mode 100644@@@ -0,0 +1,28 @@++'use strict';++++/**++ * The Watching page (spec 6): the benchmarks and models a person follows, with an unfollow button on each.++ * The buttons are plain forms, so the page works without JavaScript.++ */++++const { html } = require('../render/html');++++function watchingPage(watches, { viewer = null } = {}) {++ const items = watches.map((w) => html`<li class="wk-item">++<div class="wk-what"><b>${w.benchmark_title || w.benchmark_id}</b>++${w.model ? html`<span class="br-chip">${w.model}</span>` : html`<span class="br-chip">the whole benchmark</span>`}++<span class="bm-fine">followed since ${String(w.created_at || '').slice(0, 10)}</span></div>++<form method="post" action="/watch/unfollow" class="wk-form">++<input type="hidden" name="id" value="${w.id}">++<button class="bm-btn bm-btn-quiet bm-btn-small" type="submit">Unfollow</button>++</form>++</li>`);++ return html`<div class="bm-wrap wk" id="wk-watching">++<header class="bm-page-h"><p class="bm-kicker">OpenVibe.Bench</p><h1>Watching</h1>++<p class="bm-lede">The benchmarks and models you follow. When a score moves by more than its own interval, or a model overtakes another one, Bench tells the Network and the notice lands in your Action Center. One notice a day per watch.</p></header>++${viewer && viewer.kind === 'user' ? '' : html`<p class="bm-fine"><a class="bm-btn bm-btn-small" href="/auth/login?next=%2Fwatching">Sign in with OpenVibe</a> to follow a benchmark or a model.</p>`}++${watches.length ? html`<ul class="wk-list">${items}</ul>` : html`<div class="bm-empty"><h1>Nothing followed yet</h1><p>Follow a benchmark from its page, or a model on a benchmark from the model's page.</p></div>`}++</div>`;++}++++module.exports = { watchingPage };nnew file mode 100644@@@ -0,0 +1,266 @@++'use strict';++++/**++ * Model watch (spec 6): follow a benchmark, or a model on a benchmark, and hear when it moves.++ *++ * Two things are worth a notice:++ * - a model's style-controlled score moves by more than its own interval (the board of B9), and++ * - a model overtakes another one on that benchmark: its band sits fully above the other's.++ *++ * The check runs where the ratings are refitted, so the board it reads is the board the person would see. One++ * notice per watch per day at most, and the text names the benchmark, the two models and the numbers.++ *++ * The notice itself is a Network notification (network.notification-push-request@1): it lands in the Action Center,++ * and email or chat follow the person's own Network settings. Bench never sends email.++ */++++const { ids } = require('openvibe-contracts');++++const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++const round = (v) => (v == null ? null : Math.round(Number(v) * 10) / 10);++const dayOf = (v) => (v == null ? null : new Date(v).toISOString().slice(0, 10));++++/** The notification client: Bench's own service token, pushed at the Network. */++function notifier(ctx) {++ if (ctx.benchNotifier) return ctx.benchNotifier;++ const { createClient } = require('openvibe-sdk/core');++ const { createServiceTokenClient } = require('openvibe-sdk/auth');++ const { createNotificationsClient } = require('openvibe-sdk/notifications');++ const network = ctx.config.networkInternalUrl;++ const tokens = createServiceTokenClient({++ tokenUrl: network + '/oauth/token',++ clientId: ctx.config.oauth && ctx.config.oauth.clientId,++ clientSecret: ctx.config.oauth && ctx.config.oauth.clientSecret,++ audience: 'openvibe.network',++ scope: 'network.notifications.push',++ fetch: ctx.fetchImpl || globalThis.fetch,++ });++ const client = createClient({++ tokenProvider: tokens,++ baseUrls: { network },++ fetch: ctx.fetchImpl || globalThis.fetch,++ timeoutMs: 8000,++ });++ ctx.benchNotifier = createNotificationsClient(client, { baseUrl: network });++ return ctx.benchNotifier;++}++++/** The UTC day a timestamp falls in. */++function today(now) {++ return new Date(now || Date.now()).toISOString().slice(0, 10);++}++++/** True when the watch may get a notice: one notice per watch per day. */++function mayNotify(watch, stamp) {++ return dayOf(watch && watch.last_notified_at) !== today(stamp);++}++++function createWatches(ctx) {++ const db = ctx.s.db;++++ /** The watches of one person, with the benchmark they are on. */++ async function listFor(subject) {++ return db.many(++ `SELECT w.id, w.subject, w.benchmark_id, w.model, w.user_id, w.created_at, w.last_notified_at,++ b.title AS benchmark_title, b.kind AS benchmark_kind, b.latest_version++ FROM bench_watches w JOIN bench_benchmarks b ON b.id = w.benchmark_id++ WHERE w.subject = $1 ORDER BY w.created_at DESC`,++ [String(subject || '')]);++ }++++ /** Follow a benchmark, or a model on a benchmark. Following twice changes nothing. */++ async function follow({ subject, benchmarkId, model = null, userId = null }) {++ const benchmark = await ctx.registry.getBenchmark(String(benchmarkId || ''));++ if (!benchmark) return { ok: false, status: 404, error: 'No such benchmark.' };++ const row = await db.maybe(++ `INSERT INTO bench_watches (id, subject, benchmark_id, model, user_id, created_at)++ VALUES ($1, $2, $3, $4, $5, $6::timestamptz)++ ON CONFLICT (subject, benchmark_id, COALESCE(model, ''))++ DO UPDATE SET user_id = COALESCE(EXCLUDED.user_id, bench_watches.user_id)++ RETURNING *`,++ [ids.newId('watch'), String(subject), benchmark.id, model ? String(model) : null,++ userId == null ? null : Number(userId), ctx.s.iso()]);++ return { ok: true, watch: row };++ }++++ /** Stop watching. */++ async function unfollow({ subject, id }) {++ const row = await db.maybe('DELETE FROM bench_watches WHERE id = $1 AND subject = $2 RETURNING id',++ [String(id || ''), String(subject || '')]);++ return { ok: Boolean(row) };++ }++++ /** The style-controlled fit of a kind. */++ async function fitOf(kindId) {++ const board = await require('../rating/board').boardForKind(ctx, kindId);++ return board.fit;++ }++++ /** The board as it was last seen, so a move can be measured against it. */++ async function previous(kindId) {++ const rows = await db.many('SELECT * FROM bench_watch_ratings WHERE kind_id = $1', [String(kindId)]);++ return new Map(rows.map((r) => [r.model, r]));++ }++++ /** Remember the board we have just seen. */++ async function remember(kindId, models) {++ if (!models.length) return;++ const values = [];++ const params = [];++ for (const m of models) {++ values.push('($1, $' + (params.length + 2) + ', $' + (params.length + 3) + ', $' + (params.length + 4)++ + ', $' + (params.length + 5) + ', $' + (params.length + 6) + ', now())');++ params.push(m.model, m.rating, m.lower, m.upper, m.band);++ }++ await db.exec(++ `INSERT INTO bench_watch_ratings (kind_id, model, rating, ci_lower, ci_upper, band, updated_at)++ VALUES ${values.join(', ')}++ ON CONFLICT (kind_id, model) DO UPDATE SET rating = EXCLUDED.rating, ci_lower = EXCLUDED.ci_lower,++ ci_upper = EXCLUDED.ci_upper, band = EXCLUDED.band, updated_at = now()`,++ [String(kindId), ...params]);++ }++++ /** The notice a watch gets from the board, or null. */++ function noticeFor(watch, models, bands, before, all) {++ const byModel = new Map(models.map((m) => [m.model, m]));++ const followed = watch.model ? byModel.get(watch.model) : null;++ const title = watch.benchmark_title || watch.benchmark_id;++++ if (watch.model) {++ if (!followed) return null;++ const was = before.get(watch.model);++ if (was && was.rating != null && followed.rating != null) {++ const delta = followed.rating - num(was.rating);++ const interval = followed.upper - followed.lower;++ if (Math.abs(delta) > interval) {++ return {++ kind: 'move',++ title: 'Bench: ' + title,++ message: title + ': ' + watch.model + ' moved from ' + round(num(was.rating)) + ' to '++ + round(followed.rating) + ' (' + (delta > 0 ? '+' : '') + round(delta) + '), more than its '++ + round(interval) + '-point interval (' + round(followed.lower) + '–' + round(followed.upper) + ').',++ models: [watch.model],++ };++ }++ }++ const over = models.find((m) => m.model !== watch.model && m.lower > followed.upper++ && !all.some((w) => w.subject === watch.subject && w.benchmark_id === watch.benchmark_id && w.model === m.model));++ if (over) {++ return {++ kind: 'overtake',++ title: 'Bench: ' + title,++ message: title + ': ' + over.model + ' (' + round(over.rating) + ', ' + round(over.lower) + '–'++ + round(over.upper) + ') is now fully above ' + watch.model + ' (' + round(followed.rating) + ', '++ + round(followed.lower) + '–' + round(followed.upper) + ').',++ models: [over.model, watch.model],++ };++ }++ return null;++ }++++ // A benchmark watch: any model that moved by more than its interval, and any model that overtook another.++ for (const m of models) {++ const was = before.get(m.model);++ if (!was || was.rating == null || m.rating == null) continue;++ const delta = m.rating - num(was.rating);++ const interval = m.upper - m.lower;++ if (Math.abs(delta) > interval) {++ return {++ kind: 'move',++ title: 'Bench: ' + title,++ message: title + ': ' + m.model + ' moved from ' + round(num(was.rating)) + ' to ' + round(m.rating)++ + ' (' + (delta > 0 ? '+' : '') + round(delta) + '), more than its ' + round(interval)++ + '-point interval (' + round(m.lower) + '–' + round(m.upper) + ').',++ models: [m.model],++ };++ }++ }++ const over = models.find((m) => models.some((o) => o.model !== m.model && m.lower > o.upper));++ if (over) {++ const under = models.find((o) => o.model !== over.model && over.lower > o.upper);++ return {++ kind: 'overtake',++ title: 'Bench: ' + title,++ message: title + ': ' + over.model + ' (' + round(over.rating) + ', ' + round(over.lower) + '–'++ + round(over.upper) + ') is now fully above ' + (under ? under.model : 'the model below it') + '.',++ models: [over.model, under ? under.model : null].filter(Boolean),++ };++ }++ return null;++ }++++ /** Push the notice to the Network. The Network decides whether it reaches email or chat. */++ async function notify(watch, notice) {++ if (watch.user_id == null) return { sent: false, reason: 'no_network_user' };++ try {++ const out = await notifier(ctx).push({++ userId: Number(watch.user_id),++ type: 'WATCH_TRIGGERED',++ title: notice.title,++ message: notice.message,++ url: (ctx.config.baseUrl || 'https://bench.openvibe.services') + '/benchmarks/' + watch.benchmark_id,++ category: 'service',++ priority: 'normal',++ service: 'bench',++ });++ return { sent: Boolean(out && out.sent), skipped: Boolean(out && out.skipped) };++ } catch (err) {++ ctx.log.warn('[OpenVibe.Bench] watch notice failed: ' + (err && err.message ? err.message : err));++ return { sent: false, reason: 'push_failed' };++ }++ }++++ /**++ * Look at the board of a kind and tell the watchers what changed. Runs where the ratings are refitted.++ */++ async function check(kindId, { now = null } = {}) {++ const kind = String(kindId || '');++ if (!kind) return { notices: [], checked: 0 };++ const watches = await db.many(++ `SELECT w.*, b.title AS benchmark_title, b.kind AS benchmark_kind++ FROM bench_watches w JOIN bench_benchmarks b ON b.id = w.benchmark_id++ WHERE b.kind = $1`,++ [kind]);++ if (!watches.length) return { notices: [], checked: 0 };++++ const fit = await fitOf(kind);++ const bands = new Map();++ for (const b of ((fit.bands && fit.bands.controlled) || [])) for (const m of b.models || []) bands.set(m, b.label);++ const models = (fit.models || []).map((m) => ({++ model: m.model,++ rating: num(m.controlled.rating),++ lower: num(m.controlled.ci_lower),++ upper: num(m.controlled.ci_upper),++ band: bands.get(m.model) || null,++ votes: m.votes,++ }));++ const before = await previous(kind);++ await remember(kind, models);++++ const notices = [];++ const stamp = now || new Date().toISOString();++ for (const w of watches) {++ const notice = noticeFor(w, models, bands, before, watches);++ if (!notice) continue;++ if (!mayNotify(w, stamp)) continue; // one notice per watch per day++ const sent = await notify(w, notice);++ await db.exec('UPDATE bench_watches SET last_notified_at = $1 WHERE id = $2', [stamp, w.id]);++ notices.push({ watch: w.id, kind: notice.kind, message: notice.message, sent });++ }++ return { notices, checked: watches.length };++ }++++ /** The event Bench already publishes when the ratings change, so a watch rides on it. */++ async function publishRatingChanged(kindId, subjects) {++ const row = await ctx.bench.insertOutbox('bench.rating.changed', {++ kind_id: String(kindId || ''),++ subjects: (subjects || []).map(String),++ at: new Date().toISOString(),++ });++ return row;++ }++++ return { listFor, follow, unfollow, check, notify, publishRatingChanged, noticeFor, notifier };++}++++module.exports = { createWatches, notifier, today, dayOf, mayNotify };nnew file mode 100644@@@ -0,0 +1,52 @@++'use strict';++++/** The badge (B10): escaped text, a width from the text, the right headers, and a grey badge with a 404. */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { badgeSvg } = require('../server/http/badge');++++(async () => {++ await check('the text of the badge is escaped', () => {++ const svg = badgeSvg({ label: 'a & b <script>', message: '"x" \'y\'' });++ assert.ok(svg.includes('&'), 'the ampersand');++ assert.ok(svg.includes('<script>'), 'the angle brackets');++ assert.ok(!svg.includes('<script>'), 'no raw script tag');++ });++++ await check('the width comes from the text', () => {++ const short = badgeSvg({ label: 'ab', message: 'c' });++ const long = badgeSvg({ label: 'a much longer benchmark title', message: 'c' });++ const w = (s) => Number(/width="(\d+)"/.exec(s)[1]);++ assert.ok(w(long) > w(short), 'the longer badge is wider: ' + w(long) + ' vs ' + w(short));++ });++++ const t = await boot();++ const list = (await t.get('/api/v1/benchmarks')).json();++ const benchmark = (list.benchmarks || list)[0];++++ await check('an unknown benchmark gets a grey badge and a 404', async () => {++ const r = await t.get('/badge/bmk_nope/nope.svg');++ assert.strictEqual(r.status, 404, r.text.slice(0, 200));++ assert.ok(/image\/svg\+xml/.test(r.headers.get('content-type')), 'the type: ' + r.headers.get('content-type'));++ assert.strictEqual(r.headers.get('content-security-policy'), "default-src 'none'; style-src 'unsafe-inline'");++ assert.strictEqual(r.headers.get('x-content-type-options'), 'nosniff');++ assert.strictEqual(r.headers.get('cache-control'), 'public, max-age=3600');++ assert.ok(r.text.includes('no data'), 'it says there is no data');++ assert.ok(r.text.includes('<svg'), 'and it is still a picture');++ });++++ await check('a model with no runs gets a grey badge and a 404', async () => {++ const r = await t.get('/badge/' + benchmark.id + '/no-such-model.svg');++ assert.strictEqual(r.status, 404);++ assert.ok(r.text.includes('no data'), 'it says there is no data');++ });++++ await check('a badge may be used from another site', async () => {++ const r = await t.get('/badge/' + benchmark.id + '/no-such-model.svg');++ assert.strictEqual(r.headers.get('cross-origin-resource-policy'), 'cross-origin');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,61 @@++'use strict';++++/** The embed (B10): only that route may be framed, no cookies, and voting opens Bench in a new tab. */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++(async () => {++ const t = await boot();++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob'); // the round belongs to somebody else: a battle is never your own++ const entrySvc = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.entry.create'] });++++ const task = await t.get('/api/v1/kinds/games.play/tasks', { as: bob, json: { prompt: 'A game for an embed.', public: true } });++ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { as: bob, json: {} });++ const roundId = rr.json().round.id;++ const e1 = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: entrySvc, json: { provider: 'acme', model: 'alpha-1', payload: { html: '<h1>a</h1>' } } });++ const e2 = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: entrySvc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<h1>b</h1>' } } });++ assert.strictEqual(e1.status, 201, e1.text);++ assert.strictEqual(e2.status, 201, e2.text);++++ // A battle needs a finished round: the job would do it, the test does it directly.++ await t.ctx.s.db.exec("UPDATE bench_rounds SET status = 'finished' WHERE id = $1", [roundId]);++ const battle = await t.get('/battle', { as: alice });++ assert.strictEqual(battle.status, 200, battle.text.slice(0, 200));++ const id = /data-battle="([^"]+)"/.exec(battle.text)[1];++++ await check('the embed may be framed, and every other route may not', async () => {++ const e = await t.get('/embed/battle/' + id);++ assert.strictEqual(e.status, 200, e.text.slice(0, 200));++ assert.ok(/frame-ancestors \*/.test(e.headers.get('content-security-policy')), 'the embed may be framed: ' + e.headers.get('content-security-policy'));++ const home = await t.get('/');++ assert.ok(/frame-ancestors 'none'/.test(home.headers.get('content-security-policy')), 'the rest refuses frames: ' + home.headers.get('content-security-policy'));++ assert.strictEqual(home.headers.get('x-frame-options'), 'DENY');++ });++++ await check('the embed reads and sets no cookie', async () => {++ const e = await t.get('/embed/battle/' + id);++ assert.strictEqual(e.headers.get('set-cookie'), null, 'no cookie is set');++ const home = await t.get('/');++ assert.strictEqual(home.headers.get('set-cookie'), null, 'and none on the other pages either');++ });++++ await check('the embed keeps the frames sandboxed and sends the vote to a new tab', async () => {++ const e = await t.get('/embed/battle/' + id);++ assert.ok(e.text.includes('sandbox="allow-scripts allow-same-origin"'), 'the frames are sandboxed');++ assert.ok(e.text.includes('target="_blank"'), 'the vote opens a new tab');++ assert.ok(e.text.includes('rel="noopener"'), 'and it is safe');++ assert.ok(e.text.includes('noindex'), 'it is not indexed');++ assert.ok(e.text.includes('OpenVibe Bench'), 'it links back');++ });++++ await check('an unknown battle is a plain 404, still framable', async () => {++ const e = await t.get('/embed/battle/bbs_nope');++ assert.strictEqual(e.status, 404);++ assert.ok(/frame-ancestors \*/.test(e.headers.get('content-security-policy')));++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });@@@ -37,7 +37,7 @@ async function startNetwork() {const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });const publicPem = publicKey.export({ type: 'spki', format: 'pem' });const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' });-- const st = { users: new Map(), byUsername: new Map(), codes: new Map(), refresh: new Map(), requests: [], tokenRequests: [], down: false };++ const st = { users: new Map(), byUsername: new Map(), codes: new Map(), refresh: new Map(), requests: [], tokenRequests: [], pushes: [], down: false };let issuer = null;let nextUserId = 1;@@@ -74,6 +74,12 @@ async function startNetwork() {if (st.down) return problem(json, 503, 'network.down', 'Network is restarting');if (url.pathname === '/api/.well-known/jwks') return json(200, { public_key: publicPem, algorithm: 'RS256' });++ if (url.pathname === '/internal/notifications/push' && req.method === 'POST') {++ let body = {};++ try { body = JSON.parse(raw.toString('utf8') || '{}'); } catch { body = {}; }++ st.pushes.push(body);++ return json(200, { sent: true });++ }if (url.pathname === '/oauth/revoke') return json(200, {});if (url.pathname === '/oauth/token' && req.method === 'POST') {const ct = String(req.headers['content-type'] || '');@@@ -116,7 +122,7 @@ async function startNetwork() {issuer = srv.url;return {-- url: srv.url, publicPem, privatePem, state: st, requests: st.requests, tokenRequests: st.tokenRequests,++ url: srv.url, publicPem, privatePem, state: st, requests: st.requests, tokenRequests: st.tokenRequests, pushes: st.pushes,addUser, userToken, serviceToken, signJwt: (claims) => signJwt(claims, privatePem), issuer: () => issuer,issueCode(user, challenge) { const code = `code_${crypto.randomBytes(8).toString('hex')}`; st.codes.set(code, { user, challenge }); return code; },setDown(v) { st.down = v; },nnew file mode 100644@@@ -0,0 +1,142 @@++'use strict';++++/**++ * The pick (B10): the frontier maths, the keyword parser, thin data greyed, and the page and the API answering the++ * same thing. No model is called anywhere in here, and nothing is charged.++ */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { frontier } = require('../server/pick/frontier');++const { parsePick } = require('../server/pick/parser');++const { KINDS } = require('../server/kinds');++++(async () => {++ await check('the frontier drops the models another model beats on score and on cost', () => {++ const rows = [++ { model: 'a', score: 90, cost: 0.02 },++ { model: 'b', score: 80, cost: 0.01 },++ { model: 'c', score: 95, cost: 0.05 },++ { model: 'd', score: 70, cost: 0.03 },++ ];++ // a beats d on score and on cost, so d is not on the frontier; b is cheaper than a, so it stays.++ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);++ });++++ await check('a tie keeps both models on the frontier', () => {++ const rows = [{ model: 'a', score: 90, cost: 0.02 }, { model: 'b', score: 90, cost: 0.02 }];++ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b']);++ });++++ await check('a model with no cost is never compared, and never dominates', () => {++ const rows = [{ model: 'a', score: 90, cost: 0.02 }, { model: 'b', score: 90, cost: null }];++ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a']);++ });++++ await check('the keyword parser reads the kind, the price and fast', () => {++ const p = parsePick('landing page under 2 cents, fast', { kinds: KINDS });++ assert.strictEqual(p.kind, 'design.page');++ assert.strictEqual(p.max_cost, 0.02);++ assert.strictEqual(p.fast, true);++ });++++ await check('the keyword parser reads dollars, words and half a cent', () => {++ assert.strictEqual(parsePick('under $0.05', { kinds: KINDS }).max_cost, 0.05);++ assert.strictEqual(parsePick('ten cents', { kinds: KINDS }).max_cost, 0.1);++ assert.strictEqual(parsePick('half a cent', { kinds: KINDS }).max_cost, 0.005);++ assert.strictEqual(parsePick('two cents', { kinds: KINDS }).max_cost, 0.02);++ });++++ await check('the keyword parser changes nothing when the words say nothing', () => {++ const p = parsePick('something nice please', { kinds: KINDS });++ assert.strictEqual(p.kind, null);++ assert.strictEqual(p.max_cost, null);++ assert.strictEqual(p.fast, false);++ });++++ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' } });++ const alice = t.network.addUser('alice');++ const entrySvc = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.entry.create'] });++ const judgeSvc = t.network.serviceToken({ sub: 'svc:judge', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });++++ // alpha-1 runs three times, beta-2 once: beta-2 is too thin to stand on.++ const rounds = [];++ for (let i = 0; i < 3; i++) {++ const task = await t.get('/api/v1/kinds/design.page/tasks', { as: alice, json: { prompt: 'A page ' + i, public: true } });++ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { as: alice, json: {} });++ const id = rr.json().round.id;++ rounds.push(id);++ const a = await t.get(`/api/v1/rounds/${id}/entries`, { bearer: entrySvc, json: { provider: 'acme', model: 'alpha-1', payload: { html: '<h1>a</h1>' }, cost_usd: 0.01, ms: 1000 } });++ assert.strictEqual(a.status, 201, a.text);++ // beta-2 runs once only: fewer than three runs, so its numbers are too thin to stand on.++ if (i === 0) {++ const b = await t.get(`/api/v1/rounds/${id}/entries`, { bearer: entrySvc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<h1>b</h1>' }, cost_usd: 0.02, ms: 4000 } });++ assert.strictEqual(b.status, 201, b.text);++ const j = await t.get(`/api/v1/rounds/${id}/judgements`, { bearer: judgeSvc, json: { judge: 'game_outcome', pair: { left: a.json().entry.id, right: b.json().entry.id }, outcome: 'left' } });++ assert.strictEqual(j.status, 201, j.text);++ }++ }++++ await check('the pick page answers from the data, without JavaScript', async () => {++ const r = await t.get('/pick?kind=design.page');++ assert.strictEqual(r.status, 200, r.text.slice(0, 300));++ assert.ok(r.text.includes('method="get"'), 'the form is a GET');++ assert.ok(r.text.includes('pk-table'), 'the frontier table');++ assert.ok(r.text.includes('alpha-1'), 'the top model');++ assert.ok(r.text.includes('because no other model beats it'), 'why the top pick is first');++ assert.ok(r.text.includes('pk-card') || r.text.includes('pk-shot'), 'a sample output');++ });++++ await check('a model with fewer than three runs is greyed and says so', async () => {++ const r = await t.get('/pick?kind=design.page');++ assert.ok(r.text.includes('beta-2'), 'the thin model is shown');++ assert.ok(r.text.includes('too few runs'), 'it says the data is thin');++ assert.ok(/class="pk-thin"/.test(r.text), 'and it is greyed');++ });++++ await check('the API answers the same thing as the page', async () => {++ const api = await t.get('/api/v1/pick?kind=design.page');++ assert.strictEqual(api.status, 200, api.text.slice(0, 300));++ const json = api.json();++ assert.strictEqual(json.kind.id, 'design.page');++ assert.strictEqual(json.rows[0].model, 'alpha-1', 'the top pick');++ assert.ok(json.rows[0].cost_per_solved > 0, 'the cost a solved task');++ assert.ok(json.rows[0].median_ms > 0, 'the median time');++ assert.ok(json.rows[0].runs >= 3, 'the number of runs');++ assert.ok(json.why.includes('alpha-1'), 'the reason');++ assert.ok(json.also_ran.some((r) => r.model === 'beta-2' && r.thin), 'the thin model');++ });++++ await check('the price control keeps the models over it out', async () => {++ const json = (await t.get('/api/v1/pick?kind=design.page&max_cost=0.005')).json();++ assert.strictEqual(json.rows.length, 0, 'nothing is under half a cent');++ const ok = (await t.get('/api/v1/pick?kind=design.page&max_cost=0.02')).json();++ assert.ok(ok.rows.some((r) => r.model === 'alpha-1'), 'alpha-1 is under two cents');++ });++++ await check('fast takes its time cap from the run data', async () => {++ const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();++ assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');++ assert.ok(json.rows.every((r) => !r.median_ms || r.median_ms <= json.filters.fast_cap_ms), 'only the quick ones');++ });++++ await check('the words set the controls, and the page says how it read them', async () => {++ const r = await t.get('/pick?q=' + encodeURIComponent('landing page under 2 cents, fast'));++ assert.strictEqual(r.status, 200);++ assert.ok(r.text.includes('design.page'), 'the kind it read');++ const json = (await t.get('/api/v1/pick?q=' + encodeURIComponent('landing page under 2 cents, fast'))).json();++ assert.strictEqual(json.controls.kind, 'design.page');++ assert.strictEqual(json.controls.max_cost, 0.02);++ assert.strictEqual(json.controls.fast, true);++ assert.ok(json.controls.notes.length >= 3, 'it says what it read');++ });++++ await check('a kind with nothing in it says so, and invents nothing', async () => {++ const r = await t.get('/pick?kind=games.three');++ assert.strictEqual(r.status, 200);++ assert.ok(r.text.includes('Nothing to pick yet'), 'the empty answer');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,135 @@++'use strict';++++/**++ * The model watch (B10): the triggers, the once-a-day cap, the account data, and the notice that reaches the Network.++ */++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { createWatches, mayNotify } = require('../server/watch/watch');++const { TABLES } = require('../server/identity/account-data');++const { uncovered } = require('openvibe-sdk/account-data');++++const stub = { s: { db: null }, config: {}, log: { warn() {} } };++++(async () => {++ await check('a score that moves by more than its own interval is worth a notice', () => {++ const w = createWatches(stub);++ const notice = w.noticeFor(++ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },++ [{ model: 'alpha', rating: 1100, lower: 1060, upper: 1140 }],++ new Map(),++ new Map([['alpha', { rating: 1000 }]]),++ []);++ assert.ok(notice, 'a notice');++ assert.strictEqual(notice.kind, 'move');++ assert.ok(notice.message.includes('Playable games'), 'the benchmark is named');++ assert.ok(notice.message.includes('alpha'), 'the model is named');++ assert.ok(notice.message.includes('1000') && notice.message.includes('1100'), 'the numbers are there');++ });++++ await check('a move inside the interval is not worth a notice', () => {++ const w = createWatches(stub);++ const notice = w.noticeFor(++ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },++ [{ model: 'alpha', rating: 1010, lower: 990, upper: 1030 }],++ new Map(),++ new Map([['alpha', { rating: 1000 }]]),++ []);++ assert.strictEqual(notice, null, 'no notice');++ });++++ await check('a model that overtakes another one is worth a notice', () => {++ const w = createWatches(stub);++ const notice = w.noticeFor(++ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },++ [++ { model: 'alpha', rating: 1000, lower: 980, upper: 1020 },++ { model: 'beta', rating: 1100, lower: 1050, upper: 1150 },++ ],++ new Map(),++ new Map(),++ []);++ assert.ok(notice, 'a notice');++ assert.strictEqual(notice.kind, 'overtake');++ assert.ok(notice.message.includes('alpha') && notice.message.includes('beta'), 'both models are named');++ assert.ok(notice.message.includes('Playable games'), 'the benchmark is named');++ assert.ok(/\d/.test(notice.message), 'the numbers are there');++ });++++ await check('a model the person already follows does not count as an overtake', () => {++ const w = createWatches(stub);++ const notice = w.noticeFor(++ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },++ [++ { model: 'alpha', rating: 1000, lower: 980, upper: 1020 },++ { model: 'beta', rating: 1100, lower: 1050, upper: 1150 },++ ],++ new Map(),++ new Map(),++ [{ subject: 'usr_1', benchmark_id: 'bmk_1', model: 'beta' }]);++ assert.strictEqual(notice, null, 'no notice');++ });++++ await check('one notice per watch per day', () => {++ assert.strictEqual(mayNotify({ last_notified_at: null }, '2026-10-30T12:00:00Z'), true);++ assert.strictEqual(mayNotify({ last_notified_at: '2026-10-30T10:00:00Z' }, '2026-10-30T12:00:00Z'), false);++ assert.strictEqual(mayNotify({ last_notified_at: '2026-10-29T23:00:00Z' }, '2026-10-30T12:00:00Z'), true);++ });++++ const t = await boot();++ const alice = t.network.addUser('alice');++++ const list = (await t.get('/api/v1/benchmarks')).json();++ const benchmarks = list.benchmarks || list;++ const benchmark = benchmarks[0];++ assert.ok(benchmark && benchmark.id, 'a seeded benchmark: ' + JSON.stringify(list).slice(0, 200));++++ await check('the watches go into the account data and are deleted with the person', async () => {++ const row = TABLES.find((x) => x.table === 'bench_watches');++ assert.ok(row, 'bench_watches is in the account data');++ assert.strictEqual(row.subject, 'subject');++ assert.strictEqual(row.erase, 'delete');++ const left = await uncovered(t.ctx.s.db, { tables: TABLES });++ assert.ok(!left.includes('bench_watches'), 'the watches are covered: ' + JSON.stringify(left));++ });++++ await check('a signed-in person may follow a benchmark, and may stop', async () => {++ const r = await t.get('/watch/follow', { as: alice, method: 'POST', json: { benchmark_id: benchmark.id, back: '/watching' } });++ assert.strictEqual(r.status, 303, r.text);++ const page = await t.get('/watching', { as: alice });++ assert.strictEqual(page.status, 200, page.text.slice(0, 200));++ assert.ok(page.text.includes(benchmark.title), 'the benchmark is listed');++ assert.ok(page.text.includes('action="/watch/unfollow"'), 'an unfollow button');++ const id = /name="id" value="([^"]+)"/.exec(page.text)[1];++ const un = await t.get('/watch/unfollow', { as: alice, method: 'POST', json: { id, back: '/watching' } });++ assert.strictEqual(un.status, 303);++ const after = await t.get('/watching', { as: alice });++ assert.ok(!after.text.includes(benchmark.title), 'the watch is gone');++ });++++ await check('an anonymous visitor cannot follow anything', async () => {++ const r = await t.get('/watch/follow', { method: 'POST', origin: t.config.baseUrl, json: { benchmark_id: benchmark.id } });++ assert.strictEqual(r.status, 401, r.text);++ });++++ await check('the notice reaches the Network, naming the benchmark and the numbers', async () => {++ const out = await t.ctx.watches.follow({ subject: alice.subject, benchmarkId: benchmark.id, userId: alice.id });++ assert.ok(out.ok, out.error);++ const pushed = await t.ctx.watches.notify(out.watch, {++ title: 'Bench: ' + benchmark.title,++ message: benchmark.title + ': beta (1100, 1050-1150) is now fully above alpha (1000, 980-1020).',++ });++ assert.strictEqual(pushed.sent, true, JSON.stringify(pushed));++ const last = t.network.pushes[t.network.pushes.length - 1];++ assert.ok(last, 'the Network was called');++ assert.strictEqual(last.type, 'WATCH_TRIGGERED');++ assert.ok(last.message.includes(benchmark.title), 'the benchmark is named');++ assert.ok(last.message.includes('alpha') && last.message.includes('beta'), 'both models are named');++ assert.ok(last.message.includes('1100'), 'the numbers are there');++ assert.strictEqual(last.user_id, alice.id, 'it goes to the person');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,26 @@
@@ -0,0 +1,26 @@
+-- phase: expand
+-- The watches (spec 6): a person follows a benchmark, or a model on a benchmark. The rows go into the account export
+-- and are deleted with the person.
+CREATE TABLE IF NOT EXISTS bench_watches (
+ id TEXT PRIMARY KEY,
+ subject TEXT NOT NULL,
+ benchmark_id TEXT NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,
+ model TEXT,
+ user_id BIGINT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ last_notified_at TIMESTAMPTZ
+);
+CREATE UNIQUE INDEX IF NOT EXISTS bench_watches_once ON bench_watches (subject, benchmark_id, COALESCE(model, ''));
+CREATE INDEX IF NOT EXISTS bench_watches_benchmark ON bench_watches (benchmark_id);
+
+-- The board as it was last seen, so the next fit can be compared with it. The model is a model, not a person.
+CREATE TABLE IF NOT EXISTS bench_watch_ratings (
+ kind_id TEXT NOT NULL,
+ model TEXT NOT NULL,
+ rating DOUBLE PRECISION,
+ ci_lower DOUBLE PRECISION,
+ ci_upper DOUBLE PRECISION,
+ band TEXT,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ PRIMARY KEY (kind_id, model)
+);
@@ -360,3 +360,43 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -360,3 +360,43 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.rt-bar { height: 6px; margin-top: 4px; }
.rt-bar { height: 6px; margin-top: 4px; }
.rt-ci { display: block; font-size: .78rem; color: var(--text-secondary, #96a7c2); }
.rt-ci { display: block; font-size: .78rem; color: var(--text-secondary, #96a7c2); }
}
}
+
+/* ---- Pick for me (B10): plain controls, then the frontier ---- */
+.pk-form { display: grid; gap: 12px; margin: 18px 0 8px; padding: 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }
+.pk-form label { display: grid; gap: 6px; font-size: .8rem; font-weight: 800; color: var(--text-secondary); }
+.pk-form select, .pk-form input[type="text"] { min-height: 44px; padding: 0 12px; border-radius: 12px; border: 1px solid var(--border); background: var(--bg-primary); color: var(--text-primary); font: inherit; font-size: .95rem; font-weight: 500; width: 100%; }
+.pk-form select:focus-visible, .pk-form input:focus-visible { outline: 2px solid var(--room-accent); outline-offset: 2px; }
+.pk-check { display: flex; flex-direction: row; align-items: center; gap: 10px; min-height: 44px; }
+.pk-check input { width: 20px; height: 20px; accent-color: var(--room-accent); }
+.pk-words input { font-weight: 500; }
+.pk-notes { margin: 4px 0 0; }
+.pk-why { margin: 10px 0 14px; padding: 12px 14px; border-radius: 14px; background: color-mix(in srgb, var(--room-accent) 12%, transparent); border: 1px solid color-mix(in srgb, var(--room-accent) 30%, transparent); font-weight: 600; line-height: 1.55; }
+.pk-table td, .pk-table th { white-space: nowrap; }
+.pk-score { font-weight: 800; }
+.pk-band { font-size: .74rem; color: var(--text-muted); white-space: nowrap; }
+.pk-thin { opacity: .55; }
+.pk-thin-note { font-size: .72rem; color: var(--text-muted); }
+.pk-table .pk-thin-note { white-space: nowrap; }
+.pk-samples { display: grid; gap: 12px; grid-template-columns: repeat(auto-fill, minmax(min(100%, 280px), 1fr)); margin-top: 12px; }
+.pk-card { border: 1px solid var(--border); border-radius: 16px; overflow: hidden; background: var(--bg-card); min-width: 0; }
+.pk-card-h { margin: 0; padding: 10px 12px; font-size: .85rem; font-weight: 800; overflow-wrap: anywhere; }
+.pk-shot { aspect-ratio: 16 / 10; background: var(--bg-primary); overflow: hidden; }
+.pk-shot .ga-shot { margin: 0; height: 100%; min-width: 0; }
+.pk-shot .ga-shot figcaption { display: none; }
+.pk-shot iframe, .pk-shot img { display: block; width: 100%; height: 100%; border: 0; object-fit: cover; }
+.pk-shot .ga-frame { aspect-ratio: auto; }
+.pk-prompt { padding: 0 12px 12px; margin: 0; }
+
+/* ---- Watching (B10) ---- */
+.wk-list { list-style: none; margin: 12px 0 0; padding: 0; display: grid; gap: 10px; }
+.wk-item { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; justify-content: space-between; padding: 14px 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }
+.wk-what { display: grid; gap: 6px; min-width: 0; overflow-wrap: anywhere; }
+.wk-form { margin: 0; }
+
+/* ---- Use this (B10): the copy boxes ---- */
+.br-usethis { display: grid; gap: 12px; }
+.br-usethis-box { display: grid; gap: 10px; padding: 14px 16px; border: 1px solid var(--border); border-radius: 16px; background: var(--bg-card); }
+.br-usethis-h { margin: 0; font-size: .95rem; overflow-wrap: anywhere; }
+.br-usethis-row { display: grid; gap: 6px; min-width: 0; }
+.br-usethis-row textarea { width: 100%; min-height: 44px; padding: 10px 12px; border-radius: 12px; border: 1px solid var(--border); background: var(--bg-primary); color: var(--text-secondary); font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: .8rem; line-height: 1.45; resize: vertical; }
+.br-follow { margin: 0; }
@@ -29,6 +29,12 @@ const { createRegistryApi } = require('./http/registry');
@@ -29,6 +29,12 @@ const { createRegistryApi } = require('./http/registry');
const { createPageRoutes } = require('./http/pages');
const { createPageRoutes } = require('./http/pages');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
const { createGalleryRoutes, createGalleryPages } = require('./http/gallery');
+const { createPickPages, createPickRoutes } = require('./http/pick');
+const { createBadgeRoutes } = require('./http/badge');
+const { createEmbedRoutes } = require('./http/embed');
+const { createWatchPages } = require('./http/watch');
+const { createWatches } = require('./watch/watch');
+const { createBattles } = require('./bench/battles');
const { createBattlePages } = require('./http/battle');
const { createBattlePages } = require('./http/battle');
const { createRatingsPages } = require('./http/ratings');
const { createRatingsPages } = require('./http/ratings');
const { createRenderRoutes } = require('./http/render');
const { createRenderRoutes } = require('./http/render');
@@ -98,6 +104,15 @@ async function createApp(opts = {}) {
@@ -98,6 +104,15 @@ async function createApp(opts = {}) {
ctx.telemetry = createHttpTelemetry({ service: 'bench', sink: opts.telemetrySink || createPgTelemetrySink({ db: s.db, log }), skipPrefixes: ['/shared'], intervalMs: 60_000 });
ctx.telemetry = createHttpTelemetry({ service: 'bench', sink: opts.telemetrySink || createPgTelemetrySink({ db: s.db, log }), skipPrefixes: ['/shared'], intervalMs: 60_000 });
app.use(ctx.telemetry.middleware());
app.use(ctx.telemetry.middleware());
app.locals.ctx = ctx;
app.locals.ctx = ctx;
+ ctx.watches = createWatches(ctx);
+ ctx.battles = createBattles(ctx);
+ // Where the ratings are refitted, the watches are checked: a score that moved by more than its own interval, or
+ // a model that overtook another one, is worth one notice.
+ s.onRatingsChanged = async (kindId, models) => {
+ try { await ctx.watches.check(kindId, { models }); } catch (err) {
+ log.warn('[OpenVibe.Bench] watch check failed: ' + (err && err.message ? err.message : err));
+ }
+ };
const valkey = opts.valkey !== undefined ? opts.valkey : (config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null);
const valkey = opts.valkey !== undefined ? opts.valkey : (config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null);
ctx.valkey = valkey;
ctx.valkey = valkey;
@@ -124,6 +139,9 @@ async function createApp(opts = {}) {
@@ -124,6 +139,9 @@ async function createApp(opts = {}) {
crossOriginResourcePolicy: { policy: 'same-site' },
crossOriginResourcePolicy: { policy: 'same-site' },
referrerPolicy: { policy: 'strict-origin-when-cross-origin' },
referrerPolicy: { policy: 'strict-origin-when-cross-origin' },
}));
}));
+ // ── Badges and embeds: no session, no cookies, before the cookie parser ──
+ app.use('/badge', createBadgeRoutes(ctx));
+ app.use('/embed', createEmbedRoutes(ctx));
app.use(cookieParser());
app.use(cookieParser());
// ── Machine endpoints ───────────────────────────────────
// ── Machine endpoints ───────────────────────────────────
@@ -164,6 +182,7 @@ async function createApp(opts = {}) {
@@ -164,6 +182,7 @@ async function createApp(opts = {}) {
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
+ app.use('/api/v1', createPickRoutes(ctx));
app.use(mcpRouter(ctx));
app.use(mcpRouter(ctx));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
@@ -181,6 +200,8 @@ async function createApp(opts = {}) {
@@ -181,6 +200,8 @@ async function createApp(opts = {}) {
app.use('/vendor', require('./http/vendor').createVendorRoutes());
app.use('/vendor', require('./http/vendor').createVendorRoutes());
app.use('/showcase', createShowcasePages(ctx));
app.use('/showcase', createShowcasePages(ctx));
app.use('/gallery', createGalleryPages(ctx));
app.use('/gallery', createGalleryPages(ctx));
+ app.use('/pick', createPickPages(ctx));
+ app.use(createWatchPages(ctx));
app.use('/battle', createBattlePages(ctx));
app.use('/battle', createBattlePages(ctx));
app.use('/ratings', createRatingsPages(ctx));
app.use('/ratings', createRatingsPages(ctx));
app.use(createPageRoutes(ctx));
app.use(createPageRoutes(ctx));
new file mode 100644
new file mode 100644
@@ -0,0 +1,107 @@
@@ -0,0 +1,107 @@
+'use strict';
+
+/**
+ * The README badge (spec 3): /badge/:benchmark/:model.svg
+ *
+ * The badge carries the benchmark, its version and the model's score with its spread, plus the band the model sits in
+ * when the board knows it — never a bare rank. The text is escaped and the width comes from the text, so a long
+ * title still fits. An unknown benchmark or a model with no data gets a grey "no data" badge and a 404.
+ */
+
+const express = require('express');
+const { esc } = require('../render/html');
+
+/** The height of a badge and the width one character takes at this size. */
+const HEIGHT = 20;
+const CHAR_W = 6.9;
+const PAD = 10;
+
+/** The XML escape for text inside the SVG. */
+function xml(s) {
+ return String(s == null ? '' : s)
+ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
+ .replace(/"/g, '"').replace(/'/g, ''');
+}
+
+/** The width of a run of text, from the text itself. */
+function widthOf(text) {
+ return Math.max(20, Math.ceil(String(text).length * CHAR_W) + PAD * 2);
+}
+
+/**
+ * A shields-shaped badge. `label` on the left, `message` on the right; the width comes from both.
+ * The fills are the standard badge greys: a badge is a picture of its own, so it cannot borrow the site's theme.
+ */
+function badgeSvg({ label, message, tone = 'value', title = null }) {
+ const lw = widthOf(label);
+ const mw = widthOf(message);
+ const width = lw + mw;
+ const labelFill = tone === 'nodata' ? '#9f9f9f' : '#555';
+ const valueFill = tone === 'nodata' ? '#c8c8c8' : '#007ec6';
+ const labelText = tone === 'nodata' ? '#ffffff' : '#ffffff';
+ const name = title || (label + ' · ' + message);
+ return '<svg xmlns="http://www.w3.org/2000/svg" width="' + width + '" height="' + HEIGHT + '" role="img" aria-label="' + xml(name) + '">'
+ + '<title>' + xml(name) + '</title>'
+ + '<rect width="' + width + '" height="' + HEIGHT + '" rx="3" fill="' + valueFill + '"/>'
+ + '<rect width="' + lw + '" height="' + HEIGHT + '" rx="3" fill="' + labelFill + '"/>'
+ + '<rect x="' + (lw - 3) + '" width="3" height="' + HEIGHT + '" fill="' + valueFill + '"/>'
+ + '<g fill="' + labelText + '" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">'
+ + '<text x="' + (PAD - 0.5) + '" y="14">' + xml(label) + '</text>'
+ + '<text x="' + (lw + PAD - 0.5) + '" y="14">' + xml(message) + '</text>'
+ + '</g></svg>';
+}
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+const fmtScore = (v) => (v == null ? '—' : String(Math.round(Number(v) * 10) / 10));
+
+/** The no-data badge: grey, with the reason, and a 404 beside it. */
+function noData(reason) {
+ const svg = badgeSvg({ label: 'OpenVibe Bench', message: 'no data' + (reason ? ' (' + reason + ')' : ''), tone: 'nodata' });
+ return { svg, status: 404 };
+}
+
+/**
+ * The badge for one benchmark and one model. The score is the median of the model's verified runs on that benchmark;
+ * the spread is how much those runs differ from each other, and the band is where the model sits on that benchmark's
+ * board.
+ */
+async function badgeFor(ctx, benchmarkId, model) {
+ const registry = ctx.registry;
+ const benchmark = await registry.getBenchmark(String(benchmarkId || ''));
+ if (!benchmark) return noData('unknown benchmark');
+ const rankings = await registry.rankings({ benchmark: benchmark.id, metric: 'score' });
+ const entries = (rankings && rankings.entries) || [];
+ const row = entries.find((e) => String(e.model) === String(model || ''));
+ if (!row || row.score == null) return noData('no runs');
+ const version = num(benchmark.latest_version, 1);
+ const title = String(benchmark.title || benchmark.id);
+ const spread = row.consistency != null ? Math.round(num(row.consistency) * 10) / 10 : null;
+ let band = null;
+ try {
+ const board = await require('../rating/board').boardForKind(ctx, benchmark.kind);
+ for (const b of ((board.fit && board.fit.bands && board.fit.bands.controlled) || [])) {
+ if ((b.models || []).includes(String(model))) { band = b.label; break; }
+ }
+ } catch { /* the board is a nicety here: a badge without a band still carries the score */ }
+ const message = spread != null ? fmtScore(row.score) + ' ±' + spread : fmtScore(row.score);
+ const label = title + ' v' + version + (band ? ' · band ' + band : '');
+ return { svg: badgeSvg({ label, message, title: label + ': ' + message }), status: 200 };
+}
+
+function createBadgeRoutes(ctx) {
+ const r = express.Router();
+ r.get('/:benchmark/:model.svg', async (req, res) => {
+ const model = String(req.params.model || '').replace(/\.svg$/i, '');
+ const out = await badgeFor(ctx, String(req.params.benchmark || ''), model);
+ res.status(out.status);
+ res.set('Content-Type', 'image/svg+xml; charset=utf-8');
+ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'");
+ res.set('X-Content-Type-Options', 'nosniff');
+ res.set('Cross-Origin-Resource-Policy', 'cross-origin');
+ res.set('Cache-Control', 'public, max-age=3600');
+ return res.send(out.svg);
+ });
+ return r;
+}
+
+module.exports = { createBadgeRoutes, badgeFor, badgeSvg, xml, widthOf, noData };
@@ -15,6 +15,7 @@ const renderLib = require('../bench/render');
@@ -15,6 +15,7 @@ const renderLib = require('../bench/render');
const { kindById } = require('../kinds');
const { kindById } = require('../kinds');
const { esc, raw } = require('../render/html');
const { esc, raw } = require('../render/html');
const { send } = require('../render/layout');
const { send } = require('../render/layout');
+const { battleUseThis } = require('../views/usethis');
const { createBattles } = require('../bench/battles');
const { createBattles } = require('../bench/battles');
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
@@ -61,7 +62,7 @@ function battleView(battle) {
@@ -61,7 +62,7 @@ function battleView(battle) {
+ '</section>';
+ '</section>';
}
}
-function revealView(battle) {
+function revealView(battle, { baseUrl = '' } = {}) {
const kind = kindById(battle.kind_id);
const kind = kindById(battle.kind_id);
const vote = battle.vote || {};
const vote = battle.vote || {};
const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';
const winner = vote.outcome === 'left' ? 'A' : vote.outcome === 'right' ? 'B' : vote.outcome === 'tie' ? 'Tie' : 'Both bad';
@@ -81,6 +82,7 @@ function revealView(battle) {
@@ -81,6 +82,7 @@ function revealView(battle) {
+ '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'
+ '<div class="bt-reveal-facts">' + fact(battle.a, 'A') + fact(battle.b, 'B') + '</div>'
+ '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'
+ '<p class="bm-actions"><a class="bm-btn bm-btn-quiet" href="/battle">Next battle</a>'
+ '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'
+ '<a class="bm-btn bm-btn-quiet" href="/gallery">See the gallery</a></p>'
+ + String(battleUseThis({ baseUrl, battle, title: kind ? kind.name : battle.kind_id }))
+ '</section>';
+ '</section>';
}
}
@@ -201,7 +203,7 @@ function createBattlePages(ctx) {
@@ -201,7 +203,7 @@ function createBattlePages(ctx) {
config, viewer: viewerOf(req), path: req.originalUrl,
config, viewer: viewerOf(req), path: req.originalUrl,
title: 'Battle: the names are open',
title: 'Battle: the names are open',
description: 'The two outputs of one prompt, with their names.',
description: 'The two outputs of one prompt, with their names.',
- body: raw(revealView(battle)),
+ body: raw(revealView(battle, { baseUrl: ctx.config.baseUrl })),
});
});
});
});
new file mode 100644
new file mode 100644
@@ -0,0 +1,94 @@
@@ -0,0 +1,94 @@
+'use strict';
+
+/**
+ * The battle embed (spec 3): /embed/battle/:id
+ *
+ * A page of one battle that another site may frame. Only this route may be framed: every other route keeps refusing
+ * frames. The page reads and writes no cookie, the outputs stay in their sandboxed frames, and voting opens Bench in
+ * a new tab: the widget itself never posts anything.
+ */
+
+const express = require('express');
+const { esc } = require('../render/html');
+const renderLib = require('../bench/render');
+const { kindById } = require('../kinds');
+
+/**
+ * The style of the embed page. It stands alone on someone else's site, so it takes no theme of ours: the visitor's
+ * own light or dark setting paints it through the browser's system colours, which adapt by themselves.
+ */
+const STYLE = [
+ ':root{color-scheme:light dark;--em-bg:Canvas;--em-text:CanvasText;--em-muted:GrayText;--em-line:color-mix(in srgb,CanvasText 22%,transparent);--em-panel:color-mix(in srgb,CanvasText 6%,Canvas);--em-fill:Highlight;--em-on:HighlightText}',
+ 'body{margin:0;background:var(--em-bg);color:var(--em-text);font-family:Inter,-apple-system,BlinkMacSystemFont,Segoe UI,sans-serif;padding:12px}',
+ '.em-wrap{display:grid;gap:10px;max-width:900px;margin:0 auto}',
+ '.em-h{font-size:.95rem;font-weight:700;margin:0;overflow-wrap:anywhere}',
+ '.em-p{font-size:.86rem;margin:0;color:var(--em-muted);line-height:1.5}',
+ '.em-panes{display:grid;gap:10px;grid-template-columns:repeat(auto-fit,minmax(min(100%,260px),1fr))}',
+ '.em-pane{margin:0;border:1px solid var(--em-line);border-radius:12px;overflow:hidden;background:var(--em-panel)}',
+ '.em-pane iframe{display:block;width:100%;aspect-ratio:16/10;height:auto;border:0}',
+ '.em-cap{font-size:.78rem;padding:6px 8px;margin:0;color:var(--em-muted);font-weight:600;overflow-wrap:anywhere}',
+ '.em-links{display:flex;flex-wrap:wrap;gap:10px;align-items:center}',
+ '.em-link{display:inline-flex;align-items:center;min-height:36px;padding:0 12px;border-radius:999px;background:var(--em-fill);color:var(--em-on);font-weight:700;font-size:.86rem;text-decoration:none}',
+ '.em-back{font-size:.78rem;color:var(--em-muted);text-decoration:none}',
+ '.em-link:focus-visible,.em-back:focus-visible{outline:2px solid var(--em-fill);outline-offset:2px}',
+].join('');
+
+/** The question a battle asks, which is an object on the battle and a string on an old one. */
+function questionText(question, kind) {
+ if (typeof question === 'string' && question) return question;
+ if (question && typeof question.text === 'string' && question.text) return question.text;
+ return kind ? 'Two ' + kind.name + ' outputs, side by side.' : 'Two outputs, side by side.';
+}
+
+const pane = (entry, label) => '<figure class="em-pane">'
+ + '<iframe src="' + esc(renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(entry.id)) + '"'
+ + ' title="' + esc(label + ' output') + '" sandbox="' + esc(renderLib.WRAPPER_SANDBOX) + '"'
+ + ' referrerpolicy="no-referrer" loading="lazy"></iframe>'
+ + '<figcaption class="em-cap">' + esc(label) + '</figcaption></figure>';
+
+/** The embed page for one battle. */
+function embedHtml(battle, { baseUrl }) {
+ const kind = kindById(battle.kind_id);
+ const label = (e) => String(e.model || 'model');
+ return '<!doctype html><html lang="en"><head><meta charset="utf-8">'
+ + '<meta name="viewport" content="width=device-width, initial-scale=1">'
+ + '<meta name="robots" content="noindex, nofollow">'
+ + '<title>' + esc((kind ? kind.name : battle.kind_id) + ' — OpenVibe Bench') + '</title>'
+ + '<style>' + STYLE + '</style></head><body><div class="em-wrap">'
+ + '<h1 class="em-h">' + esc(battle.prompt || (kind ? kind.name : battle.kind_id)) + '</h1>'
+ + '<p class="em-p">' + esc(questionText(battle.question, kind)) + '</p>'
+ + '<div class="em-panes">' + pane(battle.a, label(battle.a)) + pane(battle.b, label(battle.b)) + '</div>'
+ + '<div class="em-links">'
+ + '<a class="em-link" href="' + esc(baseUrl + '/battle') + '" target="_blank" rel="noopener">Vote on OpenVibe Bench</a>'
+ + '<a class="em-back" href="' + esc(baseUrl + '/battle/revealed/' + battle.id) + '" target="_blank" rel="noopener">OpenVibe Bench</a>'
+ + '</div></div></body></html>';
+}
+
+function createEmbedRoutes(ctx) {
+ const r = express.Router();
+ r.get('/battle/:id', async (req, res) => {
+ const battle = await ctx.battles.battleOf(String(req.params.id || ''));
+ if (!battle) {
+ res.status(404);
+ res.set('Content-Type', 'text/html; charset=utf-8');
+ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors *");
+ res.set('X-Content-Type-Options', 'nosniff');
+ res.removeHeader('X-Frame-Options');
+ return res.send('<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="robots" content="noindex, nofollow">'
+ + '<title>No such battle</title><style>' + STYLE + '</style></head><body><div class="em-wrap">'
+ + '<h1 class="em-h">No such battle</h1><p class="em-p">This battle is not here.</p></div></body></html>');
+ }
+ const baseUrl = ctx.config.baseUrl || 'https://bench.openvibe.services';
+ res.status(200);
+ res.set('Content-Type', 'text/html; charset=utf-8');
+ res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; frame-src " + renderLib.RENDER_ORIGIN
+ + "; img-src data:; form-action 'none'; base-uri 'none'; frame-ancestors *");
+ res.set('X-Content-Type-Options', 'nosniff');
+ res.removeHeader('X-Frame-Options');
+ res.set('Cache-Control', 'public, max-age=60');
+ return res.send(embedHtml(battle, { baseUrl }));
+ });
+ return r;
+}
+
+module.exports = { createEmbedRoutes, embedHtml };
@@ -200,4 +200,4 @@ function createGalleryPages(ctx) {
@@ -200,4 +200,4 @@ function createGalleryPages(ctx) {
return r;
return r;
}
}
-module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, LIVE_FRAMES };
+module.exports = { createGalleryRoutes, createGalleryPages, loadGallery, facets, frame, card, screenshotOf, LIVE_FRAMES };
@@ -105,7 +105,7 @@ function createPageRoutes(ctx) {
@@ -105,7 +105,7 @@ function createPageRoutes(ctx) {
jsonLd: [seo.jsonLd.webPage({ name: out.benchmark.title, url: `${base()}/benchmarks/${encodeURIComponent(id)}`, description: out.benchmark.description || out.benchmark.title, siteUrl: base() })],
jsonLd: [seo.jsonLd.webPage({ name: out.benchmark.title, url: `${base()}/benchmarks/${encodeURIComponent(id)}`, description: out.benchmark.description || out.benchmark.title, siteUrl: base() })],
body: raw(String(results.benchmarkPage({
body: raw(String(results.benchmarkPage({
benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,
benchmark: out.benchmark, definition: out.definition, version: out.version && out.version.version,
- rankings, metric, runs, codesRunUrl: config.codesRunUrl || null,
+ rankings, metric, runs, codesRunUrl: config.codesRunUrl || null, viewer: req.viewer, baseUrl: base(),
}))),
}))),
});
});
});
});
@@ -197,7 +197,7 @@ function createPageRoutes(ctx) {
@@ -197,7 +197,7 @@ function createPageRoutes(ctx) {
index: summary.runs > 0, cache: PUBLIC_CACHE,
index: summary.runs > 0, cache: PUBLIC_CACHE,
title: model,
title: model,
description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,
description: `${model} across OpenVibe.Bench: its median score, spread, cost and time on every benchmark it ran.`,
- body: raw(String(results.modelPage({ model, summary, runs, titles }))),
+ body: raw(String(results.modelPage({ model, summary, runs, titles, viewer: req.viewer, baseUrl: base() }))),
}, summary.runs > 0 || runs.length ? 200 : 404);
}, summary.runs > 0 || runs.length ? 200 : 404);
});
});
new file mode 100644
new file mode 100644
@@ -0,0 +1,61 @@
@@ -0,0 +1,61 @@
+'use strict';
+
+/**
+ * The pick routes (spec 3): the page at /pick and the same answer as JSON at /api/v1/pick.
+ * Both read the same three controls — kind, top price per solved task and fast — and the free text that sets them.
+ */
+
+const express = require('express');
+const { asyncRouter } = require('./router');
+const { send } = require('../render/layout');
+const { raw } = require('../render/html');
+const pickLib = require('../pick/pick');
+const { pickPage } = require('../views/pick');
+
+const str = (v) => (typeof v === 'string' ? v : '');
+
+function readControls(query) {
+ const maxCost = str(query.max_cost);
+ return {
+ kind: str(query.kind).slice(0, 80),
+ maxCost: maxCost === '' ? null : Number(maxCost),
+ fast: str(query.fast) === '1' || str(query.fast) === 'true',
+ q: str(query.q).slice(0, 200),
+ };
+}
+
+const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
+ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
+ : { kind: 'anonymous' });
+
+function createPickPages(ctx) {
+ const r = express.Router();
+ r.use(ctx.principal.middleware);
+ r.get('/', async (req, res) => {
+ const controls = readControls(req.query);
+ const answer = await pickLib.answer(ctx, controls);
+ const body = pickPage(answer);
+ return send(res, 200, {
+ config: ctx.config,
+ viewer: viewerOf(req),
+ path: req.originalUrl,
+ title: 'Pick for me: the frontier of a kind',
+ description: 'Plain controls, then the models no other model beats on both score and cost per solved task.',
+ body: raw(body),
+ });
+ });
+ return r;
+}
+
+function createPickRoutes(ctx) {
+ const r = asyncRouter();
+ r.get('/pick', async (req, res) => {
+ const controls = readControls(req.query);
+ const answer = await pickLib.answer(ctx, controls);
+ res.set('Cache-Control', 'public, max-age=30');
+ return res.json(answer);
+ });
+ return r;
+}
+
+module.exports = { createPickPages, createPickRoutes, readControls };
new file mode 100644
new file mode 100644
@@ -0,0 +1,81 @@
@@ -0,0 +1,81 @@
+'use strict';
+
+/**
+ * The watch routes (spec 6): follow and unfollow, and the Watching page.
+ * Only a signed-in person may follow: the rows belong to them and are deleted with them.
+ */
+
+const express = require('express');
+const { asyncRouter } = require('./router');
+const { sameOrigin } = require('./same-origin');
+const { send } = require('../render/layout');
+const { raw } = require('../render/html');
+const { watchingPage } = require('../views/watch');
+
+const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal : null);
+/** The claims of a token, read without verifying it: the id is only used to address the notice. */
+function decodeClaims(token) {
+ const parts = String(token || '').split('.');
+ if (parts.length !== 3) return null;
+ try { return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); } catch { return null; }
+}
+
+/** The Network user id of the person, when their token carries it: the notice needs it. */
+const networkUserId = (req) => {
+ const principal = req.principal || {};
+ const claims = principal.claims || decodeClaims(principal.token);
+ const id = claims && (claims.id != null ? claims.id : claims.sub);
+ const n = Number(id);
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+
+function createWatchPages(ctx) {
+ const r = express.Router();
+ r.use(express.urlencoded({ extended: false, limit: '16kb' }));
+ r.use(express.json({ limit: '64kb' }));
+ r.use(ctx.principal.middleware);
+
+ r.get('/watching', async (req, res) => {
+ const principal = subjectOf(req);
+ const watches = principal ? await ctx.watches.listFor(principal.subject) : [];
+ const body = watchingPage(watches, { viewer: principal });
+ return send(res, 200, {
+ config: ctx.config,
+ viewer: principal ? { kind: 'user', subject: principal.subject, displayName: principal.displayName || principal.username } : { kind: 'anonymous' },
+ path: req.originalUrl,
+ title: 'Watching: the benchmarks and models you follow',
+ description: 'The benchmarks and models you follow, and what Bench tells you when they move.',
+ body: raw(body),
+ });
+ });
+
+ r.post('/watch/follow', async (req, res) => {
+ if (!sameOrigin(req, ctx.config)) return res.status(403).type('text/plain').send('That request did not come from this site.');
+ const principal = subjectOf(req);
+ if (!principal) return res.status(401).type('text/plain').send('Sign in to follow a benchmark.');
+ const body = req.body || {};
+ const benchmarkId = String(body.benchmark_id || '').trim();
+ const model = body.model ? String(body.model).trim() : null;
+ const out = await ctx.watches.follow({
+ subject: principal.subject,
+ benchmarkId,
+ model: model || null,
+ userId: networkUserId(req),
+ });
+ if (!out.ok) return res.status(out.status || 400).type('text/plain').send(out.error || 'Could not follow that.');
+ return res.redirect(303, String(body.back || '/watching'));
+ });
+
+ r.post('/watch/unfollow', async (req, res) => {
+ if (!sameOrigin(req, ctx.config)) return res.status(403).type('text/plain').send('That request did not come from this site.');
+ const principal = subjectOf(req);
+ if (!principal) return res.status(401).type('text/plain').send('Sign in to stop watching.');
+ const body = req.body || {};
+ await ctx.watches.unfollow({ subject: principal.subject, id: String(body.id || '') });
+ return res.redirect(303, String(body.back || '/watching'));
+ });
+
+ return r;
+}
+
+module.exports = { createWatchPages, networkUserId };
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+'use strict';
+
+/**
+ * The frontier (spec 3, "Pick for me"): the models no other model beats on both score and cost per solved task.
+ *
+ * The rule is deliberately narrow, because the whole point of the pick is that it never hides a model:
+ *
+ * - a model is dropped when some other model is at least as good on score AND at least as cheap per solved task,
+ * with at least one of the two strictly better (that other model dominates it);
+ * - two models that are exactly equal on both axes are a tie, and a tie keeps both (neither dominates the other);
+ * - a model with no score or no cost cannot be compared at all, so it is never on the frontier and never
+ * dominates anyone.
+ *
+ * Pure maths, no I/O: the tests are the proof.
+ */
+
+const num = (v) => (v == null || Number.isNaN(Number(v)) ? null : Number(v));
+
+/** True when `other` dominates `row`: at least as good on both axes, and strictly better on one. */
+function dominates(other, row) {
+ const os = num(other && other.score);
+ const oc = num(other && other.cost);
+ const rs = num(row && row.score);
+ const rc = num(row && row.cost);
+ if (os == null || oc == null || rs == null || rc == null) return false;
+ if (!(os >= rs && oc <= rc)) return false;
+ // Exactly equal on both axes is a tie, not domination: both models stay.
+ return !(os === rs && oc === rc);
+}
+
+/**
+ * The frontier of a list of rows. Each row needs a `score` and a `cost`; rows missing either are left out.
+ * The rows that come back keep the order they came in.
+ */
+function frontier(rows) {
+ const list = (Array.isArray(rows) ? rows : [])
+ .filter((r) => r && num(r.score) != null && num(r.cost) != null)
+ .map((r) => ({ ...r, score: num(r.score), cost: num(r.cost) }));
+ return list.filter((row) => !list.some((other) => other !== row && dominates(other, row)));
+}
+
+module.exports = { frontier, dominates };
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+'use strict';
+
+/**
+ * The plain keyword parser behind the free-text box on /pick (spec 3).
+ *
+ * There is no model here and no network call: the words in the box are read with rules, so the pick stays free and
+ * the same words always give the same answer. The parser only fills in the three controls the pick has — the kind,
+ * the top price per solved task and "fast" — and says in its notes what it read, so the reader can see the reading
+ * and change it.
+ */
+
+/** The numbers people write as words. */
+const WORD_NUMBERS = Object.freeze({
+ a: 1, an: 1, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
+ eleven: 11, twelve: 12, fifteen: 15, twenty: 20, fifty: 50, hundred: 100,
+});
+
+/** The words that mean "fast": a time cap per task. */
+const FAST_WORDS = Object.freeze(['fast', 'quicker', 'quick', 'quickly', 'snappy', 'speedy', 'speed', 'instant', 'instantly', 'low latency']);
+
+/**
+ * What people write for the kinds Bench has. The kind's own id and name are added to these at parse time, so a new
+ * kind is readable without touching this table.
+ */
+const KIND_ALIASES = Object.freeze({
+ 'games.play': ['playable games', 'playable game', 'browser game', 'game', 'games'],
+ 'games.three': ['three.js', 'three js', '3d scene', 'three'],
+ 'design.svg': ['svg', 'svg art', 'illustration', 'drawing'],
+ 'design.ui': ['ui', 'interface screens', 'screen', 'screens'],
+ 'design.dataviz': ['dataviz', 'data viz', 'data visualisation', 'chart', 'graph'],
+ 'design.motion': ['motion', 'animation', 'effects'],
+ 'design.interface': ['component system', 'component', 'design system'],
+ 'design.page': ['landing page', 'web page', 'website', 'page'],
+ 'design.shader': ['shader', 'glsl'],
+ 'design.model3d': ['3d model', 'gltf', 'model3d'],
+ 'design.component': ['component arena'],
+});
+
+const clean = (text) => String(text == null ? '' : text).toLowerCase().replace(/[,;]/g, ' ').replace(/\s+/g, ' ').trim();
+
+/** The kind the words name, or null. The longest alias wins, so "landing page" beats "page". */
+function parseKind(text, kinds) {
+ const hay = ' ' + clean(text) + ' ';
+ const list = (Array.isArray(kinds) ? kinds : []).map((k) => ({ id: k.id, name: k.name }));
+ const candidates = [];
+ for (const k of list) {
+ const aliases = new Set(KIND_ALIASES[k.id] || []);
+ if (k.name) aliases.add(String(k.name).toLowerCase());
+ aliases.add(String(k.id).toLowerCase());
+ aliases.add(String(k.id).toLowerCase().replace(/\./g, ' '));
+ for (const alias of aliases) {
+ const a = clean(alias);
+ if (!a) continue;
+ if (hay.includes(' ' + a + ' ')) candidates.push({ id: k.id, alias: a });
+ }
+ }
+ if (!candidates.length) return null;
+ candidates.sort((x, y) => y.alias.length - x.alias.length || x.id.localeCompare(y.id));
+ return candidates[0].id;
+}
+
+/** The top price per solved task the words name, in dollars, or null. */
+function parseCost(text) {
+ const t = ' ' + clean(text) + ' ';
+ // Dollars: $0.02
+ const dollar = /\$\s*(\d+(?:\.\d+)?)/.exec(t);
+ // Cents: 2 cents, 2¢, two cents, half a cent
+ const cent = /\b(\d+(?:\.\d+)?|a|an|one|two|three|four|five|six|seven|eight|nine|ten|twelve|fifteen|twenty)\s*(?:cents?|¢)/.exec(t);
+ const half = /\bhalf\s+(?:a\s+)?cent/.test(t);
+ if (half) return 0.005;
+ if (cent) {
+ const raw = cent[1];
+ const n = WORD_NUMBERS[raw] != null ? WORD_NUMBERS[raw] : Number(raw);
+ if (Number.isFinite(n)) return Math.round(n / 100 * 1e6) / 1e6;
+ }
+ if (dollar) {
+ const n = Number(dollar[1]);
+ if (Number.isFinite(n)) return Math.round(n * 1e6) / 1e6;
+ }
+ return null;
+}
+
+/** True when the words ask for fast. */
+function parseFast(text) {
+ const t = clean(text);
+ return FAST_WORDS.some((w) => new RegExp('(^|[^a-z])' + w + '([^a-z]|$)').test(t));
+}
+
+/**
+ * Read the free text into the three controls. Returns { kind, max_cost, fast, notes }.
+ * Nothing is guessed: an unknown word changes nothing.
+ */
+function parsePick(text, { kinds = [] } = {}) {
+ const raw = clean(text);
+ const notes = [];
+ if (!raw) return { kind: null, max_cost: null, fast: false, notes, text: '' };
+
+ const kind = parseKind(raw, kinds);
+ if (kind) notes.push('read the kind as ' + kind);
+ else notes.push('no kind in the words: the kind stays as it was');
+
+ const maxCost = parseCost(raw);
+ if (maxCost != null) {
+ const cents = Math.round(maxCost * 10000) / 100;
+ notes.push('read the price as at most ' + (cents + ' cents') + ' a solved task');
+ } else notes.push('no price in the words: any price');
+
+ const fast = parseFast(raw);
+ if (fast) notes.push('read "fast" as a time cap from the run times');
+ else notes.push('no "fast" in the words: no time cap');
+
+ return { kind, max_cost: maxCost, fast, notes, text: raw };
+}
+
+module.exports = { parsePick, parseKind, parseCost, parseFast, KIND_ALIASES, FAST_WORDS, WORD_NUMBERS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,177 @@
@@ -0,0 +1,177 @@
+'use strict';
+
+/**
+ * The pick (spec 3): the answer to "which model should I use for this job", read straight from the run data.
+ *
+ * Nothing here calls a model or a network service: the answer is the frontier of the kind — the models no other
+ * model beats on both score and cost per solved task — with the numbers that produced it. A model with fewer than
+ * three runs is greyed and says so; no number is ever invented.
+ */
+
+const { frontier } = require('./frontier');
+const { parsePick } = require('./parser');
+const { KINDS, kindById, SHOWCASE_KIND_IDS } = require('../kinds');
+const renderLib = require('../bench/render');
+const { screenshotOf } = require('../http/gallery');
+
+/** The price steps the form offers, in dollars a solved task. */
+const PRICE_STEPS = Object.freeze([
+ { value: 0.005, label: '0.5¢ a solved task' },
+ { value: 0.02, label: '2¢ a solved task' },
+ { value: 0.1, label: '10¢ a solved task' },
+]);
+
+/** Fewer runs than this and the numbers are too thin to stand on. */
+const MIN_RUNS = 3;
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+
+function median(list) {
+ const xs = list.map(Number).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
+ if (!xs.length) return null;
+ const mid = Math.floor(xs.length / 2);
+ return xs.length % 2 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2;
+}
+
+/** The band each model sits in, from the style-controlled board. */
+function bandMap(fit) {
+ const out = new Map();
+ const bands = (fit && fit.bands && fit.bands.controlled) || [];
+ for (const band of bands) for (const model of band.models || []) out.set(model, band.label);
+ return out;
+}
+
+/** The band a model sits in, from the board of one kind. */
+async function boardForKind(ctx, kindId) {
+ const board = require('../rating/board');
+ return board.boardForKind(ctx, kindId);
+}
+
+/** The kind with the most runs, when the reader did not name one. */
+async function busiestKind(ctx) {
+ const rows = await ctx.s.db.many(
+ `SELECT r.kind_id AS id, COUNT(e.id)::int AS n
+ FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
+ WHERE e.error IS NULL GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`);
+ if (rows && rows[0] && rows[0].id) return rows[0].id;
+ return SHOWCASE_KIND_IDS[0] || 'games.play';
+}
+
+/** One model's newest output in the kind, the way the gallery shows it. */
+async function sampleFor(ctx, kindId, model) {
+ const rows = await ctx.s.db.many(
+ `SELECT e.id, e.model, e.provider, e.cost_usd, e.ms, e.ui_check, r.id AS round_id, t.prompt
+ FROM bench_entries e
+ JOIN bench_rounds r ON r.id = e.round_id
+ JOIN bench_tasks t ON t.id = r.task_id
+ WHERE r.kind_id = $1 AND e.model = $2 AND e.error IS NULL AND r.status = 'finished'
+ ORDER BY r.created_at DESC, e.id LIMIT 1`,
+ [String(kindId), String(model)]);
+ const row = rows && rows[0];
+ if (!row) return null;
+ return {
+ id: row.id,
+ round_id: row.round_id,
+ prompt: row.prompt,
+ model: row.model,
+ provider: row.provider,
+ cost_usd: num(row.cost_usd),
+ ms: num(row.ms),
+ screenshot: screenshotOf(row.ui_check),
+ wrapper_url: renderLib.RENDER_ORIGIN + '/render/w/' + encodeURIComponent(row.id),
+ };
+}
+
+/** Why the top pick is first, in one sentence, from the numbers. */
+function whySentence(rows, kindName) {
+ const top = rows[0];
+ if (!top) return '';
+ const rivals = rows.slice(1);
+ const cents = (v) => (v == null ? 'an unknown price' : (Math.round(v * 10000) / 100) + '¢');
+ const head = top.model + ' is first on ' + kindName + ' because no other model beats it on both score and cost per solved task: '
+ + 'it scores ' + top.score + ' at ' + cents(top.cost_per_solved) + ' a solved task';
+ if (!rivals.length) return head + ', and it is the only model with enough data to stand on.';
+ const rival = rivals[0];
+ const lower = rival.score < top.score
+ ? 'scores lower (' + rival.score + ')'
+ : 'costs more (' + cents(rival.cost_per_solved) + ')';
+ return head + '. The closest, ' + rival.model + ', ' + lower + '.';
+}
+
+/**
+ * The answer. Reads the controls (with the free text read first, so the words can set them), and returns the frontier
+ * with every number that made it.
+ */
+async function answer(ctx, { kind = '', maxCost = null, fast = false, q = '', kinds = KINDS } = {}) {
+ const parsed = q ? parsePick(q, { kinds }) : { kind: null, max_cost: null, fast: false, notes: [] };
+ const kindId = parsed.kind || (kind && kindById(kind) ? kind : null) || await busiestKind(ctx);
+ const price = parsed.max_cost != null ? parsed.max_cost : (maxCost == null || maxCost === '' ? null : Number(maxCost));
+ const wantFast = parsed.fast || !!fast;
+
+ const all = await ctx.bench.valueRows(kindId);
+ const fit = await boardForKind(ctx, kindId);
+ const bands = bandMap(fit && fit.fit);
+
+ // The price comes first: a model over it is out whatever else it does.
+ const priced = price == null ? all : all.filter((r) => r.cost_per_solved != null && r.cost_per_solved <= price + 1e-9);
+
+ // "Fast" is a time cap taken from the run data: the middle of the models' median times.
+ const fastCap = wantFast ? median(priced.map((r) => r.median_ms).filter((v) => v > 0)) : null;
+ const timed = fastCap == null ? priced : priced.filter((r) => !r.median_ms || r.median_ms <= fastCap + 1e-9);
+
+ // The frontier maths works on { model, score, cost }: the cost is the cost per solved task.
+ const comparable = timed.filter((r) => r.score != null && r.cost_per_solved != null)
+ .map((r) => ({ ...r, model: r.subject, cost: r.cost_per_solved }));
+ const front = frontier(comparable);
+ const order = new Map(comparable.map((r, i) => [r.subject, i]));
+ front.sort((a, b) => (b.score - a.score)
+ || ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))
+ || a.subject.localeCompare(b.subject));
+
+ const rows = [];
+ for (const r of front) {
+ rows.push({
+ model: r.subject,
+ score: r.score,
+ band: bands.get(r.subject) || null,
+ votes: r.votes,
+ cost_per_solved: r.cost_per_solved,
+ median_ms: r.median_ms,
+ runs: r.runs,
+ thin: r.runs < MIN_RUNS,
+ sample: await sampleFor(ctx, kindId, r.subject),
+ });
+ }
+
+ const kindInfo = kindById(kindId) || { id: kindId, name: kindId };
+ const onFront = new Set(front.map((r) => r.subject));
+ const alsoRan = timed.filter((r) => !onFront.has(r.subject)).map((r) => ({
+ model: r.subject,
+ score: r.score,
+ cost_per_solved: r.cost_per_solved,
+ median_ms: r.median_ms,
+ runs: r.runs,
+ thin: r.runs < MIN_RUNS,
+ }));
+ const others = alsoRan.length;
+ const thinElsewhere = alsoRan.filter((r) => r.thin).length;
+
+ return {
+ kind: { id: kindInfo.id, name: kindInfo.name, room: kindInfo.room || null },
+ controls: {
+ kind: kindId,
+ max_cost: price == null ? null : price,
+ fast: wantFast,
+ q: String(q || ''),
+ notes: parsed.notes || [],
+ },
+ filters: { max_cost: price == null ? null : price, fast: wantFast, fast_cap_ms: fastCap, min_runs: MIN_RUNS },
+ rows,
+ also_ran: alsoRan,
+ why: whySentence(rows, kindInfo.name),
+ counts: { models: all.length, priced: priced.length, timed: timed.length, frontier: rows.length, others, thin_elsewhere: thinElsewhere },
+ generated_at: new Date().toISOString(),
+ };
+}
+
+module.exports = { answer, PRICE_STEPS, MIN_RUNS, median, bandMap, whySentence };
@@ -68,6 +68,7 @@ function gamesOf(votes) {
@@ -68,6 +68,7 @@ function gamesOf(votes) {
}
}
let cached = null;
let cached = null;
+const kindCache = new Map();
/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */
/** The board, with the fit and the per-vote weights. Cached on the shape of the vote set. */
async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
@@ -84,6 +85,22 @@ async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
@@ -84,6 +85,22 @@ async function board(ctx, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
return value;
return value;
}
}
+/**
+ * The board of one kind: the votes of that kind alone, fitted the same way as the whole board. A pick and a watch both
+ * ask about one kind at a time, and the band a model sits in there is the band that kind gives it.
+ */
+async function boardForKind(ctx, kindId, { bootstrap = style.BOOTSTRAP, force = false } = {}) {
+ const db = ctx.s.db;
+ const votes = (await loadVotes(db)).filter((v) => String(v.kind_id) === String(kindId));
+ const key = String(kindId) + ':' + votes.length + ':' + (votes.length ? votes[votes.length - 1].id : '') + ':' + num(bootstrap);
+ const hit = kindCache.get(String(kindId));
+ if (!force && hit && hit.key === key && Date.now() - hit.at < 5 * 60_000) return hit.value;
+ const fit = style.fitRatings(gamesOf(votes), { bootstrap });
+ const value = { fit, votes, generated_at: new Date().toISOString() };
+ kindCache.set(String(kindId), { key, at: Date.now(), value });
+ return value;
+}
+
/**
/**
* The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,
* The open dump: one line per vote, with no voter id anywhere. The weight the fit gave the vote is in the line,
* so the board can be re-derived from the file alone.
* so the board can be re-derived from the file alone.
@@ -120,4 +137,4 @@ function boardView(result) {
@@ -120,4 +137,4 @@ function boardView(result) {
};
};
}
}
-module.exports = { loadVotes, gamesOf, board, dumpJsonl, boardView, styleOf };
+module.exports = { loadVotes, gamesOf, board, boardForKind, dumpJsonl, boardView, styleOf };
@@ -23,7 +23,9 @@ const NAV = [
@@ -23,7 +23,9 @@ const NAV = [
{ label: 'Showcase', href: '/showcase' },
{ label: 'Showcase', href: '/showcase' },
{ label: 'Gallery', href: '/gallery' },
{ label: 'Gallery', href: '/gallery' },
{ label: 'Battles', href: '/battle' },
{ label: 'Battles', href: '/battle' },
+ { label: 'Pick for me', href: '/pick' },
{ label: 'Ratings', href: '/ratings' },
{ label: 'Ratings', href: '/ratings' },
+ { label: 'Watching', href: '/watching' },
{ label: 'Docs', href: '/docs' },
{ label: 'Docs', href: '/docs' },
];
];
@@ -456,6 +456,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -456,6 +456,12 @@ function createStore(s, { log = console, random = Math.random, config = null } =
);
);
}
}
await insertOutbox('bench.rating.changed', { kind_id: kindId, category: CATEGORY_DEFAULT, subjects: models });
await insertOutbox('bench.rating.changed', { kind_id: kindId, category: CATEGORY_DEFAULT, subjects: models });
+ // The ratings have been refitted: this is where the watches are checked.
+ if (typeof s.onRatingsChanged === 'function') {
+ try { await s.onRatingsChanged(kindId, models); } catch (err) {
+ if (log && typeof log.warn === 'function') log.warn('[OpenVibe.Bench] watch check failed: ' + (err && err.message ? err.message : err));
+ }
+ }
}
}
// ── Boards ──────────────────────────────────────────────
// ── Boards ──────────────────────────────────────────────
@@ -558,6 +564,44 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -558,6 +564,44 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return { kind_id: kindId, category, period, type, rows };
return { kind_id: kindId, category, period, type, rows };
}
}
+ /**
+ * Every model that ran a kind, with the numbers the pick reads: the score (how often its output passed), the cost
+ * per solved task, the median time a task took and how many runs it has. Every row is kept, including the ones
+ * another model beats on both score and cost: the pick works out the frontier itself.
+ */
+ async function valueRows(kindId) {
+ const cost = await costStats(kindId);
+ const pass = await passRate(kindId);
+ const eloRows = await boardElo(kindId);
+ const rows = [];
+ for (const [model, c] of cost) {
+ const p = pass.get(model);
+ const e = eloRows.find((x) => x.subject === model) || {};
+ const trials = num(p && p.trials);
+ const passed = num(p && p.passed);
+ const quality = trials ? passed / trials : (num(e.win_rate) || 0);
+ const totalCost = num(c.total_cost);
+ const solved = passed || num(c.usable);
+ rows.push({
+ subject: model,
+ score: Math.round(quality * 1000) / 10,
+ quality: Math.round(quality * 1e6) / 1e6,
+ votes: num(e.games),
+ win_rate: num(e.win_rate) || 0,
+ runs: num(c.usable),
+ entries: num(c.entries),
+ solved,
+ cost_per_solved: solved ? Math.round((totalCost / solved) * 1e6) / 1e6 : null,
+ median_cost_usd: num(c.median_cost),
+ median_ms: num(c.median_ms),
+ });
+ }
+ rows.sort((a, b) => (b.score - a.score)
+ || ((a.cost_per_solved == null ? Infinity : a.cost_per_solved) - (b.cost_per_solved == null ? Infinity : b.cost_per_solved))
+ || a.subject.localeCompare(b.subject));
+ return rows;
+ }
+
async function modelSummary(model) {
async function modelSummary(model) {
const name = String(model || '');
const name = String(model || '');
const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);
const ratings = (await db.many('SELECT * FROM bench_ratings WHERE subject = $1 ORDER BY kind_id, category', [name])).map(wireRating);
@@ -660,7 +704,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -660,7 +704,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
createTask, getTask,
createTask, getTask,
createRound, getRound, addEntry, maybeReady, finishRound,
createRound, getRound, addEntry, maybeReady, finishRound,
nextJudgement, castJudgement, pairwiseGames,
nextJudgement, castJudgement, pairwiseGames,
- board, modelSummary, dataset, datasetJsonl,
+ board, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
createEval, getEval, runEval, evalResults, evalDailyCount,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
KINDS, JUDGES, WINNERS, BASE_ELO, CATEGORY_DEFAULT, shuffle,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,92 @@
@@ -0,0 +1,92 @@
+'use strict';
+
+/**
+ * The pick page (spec 3): plain controls, then the frontier as a table, then one sample output per model.
+ * The page works without JavaScript: the form is a GET, and the words in the free-text box are read on the server.
+ */
+
+const { html, raw } = require('../render/html');
+const { frame } = require('../http/gallery');
+const { PRICE_STEPS } = require('../pick/pick');
+const { KINDS } = require('../kinds');
+
+const cents = (v) => (v == null ? '—' : (Math.round(v * 10000) / 100) + '¢');
+const msText = (v) => (v == null ? '—' : v >= 1000 ? (Math.round(v / 100) / 10) + 's' : Math.round(v) + 'ms');
+
+function pickForm(controls) {
+ const kinds = KINDS.map((k) => html`<option value="${k.id}"${k.id === controls.kind ? ' selected' : ''}>${k.name}</option>`);
+ const prices = [html`<option value=""${controls.max_cost == null ? ' selected' : ''}>Any price</option>`].concat(
+ PRICE_STEPS.map((p) => html`<option value="${p.value}"${controls.max_cost != null && Math.abs(controls.max_cost - p.value) < 1e-9 ? ' selected' : ''}>${p.label}</option>`));
+ return html`<form class="pk-form" method="get" action="/pick" data-pk-form="1">
+<label>Kind<select name="kind">${kinds}</select></label>
+<label>Top price per solved task<select name="max_cost">${prices}</select></label>
+<label class="pk-check"><input type="checkbox" name="fast" value="1"${controls.fast ? ' checked' : ''}> Fast</label>
+<label class="pk-words">Or say it in words<input type="text" name="q" value="${controls.q || ''}" placeholder="landing page under 2 cents, fast"></label>
+<button class="bm-btn" type="submit">Pick</button>
+</form>`;
+}
+
+function pickTable(a) {
+ if (!a.rows.length) {
+ return html`<div class="bm-empty"><h1>Nothing to pick yet</h1><p>${a.counts.models
+ ? 'No model on this frontier has both a score and a cost per solved task yet, so there is no answer to give. Nothing is guessed here.'
+ : 'No model has run this kind yet, so there is nothing to compare. The answer appears as soon as runs are published.'}</p></div>`;
+ }
+ const rows = a.rows.map((r) => html`<tr class="${r.thin ? 'pk-thin' : ''}">
+<td><a class="br-model" href="/models/${r.model}">${r.model}</a></td>
+<td><span class="pk-score">${r.score}</span> <span class="pk-band">${r.band ? 'band ' + r.band : 'no band yet'}</span></td>
+<td class="br-num">${cents(r.cost_per_solved)}</td>
+<td class="br-num">${msText(r.median_ms)}</td>
+<td class="br-num">${r.runs}${r.thin ? html` <span class="pk-thin-note">too few runs</span>` : ''}</td>
+</tr>`);
+ return html`<div class="br-table-wrap"><table class="br-table pk-table"><thead><tr>
+<th scope="col">Model</th><th scope="col">Score</th><th scope="col" class="br-num">Cost per solved task</th>
+<th scope="col" class="br-num">Median time</th><th scope="col" class="br-num">Runs</th>
+</tr></thead><tbody>${rows}</tbody></table></div>`;
+}
+
+function pickSamples(a) {
+ const withSample = a.rows.filter((r) => r.sample);
+ if (!withSample.length) return '';
+ return html`<section class="bm-section" aria-labelledby="pk-samples-h"><h2 id="pk-samples-h">One output each</h2>
+<p class="bm-fine">The newest published output of every model on the frontier, live in its sandboxed frame.</p>
+<div class="pk-samples">${withSample.map((r) => html`<article class="pk-card">
+<h3 class="pk-card-h">${r.model}</h3>
+<div class="pk-shot">${raw(frame(r.sample, !r.sample.screenshot))}</div>
+<p class="bm-fine pk-prompt">${r.sample.prompt}</p>
+</article>`)}</div></section>`;
+}
+
+function pickPage(a) {
+ const notes = (a.controls.notes || []).length
+ ? html`<p class="pk-notes bm-fine">Read from the words: ${a.controls.notes.join('; ')}.</p>`
+ : '';
+ const thin = a.rows.filter((r) => r.thin).length;
+ const thinNote = thin
+ ? html`<p class="pk-thin-note bm-fine">${thin} model${thin === 1 ? '' : 's'} on this frontier have fewer than ${a.filters.min_runs} runs. Their numbers are shown greyed, because that is all the data there is.</p>`
+ : '';
+ const fastNote = a.filters.fast && a.filters.fast_cap_ms != null
+ ? html`<p class="bm-fine">Fast: a median time of ${msText(a.filters.fast_cap_ms)} a task or under, the middle of the run times.</p>`
+ : '';
+ const also = (a.also_ran || []).length ? html`<section class="bm-section" aria-labelledby="pk-also-h"><h2 id="pk-also-h">Also ran this kind</h2>
+<ul class="pk-also">${(a.also_ran || []).map((r) => html`<li class="${r.thin ? 'pk-thin' : ''}"><b>${r.model}</b> · score ${r.score} · ${cents(r.cost_per_solved)} a solved task · ${r.runs} run${r.runs === 1 ? '' : 's'}${r.thin ? ' — too few runs to stand on, so these numbers are greyed' : ' — another model is at least as good on score and at least as cheap'}</li>`)}</ul>
+</section>` : '';
+ return html`<div class="bm-wrap pk" id="pk-pick">
+<header class="bm-page-h"><p class="bm-kicker">OpenVibe.Bench</p><h1>Pick for me</h1>
+<p class="bm-lede">Say the job with plain controls. No model is called and nothing is charged: the answer comes from the runs Bench has published.</p></header>
+${pickForm(a.controls)}
+<p class="bm-fine">The free-text box is read by a plain parser, never by a model. It only sets the three controls above.</p>
+${notes}
+<section class="bm-section pk-answer" aria-labelledby="pk-answer-h">
+<h2 id="pk-answer-h">The frontier for ${a.kind.name}</h2>
+<p class="pk-why">${a.why}</p>
+${fastNote}
+${pickTable(a)}
+${thinNote}
+</section>
+${also}
+${pickSamples(a)}
+</div>`;
+}
+
+module.exports = { pickPage, pickForm, pickTable, pickSamples };
@@ -7,6 +7,7 @@
@@ -7,6 +7,7 @@
* Ids and classes start with `br-`.
* Ids and classes start with `br-`.
*/
*/
const { html, raw, render } = require('../render/html');
const { html, raw, render } = require('../render/html');
+const { benchmarkUseThis, modelUseThis } = require('./usethis');
const renderLib = require('../bench/render');
const renderLib = require('../bench/render');
const MEDIA_ORIGIN = 'https://openvibe.media';
const MEDIA_ORIGIN = 'https://openvibe.media';
@@ -201,7 +202,7 @@ function benchmarkCard(b) {
@@ -201,7 +202,7 @@ function benchmarkCard(b) {
return html`<li class="br-card"><a class="br-card-link" href="${benchmarkHref(b.id)}"><span class="br-card-top">${kindChip(b.kind)}<span class="br-card-v">v${b.latest_version || 0}</span></span><span class="br-card-title">${b.title}</span><span class="br-card-desc">${b.description}</span><span class="br-card-foot">${plural(tasks, 'task')}${b.runs != null ? html` · ${plural(b.runs, 'run')}` : ''}${b.parent ? ' · a fork' : ''}</span></a></li>`;
return html`<li class="br-card"><a class="br-card-link" href="${benchmarkHref(b.id)}"><span class="br-card-top">${kindChip(b.kind)}<span class="br-card-v">v${b.latest_version || 0}</span></span><span class="br-card-title">${b.title}</span><span class="br-card-desc">${b.description}</span><span class="br-card-foot">${plural(tasks, 'task')}${b.runs != null ? html` · ${plural(b.runs, 'run')}` : ''}${b.parent ? ' · a fork' : ''}</span></a></li>`;
}
}
-function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null }) {
+function benchmarkPage({ benchmark, definition, version, rankings, metric = 'score', runs = [], codesRunUrl = null, viewer = null, baseUrl = '' }) {
const d = definition || {};
const d = definition || {};
const tasks = Array.isArray(d.tasks) ? d.tasks : [];
const tasks = Array.isArray(d.tasks) ? d.tasks : [];
const shown = tasks.filter((t) => !t.holdout && t.prompt);
const shown = tasks.filter((t) => !t.holdout && t.prompt);
@@ -227,6 +228,7 @@ ${entries.length ? rankingsTable(entries, metric) : empty('No verified runs yet'
@@ -227,6 +228,7 @@ ${entries.length ? rankingsTable(entries, metric) : empty('No verified runs yet'
<ol class="br-tasks">${shown.map((t) => html`<li class="br-task"><span class="br-task-id br-mono">${t.id}</span><p>${t.prompt}</p></li>`)}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section>
<ol class="br-tasks">${shown.map((t) => html`<li class="br-task"><span class="br-task-id br-mono">${t.id}</span><p>${t.prompt}</p></li>`)}${hidden ? html`<li class="br-task is-hidden"><span class="br-task-id br-mono">${plural(hidden, 'task')}</span><p>Held out: only the executor sees ${hidden === 1 ? 'it' : 'them'}, so nobody can tune for the answer.</p></li>` : ''}</ol></section>
<section class="bm-section" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>
<section class="bm-section" aria-labelledby="br-latest-h"><div class="bm-head-row"><h2 id="br-latest-h">Latest runs</h2>${runs.length > 1 ? html`<a class="bm-more" href="${compareHref(runs.slice(0, 2).map((r) => r.id))}">Compare the newest two</a>` : ''}</div>
${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section>
${runs.length ? runsList(runs) : empty('No runs yet', 'The first run shows up here, verified or submitted.')}</section>
+${benchmarkUseThis({ baseUrl, benchmark, entries, viewer })}
<p class="br-fine br-api">The same data as JSON: <a href="/api/v1/benchmarks/${encodeURIComponent(benchmark.id)}">definition</a> · <a href="/api/v1/rankings?benchmark=${encodeURIComponent(benchmark.id)}&metric=${metric}">rankings</a></p>
<p class="br-fine br-api">The same data as JSON: <a href="/api/v1/benchmarks/${encodeURIComponent(benchmark.id)}">definition</a> · <a href="/api/v1/rankings?benchmark=${encodeURIComponent(benchmark.id)}&metric=${metric}">rankings</a></p>
</div>`;
</div>`;
}
}
@@ -322,7 +324,7 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta
@@ -322,7 +324,7 @@ ${rows.some((row) => runs.some((r) => ((r.tasks.find((t) => t.task_id === row.ta
</div>`;
</div>`;
}
}
-function modelPage({ model, summary, runs = [], titles = {} }) {
+function modelPage({ model, summary, runs = [], titles = {}, viewer = null, baseUrl = '' }) {
const rows = (summary && summary.benchmarks) || [];
const rows = (summary && summary.benchmarks) || [];
return html`<div class="bm-wrap br" id="br-model">
return html`<div class="bm-wrap br" id="br-model">
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Models</span></nav>
@@ -330,6 +332,7 @@ function modelPage({ model, summary, runs = [], titles = {} }) {
@@ -330,6 +332,7 @@ function modelPage({ model, summary, runs = [], titles = {} }) {
<section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>
<section class="bm-section" aria-labelledby="br-mb-h"><h2 id="br-mb-h">By benchmark</h2>
${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>
${rows.length ? html`<div class="br-table-wrap"><table class="br-table"><thead><tr><th scope="col">Benchmark</th><th scope="col">Median</th><th scope="col" class="br-num">Spread</th><th scope="col" class="br-num">Cost / run</th><th scope="col" class="br-num">Time</th><th scope="col" class="br-num">Runs</th></tr></thead><tbody>${rows.map((b) => html`<tr><td><a href="${benchmarkHref(b.benchmark_id)}">${b.title}</a> ${kindChip(b.kind)}</td><td>${scoreBar(b.score)}</td><td class="br-num">${b.runs > 1 ? `${fmtScore(b.worst)}–${fmtScore(b.best)}` : '—'}</td><td class="br-num">${fmtUsd(b.cost_usd)}</td><td class="br-num">${fmtMs(b.ms)}</td><td class="br-num">${b.runs}</td></tr>`)}</tbody></table></div>` : empty('Nothing measured yet', 'This model shows up here after its first verified run.', html`<a class="bm-btn bm-btn-small" href="/benchmarks">Pick a benchmark</a>`)}</section>
${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}
${runs.length ? html`<section class="bm-section" aria-labelledby="br-mr-h"><h2 id="br-mr-h">Latest runs</h2>${runsList(runs, { showBenchmark: true, titles })}</section>` : ''}
+${modelUseThis({ baseUrl, model, rows, viewer })}
</div>`;
</div>`;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,88 @@
@@ -0,0 +1,88 @@
+'use strict';
+
+/**
+ * "Use this" (spec 3): the copy boxes. A benchmark and a model page carry the badge's Markdown and HTML; the battle
+ * page carries the embed's iframe snippet. The boxes are plain textareas, so copying works without JavaScript.
+ */
+
+const { html } = require('../render/html');
+
+/** The badge's URL for a benchmark and a model. */
+function badgeUrl(baseUrl, benchmarkId, model) {
+ return String(baseUrl || '').replace(/\/+$/, '') + '/badge/' + encodeURIComponent(benchmarkId) + '/' + encodeURIComponent(model) + '.svg';
+}
+
+/** The Markdown and the HTML of the badge. */
+function badgeSnippets(baseUrl, benchmarkId, model, title) {
+ const url = badgeUrl(baseUrl, benchmarkId, model);
+ const alt = (title ? title + ' on OpenVibe Bench' : 'OpenVibe Bench') + ' badge';
+ return { markdown: '', html: '<img src="' + url + '" alt="' + alt + '">' };
+}
+
+/** The embed's snippet: an iframe that may be framed by other sites. */
+function embedSnippets(baseUrl, battleId, title) {
+ const url = String(baseUrl || '').replace(/\/+$/, '') + '/embed/battle/' + encodeURIComponent(battleId);
+ const name = title || 'OpenVibe Bench battle';
+ return {
+ markdown: '[' + name + '](' + url + ')',
+ html: '<iframe src="' + url + '" width="100%" height="560" style="border:0" loading="lazy" title="' + name + '"></iframe>',
+ };
+}
+
+/** A copy box: one textarea per snippet. */
+function copyBox({ heading, pairs }) {
+ return html`<div class="br-usethis-box"><h3 class="br-usethis-h">${heading}</h3>
+${pairs.map((p) => html`<label class="br-usethis-row"><span class="br-fine">${p.label}</span><textarea readonly rows="2" aria-label="${p.label}">${p.value}</textarea></label>`)}
+</div>`;
+}
+
+/** The follow form: a plain POST, so it works without JavaScript. */
+function followForm({ benchmarkId, model = null, back = '/watching', signedIn = false }) {
+ if (!signedIn) return html`<p class="br-fine">Sign in to follow this, and Bench will tell you when it moves.</p>`;
+ return html`<form class="br-follow" method="post" action="/watch/follow">
+<input type="hidden" name="benchmark_id" value="${benchmarkId}">
+${model ? html`<input type="hidden" name="model" value="${model}">` : ''}
+<input type="hidden" name="back" value="${back}">
+<button class="bm-btn bm-btn-small" type="submit">${model ? 'Follow this model' : 'Follow this benchmark'}</button>
+</form>`;
+}
+
+/** The benchmark page: a badge for each model it ranks, and the follow button. */
+function benchmarkUseThis({ baseUrl, benchmark, entries = [], viewer = null, back = '' }) {
+ const boxes = entries.slice(0, 4).map((e) => {
+ const s = badgeSnippets(baseUrl, benchmark.id, e.model, benchmark.title);
+ return copyBox({ heading: e.model, pairs: [{ label: 'Markdown', value: s.markdown }, { label: 'HTML', value: s.html }] });
+ });
+ return html`<section class="bm-section" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>
+<p class="br-fine">A badge for your README, and a watch that tells you when this benchmark moves.</p>
+${boxes}
+${followForm({ benchmarkId: benchmark.id, back: back || ('/benchmarks/' + benchmark.id), signedIn: Boolean(viewer && viewer.kind === 'user') })}
+</section>`;
+}
+
+/** The model page: a badge for each benchmark the model ran, and a watch for each. */
+function modelUseThis({ baseUrl, model, rows = [], viewer = null, back = '' }) {
+ const boxes = rows.slice(0, 4).map((b) => {
+ const s = badgeSnippets(baseUrl, b.benchmark_id, model, b.title);
+ return html`<div class="br-usethis-box"><h3 class="br-usethis-h">${b.title}</h3>
+<label class="br-usethis-row"><span class="br-fine">Markdown</span><textarea readonly rows="2" aria-label="Markdown">${s.markdown}</textarea></label>
+<label class="br-usethis-row"><span class="br-fine">HTML</span><textarea readonly rows="2" aria-label="HTML">${s.html}</textarea></label>
+${followForm({ benchmarkId: b.benchmark_id, model, back: back || ('/models/' + encodeURIComponent(model)), signedIn: Boolean(viewer && viewer.kind === 'user') })}
+</div>`;
+ });
+ return html`<section class="bm-section" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>
+<p class="br-fine">A badge for your README, and a watch for each benchmark this model ran.</p>
+${boxes}
+</section>`;
+}
+
+/** The battle page: the embed's iframe snippet. */
+function battleUseThis({ baseUrl, battle, title }) {
+ const s = embedSnippets(baseUrl, battle.id, title);
+ return html`<section class="bm-section br-usethis" aria-labelledby="br-use-h"><h2 id="br-use-h">Use this</h2>
+<p class="br-fine">An embed of this battle for another site. It may be framed; voting opens Bench in a new tab.</p>
+<label class="br-usethis-row"><span class="br-fine">HTML</span><textarea readonly rows="3" aria-label="HTML">${s.html}</textarea></label>
+</section>`;
+}
+
+module.exports = { badgeUrl, badgeSnippets, embedSnippets, copyBox, followForm, benchmarkUseThis, modelUseThis, battleUseThis };
new file mode 100644
new file mode 100644
@@ -0,0 +1,28 @@
@@ -0,0 +1,28 @@
+'use strict';
+
+/**
+ * The Watching page (spec 6): the benchmarks and models a person follows, with an unfollow button on each.
+ * The buttons are plain forms, so the page works without JavaScript.
+ */
+
+const { html } = require('../render/html');
+
+function watchingPage(watches, { viewer = null } = {}) {
+ const items = watches.map((w) => html`<li class="wk-item">
+<div class="wk-what"><b>${w.benchmark_title || w.benchmark_id}</b>
+${w.model ? html`<span class="br-chip">${w.model}</span>` : html`<span class="br-chip">the whole benchmark</span>`}
+<span class="bm-fine">followed since ${String(w.created_at || '').slice(0, 10)}</span></div>
+<form method="post" action="/watch/unfollow" class="wk-form">
+<input type="hidden" name="id" value="${w.id}">
+<button class="bm-btn bm-btn-quiet bm-btn-small" type="submit">Unfollow</button>
+</form>
+</li>`);
+ return html`<div class="bm-wrap wk" id="wk-watching">
+<header class="bm-page-h"><p class="bm-kicker">OpenVibe.Bench</p><h1>Watching</h1>
+<p class="bm-lede">The benchmarks and models you follow. When a score moves by more than its own interval, or a model overtakes another one, Bench tells the Network and the notice lands in your Action Center. One notice a day per watch.</p></header>
+${viewer && viewer.kind === 'user' ? '' : html`<p class="bm-fine"><a class="bm-btn bm-btn-small" href="/auth/login?next=%2Fwatching">Sign in with OpenVibe</a> to follow a benchmark or a model.</p>`}
+${watches.length ? html`<ul class="wk-list">${items}</ul>` : html`<div class="bm-empty"><h1>Nothing followed yet</h1><p>Follow a benchmark from its page, or a model on a benchmark from the model's page.</p></div>`}
+</div>`;
+}
+
+module.exports = { watchingPage };
new file mode 100644
new file mode 100644
@@ -0,0 +1,266 @@
@@ -0,0 +1,266 @@
+'use strict';
+
+/**
+ * Model watch (spec 6): follow a benchmark, or a model on a benchmark, and hear when it moves.
+ *
+ * Two things are worth a notice:
+ * - a model's style-controlled score moves by more than its own interval (the board of B9), and
+ * - a model overtakes another one on that benchmark: its band sits fully above the other's.
+ *
+ * The check runs where the ratings are refitted, so the board it reads is the board the person would see. One
+ * notice per watch per day at most, and the text names the benchmark, the two models and the numbers.
+ *
+ * The notice itself is a Network notification (network.notification-push-request@1): it lands in the Action Center,
+ * and email or chat follow the person's own Network settings. Bench never sends email.
+ */
+
+const { ids } = require('openvibe-contracts');
+
+const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+const round = (v) => (v == null ? null : Math.round(Number(v) * 10) / 10);
+const dayOf = (v) => (v == null ? null : new Date(v).toISOString().slice(0, 10));
+
+/** The notification client: Bench's own service token, pushed at the Network. */
+function notifier(ctx) {
+ if (ctx.benchNotifier) return ctx.benchNotifier;
+ const { createClient } = require('openvibe-sdk/core');
+ const { createServiceTokenClient } = require('openvibe-sdk/auth');
+ const { createNotificationsClient } = require('openvibe-sdk/notifications');
+ const network = ctx.config.networkInternalUrl;
+ const tokens = createServiceTokenClient({
+ tokenUrl: network + '/oauth/token',
+ clientId: ctx.config.oauth && ctx.config.oauth.clientId,
+ clientSecret: ctx.config.oauth && ctx.config.oauth.clientSecret,
+ audience: 'openvibe.network',
+ scope: 'network.notifications.push',
+ fetch: ctx.fetchImpl || globalThis.fetch,
+ });
+ const client = createClient({
+ tokenProvider: tokens,
+ baseUrls: { network },
+ fetch: ctx.fetchImpl || globalThis.fetch,
+ timeoutMs: 8000,
+ });
+ ctx.benchNotifier = createNotificationsClient(client, { baseUrl: network });
+ return ctx.benchNotifier;
+}
+
+/** The UTC day a timestamp falls in. */
+function today(now) {
+ return new Date(now || Date.now()).toISOString().slice(0, 10);
+}
+
+/** True when the watch may get a notice: one notice per watch per day. */
+function mayNotify(watch, stamp) {
+ return dayOf(watch && watch.last_notified_at) !== today(stamp);
+}
+
+function createWatches(ctx) {
+ const db = ctx.s.db;
+
+ /** The watches of one person, with the benchmark they are on. */
+ async function listFor(subject) {
+ return db.many(
+ `SELECT w.id, w.subject, w.benchmark_id, w.model, w.user_id, w.created_at, w.last_notified_at,
+ b.title AS benchmark_title, b.kind AS benchmark_kind, b.latest_version
+ FROM bench_watches w JOIN bench_benchmarks b ON b.id = w.benchmark_id
+ WHERE w.subject = $1 ORDER BY w.created_at DESC`,
+ [String(subject || '')]);
+ }
+
+ /** Follow a benchmark, or a model on a benchmark. Following twice changes nothing. */
+ async function follow({ subject, benchmarkId, model = null, userId = null }) {
+ const benchmark = await ctx.registry.getBenchmark(String(benchmarkId || ''));
+ if (!benchmark) return { ok: false, status: 404, error: 'No such benchmark.' };
+ const row = await db.maybe(
+ `INSERT INTO bench_watches (id, subject, benchmark_id, model, user_id, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6::timestamptz)
+ ON CONFLICT (subject, benchmark_id, COALESCE(model, ''))
+ DO UPDATE SET user_id = COALESCE(EXCLUDED.user_id, bench_watches.user_id)
+ RETURNING *`,
+ [ids.newId('watch'), String(subject), benchmark.id, model ? String(model) : null,
+ userId == null ? null : Number(userId), ctx.s.iso()]);
+ return { ok: true, watch: row };
+ }
+
+ /** Stop watching. */
+ async function unfollow({ subject, id }) {
+ const row = await db.maybe('DELETE FROM bench_watches WHERE id = $1 AND subject = $2 RETURNING id',
+ [String(id || ''), String(subject || '')]);
+ return { ok: Boolean(row) };
+ }
+
+ /** The style-controlled fit of a kind. */
+ async function fitOf(kindId) {
+ const board = await require('../rating/board').boardForKind(ctx, kindId);
+ return board.fit;
+ }
+
+ /** The board as it was last seen, so a move can be measured against it. */
+ async function previous(kindId) {
+ const rows = await db.many('SELECT * FROM bench_watch_ratings WHERE kind_id = $1', [String(kindId)]);
+ return new Map(rows.map((r) => [r.model, r]));
+ }
+
+ /** Remember the board we have just seen. */
+ async function remember(kindId, models) {
+ if (!models.length) return;
+ const values = [];
+ const params = [];
+ for (const m of models) {
+ values.push('($1, $' + (params.length + 2) + ', $' + (params.length + 3) + ', $' + (params.length + 4)
+ + ', $' + (params.length + 5) + ', $' + (params.length + 6) + ', now())');
+ params.push(m.model, m.rating, m.lower, m.upper, m.band);
+ }
+ await db.exec(
+ `INSERT INTO bench_watch_ratings (kind_id, model, rating, ci_lower, ci_upper, band, updated_at)
+ VALUES ${values.join(', ')}
+ ON CONFLICT (kind_id, model) DO UPDATE SET rating = EXCLUDED.rating, ci_lower = EXCLUDED.ci_lower,
+ ci_upper = EXCLUDED.ci_upper, band = EXCLUDED.band, updated_at = now()`,
+ [String(kindId), ...params]);
+ }
+
+ /** The notice a watch gets from the board, or null. */
+ function noticeFor(watch, models, bands, before, all) {
+ const byModel = new Map(models.map((m) => [m.model, m]));
+ const followed = watch.model ? byModel.get(watch.model) : null;
+ const title = watch.benchmark_title || watch.benchmark_id;
+
+ if (watch.model) {
+ if (!followed) return null;
+ const was = before.get(watch.model);
+ if (was && was.rating != null && followed.rating != null) {
+ const delta = followed.rating - num(was.rating);
+ const interval = followed.upper - followed.lower;
+ if (Math.abs(delta) > interval) {
+ return {
+ kind: 'move',
+ title: 'Bench: ' + title,
+ message: title + ': ' + watch.model + ' moved from ' + round(num(was.rating)) + ' to '
+ + round(followed.rating) + ' (' + (delta > 0 ? '+' : '') + round(delta) + '), more than its '
+ + round(interval) + '-point interval (' + round(followed.lower) + '–' + round(followed.upper) + ').',
+ models: [watch.model],
+ };
+ }
+ }
+ const over = models.find((m) => m.model !== watch.model && m.lower > followed.upper
+ && !all.some((w) => w.subject === watch.subject && w.benchmark_id === watch.benchmark_id && w.model === m.model));
+ if (over) {
+ return {
+ kind: 'overtake',
+ title: 'Bench: ' + title,
+ message: title + ': ' + over.model + ' (' + round(over.rating) + ', ' + round(over.lower) + '–'
+ + round(over.upper) + ') is now fully above ' + watch.model + ' (' + round(followed.rating) + ', '
+ + round(followed.lower) + '–' + round(followed.upper) + ').',
+ models: [over.model, watch.model],
+ };
+ }
+ return null;
+ }
+
+ // A benchmark watch: any model that moved by more than its interval, and any model that overtook another.
+ for (const m of models) {
+ const was = before.get(m.model);
+ if (!was || was.rating == null || m.rating == null) continue;
+ const delta = m.rating - num(was.rating);
+ const interval = m.upper - m.lower;
+ if (Math.abs(delta) > interval) {
+ return {
+ kind: 'move',
+ title: 'Bench: ' + title,
+ message: title + ': ' + m.model + ' moved from ' + round(num(was.rating)) + ' to ' + round(m.rating)
+ + ' (' + (delta > 0 ? '+' : '') + round(delta) + '), more than its ' + round(interval)
+ + '-point interval (' + round(m.lower) + '–' + round(m.upper) + ').',
+ models: [m.model],
+ };
+ }
+ }
+ const over = models.find((m) => models.some((o) => o.model !== m.model && m.lower > o.upper));
+ if (over) {
+ const under = models.find((o) => o.model !== over.model && over.lower > o.upper);
+ return {
+ kind: 'overtake',
+ title: 'Bench: ' + title,
+ message: title + ': ' + over.model + ' (' + round(over.rating) + ', ' + round(over.lower) + '–'
+ + round(over.upper) + ') is now fully above ' + (under ? under.model : 'the model below it') + '.',
+ models: [over.model, under ? under.model : null].filter(Boolean),
+ };
+ }
+ return null;
+ }
+
+ /** Push the notice to the Network. The Network decides whether it reaches email or chat. */
+ async function notify(watch, notice) {
+ if (watch.user_id == null) return { sent: false, reason: 'no_network_user' };
+ try {
+ const out = await notifier(ctx).push({
+ userId: Number(watch.user_id),
+ type: 'WATCH_TRIGGERED',
+ title: notice.title,
+ message: notice.message,
+ url: (ctx.config.baseUrl || 'https://bench.openvibe.services') + '/benchmarks/' + watch.benchmark_id,
+ category: 'service',
+ priority: 'normal',
+ service: 'bench',
+ });
+ return { sent: Boolean(out && out.sent), skipped: Boolean(out && out.skipped) };
+ } catch (err) {
+ ctx.log.warn('[OpenVibe.Bench] watch notice failed: ' + (err && err.message ? err.message : err));
+ return { sent: false, reason: 'push_failed' };
+ }
+ }
+
+ /**
+ * Look at the board of a kind and tell the watchers what changed. Runs where the ratings are refitted.
+ */
+ async function check(kindId, { now = null } = {}) {
+ const kind = String(kindId || '');
+ if (!kind) return { notices: [], checked: 0 };
+ const watches = await db.many(
+ `SELECT w.*, b.title AS benchmark_title, b.kind AS benchmark_kind
+ FROM bench_watches w JOIN bench_benchmarks b ON b.id = w.benchmark_id
+ WHERE b.kind = $1`,
+ [kind]);
+ if (!watches.length) return { notices: [], checked: 0 };
+
+ const fit = await fitOf(kind);
+ const bands = new Map();
+ for (const b of ((fit.bands && fit.bands.controlled) || [])) for (const m of b.models || []) bands.set(m, b.label);
+ const models = (fit.models || []).map((m) => ({
+ model: m.model,
+ rating: num(m.controlled.rating),
+ lower: num(m.controlled.ci_lower),
+ upper: num(m.controlled.ci_upper),
+ band: bands.get(m.model) || null,
+ votes: m.votes,
+ }));
+ const before = await previous(kind);
+ await remember(kind, models);
+
+ const notices = [];
+ const stamp = now || new Date().toISOString();
+ for (const w of watches) {
+ const notice = noticeFor(w, models, bands, before, watches);
+ if (!notice) continue;
+ if (!mayNotify(w, stamp)) continue; // one notice per watch per day
+ const sent = await notify(w, notice);
+ await db.exec('UPDATE bench_watches SET last_notified_at = $1 WHERE id = $2', [stamp, w.id]);
+ notices.push({ watch: w.id, kind: notice.kind, message: notice.message, sent });
+ }
+ return { notices, checked: watches.length };
+ }
+
+ /** The event Bench already publishes when the ratings change, so a watch rides on it. */
+ async function publishRatingChanged(kindId, subjects) {
+ const row = await ctx.bench.insertOutbox('bench.rating.changed', {
+ kind_id: String(kindId || ''),
+ subjects: (subjects || []).map(String),
+ at: new Date().toISOString(),
+ });
+ return row;
+ }
+
+ return { listFor, follow, unfollow, check, notify, publishRatingChanged, noticeFor, notifier };
+}
+
+module.exports = { createWatches, notifier, today, dayOf, mayNotify };
new file mode 100644
new file mode 100644
@@ -0,0 +1,52 @@
@@ -0,0 +1,52 @@
+'use strict';
+
+/** The badge (B10): escaped text, a width from the text, the right headers, and a grey badge with a 404. */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { badgeSvg } = require('../server/http/badge');
+
+(async () => {
+ await check('the text of the badge is escaped', () => {
+ const svg = badgeSvg({ label: 'a & b <script>', message: '"x" \'y\'' });
+ assert.ok(svg.includes('&'), 'the ampersand');
+ assert.ok(svg.includes('<script>'), 'the angle brackets');
+ assert.ok(!svg.includes('<script>'), 'no raw script tag');
+ });
+
+ await check('the width comes from the text', () => {
+ const short = badgeSvg({ label: 'ab', message: 'c' });
+ const long = badgeSvg({ label: 'a much longer benchmark title', message: 'c' });
+ const w = (s) => Number(/width="(\d+)"/.exec(s)[1]);
+ assert.ok(w(long) > w(short), 'the longer badge is wider: ' + w(long) + ' vs ' + w(short));
+ });
+
+ const t = await boot();
+ const list = (await t.get('/api/v1/benchmarks')).json();
+ const benchmark = (list.benchmarks || list)[0];
+
+ await check('an unknown benchmark gets a grey badge and a 404', async () => {
+ const r = await t.get('/badge/bmk_nope/nope.svg');
+ assert.strictEqual(r.status, 404, r.text.slice(0, 200));
+ assert.ok(/image\/svg\+xml/.test(r.headers.get('content-type')), 'the type: ' + r.headers.get('content-type'));
+ assert.strictEqual(r.headers.get('content-security-policy'), "default-src 'none'; style-src 'unsafe-inline'");
+ assert.strictEqual(r.headers.get('x-content-type-options'), 'nosniff');
+ assert.strictEqual(r.headers.get('cache-control'), 'public, max-age=3600');
+ assert.ok(r.text.includes('no data'), 'it says there is no data');
+ assert.ok(r.text.includes('<svg'), 'and it is still a picture');
+ });
+
+ await check('a model with no runs gets a grey badge and a 404', async () => {
+ const r = await t.get('/badge/' + benchmark.id + '/no-such-model.svg');
+ assert.strictEqual(r.status, 404);
+ assert.ok(r.text.includes('no data'), 'it says there is no data');
+ });
+
+ await check('a badge may be used from another site', async () => {
+ const r = await t.get('/badge/' + benchmark.id + '/no-such-model.svg');
+ assert.strictEqual(r.headers.get('cross-origin-resource-policy'), 'cross-origin');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,61 @@
@@ -0,0 +1,61 @@
+'use strict';
+
+/** The embed (B10): only that route may be framed, no cookies, and voting opens Bench in a new tab. */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+(async () => {
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob'); // the round belongs to somebody else: a battle is never your own
+ const entrySvc = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
+
+ const task = await t.get('/api/v1/kinds/games.play/tasks', { as: bob, json: { prompt: 'A game for an embed.', public: true } });
+ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { as: bob, json: {} });
+ const roundId = rr.json().round.id;
+ const e1 = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: entrySvc, json: { provider: 'acme', model: 'alpha-1', payload: { html: '<h1>a</h1>' } } });
+ const e2 = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: entrySvc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<h1>b</h1>' } } });
+ assert.strictEqual(e1.status, 201, e1.text);
+ assert.strictEqual(e2.status, 201, e2.text);
+
+ // A battle needs a finished round: the job would do it, the test does it directly.
+ await t.ctx.s.db.exec("UPDATE bench_rounds SET status = 'finished' WHERE id = $1", [roundId]);
+ const battle = await t.get('/battle', { as: alice });
+ assert.strictEqual(battle.status, 200, battle.text.slice(0, 200));
+ const id = /data-battle="([^"]+)"/.exec(battle.text)[1];
+
+ await check('the embed may be framed, and every other route may not', async () => {
+ const e = await t.get('/embed/battle/' + id);
+ assert.strictEqual(e.status, 200, e.text.slice(0, 200));
+ assert.ok(/frame-ancestors \*/.test(e.headers.get('content-security-policy')), 'the embed may be framed: ' + e.headers.get('content-security-policy'));
+ const home = await t.get('/');
+ assert.ok(/frame-ancestors 'none'/.test(home.headers.get('content-security-policy')), 'the rest refuses frames: ' + home.headers.get('content-security-policy'));
+ assert.strictEqual(home.headers.get('x-frame-options'), 'DENY');
+ });
+
+ await check('the embed reads and sets no cookie', async () => {
+ const e = await t.get('/embed/battle/' + id);
+ assert.strictEqual(e.headers.get('set-cookie'), null, 'no cookie is set');
+ const home = await t.get('/');
+ assert.strictEqual(home.headers.get('set-cookie'), null, 'and none on the other pages either');
+ });
+
+ await check('the embed keeps the frames sandboxed and sends the vote to a new tab', async () => {
+ const e = await t.get('/embed/battle/' + id);
+ assert.ok(e.text.includes('sandbox="allow-scripts allow-same-origin"'), 'the frames are sandboxed');
+ assert.ok(e.text.includes('target="_blank"'), 'the vote opens a new tab');
+ assert.ok(e.text.includes('rel="noopener"'), 'and it is safe');
+ assert.ok(e.text.includes('noindex'), 'it is not indexed');
+ assert.ok(e.text.includes('OpenVibe Bench'), 'it links back');
+ });
+
+ await check('an unknown battle is a plain 404, still framable', async () => {
+ const e = await t.get('/embed/battle/bbs_nope');
+ assert.strictEqual(e.status, 404);
+ assert.ok(/frame-ancestors \*/.test(e.headers.get('content-security-policy')));
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -37,7 +37,7 @@ async function startNetwork() {
@@ -37,7 +37,7 @@ async function startNetwork() {
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });
const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 });
const publicPem = publicKey.export({ type: 'spki', format: 'pem' });
const publicPem = publicKey.export({ type: 'spki', format: 'pem' });
const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' });
const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' });
- const st = { users: new Map(), byUsername: new Map(), codes: new Map(), refresh: new Map(), requests: [], tokenRequests: [], down: false };
+ const st = { users: new Map(), byUsername: new Map(), codes: new Map(), refresh: new Map(), requests: [], tokenRequests: [], pushes: [], down: false };
let issuer = null;
let issuer = null;
let nextUserId = 1;
let nextUserId = 1;
@@ -74,6 +74,12 @@ async function startNetwork() {
@@ -74,6 +74,12 @@ async function startNetwork() {
if (st.down) return problem(json, 503, 'network.down', 'Network is restarting');
if (st.down) return problem(json, 503, 'network.down', 'Network is restarting');
if (url.pathname === '/api/.well-known/jwks') return json(200, { public_key: publicPem, algorithm: 'RS256' });
if (url.pathname === '/api/.well-known/jwks') return json(200, { public_key: publicPem, algorithm: 'RS256' });
+ if (url.pathname === '/internal/notifications/push' && req.method === 'POST') {
+ let body = {};
+ try { body = JSON.parse(raw.toString('utf8') || '{}'); } catch { body = {}; }
+ st.pushes.push(body);
+ return json(200, { sent: true });
+ }
if (url.pathname === '/oauth/revoke') return json(200, {});
if (url.pathname === '/oauth/revoke') return json(200, {});
if (url.pathname === '/oauth/token' && req.method === 'POST') {
if (url.pathname === '/oauth/token' && req.method === 'POST') {
const ct = String(req.headers['content-type'] || '');
const ct = String(req.headers['content-type'] || '');
@@ -116,7 +122,7 @@ async function startNetwork() {
@@ -116,7 +122,7 @@ async function startNetwork() {
issuer = srv.url;
issuer = srv.url;
return {
return {
- url: srv.url, publicPem, privatePem, state: st, requests: st.requests, tokenRequests: st.tokenRequests,
+ url: srv.url, publicPem, privatePem, state: st, requests: st.requests, tokenRequests: st.tokenRequests, pushes: st.pushes,
addUser, userToken, serviceToken, signJwt: (claims) => signJwt(claims, privatePem), issuer: () => issuer,
addUser, userToken, serviceToken, signJwt: (claims) => signJwt(claims, privatePem), issuer: () => issuer,
issueCode(user, challenge) { const code = `code_${crypto.randomBytes(8).toString('hex')}`; st.codes.set(code, { user, challenge }); return code; },
issueCode(user, challenge) { const code = `code_${crypto.randomBytes(8).toString('hex')}`; st.codes.set(code, { user, challenge }); return code; },
setDown(v) { st.down = v; },
setDown(v) { st.down = v; },
new file mode 100644
new file mode 100644
@@ -0,0 +1,142 @@
@@ -0,0 +1,142 @@
+'use strict';
+
+/**
+ * The pick (B10): the frontier maths, the keyword parser, thin data greyed, and the page and the API answering the
+ * same thing. No model is called anywhere in here, and nothing is charged.
+ */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { frontier } = require('../server/pick/frontier');
+const { parsePick } = require('../server/pick/parser');
+const { KINDS } = require('../server/kinds');
+
+(async () => {
+ await check('the frontier drops the models another model beats on score and on cost', () => {
+ const rows = [
+ { model: 'a', score: 90, cost: 0.02 },
+ { model: 'b', score: 80, cost: 0.01 },
+ { model: 'c', score: 95, cost: 0.05 },
+ { model: 'd', score: 70, cost: 0.03 },
+ ];
+ // a beats d on score and on cost, so d is not on the frontier; b is cheaper than a, so it stays.
+ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b', 'c']);
+ });
+
+ await check('a tie keeps both models on the frontier', () => {
+ const rows = [{ model: 'a', score: 90, cost: 0.02 }, { model: 'b', score: 90, cost: 0.02 }];
+ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a', 'b']);
+ });
+
+ await check('a model with no cost is never compared, and never dominates', () => {
+ const rows = [{ model: 'a', score: 90, cost: 0.02 }, { model: 'b', score: 90, cost: null }];
+ assert.deepStrictEqual(frontier(rows).map((r) => r.model), ['a']);
+ });
+
+ await check('the keyword parser reads the kind, the price and fast', () => {
+ const p = parsePick('landing page under 2 cents, fast', { kinds: KINDS });
+ assert.strictEqual(p.kind, 'design.page');
+ assert.strictEqual(p.max_cost, 0.02);
+ assert.strictEqual(p.fast, true);
+ });
+
+ await check('the keyword parser reads dollars, words and half a cent', () => {
+ assert.strictEqual(parsePick('under $0.05', { kinds: KINDS }).max_cost, 0.05);
+ assert.strictEqual(parsePick('ten cents', { kinds: KINDS }).max_cost, 0.1);
+ assert.strictEqual(parsePick('half a cent', { kinds: KINDS }).max_cost, 0.005);
+ assert.strictEqual(parsePick('two cents', { kinds: KINDS }).max_cost, 0.02);
+ });
+
+ await check('the keyword parser changes nothing when the words say nothing', () => {
+ const p = parsePick('something nice please', { kinds: KINDS });
+ assert.strictEqual(p.kind, null);
+ assert.strictEqual(p.max_cost, null);
+ assert.strictEqual(p.fast, false);
+ });
+
+ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' } });
+ const alice = t.network.addUser('alice');
+ const entrySvc = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
+ const judgeSvc = t.network.serviceToken({ sub: 'svc:judge', audience: 'openvibe.bench', cap: ['bench.judgement.cast'] });
+
+ // alpha-1 runs three times, beta-2 once: beta-2 is too thin to stand on.
+ const rounds = [];
+ for (let i = 0; i < 3; i++) {
+ const task = await t.get('/api/v1/kinds/design.page/tasks', { as: alice, json: { prompt: 'A page ' + i, public: true } });
+ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { as: alice, json: {} });
+ const id = rr.json().round.id;
+ rounds.push(id);
+ const a = await t.get(`/api/v1/rounds/${id}/entries`, { bearer: entrySvc, json: { provider: 'acme', model: 'alpha-1', payload: { html: '<h1>a</h1>' }, cost_usd: 0.01, ms: 1000 } });
+ assert.strictEqual(a.status, 201, a.text);
+ // beta-2 runs once only: fewer than three runs, so its numbers are too thin to stand on.
+ if (i === 0) {
+ const b = await t.get(`/api/v1/rounds/${id}/entries`, { bearer: entrySvc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<h1>b</h1>' }, cost_usd: 0.02, ms: 4000 } });
+ assert.strictEqual(b.status, 201, b.text);
+ const j = await t.get(`/api/v1/rounds/${id}/judgements`, { bearer: judgeSvc, json: { judge: 'game_outcome', pair: { left: a.json().entry.id, right: b.json().entry.id }, outcome: 'left' } });
+ assert.strictEqual(j.status, 201, j.text);
+ }
+ }
+
+ await check('the pick page answers from the data, without JavaScript', async () => {
+ const r = await t.get('/pick?kind=design.page');
+ assert.strictEqual(r.status, 200, r.text.slice(0, 300));
+ assert.ok(r.text.includes('method="get"'), 'the form is a GET');
+ assert.ok(r.text.includes('pk-table'), 'the frontier table');
+ assert.ok(r.text.includes('alpha-1'), 'the top model');
+ assert.ok(r.text.includes('because no other model beats it'), 'why the top pick is first');
+ assert.ok(r.text.includes('pk-card') || r.text.includes('pk-shot'), 'a sample output');
+ });
+
+ await check('a model with fewer than three runs is greyed and says so', async () => {
+ const r = await t.get('/pick?kind=design.page');
+ assert.ok(r.text.includes('beta-2'), 'the thin model is shown');
+ assert.ok(r.text.includes('too few runs'), 'it says the data is thin');
+ assert.ok(/class="pk-thin"/.test(r.text), 'and it is greyed');
+ });
+
+ await check('the API answers the same thing as the page', async () => {
+ const api = await t.get('/api/v1/pick?kind=design.page');
+ assert.strictEqual(api.status, 200, api.text.slice(0, 300));
+ const json = api.json();
+ assert.strictEqual(json.kind.id, 'design.page');
+ assert.strictEqual(json.rows[0].model, 'alpha-1', 'the top pick');
+ assert.ok(json.rows[0].cost_per_solved > 0, 'the cost a solved task');
+ assert.ok(json.rows[0].median_ms > 0, 'the median time');
+ assert.ok(json.rows[0].runs >= 3, 'the number of runs');
+ assert.ok(json.why.includes('alpha-1'), 'the reason');
+ assert.ok(json.also_ran.some((r) => r.model === 'beta-2' && r.thin), 'the thin model');
+ });
+
+ await check('the price control keeps the models over it out', async () => {
+ const json = (await t.get('/api/v1/pick?kind=design.page&max_cost=0.005')).json();
+ assert.strictEqual(json.rows.length, 0, 'nothing is under half a cent');
+ const ok = (await t.get('/api/v1/pick?kind=design.page&max_cost=0.02')).json();
+ assert.ok(ok.rows.some((r) => r.model === 'alpha-1'), 'alpha-1 is under two cents');
+ });
+
+ await check('fast takes its time cap from the run data', async () => {
+ const json = (await t.get('/api/v1/pick?kind=design.page&fast=1')).json();
+ assert.ok(json.filters.fast_cap_ms > 0, 'a cap from the run times');
+ assert.ok(json.rows.every((r) => !r.median_ms || r.median_ms <= json.filters.fast_cap_ms), 'only the quick ones');
+ });
+
+ await check('the words set the controls, and the page says how it read them', async () => {
+ const r = await t.get('/pick?q=' + encodeURIComponent('landing page under 2 cents, fast'));
+ assert.strictEqual(r.status, 200);
+ assert.ok(r.text.includes('design.page'), 'the kind it read');
+ const json = (await t.get('/api/v1/pick?q=' + encodeURIComponent('landing page under 2 cents, fast'))).json();
+ assert.strictEqual(json.controls.kind, 'design.page');
+ assert.strictEqual(json.controls.max_cost, 0.02);
+ assert.strictEqual(json.controls.fast, true);
+ assert.ok(json.controls.notes.length >= 3, 'it says what it read');
+ });
+
+ await check('a kind with nothing in it says so, and invents nothing', async () => {
+ const r = await t.get('/pick?kind=games.three');
+ assert.strictEqual(r.status, 200);
+ assert.ok(r.text.includes('Nothing to pick yet'), 'the empty answer');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,135 @@
@@ -0,0 +1,135 @@
+'use strict';
+
+/**
+ * The model watch (B10): the triggers, the once-a-day cap, the account data, and the notice that reaches the Network.
+ */
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { createWatches, mayNotify } = require('../server/watch/watch');
+const { TABLES } = require('../server/identity/account-data');
+const { uncovered } = require('openvibe-sdk/account-data');
+
+const stub = { s: { db: null }, config: {}, log: { warn() {} } };
+
+(async () => {
+ await check('a score that moves by more than its own interval is worth a notice', () => {
+ const w = createWatches(stub);
+ const notice = w.noticeFor(
+ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },
+ [{ model: 'alpha', rating: 1100, lower: 1060, upper: 1140 }],
+ new Map(),
+ new Map([['alpha', { rating: 1000 }]]),
+ []);
+ assert.ok(notice, 'a notice');
+ assert.strictEqual(notice.kind, 'move');
+ assert.ok(notice.message.includes('Playable games'), 'the benchmark is named');
+ assert.ok(notice.message.includes('alpha'), 'the model is named');
+ assert.ok(notice.message.includes('1000') && notice.message.includes('1100'), 'the numbers are there');
+ });
+
+ await check('a move inside the interval is not worth a notice', () => {
+ const w = createWatches(stub);
+ const notice = w.noticeFor(
+ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },
+ [{ model: 'alpha', rating: 1010, lower: 990, upper: 1030 }],
+ new Map(),
+ new Map([['alpha', { rating: 1000 }]]),
+ []);
+ assert.strictEqual(notice, null, 'no notice');
+ });
+
+ await check('a model that overtakes another one is worth a notice', () => {
+ const w = createWatches(stub);
+ const notice = w.noticeFor(
+ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },
+ [
+ { model: 'alpha', rating: 1000, lower: 980, upper: 1020 },
+ { model: 'beta', rating: 1100, lower: 1050, upper: 1150 },
+ ],
+ new Map(),
+ new Map(),
+ []);
+ assert.ok(notice, 'a notice');
+ assert.strictEqual(notice.kind, 'overtake');
+ assert.ok(notice.message.includes('alpha') && notice.message.includes('beta'), 'both models are named');
+ assert.ok(notice.message.includes('Playable games'), 'the benchmark is named');
+ assert.ok(/\d/.test(notice.message), 'the numbers are there');
+ });
+
+ await check('a model the person already follows does not count as an overtake', () => {
+ const w = createWatches(stub);
+ const notice = w.noticeFor(
+ { subject: 'usr_1', benchmark_id: 'bmk_1', model: 'alpha', benchmark_title: 'Playable games' },
+ [
+ { model: 'alpha', rating: 1000, lower: 980, upper: 1020 },
+ { model: 'beta', rating: 1100, lower: 1050, upper: 1150 },
+ ],
+ new Map(),
+ new Map(),
+ [{ subject: 'usr_1', benchmark_id: 'bmk_1', model: 'beta' }]);
+ assert.strictEqual(notice, null, 'no notice');
+ });
+
+ await check('one notice per watch per day', () => {
+ assert.strictEqual(mayNotify({ last_notified_at: null }, '2026-10-30T12:00:00Z'), true);
+ assert.strictEqual(mayNotify({ last_notified_at: '2026-10-30T10:00:00Z' }, '2026-10-30T12:00:00Z'), false);
+ assert.strictEqual(mayNotify({ last_notified_at: '2026-10-29T23:00:00Z' }, '2026-10-30T12:00:00Z'), true);
+ });
+
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+
+ const list = (await t.get('/api/v1/benchmarks')).json();
+ const benchmarks = list.benchmarks || list;
+ const benchmark = benchmarks[0];
+ assert.ok(benchmark && benchmark.id, 'a seeded benchmark: ' + JSON.stringify(list).slice(0, 200));
+
+ await check('the watches go into the account data and are deleted with the person', async () => {
+ const row = TABLES.find((x) => x.table === 'bench_watches');
+ assert.ok(row, 'bench_watches is in the account data');
+ assert.strictEqual(row.subject, 'subject');
+ assert.strictEqual(row.erase, 'delete');
+ const left = await uncovered(t.ctx.s.db, { tables: TABLES });
+ assert.ok(!left.includes('bench_watches'), 'the watches are covered: ' + JSON.stringify(left));
+ });
+
+ await check('a signed-in person may follow a benchmark, and may stop', async () => {
+ const r = await t.get('/watch/follow', { as: alice, method: 'POST', json: { benchmark_id: benchmark.id, back: '/watching' } });
+ assert.strictEqual(r.status, 303, r.text);
+ const page = await t.get('/watching', { as: alice });
+ assert.strictEqual(page.status, 200, page.text.slice(0, 200));
+ assert.ok(page.text.includes(benchmark.title), 'the benchmark is listed');
+ assert.ok(page.text.includes('action="/watch/unfollow"'), 'an unfollow button');
+ const id = /name="id" value="([^"]+)"/.exec(page.text)[1];
+ const un = await t.get('/watch/unfollow', { as: alice, method: 'POST', json: { id, back: '/watching' } });
+ assert.strictEqual(un.status, 303);
+ const after = await t.get('/watching', { as: alice });
+ assert.ok(!after.text.includes(benchmark.title), 'the watch is gone');
+ });
+
+ await check('an anonymous visitor cannot follow anything', async () => {
+ const r = await t.get('/watch/follow', { method: 'POST', origin: t.config.baseUrl, json: { benchmark_id: benchmark.id } });
+ assert.strictEqual(r.status, 401, r.text);
+ });
+
+ await check('the notice reaches the Network, naming the benchmark and the numbers', async () => {
+ const out = await t.ctx.watches.follow({ subject: alice.subject, benchmarkId: benchmark.id, userId: alice.id });
+ assert.ok(out.ok, out.error);
+ const pushed = await t.ctx.watches.notify(out.watch, {
+ title: 'Bench: ' + benchmark.title,
+ message: benchmark.title + ': beta (1100, 1050-1150) is now fully above alpha (1000, 980-1020).',
+ });
+ assert.strictEqual(pushed.sent, true, JSON.stringify(pushed));
+ const last = t.network.pushes[t.network.pushes.length - 1];
+ assert.ok(last, 'the Network was called');
+ assert.strictEqual(last.type, 'WATCH_TRIGGERED');
+ assert.ok(last.message.includes(benchmark.title), 'the benchmark is named');
+ assert.ok(last.message.includes('alpha') && last.message.includes('beta'), 'both models are named');
+ assert.ok(last.message.includes('1100'), 'the numbers are there');
+ assert.strictEqual(last.user_id, alice.id, 'it goes to the person');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });