openvibers/openvibe.bench Commits
B12: import results from the eval tools people already use, and mirror standard results with provenance (#43)
Changes
nnew file mode 100644@@@ -0,0 +1,68 @@++-- phase: expand++-- B12: imported results. An upload is the owner's own file (private until they share it), the runs it becomes are++-- submitted (never verified) and carry the provenance of where they came from: a tool, or a mirror's source.++CREATE TABLE bench_imports (++ id text COLLATE "C" PRIMARY KEY, -- bim_ + ULID++ owner text COLLATE "C" NOT NULL,++ tool text COLLATE "C" NOT NULL, -- inspect | harbor | lm-eval | promptfoo++ tool_version text COLLATE "C" NOT NULL DEFAULT '',++ filename text COLLATE "C" NOT NULL DEFAULT '', -- the name the person gave it; never a path++ sha256 text COLLATE "C" NOT NULL,++ size integer NOT NULL DEFAULT 0,++ status text COLLATE "C" NOT NULL DEFAULT 'preview', -- preview | imported | failed++ summary jsonb NOT NULL DEFAULT '{}'::jsonb,++ benchmark_id text COLLATE "C" REFERENCES bench_benchmarks (id) ON DELETE SET NULL,++ run_ids jsonb NOT NULL DEFAULT '[]'::jsonb,++ created_by text COLLATE "C" NOT NULL,++ created_at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT bench_imports_id CHECK (id ~ '^bim_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_imports_tool CHECK (tool IN ('inspect', 'harbor', 'lm-eval', 'promptfoo')),++ CONSTRAINT bench_imports_status CHECK (status IN ('preview', 'imported', 'failed')),++ CONSTRAINT bench_imports_sha256 CHECK (sha256 ~ '^[0-9a-f]{64}$'),++ CONSTRAINT bench_imports_size CHECK (size >= 0)++);++CREATE INDEX bench_imports_by_owner ON bench_imports (owner, created_at DESC);++++-- The uploaded file itself. It hangs off the import, so erasing the person erases the file with it.++CREATE TABLE bench_import_files (++ import_id text COLLATE "C" PRIMARY KEY REFERENCES bench_imports (id) ON DELETE CASCADE,++ content bytea NOT NULL,++ size integer NOT NULL DEFAULT 0,++ CONSTRAINT bench_import_files_size CHECK (size >= 0)++);++++-- Where a run came from: an upload (tool and its version) or a mirror (source url, licence, the day it was read).++-- It sits beside the run, not inside the published bench.run document, which the contract closes to extras.++ALTER TABLE bench_runs ADD COLUMN source jsonb NOT NULL DEFAULT '{}'::jsonb;++++-- The mirror's ledger: one row per source read, with the licence it was read under.++CREATE TABLE bench_mirror_reads (++ id text COLLATE "C" PRIMARY KEY, -- bmr_ + ULID++ source_key text COLLATE "C" NOT NULL,++ source_url text COLLATE "C" NOT NULL,++ licence text COLLATE "C" NOT NULL,++ licence_url text COLLATE "C" NOT NULL DEFAULT '',++ attribution text COLLATE "C" NOT NULL DEFAULT '',++ read_at timestamptz NOT NULL DEFAULT now(),++ rows integer NOT NULL DEFAULT 0,++ CONSTRAINT bench_mirror_reads_id CHECK (id ~ '^bmr_[0-9A-HJKMNP-TV-Z]{26}$')++);++CREATE UNIQUE INDEX bench_mirror_reads_source ON bench_mirror_reads (source_key, source_url);++++-- One mirrored result per model, benchmark version and source: the same one never lands twice.++CREATE TABLE bench_mirror_runs (++ id text COLLATE "C" PRIMARY KEY, -- bmr_ + ULID++ source_key text COLLATE "C" NOT NULL,++ source_url text COLLATE "C" NOT NULL,++ licence text COLLATE "C" NOT NULL,++ licence_url text COLLATE "C" NOT NULL DEFAULT '',++ read_at timestamptz NOT NULL DEFAULT now(),++ benchmark_id text COLLATE "C" NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,++ benchmark_version integer NOT NULL,++ model text COLLATE "C" NOT NULL,++ run_id text COLLATE "C" REFERENCES bench_runs (id) ON DELETE SET NULL,++ created_at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT bench_mirror_runs_id CHECK (id ~ '^bmr_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_mirror_runs_version CHECK (benchmark_version >= 1)++);++CREATE UNIQUE INDEX bench_mirror_runs_dedupe ON bench_mirror_runs (source_key, benchmark_id, benchmark_version, model);@@@ -108,6 +108,12 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex.br-prov.is-verified { background: color-mix(in srgb, var(--success, #22c55e) 15%, transparent); color: var(--success, #22c55e); }.br-prov.is-verified::before { content: ""; width: 6px; height: 6px; border-radius: 50%; background: currentColor; }.br-prov.is-submitted { border: 1px solid var(--border, #1f2d47); color: var(--text-muted, #7386a3); }++/* A source url is long: it breaks inside the line, so the page never scrolls sideways */++.br-source, .br-source a { overflow-wrap: anywhere; word-break: break-word; }++++/* Imported (someone's own file) and mirrored (a published result): both are never verified */++.br-prov.is-imported { border: 1px dashed var(--warning, #f59e0b); color: var(--warning, #f59e0b); }++.br-prov.is-mirrored { border: 1px dotted var(--info, #38bdf8); color: var(--info, #38bdf8); }/* Rank-by tabs: links, so every metric has its own URL */.br-tabs { display: flex; flex-wrap: wrap; gap: 4px; padding: 3px; border-radius: 999px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }nnew file mode 100644@@@ -0,0 +1,66 @@++'use strict';++++/**++ * Mirror published results into Bench (B12).++ *++ * node scripts/mirror-import.js # what HELM would bring (writes nothing)++ * node scripts/mirror-import.js --source helm --group mmlu --release v1.13.0++ * node scripts/mirror-import.js --apply # write it++ *++ * A source is read only when its licence clears, and every row that lands says where it came from, under which++ * licence and the day it was read. The same model, benchmark version and source never land twice.++ *++ * Run it as the service user, with the service's own environment (never printed):++ *++ * sudo systemd-run --wait --pipe --uid=ubuntu --gid=ubuntu \++ * --property=EnvironmentFile=/etc/openvibe/bench.env \++ * --working-directory=/opt/openvibe.bench /usr/bin/env node scripts/mirror-import.js++ */++const { load } = require('../server/config');++const { openStore } = require('../server/db');++const { createRegistry } = require('../server/bench/registry');++const { createMirror } = require('../server/bench/mirror');++++const argv = process.argv.slice(2);++const flag = (name) => argv.includes('--' + name);++const value = (name, fallback) => {++ const at = argv.indexOf('--' + name);++ return at >= 0 && argv[at + 1] ? argv[at + 1] : fallback;++};++++(async () => {++ const config = load();++ const s = await openStore(config, { log: console });++ const registry = createRegistry(s, { log: console });++ const mirror = createMirror({ s, registry, fetchImpl: globalThis.fetch, log: console });++ const source = value('source', 'helm');++ const options = {++ suite: value('suite', undefined),++ release: value('release', undefined),++ group: value('group', undefined),++ limit: value('limit', undefined),++ };++ const out = flag('apply')++ ? await mirror.apply(source, options)++ : await mirror.preview(source, options);++ if (!out.ok) {++ process.stdout.write(JSON.stringify(out, null, 2) + '\n');++ process.exitCode = 1;++ return;++ }++ if (flag('apply')) {++ process.stdout.write(JSON.stringify({++ source: out.source, licence: out.licence, licence_url: out.licence_url, read_at: out.read_at,++ written: out.written.length, skipped: out.skipped.length,++ }, null, 2) + '\n');++ return;++ }++ const sample = (out.rows || []).slice(0, 10).map((r) => ({ benchmark: r.benchmark, model: r.model, score: Math.round(r.score * 100) / 100, source_url: r.source_url }));++ process.stdout.write(JSON.stringify({++ source: out.source, title: out.title, url: out.url, licence: out.licence, licence_url: out.licence_url,++ licence_quote: out.licence_quote, attribution: out.attribution, rows: out.count, sample,++ }, null, 2) + '\n');++})().catch((err) => {++ process.stderr.write('The mirror failed: ' + ((err && err.message) || err) + '\n');++ process.exitCode = 1;++});@@@ -75,6 +75,9 @@ async function createApp(opts = {}) {const bench = opts.bench || createStore(s, { config, log, random: opts.random });const registry = opts.registry || createRegistry(s, { log });++ // B12: bringing your own results, and the operator's mirror.++ const imports = require('./bench/imports').createImports({ s, registry, log });++ const mirror = require('./bench/mirror').createMirror({ s, registry, fetchImpl, log });const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });const jobs = createBenchJobs({ s, bench, ai, config, log });const publisher = createPublisher({ config, s, fetchImpl, log });@@@ -91,6 +94,7 @@ async function createApp(opts = {}) {const launcher = createLauncher({ config, registry, fetchImpl: fetchImpl || globalThis.fetch, log });const ctx = {config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,++ imports, mirror,random: opts.random || Math.random,};@@@ -183,6 +187,7 @@ async function createApp(opts = {}) {app.use('/api/v1/showcase', createShowcaseRoutes(ctx));app.use('/api/v1/gallery', createGalleryRoutes(ctx));app.use('/api/v1', createPickRoutes(ctx));++ app.use(require('./http/mirror').createMirrorRoutes(ctx));app.use(mcpRouter(ctx));app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));@@@ -204,6 +209,7 @@ async function createApp(opts = {}) {app.use(createWatchPages(ctx));app.use('/battle', createBattlePages(ctx));app.use('/ratings', createRatingsPages(ctx));++ app.use(require('./http/imports').createImportRoutes(ctx));app.use(createPageRoutes(ctx));app.use((req, res) => send(res, 404, {viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',nnew file mode 100644@@@ -0,0 +1,93 @@++'use strict';++++/**++ * Harbor (harbor-framework/harbor) results: the results.json of a job.++ *++ * A Harbor job result (src/harbor/models/job/result.py) is { id, started_at, finished_at, n_total_trials, stats,++ * trial_results }. Every trial carries the agent that ran it (agent_info.model_info.name is the model) and the++ * verifier's reward (verifier_result.reward), so one model here is one agent over a set of tasks.++ */++const { parseJson, asText } = require('./limits');++++const TOOL = 'harbor';++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++function trialsOf(doc) {++ if (Array.isArray(doc)) return doc;++ if (doc && Array.isArray(doc.trial_results)) return doc.trial_results;++ if (doc && Array.isArray(doc.trials)) return doc.trials;++ return [];++}++++function agentOf(trial) {++ const agent = (trial && trial.agent_info) || (trial && trial.agent) || {};++ const model = (agent.model_info && agent.model_info.name) || agent.model || trial.model || '';++ const provider = (agent.model_info && agent.model_info.provider) || agent.provider || 'harbor';++ return { model: String(model || '').trim(), provider: String(provider || 'harbor').trim(), agent: agent };++}++++function rewardOf(trial) {++ const verifier = trial && (trial.verifier_result || trial.verifier || trial.rewards || {});++ const value = num(verifier && (verifier.reward != null ? verifier.reward : verifier.score != null ? verifier.score : verifier.value));++ return value == null ? null : value;++}++++function parse(bytes) {++ const doc = parseJson(asText(bytes), { what: 'the results' });++ const trials = trialsOf(doc);++ if (!trials.length) throw new Error('That file holds no trials.');++ const stats = (doc && doc.stats) || {};++ const byModel = new Map();++ for (const trial of trials) {++ const { model, provider, agent } = agentOf(trial);++ const key = model + '|' + provider;++ if (!byModel.has(key)) {++ byModel.set(key, {++ model: model || 'unknown',++ provider: provider || 'harbor',++ score: null,++ cost_usd: 0,++ hasCost: false,++ tasks: [],++ agentName: String(agent.name || ''),++ agentVersion: String(agent.version || ''),++ });++ }++ const row = byModel.get(key);++ const reward = rewardOf(trial);++ const taskId = String((trial && (trial.task_id || trial.task_name || trial.name)) || ('trial-' + row.tasks.length));++ row.tasks.push({ task_id: taskId, score: reward == null ? null : reward * 100 });++ const cost = num(trial && (trial.cost_usd != null ? trial.cost_usd : trial.cost));++ if (cost != null) { row.cost_usd += cost; row.hasCost = true; }++ if (reward != null) row.score = row.score == null ? reward * 100 : (row.score + reward * 100) / 2;++ }++ const models = [...byModel.values()].map((row) => ({++ model: row.model,++ provider: row.provider,++ score: row.score,++ cost_usd: row.hasCost ? row.cost_usd : null,++ tasks: row.tasks,++ }));++ const totalCost = num(stats.cost_usd != null ? stats.cost_usd : stats.total_cost);++ if (totalCost != null && models.length === 1) models[0].cost_usd = totalCost;++ const date = doc.started_at || (doc.stats && doc.stats.started_at) || doc.created_at || null;++ const version = String((doc && (doc.harbor_version || doc.version)) || (stats.harness_version) || '').trim();++ const agentVersion = models.length && models[0] ? '' : '';++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'harbor', version: version || 'unknown' },++ date: date ? new Date(date).toISOString() : null,++ suite: null,++ models,++ };++}++++function looksLike(doc) {++ if (!doc || typeof doc !== 'object') return false;++ if (Array.isArray(doc.trial_results)) return true;++ if (doc.n_total_trials != null && doc.stats) return true;++ return false;++}++++module.exports = { TOOL, parse, looksLike };nnew file mode 100644@@@ -0,0 +1,74 @@++'use strict';++++/**++ * The four tools whose results a person can bring to Bench (B12): Inspect AI, Harbor, the EleutherAI++ * lm-evaluation-harness and promptfoo. The file says which one it is; when it does not, the shape of the++ * document says it.++ */++const { ImportError, MAX_UPLOAD_BYTES } = require('./limits');++const inspectTool = require('./inspect');++const harborTool = require('./harbor');++const lmEvalTool = require('./lm-eval');++const promptfooTool = require('./promptfoo');++++const TOOLS = {++ inspect: inspectTool,++ harbor: harborTool,++ 'lm-eval': lmEvalTool,++ promptfoo: promptfooTool,++};++++const TOOL_LABELS = {++ inspect: 'Inspect AI',++ harbor: 'Harbor',++ 'lm-eval': 'lm-evaluation-harness',++ promptfoo: 'promptfoo',++};++++/** The tool that wrote this file, or null when it is none of ours. */++function detect(doc, filename = '') {++ const name = String(filename || '').toLowerCase();++ if (name.endsWith('.eval')) return 'inspect';++ if (inspectTool.looksLike(doc)) return 'inspect';++ if (harborTool.looksLike(doc)) return 'harbor';++ if (lmEvalTool.looksLike(doc)) return 'lm-eval';++ if (promptfooTool.looksLike(doc)) return 'promptfoo';++ return null;++}++++/**++ * Read a results file: check its size, find the tool, read it.++ * The tool's own parser throws when the file is not its shape; that error is passed on as a plain message.++ */++function parse(bytes, { filename = '', tool = null } = {}) {++ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');++ if (!buf.length) throw new ImportError(422, 'bench.import_empty', 'That file is empty.');++ if (buf.length > MAX_UPLOAD_BYTES) {++ throw new ImportError(413, 'bench.import_too_large', 'That file is larger than 20 MB.');++ }++ const wanted = tool ? String(tool) : null;++ if (wanted && !TOOLS[wanted]) {++ throw new ImportError(422, 'bench.import_unknown_tool', 'I do not know that tool.');++ }++ if (wanted) {++ const out = TOOLS[wanted].parse(buf);++ out.filename = String(filename || '');++ return out;++ }++ // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others.++ const errors = [];++ for (const key of ['inspect', 'harbor', 'lm-eval', 'promptfoo']) {++ try {++ const out = TOOLS[key].parse(buf);++ out.filename = String(filename || '');++ return out;++ } catch (err) {++ if (err instanceof ImportError) throw err; // a size or a shape problem is the answer++ errors.push(key + ': ' + ((err && err.message) || 'unknown'));++ }++ }++ throw new ImportError(422, 'bench.import_unknown_format',++ 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, an lm-evaluation-harness results.json or a promptfoo output.json.');++}++++module.exports = { TOOLS, TOOL_LABELS, detect, parse, ImportError, MAX_UPLOAD_BYTES };nnew file mode 100644@@@ -0,0 +1,139 @@++'use strict';++++/**++ * Inspect AI (inspect_ai) results: the .eval log (a zip of the log directory) or its JSON log.++ *++ * The JSON log of inspect_ai 0.3.x is an EvalLog: eval (the task, the model, the samples), results (the scores of++ * the scorers), stats (the model usage, with the cost) and reductions (the per-sample scores). Nothing in it is++ * executed: it is read as JSON and nothing else.++ */++const { parseJson, asText, jsonPartsOfZip } = require('./limits');++++const TOOL = 'inspect';++++function isZip(buffer) {++ return buffer.length > 4 && buffer[0] === 0x50 && buffer[1] === 0x4b && (buffer[2] === 0x03 || buffer[2] === 0x05 || buffer[2] === 0x07);++}++++/** The log inside the file: a zip (an .eval) or the JSON log itself. */++function readLog(bytes) {++ if (isZip(bytes)) {++ const parts = jsonPartsOfZip(bytes);++ for (const part of parts) {++ const doc = parseJson(asText(part.data), { what: 'the log' });++ if (doc && typeof doc === 'object' && doc.eval) return doc;++ }++ const doc = parseJson(asText(parts[0].data), { what: 'the log' });++ return doc;++ }++ return parseJson(asText(bytes), { what: 'the log' });++}++++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++/** The overall score: the first scorer's accuracy, or the first metric that is not an error bar. */++function overallScore(results) {++ const scores = results && Array.isArray(results.scores) ? results.scores : [];++ for (const score of scores) {++ const metrics = score && score.metrics ? score.metrics : {};++ if (metrics.accuracy && num(metrics.accuracy.value) != null) return num(metrics.accuracy.value);++ }++ for (const score of scores) {++ const metrics = score && score.metrics ? score.metrics : {};++ for (const [name, metric] of Object.entries(metrics)) {++ if (/stderr|error|nan/i.test(name)) continue;++ if (num(metric && metric.value) != null) return num(metric.value);++ }++ }++ return null;++}++++/** The per-sample scores: the reductions of the log, or the samples themselves. */++function taskScores(doc) {++ const out = [];++ const reductions = doc && doc.reductions ? doc.reductions : null;++ const samples = reductions && Array.isArray(reductions.samples) ? reductions.samples : null;++ if (samples) {++ for (const row of samples) {++ if (!row || typeof row !== 'object') continue;++ const value = num(row.value);++ if (value == null) continue;++ out.push({ task_id: String(row.sample_id != null ? row.sample_id : row.id), score: value * 100 });++ }++ if (out.length) return out;++ }++ const list = doc && Array.isArray(doc.samples) ? doc.samples : [];++ for (const sample of list) {++ if (!sample || typeof sample !== 'object') continue;++ const scores = sample.scores && typeof sample.scores === 'object' ? sample.scores : {};++ let value = null;++ for (const entry of Object.values(scores)) {++ const inner = entry && typeof entry === 'object' ? entry : {};++ if (num(inner.value) != null) { value = num(inner.value); break; }++ if (num(inner.accuracy) != null) { value = num(inner.accuracy); break; }++ }++ if (value == null) continue;++ out.push({ task_id: String(sample.id != null ? sample.id : sample.sample_id), score: value * 100 });++ }++ return out;++}++++/** The cost the log carries: the model usage of the run. */++function costOf(doc) {++ const stats = doc && doc.stats ? doc.stats : {};++ const usage = stats.model_usage && typeof stats.model_usage === 'object' ? stats.model_usage : {};++ let total = 0;++ let found = false;++ const add = (v) => {++ const n = num(v);++ if (n != null) { total += n; found = true; }++ };++ for (const entry of Object.values(usage)) {++ if (entry == null) continue;++ if (typeof entry === 'number') { add(entry); continue; }++ if (typeof entry === 'object') {++ add(entry.cost);++ add(entry.total_cost);++ }++ }++ if (!found) {++ add(stats.total_cost);++ add(stats.cost);++ }++ return found ? total : null;++}++++function parse(bytes) {++ const doc = readLog(bytes);++ if (!looksLike(doc)) throw new Error('That file is not an Inspect AI log.');++ const evalSpec = doc && doc.eval ? doc.eval : {};++ const model = String(evalSpec.model || '').trim();++ const provider = model.includes('/')++ ? model.split('/')[0]++ : String(evalSpec.model_api || evalSpec.provider || 'inspect');++ const version = String(doc.version || evalSpec.eval_version || '').trim();++ const score = overallScore(doc.results);++ const tasks = taskScores(doc);++ const stats = doc.stats && typeof doc.stats === 'object' ? doc.stats : {};++ const date = stats.started_at || evalSpec.created || doc.created_at || null;++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'inspect-ai', version: version || 'unknown' },++ date: date ? new Date(date).toISOString() : null,++ suite: evalSpec.task ? { name: String(evalSpec.task), version: evalSpec.task_version == null ? null : String(evalSpec.task_version) } : null,++ models: [{++ model: model || 'unknown',++ provider: provider || 'inspect',++ score: score == null ? null : score * 100,++ cost_usd: costOf(doc),++ tasks,++ }],++ };++}++++function looksLike(doc) {++ return Boolean(doc && typeof doc === 'object' && doc.eval && typeof doc.eval === 'object' && (doc.eval.model || doc.eval.task));++}++++module.exports = { TOOL, parse, looksLike, readLog, isZip };nnew file mode 100644@@@ -0,0 +1,199 @@++'use strict';++++/**++ * What an uploaded file may be (B12). The rules are the same for every tool:++ *++ * 20 MB the file as it arrives++ * 100 MB everything it unpacks to (a zip that says more is refused, so a bomb never fills the disk)++ * 64 how deep a JSON document may nest++ *++ * Nothing in the file is ever run. A name in the file is never used as a path: the entries of a zip are read in++ * memory, and only their bytes matter.++ */++const zlib = require('zlib');++++const MAX_UPLOAD_BYTES = 20 * 1024 * 1024;++const MAX_DECOMPRESSED_BYTES = 100 * 1024 * 1024;++const MAX_JSON_DEPTH = 64;++const MAX_ZIP_ENTRIES = 2000;++const MAX_COMPRESSION_RATIO = 200;++++class ImportError extends Error {++ constructor(status, code, detail) {++ super(detail);++ this.name = 'ImportError';++ this.status = status;++ this.code = code;++ this.detail = detail;++ }++}++++/** How deep a parsed document nests, counted with an explicit stack so a deep file cannot blow the call stack. */++function depthOf(value) {++ let max = 0;++ const stack = [[value, 1]];++ while (stack.length) {++ const [node, depth] = stack.pop();++ if (depth > max) max = depth;++ if (node && typeof node === 'object') {++ for (const child of Object.values(node)) {++ if (child && typeof child === 'object') stack.push([child, depth + 1]);++ }++ }++ if (stack.length > 200000) return max + 1; // a document this wide is refused on its size alone++ }++ return max;++}++++/** JSON with a size and a depth limit: a file that is not JSON, or nests too deep, is refused, never guessed at. */++function parseJson(text, { maxDepth = MAX_JSON_DEPTH, what = 'file' } = {}) {++ const body = String(text == null ? '' : text);++ if (!body.trim()) throw new ImportError(422, 'bench.import_empty', 'That ' + what + ' is empty.');++ let value;++ try {++ value = JSON.parse(body);++ } catch (err) {++ throw new ImportError(422, 'bench.import_not_json', 'That ' + what + ' is not JSON I can read.');++ }++ if (depthOf(value) > maxDepth) {++ throw new ImportError(422, 'bench.import_too_deep', 'That ' + what + ' nests too deeply for me to read.');++ }++ return value;++}++++/** The bytes as text, refusing anything that is not text (a binary file is not a results file). */++function asText(bytes, { what = 'file' } = {}) {++ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');++ // A NUL byte or a lone high-byte run means binary: refuse rather than mangle it.++ let nul = 0;++ for (let i = 0; i < buf.length && nul < 2; i += 1) if (buf[i] === 0) nul += 1;++ if (nul >= 2) throw new ImportError(422, 'bench.import_not_text', 'That ' + what + ' is not a text file.');++ return buf.toString('utf8');++}++++// ── zip ─────────────────────────────────────────────────────────────────────────────────────────────────────────++++const EOCD = 0x06054b50;++const CENTRAL = 0x02014b50;++const LOCAL = 0x04034b50;++++/**++ * The entries of a zip, in memory. Only what a results file needs: the names and the bytes. A zip that claims++ * more than it may unpack to, or that hides a thousand parts, is refused before anything is inflated.++ */++function readZip(buffer) {++ const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer || '');++ // The end-of-central-directory record sits at the end, behind a comment of up to 64 KB.++ let eocd = -1;++ const from = Math.max(0, buf.length - (22 + 0xffff));++ for (let i = buf.length - 22; i >= from; i -= 1) {++ if (buf.readUInt32LE(i) === EOCD) { eocd = i; break; }++ }++ if (eocd < 0) throw new ImportError(422, 'bench.import_not_zip', 'That file is not a zip I can read.');++ const count = buf.readUInt16LE(eocd + 10);++ const size = buf.readUInt32LE(eocd + 12);++ const offset = buf.readUInt32LE(eocd + 16);++ if (count > MAX_ZIP_ENTRIES) throw new ImportError(413, 'bench.import_too_many_parts', 'That zip holds too many parts.');++ if (size > buf.length) throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');++ const entries = [];++ let at = offset;++ let total = 0;++ for (let n = 0; n < count; n += 1) {++ if (at + 46 > buf.length || buf.readUInt32LE(at) !== CENTRAL) {++ throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');++ }++ const method = buf.readUInt16LE(at + 10);++ const compressed = buf.readUInt32LE(at + 20);++ const uncompressed = buf.readUInt32LE(at + 24);++ const nameLen = buf.readUInt16LE(at + 28);++ const extraLen = buf.readUInt16LE(at + 30);++ const commentLen = buf.readUInt16LE(at + 32);++ const localOffset = buf.readUInt32LE(at + 42);++ const name = buf.subarray(at + 46, at + 46 + nameLen).toString('utf8');++ at += 46 + nameLen + extraLen + commentLen;++ if (uncompressed > MAX_DECOMPRESSED_BYTES) {++ throw new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');++ }++ if (compressed > 0 && uncompressed / compressed > MAX_COMPRESSION_RATIO) {++ throw new ImportError(413, 'bench.import_zip_bomb', 'That zip packs far too much into too little; I will not open it.');++ }++ total += uncompressed;++ if (total > MAX_DECOMPRESSED_BYTES) {++ throw new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');++ }++ entries.push({ name, method, compressed, uncompressed, localOffset });++ }++ return entries;++}++++/**++ * The bytes of one entry of a zip, inflated in memory.++ *++ * The cap is the real one: the inflater is given the room that is left in the whole zip, so a part that lies++ * about its size cannot fill the memory. A zip that says it holds a kilobyte and really holds sixty megabytes is++ * cut off at the budget, not believed.++ */++function readZipEntry(buffer, entry, { maxOutput = MAX_DECOMPRESSED_BYTES, used = 0 } = {}) {++ const buf = buffer;++ const at = entry.localOffset;++ if (at + 30 > buf.length || buf.readUInt32LE(at) !== LOCAL) {++ throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');++ }++ const nameLen = buf.readUInt16LE(at + 26);++ const extraLen = buf.readUInt16LE(at + 28);++ const start = at + 30 + nameLen + extraLen;++ const raw = buf.subarray(start, start + entry.compressed);++ if (entry.method === 0) {++ const out = Buffer.from(raw);++ if (out.length > maxOutput) throw unpackedTooMuch(used + out.length);++ return out;++ }++ if (entry.method !== 8) throw new ImportError(422, 'bench.import_bad_zip', 'That zip uses a compression I do not know.');++ let out;++ try {++ out = zlib.inflateRawSync(raw, { maxOutputLength: maxOutput });++ } catch (err) {++ // The inflater stops at the budget: that is the cap working, not a broken file.++ if (err && (err.code === 'ERR_BUFFER_TOO_LARGE' || /maxOutputLength/i.test(String(err.message)))) {++ throw unpackedTooMuch(used + maxOutput);++ }++ throw new ImportError(422, 'bench.import_bad_zip', 'That zip could not be unpacked.');++ }++ if (out.length > maxOutput) throw unpackedTooMuch(used + out.length);++ return out;++}++++/** The refusal, with the bytes that were really unpacked: the cap is on the memory, not on the claim. */++function unpackedTooMuch(inflated) {++ const err = new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');++ err.inflated = Number(inflated) || 0;++ return err;++}++++/**++ * Every JSON part of a zip, biggest first: the results log of an .eval file is the one that matters.++ *++ * The running total is of the bytes really inflated, and each part is only given the room that is left. When the++ * room runs out the import is refused, so a zip of many parts cannot unpack to more than the cap in all.++ */++function jsonPartsOfZip(buffer) {++ const entries = readZip(buffer);++ const out = [];++ let used = 0;++ for (const entry of entries) {++ if (!/\.json$/i.test(entry.name)) continue;++ const budget = MAX_DECOMPRESSED_BYTES - used;++ if (budget <= 0) throw unpackedTooMuch(used);++ const data = readZipEntry(buffer, entry, { maxOutput: budget, used });++ used += data.length;++ out.push({ name: entry.name, data });++ }++ if (!out.length) throw new ImportError(422, 'bench.import_no_json', 'That zip holds no JSON file I can read.');++ out.sort((a, b) => b.data.length - a.data.length);++ return out;++}++++module.exports = {++ MAX_UPLOAD_BYTES, MAX_DECOMPRESSED_BYTES, MAX_JSON_DEPTH, MAX_ZIP_ENTRIES, MAX_COMPRESSION_RATIO,++ ImportError, depthOf, parseJson, asText, readZip, readZipEntry, jsonPartsOfZip,++};nnew file mode 100644@@@ -0,0 +1,134 @@++'use strict';++++/**++ * EleutherAI lm-evaluation-harness results: the results.json that lm_eval writes.++ *++ * The file is { results: { <task>: { "acc,none": 0.61, "acc_stderr,none": 0.006, ... } }, configs: { <task>: {...} },++ * versions: { <task>: 0 }, "n-shot": {...}, higher_is_better: {...} }. One config is one model, so one model here is++ * one run; every task of the file is a task of that run.++ */++const { parseJson, asText } = require('./limits');++++const TOOL = 'lm-eval';++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++const METRIC_ORDER = ['acc,none', 'acc_norm,none', 'exact_match,none', 'normalized_accuracy,none', 'f1,none', 'acc', 'acc_norm', 'exact_match'];++const BACKENDS = {++ hf: 'huggingface', 'huggingface': 'huggingface', openai: 'openai', 'openai-chat-completions': 'openai',++ anthropic: 'anthropic', 'anthropic-chat': 'anthropic', google: 'google', 'gemini': 'google', mistral: 'mistral',++ local: 'local', 'local-completions': 'local', dummy: 'dummy',++};++++/** The metric of a result that means "how good was it", skipping the error bars and the losses. */++function metricOf(entry, higherIsBetter) {++ if (!entry || typeof entry !== 'object') return null;++ for (const key of METRIC_ORDER) {++ if (num(entry[key]) != null) return { metric: key, value: num(entry[key]) };++ }++ for (const [key, value] of Object.entries(entry)) {++ if (/stderr|stderr|nan|perplexity|word_perplexity|byte_perplexity|bits_per_byte/i.test(key)) continue;++ const n = num(value);++ if (n == null) continue;++ if (higherIsBetter && higherIsBetter[key] === false) continue;++ return { metric: key, value: n };++ }++ return null;++}++++/** The subtasks of a task, when the harness wrote them down (MMLU's 57 subjects). */++function subtasksOf(results, task, entry) {++ const out = [];++ if (entry && entry.acc_by_task && typeof entry.acc_by_task === 'object') {++ for (const [name, value] of Object.entries(entry.acc_by_task)) {++ const n = num(value);++ if (n != null) out.push({ task_id: task + '/' + name, score: n * 100 });++ }++ if (out.length) return out;++ }++ const grouped = results[task + '_subtasks'] || results[task + '_groups'];++ if (grouped && typeof grouped === 'object') {++ for (const [name, sub] of Object.entries(grouped)) {++ const m = metricOf(sub, null);++ if (m) out.push({ task_id: task + '/' + name, score: m.value * 100 });++ }++ if (out.length) return out;++ }++ return out;++}++++function modelOf(config) {++ const args = (config && config.model_args) || {};++ const name = args.model_name || args.pretrained || args.model || args.name || (config && config.model) || '';++ const backend = (config && config.model) || '';++ return { name: String(name || '').trim(), backend: String(backend || '').trim() };++}++++function providerOf(name, backend) {++ const key = String(backend || '').toLowerCase();++ if (BACKENDS[key]) return BACKENDS[key];++ if (name.includes('/')) return name.split('/')[0];++ return key || 'lm-eval';++}++++function parse(bytes) {++ const doc = parseJson(asText(bytes), { what: 'the results' });++ const results = doc && doc.results && typeof doc.results === 'object' && !Array.isArray(doc.results) ? doc.results : null;++ if (!results) throw new Error('That file has no results.');++ // The harness's own markers: its results file always carries the configs and the versions of the tasks.++ if (!doc.configs && !doc.versions && !doc['n-shot'] && !doc.higher_is_better) {++ throw new Error('That file is not an lm-evaluation-harness results file.');++ }++ const configs = doc.configs && typeof doc.configs === 'object' ? doc.configs : {};++ const versions = doc.versions && typeof doc.versions === 'object' ? doc.versions : {};++ const higher = doc.higher_is_better && typeof doc.higher_is_better === 'object' ? doc.higher_is_better : {};++++ const tasks = [];++ let suiteName = null;++ let suiteVersion = null;++ for (const [task, entry] of Object.entries(results)) {++ if (task.endsWith('_subtasks') || task.endsWith('_groups')) continue;++ const subtasks = subtasksOf(results, task, entry);++ if (subtasks.length) {++ for (const t of subtasks) tasks.push(t);++ } else {++ const m = metricOf(entry, higher);++ if (m) tasks.push({ task_id: task, score: m.value * 100 });++ }++ if (!suiteName) { suiteName = task; suiteVersion = versions[task] == null ? null : String(versions[task]); }++ }++ if (Object.keys(results).filter((t) => !t.endsWith('_subtasks') && !t.endsWith('_groups')).length > 1) {++ suiteName = null; // a file with many tasks is a mixed file: it becomes its own private benchmark++ }++ if (!tasks.length) throw new Error('That file holds no scores I can read.');++++ const config = configs[suiteName] || configs[Object.keys(configs)[0]] || doc.config || {};++ const { name, backend } = modelOf(config);++ const provider = providerOf(name, backend);++ const model = name || backend || 'unknown';++ const overall = tasks.reduce((n, t) => n + t.score, 0) / tasks.length;++ const version = String(doc.harness_version || doc.lm_eval_version || doc.version || '').trim();++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'lm-evaluation-harness', version: version || 'unknown' },++ date: doc.created_at || doc.date || null,++ suite: suiteName ? { name: suiteName, version: suiteVersion } : null,++ models: [{++ model,++ provider,++ score: overall,++ cost_usd: num(doc.cost_usd),++ tasks,++ }],++ };++}++++function looksLike(doc) {++ if (!doc || typeof doc !== 'object') return false;++ if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) return false;++ if (doc.configs || doc.versions || doc['n-shot'] || doc.higher_is_better) return true;++ const keys = Object.keys(doc.results);++ return keys.length > 0 && keys.every((k) => doc.results[k] && typeof doc.results[k] === 'object');++}++++module.exports = { TOOL, parse, looksLike };nnew file mode 100644@@@ -0,0 +1,137 @@++'use strict';++++/**++ * promptfoo results: the JSON that "promptfoo eval -o output.json" writes.++ *++ * The file is { version, timestamp, results, stats }. The results are either the rows of the run (one row per test++ * case, each with the provider that answered and the score it got) or the newer shape, where results.outputs holds++ * one entry per provider with its metrics. Both are read; each row becomes a task.++ */++const { parseJson, asText } = require('./limits');++++const TOOL = 'promptfoo';++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++function providerIdOf(row) {++ const p = row && (row.provider != null ? row.provider : row.providerId != null ? row.providerId : row.target);++ if (p == null) return 'promptfoo';++ if (typeof p === 'string') return p;++ if (typeof p === 'object') return String(p.id || p.label || p.name || 'promptfoo');++ return 'promptfoo';++}++++/** "openai:gpt-4o" -> provider "openai", model "openai:gpt-4o". */++function splitProvider(id) {++ const at = id.indexOf(':');++ if (at <= 0) return { provider: 'promptfoo', model: id };++ return { provider: id.slice(0, at), model: id };++}++++function scoreOf(row) {++ const grading = row && row.gradingResult;++ if (grading && num(grading.score) != null) return num(grading.score);++ if (num(row && row.score) != null) return num(row.score);++ if (row && typeof row.success === 'boolean') return row.success ? 1 : 0;++ return null;++}++++function costOf(row) {++ const response = (row && row.response) || {};++ const c = num(response.cost != null ? response.cost : row && row.cost);++ return c;++}++++function tokensOf(row) {++ const response = (row && row.response) || {};++ const usage = response.tokenUsage || response.tokens || null;++ if (!usage || typeof usage !== 'object') return null;++ return num(usage.total != null ? usage.total : usage.total_tokens);++}++++function rowId(row, index) {++ if (row && row.id != null) return String(row.id);++ if (row && row.testIdx != null) return 'test-' + row.testIdx;++ if (row && row.testCase && row.testCase.vars) return 'vars-' + index;++ return 'case-' + index;++}++++function parse(bytes) {++ const doc = parseJson(asText(bytes), { what: 'the results' });++ const results = doc && doc.results != null ? doc.results : null;++ if (!results) throw new Error('That file has no results.');++++ const rows = [];++ if (Array.isArray(results)) {++ rows.push(...results);++ } else if (Array.isArray(results.outputs)) {++ // The newer shape: one entry per provider, with the test results inside it.++ results.outputs.forEach((output, index) => {++ const cases = Array.isArray(output.tests) ? output.tests : Array.isArray(output.testCases) ? output.testCases : [];++ if (cases.length) {++ for (const c of cases) rows.push({ ...c, provider: output.provider, id: (c && c.id) || 'case-' + index });++ } else {++ const named = output.metrics && output.metrics.namedScores ? output.metrics.namedScores : {};++ const score = num(output.score) != null ? num(output.score)++ : (Object.values(named).map(num).find((v) => v != null) != null ? Object.values(named).map(num).find((v) => v != null) : null);++ rows.push({++ id: output.id != null ? String(output.id) : 'output-' + index,++ provider: output.provider,++ score: score == null ? null : score,++ gradingResult: null,++ });++ }++ });++ } else {++ throw new Error('That file has no results I can read.');++ }++ if (!rows.length) throw new Error('That file holds no results I can read.');++++ const byProvider = new Map();++ for (let i = 0; i < rows.length; i += 1) {++ const row = rows[i];++ const id = providerIdOf(row);++ const { provider, model } = splitProvider(id);++ if (!byProvider.has(id)) {++ byProvider.set(id, { model, provider, score: null, cost_usd: 0, hasCost: false, tokens: 0, hasTokens: false, tasks: [] });++ }++ const entry = byProvider.get(id);++ const score = scoreOf(row);++ entry.tasks.push({ task_id: rowId(row, i), score: score == null ? null : score * 100 });++ const cost = costOf(row);++ if (cost != null) { entry.cost_usd += cost; entry.hasCost = true; }++ const tokens = tokensOf(row);++ if (tokens != null) { entry.tokens += tokens; entry.hasTokens = true; }++ }++++ const models = [...byProvider.values()].map((entry) => {++ const scored = entry.tasks.filter((t) => t.score != null);++ return {++ model: entry.model,++ provider: entry.provider,++ score: scored.length ? scored.reduce((n, t) => n + t.score, 0) / scored.length : null,++ cost_usd: entry.hasCost ? entry.cost_usd : null,++ tokens: entry.hasTokens ? entry.tokens : null,++ tasks: entry.tasks,++ };++ });++ const version = String(doc.promptfoo_version || doc.tool_version || doc.promptfooVersion || '').trim();++ const date = doc.timestamp || doc.created_at || null;++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'promptfoo', version: version || 'unknown' },++ date: date ? new Date(date).toISOString() : null,++ suite: null,++ models,++ };++}++++function looksLike(doc) {++ if (!doc || typeof doc !== 'object') return false;++ if (Array.isArray(doc.results)) return true;++ if (doc.results && Array.isArray(doc.results.outputs)) return true;++ if (doc.results && typeof doc.results === 'object' && (doc.results.stats || doc.results.prompts)) return true;++ return false;++}++++module.exports = { TOOL, parse, looksLike };nnew file mode 100644@@@ -0,0 +1,269 @@++'use strict';++++/**++ * Importing results a person already has (B12).++ *++ * A person brings a file from Inspect AI, Harbor, the lm-evaluation-harness or promptfoo. The file becomes:++ *++ * a benchmark matched to a suite Bench knows (MMLU, GSM8K, HumanEval, ...) when the file names one, and a++ * private benchmark of their own otherwise++ * runs one per model in the file, always submitted (never verified: nobody re-ran them), with the++ * per-task scores the file carries, the harness and its version, the date and the cost++ *++ * The uploaded file stays the owner's own: private until they share the benchmark it became.++ */++const crypto = require('crypto');++const { ImportError, MAX_UPLOAD_BYTES } = require('./importers/limits');++const importers = require('./importers');++const { matchSuite, suiteTag, IMPORT_TAG } = require('./suites');++const { validateBenchmark } = require('./validate');++++const MAX_MODELS = 200;++const MAX_TASKS_PER_MODEL = 10000;++++const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');++const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));++++/** The name of the file, without any path: a name is never a path here. */++function safeFilename(name) {++ const raw = String(name == null ? '' : name);++ const base = raw.split(/[\\/]/).pop() || '';++ return base.replace(/[\u0000-\u001f]/g, '').trim().slice(0, 160) || 'results';++}++++function createImports({ s, registry, log = console }) {++ const db = s.db;++++ const wireImport = (row) => {++ if (!row) return null;++ const summary = typeof row.summary === 'string' ? JSON.parse(row.summary) : (row.summary || {});++ return {++ id: row.id,++ owner: row.owner,++ tool: row.tool,++ tool_version: row.tool_version,++ filename: row.filename,++ sha256: row.sha256,++ size: num(row.size),++ status: row.status,++ summary,++ benchmark_id: row.benchmark_id || null,++ run_ids: Array.isArray(row.run_ids) ? row.run_ids : [],++ created_at: row.created_at instanceof Date ? row.created_at.toISOString() : row.created_at,++ };++ };++++ /** Read the file, work out which tool wrote it, and keep it (and what it says) for the preview. */++ async function store({ owner, subject, filename, bytes, tool = null }) {++ const subjectId = String(subject || owner || '');++ if (!subjectId) throw new ImportError(401, 'bench.import_signed_in', 'Sign in to bring your own results.');++ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');++ if (!buf.length) throw new ImportError(422, 'bench.import_empty', 'That file is empty.');++ if (buf.length > MAX_UPLOAD_BYTES) {++ throw new ImportError(413, 'bench.import_too_large', 'That file is larger than 20 MB.');++ }++ let parsed;++ try {++ parsed = importers.parse(buf, { filename: safeFilename(filename), tool });++ } catch (err) {++ if (err instanceof ImportError) throw err;++ throw new ImportError(422, 'bench.import_unreadable', (err && err.message) || 'I could not read that file.');++ }++ const models = (parsed.models || []).slice(0, MAX_MODELS);++ for (const model of models) {++ if ((model.tasks || []).length > MAX_TASKS_PER_MODEL) {++ throw new ImportError(413, 'bench.import_too_many_tasks', 'That file holds more tasks than I can import.');++ }++ }++ const id = s.newId('bim');++ const name = safeFilename(filename);++ const summary = {++ tool: parsed.tool,++ tool_version: parsed.toolVersion,++ harness: parsed.harness,++ date: parsed.date,++ suite: parsed.suite || null,++ models: models.map((m) => ({++ model: m.model,++ provider: m.provider,++ score: m.score == null ? null : num(m.score),++ cost_usd: m.cost_usd == null ? null : num(m.cost_usd),++ tokens: m.tokens == null ? null : num(m.tokens),++ tasks: (m.tasks || []).map((t) => ({ task_id: String(t.task_id), score: t.score == null ? null : num(t.score) })),++ })),++ };++ await db.exec(++ `INSERT INTO bench_imports (id, owner, tool, tool_version, filename, sha256, size, status, summary, created_by, created_at)++ VALUES ($1, $2, $3, $4, $5, $6, $7, 'preview', $8::jsonb, $9, $10::timestamptz)`,++ [id, subjectId, parsed.tool, String(parsed.toolVersion || ''), name, sha256(buf), buf.length,++ JSON.stringify(summary), subjectId, s.iso()],++ );++ await db.exec('INSERT INTO bench_import_files (import_id, content, size) VALUES ($1, $2, $3)', [id, buf, buf.length]);++ return get(id);++ }++++ async function get(id) {++ return wireImport(await db.maybe('SELECT * FROM bench_imports WHERE id = $1', [String(id || '')]));++ }++++ /** The imports of a person, newest first. */++ async function listFor(owner, { limit = 20 } = {}) {++ const rows = await db.many(++ 'SELECT * FROM bench_imports WHERE owner = $1 ORDER BY created_at DESC LIMIT $2',++ [String(owner || ''), Math.max(1, Math.min(100, num(limit, 20)))],++ );++ return rows.map(wireImport);++ }++++ /** The task ids of an import: every task any model in the file scored. */++ function taskIdsOf(summary) {++ const ids = [];++ for (const model of summary.models || []) {++ for (const task of model.tasks || []) {++ if (!ids.includes(task.task_id)) ids.push(task.task_id);++ }++ }++ return ids;++ }++++ /** The benchmark an import becomes: the one of a suite Bench knows, or a new private one. */++ async function benchmarkFor(summary, { subject }) {++ const suite = summary.suite ? matchSuite(summary.suite.name) : null;++ const tag = suite ? suiteTag(suite.key) : null;++ if (tag) {++ const list = await registry.listBenchmarks({ tag, owner: subject, limit: 50 });++ const found = list.find((b) => Array.isArray(b.tags) && b.tags.includes(tag));++ if (found) return { benchmark: found, suite, matched: true };++ }++ const title = suite++ ? suite.title++ : (summary.suite && summary.suite.name) || (importers.TOOL_LABELS[summary.tool] + ' results');++ const toolLabel = importers.TOOL_LABELS[summary.tool] || summary.tool;++ const made = await registry.createBenchmark({++ owner: subject,++ title,++ description: 'Imported from ' + toolLabel + ' ' + (summary.tool_version || '') + '.',++ kind: 'text.answer',++ tags: [IMPORT_TAG].concat(tag ? [tag] : []),++ visibility: 'private',++ subject,++ });++ if (!made.ok) throw new ImportError(made.status || 422, made.code || 'bench.import_failed', made.message || 'The benchmark could not be made.');++ return { benchmark: made.benchmark, suite, matched: false };++ }++++ /** The version the runs hang on: one that has the tasks, or a new one frozen with them. */++ async function versionFor(benchmark, summary, { subject, toolLabel }) {++ const taskIds = taskIdsOf(summary);++ const latest = await registry.latestVersion(benchmark.id);++ if (latest && latest.definition) {++ const known = new Set((latest.definition.tasks || []).map((t) => String(t.id)));++ if (taskIds.length && taskIds.every((id) => known.has(id))) return latest.version;++ }++ const title = benchmark.title;++ const definition = {++ title,++ description: benchmark.description || ('Imported from ' + toolLabel + '.'),++ kind: benchmark.kind,++ tags: benchmark.tags || [],++ visibility: 'unlisted',++ tasks: taskIds.map((id) => ({++ id,++ prompt: 'Imported task ' + id + '. The file carried the score, not the prompt.',++ holdout: false,++ })),++ judges: [],++ scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: {},++ matrix: {},++ };++ const frozen = await registry.freezeVersion(benchmark.id, { definition, subject, validate: validateBenchmark });++ if (!frozen.ok) throw new ImportError(frozen.status || 422, frozen.code || 'bench.import_failed', frozen.message || 'The benchmark could not be frozen.');++ return frozen.version.version;++ }++++ /** Turn the upload into runs: one per model, always submitted. */++ async function commit(id, { subject }) {++ const row = await get(id);++ if (!row) throw new ImportError(404, 'bench.import_not_found', 'No such upload.');++ if (String(row.owner) !== String(subject)) throw new ImportError(404, 'bench.import_not_found', 'No such upload.');++ if (row.status === 'imported') {++ const benchmark = row.benchmark_id ? await registry.getBenchmark(row.benchmark_id) : null;++ return { ok: true, benchmark, runs: row.run_ids.map((rid) => ({ id: rid })), already: true };++ }++ const summary = row.summary || {};++ const toolLabel = importers.TOOL_LABELS[summary.tool] || summary.tool;++ const { benchmark } = await benchmarkFor(summary, { subject });++ const version = await versionFor(benchmark, summary, { subject, toolLabel });++ const known = new Set(taskIdsOf(summary));++ const runIds = [];++ for (const model of summary.models || []) {++ const tasks = (model.tasks || [])++ .filter((t) => known.has(t.task_id))++ .map((t) => ({ task_id: t.task_id, score: t.score == null ? null : num(t.score), cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }));++ if (!tasks.length) continue;++ const published = await registry.publishRun({++ benchmark: benchmark.id + '@' + version,++ model: String(model.model || 'unknown').slice(0, 160),++ provider: String(model.provider || 'unknown').slice(0, 160),++ route: {},++ harness: summary.harness || { name: toolLabel, version: summary.tool_version || 'unknown' },++ settings: {},++ tasks,++ score: model.score == null ? null : num(model.score),++ cost_usd: num(model.cost_usd),++ ms: 0,++ tokens: num(model.tokens),++ source: {++ kind: 'import',++ tool: summary.tool,++ tool_version: summary.tool_version || '',++ import_id: row.id,++ filename: row.filename,++ date: summary.date || null,++ },++ }, { provenance: 'submitted', subject });++ if (!published.ok) {++ log.warn('[OpenVibe.Bench] an imported run was refused:', published.code || '', published.message || '');++ continue;++ }++ runIds.push(published.run.id);++ }++ await db.exec(++ "UPDATE bench_imports SET status = 'imported', benchmark_id = $1, run_ids = $2::jsonb WHERE id = $3",++ [benchmark.id, JSON.stringify(runIds), row.id],++ );++ return { ok: true, benchmark, runs: runIds.map((rid) => ({ id: rid })), import: await get(row.id) };++ }++++ /**++ * The uploaded file itself. The owner may always read it. Once the benchmark it became is public, anyone may;++ * an unlisted benchmark shares it with the people its link reaches.++ */++ async function file(id, { subject = null, shareToken = null } = {}) {++ const row = await db.maybe('SELECT * FROM bench_imports WHERE id = $1', [String(id || '')]);++ if (!row) return null;++ const owner = subject != null && String(row.owner) === String(subject);++ if (!owner) {++ const benchmark = row.benchmark_id ? await registry.getBenchmark(row.benchmark_id) : null;++ let allowed = Boolean(benchmark && benchmark.visibility === 'public');++ if (!allowed && benchmark && benchmark.visibility === 'unlisted' && benchmark.shared && shareToken) {++ const shared = await registry.findShared(String(shareToken));++ allowed = Boolean(shared && shared.id === benchmark.id);++ }++ if (!allowed) return null;++ }++ const content = await db.maybe('SELECT content, size FROM bench_import_files WHERE import_id = $1', [row.id]);++ if (!content) return null;++ return {++ import: wireImport(row),++ content: content.content,++ filename: row.filename || 'results',++ size: num(content.size),++ };++ }++++ return { store, get, listFor, commit, file, taskIdsOf };++}++++module.exports = { createImports, safeFilename, MAX_MODELS, MAX_TASKS_PER_MODEL };nnew file mode 100644@@@ -0,0 +1,290 @@++'use strict';++++/**++ * Mirroring standard results (B12).++ *++ * A mirror is an operator's job: it reads a published leaderboard from an open source and writes it into Bench++ * with its provenance. Two rules hold it together:++ *++ * the licence a source is read only when its licence allows the reuse, and the licence (with its text and++ * where it was read) travels with every row it brings++ * the label a mirrored run is never verified: it is submitted, and it says where it came from++ *++ * The same model, benchmark version and source never land twice: bench_mirror_runs holds one row for each, with a++ * unique index over the three.++ */++const { matchSuite, suiteTag, IMPORT_TAG } = require('./suites');++++const MIRROR_TAG = 'mirrored';++++const SOURCES = {++ helm: {++ key: 'helm',++ title: 'HELM',++ base: 'https://storage.googleapis.com/crfm-helm-public',++ page: 'https://www.helm.ai/',++ licence: 'Apache-2.0',++ licence_url: 'https://github.com/stanford-crfm/helm/blob/main/LICENSE',++ attribution: 'HELM (Stanford Center for Research on Foundation Models), stanford-crfm/helm',++ },++ 'open-llm-leaderboard': {++ key: 'open-llm-leaderboard',++ title: 'Open LLM Leaderboard',++ base: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',++ page: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',++ licence: '',++ licence_url: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',++ attribution: 'Open LLM Leaderboard (Hugging Face)',++ },++};++++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++/**++ * The licence of a source, read at the moment of the import. A source whose licence cannot be read does not++ * clear, and nothing of it is imported.++ */++async function checkLicence(source, { fetchImpl = globalThis.fetch } = {}) {++ if (source.key === 'helm') {++ const url = 'https://raw.githubusercontent.com/stanford-crfm/helm/main/LICENSE';++ const res = await fetchImpl(url);++ if (!res.ok) return { cleared: false, reason: 'the licence file could not be read (' + res.status + ')' };++ const text = await res.text();++ const line = text.split('\n').map((l) => l.trim()).find((l) => /Apache License/i.test(l)) || '';++ if (!line) return { cleared: false, reason: 'the licence file names no licence I recognise' };++ return { cleared: true, licence: 'Apache-2.0', licence_url: url, quote: line };++ }++ if (source.key === 'open-llm-leaderboard') {++ const url = 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard/raw/main/README.md';++ const res = await fetchImpl(url);++ if (!res.ok) return { cleared: false, reason: 'the dataset card could not be read (' + res.status + ')' };++ const text = await res.text();++ const line = text.split('\n').map((l) => l.trim()).find((l) => /^license:\s*/i.test(l)) || '';++ if (!line) return { cleared: false, reason: 'the dataset card states no licence' };++ const licence = line.replace(/^license:\s*/i, '').replace(/^["']|["']$/g, '').trim();++ return { cleared: true, licence, licence_url: url, quote: line };++ }++ return { cleared: false, reason: 'no licence is known for this source' };++}++++/** The rows of a HELM group table: one per model, with the score of the group. */++function helmRows(table, { group, release }) {++ const out = [];++ const tables = Array.isArray(table) ? table : [table];++ for (const one of tables) {++ if (!one || !Array.isArray(one.header) || !Array.isArray(one.rows)) continue;++ const header = one.header;++ const wanted = ['EM', 'Accuracy', 'exact_match', 'Normalized accuracy', 'acc'];++ let index = -1;++ for (const name of wanted) {++ const at = header.findIndex((h) => h && h.metadata && String(h.metadata.metric || '') === name);++ if (at > 0) { index = at; break; }++ }++ if (index < 0) continue;++ for (const row of one.rows) {++ if (!Array.isArray(row) || !row.length) continue;++ const model = row[0] && row[0].value != null ? String(row[0].value) : '';++ const score = num(row[index] && row[index].value);++ if (!model || score == null) continue;++ const group_ = (one.header[index] && one.header[index].metadata && one.header[index].metadata.run_group) || group;++ out.push({++ benchmark: group_,++ benchmark_version: release,++ model,++ score: score * 100,++ task_id: String(group_).toLowerCase().replace(/[^a-z0-9]+/g, '_'),++ source_url: sourceUrlHelm(group, release),++ });++ }++ }++ return out;++}++++function sourceUrlHelm(group, release) {++ return 'https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/' + release + '/groups/' + group + '.json';++}++++/** Read a HELM group: the leaderboard table of one scenario group of one release. */++async function loadHelm(source, { fetchImpl = globalThis.fetch, suite = 'lite', release = 'v1.13.0', group = 'mmlu' } = {}) {++ const url = source.base + '/' + suite + '/benchmark_output/releases/' + release + '/groups/' + group + '.json';++ const res = await fetchImpl(url);++ if (!res.ok) throw new Error('the source answered ' + res.status + ' for ' + url);++ const table = await res.json();++ return { url, rows: helmRows(table, { group, release }) };++}++++/** Read the Open LLM Leaderboard dataset: one row per model and task. */++async function loadOpenLlmLeaderboard(source, { fetchImpl = globalThis.fetch, limit = 200 } = {}) {++ const rows = [];++ const configsUrl = 'https://datasets-server.huggingface.co/splits?dataset=' + encodeURIComponent('open-llm-leaderboard/open_llm_leaderboard');++ const res = await fetchImpl(configsUrl);++ if (!res.ok) throw new Error('the source answered ' + res.status + ' for the dataset splits');++ const splits = await res.json();++ const configs = (splits.splits || []).slice(0, 6);++ for (const config of configs) {++ const url = 'https://datasets-server.huggingface.co/rows?dataset='++ + encodeURIComponent('open-llm-leaderboard/open_llm_leaderboard')++ + '&config=' + encodeURIComponent(config.config) + '&split=' + encodeURIComponent(config.split)++ + '&offset=0&length=' + Math.max(1, Math.min(100, limit));++ const r2 = await fetchImpl(url);++ if (!r2.ok) continue;++ const body = await r2.json();++ for (const item of body.rows || []) {++ const row = item.row || {};++ const model = row.model_name || row.model || row.Model || row.model_full_name;++ const score = num(row.accuracy != null ? row.accuracy : row.acc != null ? row.acc : row.score);++ if (!model || score == null) continue;++ rows.push({++ benchmark: config.config,++ benchmark_version: 'leaderboard',++ model: String(model),++ score: score * 100,++ task_id: String(config.config),++ source_url: source.page,++ });++ }++ }++ return { url: source.page, rows };++}++++/**++ * The mirror: read the sources, work out what they would bring, and (only when asked) write it.++ * Every mirrored run is submitted and carries the source, its licence and the day it was read.++ */++function createMirror({ s, registry, fetchImpl = globalThis.fetch, log = console } = {}) {++ const db = s.db;++++ /** What a source would bring, without writing anything. */++ async function preview(sourceKey, options = {}) {++ const source = SOURCES[sourceKey];++ if (!source) return { ok: false, error: 'No such source.' };++ const licence = await checkLicence(source, { fetchImpl });++ if (!licence.cleared) {++ return { ok: false, source: source.key, blocked: true, reason: 'the licence does not clear: ' + licence.reason };++ }++ let loaded;++ try {++ loaded = source.key === 'helm'++ ? await loadHelm(source, { fetchImpl, ...options })++ : await loadOpenLlmLeaderboard(source, { fetchImpl, ...options });++ } catch (err) {++ return { ok: false, source: source.key, blocked: true, reason: (err && err.message) || 'the source could not be read' };++ }++ return {++ ok: true,++ source: source.key,++ title: source.title,++ url: loaded.url,++ licence: licence.licence,++ licence_url: licence.licence_url,++ licence_quote: licence.quote,++ attribution: source.attribution,++ rows: loaded.rows,++ count: loaded.rows.length,++ };++ }++++ /** The benchmark a mirrored group becomes: one per source and group, public, so anybody can read it. */++ async function benchmarkFor(row, { source, licence }) {++ const suite = matchSuite(row.benchmark);++ const tags = [MIRROR_TAG, IMPORT_TAG].concat(suite ? [suiteTag(suite.key)] : []);++ const tag = 'mirror:' + source.key + ':' + String(row.benchmark).toLowerCase().replace(/[^a-z0-9]+/g, '_');++ const list = await registry.listBenchmarks({ tag, limit: 20 });++ const found = list.find((b) => Array.isArray(b.tags) && b.tags.includes(tag));++ if (found) return found;++ const made = await registry.createBenchmark({++ owner: 'openvibe',++ title: (suite ? suite.title : row.benchmark) + ' (' + source.title + ')',++ description: 'Mirrored from ' + source.title + '. Licence: ' + licence.licence + '. ' + source.attribution + '.',++ kind: 'text.answer',++ tags: tags.concat([tag]),++ visibility: 'public',++ subject: 'openvibe',++ });++ if (!made.ok) throw new Error(made.message || 'the benchmark could not be made');++ return made.benchmark;++ }++++ /** Write the rows: one mirrored run per model, benchmark version and source; the same one never lands twice. */++ async function apply(sourceKey, options = {}) {++ const looked = await preview(sourceKey, options);++ if (!looked.ok) return looked;++ const source = SOURCES[sourceKey];++ const readAt = s.iso();++ const written = [];++ const skipped = [];++ for (const row of looked.rows) {++ const benchmark = await benchmarkFor(row, { source, licence: looked.licence });++ const definition = {++ title: benchmark.title,++ description: benchmark.description,++ kind: benchmark.kind,++ tags: benchmark.tags || [],++ visibility: 'public',++ tasks: [{ id: row.task_id, prompt: 'Mirrored result: ' + row.benchmark + '.', holdout: false }],++ judges: [],++ scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: {},++ matrix: {},++ };++ const latest = await registry.latestVersion(benchmark.id);++ const known = latest && latest.definition++ ? new Set((latest.definition.tasks || []).map((t) => String(t.id)))++ : new Set();++ let version;++ if (latest && known.has(row.task_id)) {++ version = latest.version;++ } else {++ const frozen = await registry.freezeVersion(benchmark.id, { definition, subject: 'openvibe' });++ if (!frozen.ok) { skipped.push({ model: row.model, why: frozen.message }); continue; }++ version = frozen.version.version;++ }++ const prior = await db.maybe(++ 'SELECT id FROM bench_mirror_runs WHERE source_key = $1 AND benchmark_id = $2 AND benchmark_version = $3 AND model = $4',++ [source.key, benchmark.id, version, row.model],++ );++ if (prior) { skipped.push({ model: row.model, why: 'already mirrored' }); continue; }++ const published = await registry.publishRun({++ benchmark: benchmark.id + '@' + version,++ model: String(row.model).slice(0, 160),++ provider: source.key,++ route: {},++ harness: { name: source.title, version: String(row.benchmark_version || 'published') },++ settings: {},++ tasks: [{ task_id: row.task_id, score: row.score, cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }],++ score: row.score,++ cost_usd: 0,++ ms: 0,++ tokens: 0,++ source: {++ kind: 'mirror',++ source_key: source.key,++ source_url: row.source_url,++ licence: looked.licence,++ licence_url: looked.licence_url,++ attribution: looked.attribution,++ read_at: readAt,++ },++ }, { provenance: 'submitted', subject: 'openvibe' });++ if (!published.ok) { skipped.push({ model: row.model, why: published.message }); continue; }++ await db.exec(++ `INSERT INTO bench_mirror_runs (id, source_key, source_url, licence, licence_url, read_at, benchmark_id, benchmark_version, model, run_id)++ VALUES ($1, $2, $3, $4, $5, $6::timestamptz, $7, $8, $9, $10)`,++ [s.newId('bmr'), source.key, row.source_url, looked.licence, looked.licence_url, readAt,++ benchmark.id, version, row.model, published.run.id],++ );++ written.push({ model: row.model, benchmark: benchmark.id, run: published.run.id, score: row.score });++ }++ await db.exec(++ `INSERT INTO bench_mirror_reads (id, source_key, source_url, licence, licence_url, attribution, read_at, rows)++ VALUES ($1, $2, $3, $4, $5, $6, $7::timestamptz, $8)++ ON CONFLICT (source_key, source_url) DO UPDATE SET read_at = EXCLUDED.read_at, rows = EXCLUDED.rows`,++ [s.newId('bmr'), source.key, looked.url, looked.licence, looked.licence_url, looked.attribution, readAt, written.length],++ );++ return { ok: true, source: source.key, licence: looked.licence, licence_url: looked.licence_url, written, skipped, read_at: readAt };++ }++++ return { preview, apply, SOURCES, checkLicence, MIRROR_TAG };++}++++module.exports = { createMirror, SOURCES, checkLicence, MIRROR_TAG };@@@ -137,6 +137,9 @@ function wireRun(row, tasks = null) {ms: num(row.ms),tokens: num(row.tokens),provenance: row.provenance,++ // Where the run came from: an upload, or a mirror's source with its licence. It is Bench's own field:++ // the published bench.run document closes its shape to the contract, so this sits beside it.++ source: json(row.source, {}),session: row.session || null,created_by: row.created_by,created_at: iso(row.created_at),@@@ -520,11 +523,12 @@ function createRegistry(s, { log = console } = {}) {await db.tx(async () => {await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, route, provider, harness, settings, executor,-- hardware, score, cost_usd, ms, tokens, provenance, session, created_by, created_at)-- VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15, $16, $17, $18::timestamptz)`,++ hardware, score, cost_usd, ms, tokens, provenance, source, session, created_by, created_at)++ VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15, $16::jsonb, $17, $18, $19::timestamptz)`,[id, benchmarkId, version, model, JSON.stringify(doc.route || {}), provider,JSON.stringify(doc.harness || {}), JSON.stringify(doc.settings || {}), JSON.stringify(executor),doc.hardware == null ? null : JSON.stringify(doc.hardware), score, cost, ms, tokens, provenance,++ JSON.stringify(doc.source || {}),doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],);for (const t of tasks) {nnew file mode 100644@@@ -0,0 +1,56 @@++'use strict';++++/**++ * The suites Bench knows by name (B12). A results file that names one of them is matched to the benchmark of that++ * name in the registry; anything else becomes a new private benchmark of the person's own.++ *++ * A suite is a name, a version and the aliases the tools use for it. The alias list is what makes "mmlu",++ * "MMLU" and "massively multitask language understanding" the same suite.++ */++const SUITES = [++ { key: 'mmlu', title: 'MMLU', aliases: ['mmlu', 'massively multitask language understanding', 'massive multitask language understanding'] },++ { key: 'mmlu_pro', title: 'MMLU-Pro', aliases: ['mmlu_pro', 'mmlu-pro', 'mmlu pro'] },++ { key: 'gsm8k', title: 'GSM8K', aliases: ['gsm8k', 'gsm-8k', 'grade school math 8k'] },++ { key: 'humaneval', title: 'HumanEval', aliases: ['humaneval', 'human_eval', 'human-eval'] },++ { key: 'hellaswag', title: 'HellaSwag', aliases: ['hellaswag'] },++ { key: 'arc_challenge', title: 'ARC-Challenge', aliases: ['arc_challenge', 'arc-challenge', 'arc challenge'] },++ { key: 'arc_easy', title: 'ARC-Easy', aliases: ['arc_easy', 'arc-easy'] },++ { key: 'truthful_qa', title: 'TruthfulQA', aliases: ['truthful_qa', 'truthful-qa', 'truthfulqa'] },++ { key: 'winogrande', title: 'WinoGrande', aliases: ['winogrande', 'winogrande_xl'] },++ { key: 'hendrycks_math', title: 'Hendrycks MATH', aliases: ['hendrycks_math', 'math', 'hendrycks-math'] },++ { key: 'gpqa', title: 'GPQA', aliases: ['gpqa', 'gpqa_main', 'gpqa_diamond'] },++ { key: 'bigbench', title: 'BIG-bench', aliases: ['bigbench', 'big_bench', 'big-bench'] },++ { key: 'tau', title: 'tau-bench', aliases: ['tau', 'tau_bench', 'tau-bench', 'taubench'] },++ { key: 'swe_bench', title: 'SWE-bench', aliases: ['swe_bench', 'swe-bench', 'swebench'] },++ { key: 'bfcl', title: 'Berkeley Function Calling Leaderboard', aliases: ['bfcl', 'bfcl_v3', 'function_calling'] },++ { key: 'mt_bench', title: 'MT-Bench', aliases: ['mt_bench', 'mt-bench', 'mtbench'] },++ { key: 'hle', title: "Humanity's Last Exam", aliases: ['hle', 'humanitys_last_exam'] },++];++++/** "ARC-Challenge" -> "arcchallenge": the name without its punctuation. */++function normalize(name) {++ return String(name == null ? '' : name).toLowerCase().replace(/[^a-z0-9]+/g, '');++}++++/** The suite a name means, or null. */++function matchSuite(name) {++ const key = normalize(name);++ if (!key) return null;++ for (const suite of SUITES) {++ if (normalize(suite.key) === key) return suite;++ for (const alias of suite.aliases) {++ if (normalize(alias) === key) return suite;++ }++ }++ return null;++}++++/** The tag a benchmark of that suite carries, so a later import finds it again. */++function suiteTag(key) {++ return 'suite:' + String(key || '').toLowerCase();++}++++/** The tag every imported benchmark carries. */++const IMPORT_TAG = 'imported';++++module.exports = { SUITES, normalize, matchSuite, suiteTag, IMPORT_TAG };nnew file mode 100644@@@ -0,0 +1,114 @@++'use strict';++++/**++ * Bringing your own results (B12): the upload form, the preview and the file itself.++ *++ * The upload is a plain form post, same-origin (CSRF), signed in, and capped at 20 MB by the body parser. The++ * file is the person's own: the page that shows it is theirs, and the file stays private until they share the++ * benchmark it became.++ */++const express = require('express');++const { asyncRouter } = require('./router');++const { sameOrigin } = require('./same-origin');++const { parseMultipart } = require('./multipart');++const { raw } = require('../render/html');++const { send } = require('../render/layout');++const views = require('../views/imports');++const { ImportError } = require('../bench/importers/limits');++++function createImportRoutes(ctx) {++ const { config } = ctx;++ const r = asyncRouter();++ const subjectOf = (req) => (req.viewer && req.viewer.kind === 'user' ? (req.viewer.subject || req.viewer.id || null) : null);++ const page = (req, res, o, status = 200) => send(res, status, { viewer: req.viewer, config, path: req.originalUrl, ...o });++++ const upload = async (req, res) => {++ const subject = subjectOf(req);++ const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');++ const imports = subject ? await ctx.imports.listFor(subject, { limit: 10 }) : [];++ return page(req, res, {++ index: false,++ title: 'Import a results file',++ description: 'Bring the results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo, and see the runs it holds before they are imported.',++ body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, imports, error: req.query.error ? String(req.query.error).slice(0, 200) : null }))),++ });++ };++ r.get('/imports', upload);++++ r.post('/imports', express.raw({ type: 'multipart/form-data', limit: '21mb' }), ctx.principal.middleware, async (req, res) => {++ const subject = subjectOf(req);++ const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');++ const fail = (status, error) => page(req, res, {++ index: false, title: 'Import a results file',++ description: 'Bring a results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo.',++ body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, error }))),++ }, status);++ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');++ if (!subject) return fail(401, 'Sign in to bring your own results.');++ const parts = parseMultipart(req.body, req.get('content-type'));++ if (!parts) return fail(422, 'That upload did not arrive as a form. Try again.');++ const file = parts.parts.find((p) => p.name === 'file' && p.data && p.data.length);++ if (!file) return fail(422, 'Pick a results file to read.');++ const toolPart = parts.parts.find((p) => p.name === 'tool');++ const tool = toolPart ? String(toolPart.data.toString('utf8')).trim() : '';++ try {++ const imp = await ctx.imports.store({++ owner: subject, subject, filename: file.filename || 'results', bytes: file.data, tool: tool || null,++ });++ return res.redirect(303, '/imports/' + encodeURIComponent(imp.id));++ } catch (err) {++ const detail = err instanceof ImportError ? err.detail : 'That file could not be read.';++ return fail(err instanceof ImportError ? err.status : 422, detail);++ }++ });++++ /** The preview: what the file says, before anything is written. */++ r.get('/imports/:id', async (req, res) => {++ const subject = subjectOf(req);++ const imp = await ctx.imports.get(String(req.params.id || ''));++ if (!imp || String(imp.owner) !== String(subject)) {++ return page(req, res, { index: false, title: 'No such upload', body: raw(String(require('../views').notFound())) }, 404);++ }++ return page(req, res, {++ index: false,++ title: 'Import preview',++ description: 'What this results file holds, before it is imported.',++ body: raw(String(views.previewPage({ imp }))),++ });++ });++++ r.post('/imports/:id/confirm', ctx.principal.middleware, async (req, res) => {++ const subject = subjectOf(req);++ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');++ const imp = await ctx.imports.get(String(req.params.id || ''));++ if (!imp || String(imp.owner) !== String(subject)) {++ return page(req, res, { index: false, title: 'No such upload', body: raw(String(require('../views').notFound())) }, 404);++ }++ try {++ const done = await ctx.imports.commit(imp.id, { subject });++ if (done.benchmark) return res.redirect(303, '/benchmarks/' + encodeURIComponent(done.benchmark.id));++ return page(req, res, { index: false, title: 'Import preview', body: raw(String(views.previewPage({ imp, error: 'Nothing could be imported from that file.' }))) }, 422);++ } catch (err) {++ const detail = err instanceof ImportError ? err.detail : 'The import did not go through.';++ return page(req, res, { index: false, title: 'Import preview', body: raw(String(views.previewPage({ imp, error: detail }))) }, 422);++ }++ });++++ /** The uploaded file: the owner's own, and anyone else's once the benchmark it became is shared. */++ r.get('/imports/:id/file', async (req, res) => {++ const subject = subjectOf(req);++ const shareToken = req.query.share ? String(req.query.share) : null;++ const found = await ctx.imports.file(String(req.params.id || ''), { subject, shareToken });++ if (!found) {++ return page(req, res, { index: false, title: 'No such file', body: raw(String(require('../views').notFound())) }, 404);++ }++ res.setHeader('Content-Type', 'application/octet-stream');++ res.setHeader('Content-Disposition', 'attachment; filename="' + String(found.filename).replace(/[^A-Za-z0-9._-]+/g, '_') + '"');++ res.setHeader('Cache-Control', 'private, max-age=0, no-store');++ return res.send(found.content);++ });++++ return r;++}++++module.exports = { createImportRoutes };nnew file mode 100644@@@ -0,0 +1,53 @@++'use strict';++++/**++ * The operator's mirror route (B12). It is not a public API: it needs the operator's own token, and it reads++ * the sources' licences before it writes anything. A dry run (the default) says what it would import and from++ * where, and writes nothing.++ */++const crypto = require('crypto');++const { asyncRouter } = require('./router');++++function operatorToken(req) {++ const h = String(req.get('authorization') || '');++ return /^Bearer\s+\S/i.test(h) ? h.replace(/^Bearer\s+/i, '').trim() : null;++}++++/**++ * The operator's token, compared in constant time: the digests of both are compared with timingSafeEqual, so++ * the answer does not depend on how many characters matched.++ */++function tokenMatches(given, expected) {++ if (!given || !expected) return false;++ const a = crypto.createHash('sha256').update(String(given), 'utf8').digest();++ const b = crypto.createHash('sha256').update(String(expected), 'utf8').digest();++ return a.length === b.length && crypto.timingSafeEqual(a, b);++}++++function createMirrorRoutes(ctx) {++ const r = asyncRouter();++ const token = process.env.BENCH_ADMIN_TOKEN || '';++++ r.post('/api/v1/admin/mirror', async (req, res) => {++ if (!token) return res.status(404).type('text').send('No such route.');++ const given = operatorToken(req);++ if (!tokenMatches(given, token)) return res.status(403).type('text').send('The operator token is required.');++ const body = req.body && typeof req.body === 'object' ? req.body : {};++ const source = String(body.source || 'helm');++ const options = {++ suite: body.suite ? String(body.suite) : undefined,++ release: body.release ? String(body.release) : undefined,++ group: body.group ? String(body.group) : undefined,++ limit: body.limit ? Number(body.limit) : undefined,++ };++ const apply = body.apply === true || body.apply === 'true';++ const out = apply++ ? await ctx.mirror.apply(source, options)++ : await ctx.mirror.preview(source, options);++ return res.json(out);++ });++++ return r;++}++++module.exports = { createMirrorRoutes, tokenMatches };nnew file mode 100644@@@ -0,0 +1,47 @@++'use strict';++++/**++ * A multipart/form-data body, read in memory (B12). Express has no body parser for it and a results file needs++ * one: the upload form posts one file and nothing else.++ *++ * The body is already capped by the caller (express.raw with a limit), so this only has to find the parts.++ * A part's name is used as a field name; a part's filename is only ever shown, never used as a path.++ */++function parseMultipart(buffer, contentType) {++ const type = String(contentType || '');++ const at = type.toLowerCase().indexOf('multipart/form-data');++ if (at < 0) return null;++ const match = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(type.slice(at));++ if (!match) return null;++ const boundary = (match[1] || match[2] || '').trim();++ if (!boundary || boundary.length > 200) return null;++ const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer || '');++ const delimiter = Buffer.from('--' + boundary);++ const parts = [];++ let start = buf.indexOf(delimiter);++ if (start < 0) return null;++ start += delimiter.length;++ for (let guard = 0; guard < 50; guard += 1) {++ if (start + 2 <= buf.length && buf[start] === 0x2d && buf[start + 1] === 0x2d) break; // the closing --++ // The part's headers run to the first blank line.++ const headEnd = buf.indexOf('\r\n\r\n', start, 'latin1');++ if (headEnd < 0) break;++ const head = buf.subarray(start, headEnd).toString('latin1');++ const bodyStart = headEnd + 4;++ const next = buf.indexOf(delimiter, bodyStart);++ if (next < 0) break;++ const bodyEnd = Math.max(bodyStart, next - 2); // the CRLF before the boundary belongs to the delimiter++ const name = /name="([^"]*)"/i.exec(head);++ const filename = /filename="([^"]*)"/i.exec(head);++ parts.push({++ name: name ? name[1] : '',++ filename: filename ? filename[1] : null,++ contentType: (/content-type:\s*([^\r\n]+)/i.exec(head) || [])[1] || null,++ data: buf.subarray(bodyStart, bodyEnd),++ });++ start = next + delimiter.length;++ }++ return { parts };++}++++module.exports = { parseMultipart };@@@ -35,6 +35,8 @@ const TABLES = [// The runs they submitted and the benchmarks they watch: theirs, so they go.{ table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },++ // The results files they brought (B12): theirs, so they go with them. The bytes hang off the row and go with it.++ { table: 'bench_imports', subject: 'owner', file: 'bench_imports.json', erase: 'delete' },];// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang@@@ -59,6 +61,11 @@ async function extraExport(db, subject) {AND (visibility = 'private' OR tags @> '["your-prompts"]'::jsonb))ORDER BY created_at DESC LIMIT 5000`, [subject]);if (runs.length) files.push({ name: 'benchmark_runs.json', content: runs });++ // The files they uploaded: the bytes are theirs too, so the export carries them.++ const uploads = await db.many(`SELECT f.import_id, i.filename, i.sha256, i.size, encode(f.content, 'base64') AS content_base64++ FROM bench_import_files f JOIN bench_imports i ON i.id = f.import_id++ WHERE i.owner = $1 OR i.created_by = $1 ORDER BY i.created_at DESC LIMIT 5000`, [subject]);++ if (uploads.length) files.push({ name: 'import_files.json', content: uploads });return files;}nnew file mode 100644@@@ -0,0 +1,68 @@++'use strict';++++/**++ * The pages of an import (B12): the upload form and the preview. Both are plain forms: they work without++ * JavaScript, they are narrow enough for a phone, and they say plainly what will happen.++ */++const { html, raw } = require('../render/html');++++const TOOLS = [++ { key: 'inspect', label: 'Inspect AI', hint: 'an .eval log (a zip) or its JSON log' },++ { key: 'harbor', label: 'Harbor', hint: 'the results.json of a Harbor job' },++ { key: 'lm-eval', label: 'lm-evaluation-harness', hint: 'the results.json of a run' },++ { key: 'promptfoo', label: 'promptfoo', hint: 'the output.json of promptfoo eval -o' },++];++++const fmtDate = (v) => {++ if (!v) return '';++ const d = new Date(v);++ return Number.isNaN(d.getTime()) ? '' : d.toISOString().slice(0, 10);++};++++function uploadPage({ signedIn = false, loginUrl = '/auth/login', error = null, imports = [] } = {}) {++ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Bring your results</span></nav>++<header class="bm-page-h"><p class="bm-kicker">Bring your results</p><h1>Import a results file</h1><p class="bm-lede">Run your evals with the tools you already use, then bring the file here. Every run it holds becomes a submitted run: Bench never calls them verified, because nobody re-ran them.</p></header>`;++ if (!signedIn) {++ return html`<div class="bm-wrap br bm-narrow" id="br-import">${head}<div class="br-note" role="alert">Sign in first: the file is your own, and it stays yours until you share it.</div><p><a class="bm-btn" href="${loginUrl}">Sign in with OpenVibe</a></p></div>`;++ }++ const rows = imports.map((i) => html`<li class="br-import-row"><a href="/imports/${encodeURIComponent(i.id)}">${i.filename || 'results'}</a><span class="br-fine">${i.tool} ${i.tool_version} · ${fmtDate(i.created_at)} · ${i.status === 'imported' ? 'imported' : 'waiting for you'}</span></li>`);++ return html`<div class="bm-wrap br bm-narrow" id="br-import">${head}++${error ? html`<div class="br-note is-bad" role="alert">${error}</div>` : ''}++<form method="post" action="/imports" enctype="multipart/form-data" class="bm-form">++ <p class="bm-field"><label for="br-file">The results file</label>++ <input id="br-file" name="file" type="file" accept=".json,.eval,application/json,application/zip" required></p>++ <p class="bm-field"><label for="br-tool">Which tool wrote it</label>++ <select id="br-tool" name="tool">++ <option value="">Let me see (I will work it out)</option>++ ${TOOLS.map((t) => html`<option value="${t.key}">${t.label} — ${t.hint}</option>`)}++ </select></p>++ <p class="br-fine">Up to 20 MB. A zip is opened in memory and never unpacked onto the disk, and nothing inside it is ever run.</p>++ <p><button class="bm-btn" type="submit">Read the file</button></p>++</form>++${rows.length ? html`<section class="bm-section"><h2>Your uploads</h2><ul class="br-imports">${rows}</ul></section>` : ''}++</div>`;++}++++function previewPage({ imp, benchmark = null, error = null } = {}) {++ const summary = imp.summary || {};++ const models = summary.models || [];++ const taskCount = (imp.summary && imp.summary.models || []).reduce((n, m) => n + ((m.tasks || []).length), 0);++ const suite = summary.suite ? summary.suite.name : null;++ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/imports">Bring your results</a><span aria-hidden="true">/</span><span>Preview</span></nav>++<header class="bm-page-h"><p class="bm-kicker">Preview</p><h1>${models.length} model${models.length === 1 ? '' : 's'}, ${taskCount} task${taskCount === 1 ? '' : 's'}, from ${imp.tool} ${imp.tool_version}</h1><p class="bm-lede">Import? Every run below is submitted, not verified, and says which tool it came from.</p></header>`;++ const list = models.map((m) => html`<li><b>${m.model}</b><span class="br-fine">${m.score == null ? '' : ' · score ' + Math.round(m.score)} · ${(m.tasks || []).length} task${(m.tasks || []).length === 1 ? '' : 's'}${m.cost_usd ? ' · $' + m.cost_usd : ''}</span></li>`);++ return html`<div class="bm-wrap br bm-narrow" id="br-import-preview">${head}++${error ? html`<div class="br-note is-bad" role="alert">${error}</div>` : ''}++<ul class="br-plain">${list}</ul>++<dl class="br-env"><div><dt>File</dt><dd>${imp.filename}</dd></div><div><dt>Tool</dt><dd>${imp.tool} ${imp.tool_version}</dd></div><div><dt>Suite</dt><dd>${suite || 'none named'}</dd></div><div><dt>Date in the file</dt><dd>${fmtDate(summary.date) || 'not stated'}</dd></div></dl>++${suite++ ? html`<p class="br-fine">Bench knows this suite: the runs join its benchmark.</p>`++ : html`<p class="br-fine">This becomes a private benchmark of your own, which you can share later.</p>`}++<form method="post" action="/imports/${encodeURIComponent(imp.id)}/confirm" class="bm-form">++ <p><button class="bm-btn" type="submit">Import these runs</button></p>++</form>++<p class="br-fine"><a href="/imports">Cancel</a></p>++</div>`;++}++++module.exports = { uploadPage, previewPage, TOOLS };@@@ -56,10 +56,40 @@ const compareHref = (ids) => `/compare?runs=${ids.map(encodeURIComponent).join('const kindChip = (kind) => (kind ? html`<span class="br-chip br-mono">${kind}</span>` : '');--function provenance(p) {-- return p === 'verified'++function provenance(p, source = null) {++ const badge = p === 'verified'? html`<span class="br-prov is-verified" title="Run by openvibe.codes, Run or a Node worker, and published by that service">Verified</span>`: html`<span class="br-prov is-submitted" title="Published by a person; nobody re-ran it">Submitted</span>`;++ const kind = source && typeof source === 'object' ? source.kind : null;++ if (kind === 'import') {++ const tool = String(source.tool || 'a tool');++ const version = String(source.tool_version || '');++ return html`${badge}<span class="br-prov is-imported" title="Imported from ${tool} ${version}: the file was read, never re-run">Imported from ${tool}${version ? ' ' + version : ''}</span>`;++ }++ if (kind === 'mirror') {++ const url = String(source.source_url || '');++ const licence = String(source.licence || '');++ return html`${badge}<span class="br-prov is-mirrored" title="Mirrored from ${url}${licence ? ' (' + licence + ')' : ''}: a published result, never re-run">Mirrored</span>`;++ }++ return badge;++}++++/** Where a run came from, in a line of its own: an upload with its file, or a mirror with its source. */++function sourceLine(run) {++ const source = run && run.source && typeof run.source === 'object' ? run.source : null;++ if (!source) return '';++ if (source.kind === 'import') {++ const tool = String(source.tool || 'a tool');++ const version = String(source.tool_version || '');++ const file = source.import_id ? html` · <a href="/imports/${encodeURIComponent(source.import_id)}/file">the original file</a>` : '';++ const said = source.date ? html` · the file says ${fmtDate(source.date)}` : '';++ return html`<p class="br-fine">Imported from ${tool}${version ? ' ' + version : ''}${said}. The original file is the owner's, and stays private until they share it${file}.</p>`;++ }++ if (source.kind === 'mirror') {++ const read = source.read_at ? html` · read ${fmtDate(source.read_at)}` : '';++ return html`<p class="br-fine br-source">Mirrored from <a href="${String(source.source_url || '')}" rel="nofollow noopener">${String(source.source_url || 'the source')}</a>${source.licence ? ' · ' + source.licence : ''}${read}. A published result, never re-run here.</p>`;++ }++ return '';}/** A score out of 100 with a thin bar under it. */@@@ -201,7 +231,7 @@ ${publicEntries.lengthfunction runsList(runs, { showBenchmark = false, titles = {} } = {}) {if (!runs.length) return '';-- return html`<ul class="br-runs">${runs.map((r) => html`<li class="br-run"><a class="br-run-link" href="${runHref(r.id)}"><span class="br-run-main"><span class="br-run-model">${r.model}</span><span class="br-run-sub">${showBenchmark ? html`${titles[r.benchmark_id] || r.benchmark_id} · ` : ''}${harnessLabel(r.harness)} · ${fmtDate(r.created_at)}</span></span>${scoreBar(r.score)}</a>${provenance(r.provenance)}</li>`)}</ul>`;++ return html`<ul class="br-runs">${runs.map((r) => html`<li class="br-run"><a class="br-run-link" href="${runHref(r.id)}"><span class="br-run-main"><span class="br-run-model">${r.model}</span><span class="br-run-sub">${showBenchmark ? html`${titles[r.benchmark_id] || r.benchmark_id} · ` : ''}${harnessLabel(r.harness)} · ${fmtDate(r.created_at)}</span></span>${scoreBar(r.score)}</a>${provenance(r.provenance, r.source)}</li>`)}</ul>`;}// ---- outputs -------------------------------------------------------------------------------------------------@@@ -414,9 +444,10 @@ function runPage({ run, benchmark, definition, pct }) {: html`<p class="br-fine">Where it sits shows once ${run.model} has more verified runs here.</p>`;return html`<div class="bm-wrap br" id="br-run"><nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${benchmarkHref(run.benchmark_id)}">${benchmark ? benchmark.title : run.benchmark_id}</a><span aria-hidden="true">/</span><span class="br-mono">${run.id}</span></nav>--<header class="br-head br-run-head"><div class="br-head-main"><p class="br-head-meta">${provenance(run.provenance)}<span>${fmtDate(run.created_at)}</span></p><h1>${run.model}</h1><p class="bm-lede">on ${benchmark ? benchmark.title : run.benchmark_id}, through ${harnessLabel(run.harness)}</p></div>++<header class="br-head br-run-head"><div class="br-head-main"><p class="br-head-meta">${provenance(run.provenance, run.source)}<span>${fmtDate(run.created_at)}</span></p><h1>${run.model}</h1><p class="bm-lede">on ${benchmark ? benchmark.title : run.benchmark_id}, through ${harnessLabel(run.harness)}</p></div><div class="br-head-score">${scoreBar(run.score, { big: true })}<dl class="br-totals"><div><dt>Cost</dt><dd>${fmtUsd(run.cost_usd)}</dd></div><div><dt>Time</dt><dd>${fmtMs(run.ms)}</dd></div><div><dt>Tokens</dt><dd>${fmtTokens(run.tokens)}</dd></div></dl></div></header>${pctLine}++${sourceLine(run)}<section class="bm-section" aria-labelledby="br-env-h"><h2 id="br-env-h">Environment</h2><dl class="br-env">${env.map(([k, v]) => html`<div><dt>${k}</dt><dd>${v}</dd></div>`)}</dl></section><section class="bm-section" aria-labelledby="br-pt-h"><div class="bm-head-row"><h2 id="br-pt-h">Tasks</h2><span class="bm-count">${(run.tasks || []).length}</span></div>${(run.tasks || []).length ? html`<ol class="br-ptasks">${run.tasks.map((t) => {@@@ -438,7 +469,7 @@ function comparePage({ cmp, titles = {} }) {const line = (label, vals, f, hi = true) => { const b = best(vals, hi); return html`<tr><th scope="row">${label}</th>${vals.map((v) => cell(v, b, f))}</tr>`; };return html`<div class="bm-wrap br" id="br-compare"><header class="bm-page-h"><p class="bm-kicker">Compare</p><h1>${runs.map((r) => r.model).join(' vs ')}</h1><p class="bm-lede">${[...new Set(runs.map((r) => titles[r.benchmark_id] || r.benchmark_id))].join(', ')}. The best value in each row is marked.</p></header>--<div class="br-table-wrap"><table class="br-table br-cmp"><thead><tr><th scope="col"><span class="br-sr">Measure</span></th>${runs.map((r) => html`<th scope="col"><a href="${runHref(r.id)}">${r.model}</a><span class="br-cmp-sub">${harnessLabel(r.harness)}</span>${provenance(r.provenance)}</th>`)}</tr></thead><tbody>++<div class="br-table-wrap"><table class="br-table br-cmp"><thead><tr><th scope="col"><span class="br-sr">Measure</span></th>${runs.map((r) => html`<th scope="col"><a href="${runHref(r.id)}">${r.model}</a><span class="br-cmp-sub">${harnessLabel(r.harness)}</span>${provenance(r.provenance, r.source)}</th>`)}</tr></thead><tbody>${line('Score', runs.map((r) => r.score), fmtScore)}${line('Cost', runs.map((r) => r.cost_usd), fmtUsd, false)}${line('Time', runs.map((r) => r.ms), fmtMs, false)}${line('Tokens', runs.map((r) => r.tokens), fmtTokens, false)}${rows.map((row) => line(html`<span class="br-mono">${row.task_id}</span>`, row.scores, fmtScore))}</tbody></table></div>nnew file mode 100644@@@ -0,0 +1,25 @@++{++ "id": "0f1c9a52-0000-4000-8000-000000000001",++ "started_at": "2026-10-02T09:00:00Z",++ "finished_at": "2026-10-02T09:10:00Z",++ "n_total_trials": 2,++ "stats": { "n_completed_trials": 2, "cost_usd": 0.31 },++ "trial_results": [++ {++ "id": "trial-1",++ "task_name": "write-a-python-parser",++ "task_id": "task-parser",++ "agent_info": { "name": "claude-code", "version": "1.2.3", "model_info": { "name": "claude-sonnet-4", "provider": "anthropic" } },++ "verifier_result": { "reward": 1.0 },++ "cost_usd": 0.2++ },++ {++ "id": "trial-2",++ "task_name": "fix-the-failing-test",++ "task_id": "task-test",++ "agent_info": { "name": "claude-code", "version": "1.2.3", "model_info": { "name": "claude-sonnet-4", "provider": "anthropic" } },++ "verifier_result": { "reward": 0.0 },++ "cost_usd": 0.11++ }++ ]++}nnew file mode 100644@@@ -0,0 +1,31 @@++{++ "version": "0.3.286",++ "status": "success",++ "eval": {++ "task": "mmlu",++ "task_version": "1",++ "dataset": { "samples": [1, 2, 3] },++ "model": "openai/gpt-4o",++ "model_api": "openai",++ "created": "2026-10-01T10:00:00Z"++ },++ "results": {++ "total_samples": 3,++ "completed_samples": 3,++ "scores": [++ { "name": "accuracy", "scorer": "choice", "metrics": { "accuracy": { "value": 0.6666666666666666, "stderr": 0.1 } } }++ ]++ },++ "stats": {++ "started_at": "2026-10-01T10:00:00Z",++ "completed_at": "2026-10-01T10:05:00Z",++ "model_usage": { "openai/gpt-4o": { "input_tokens": 1200, "output_tokens": 300, "total_tokens": 1500, "cost": 0.42 } }++ },++ "reductions": {++ "samples": [++ { "sample_id": 1, "value": 1 },++ { "sample_id": 2, "value": 0 },++ { "sample_id": 3, "value": 1 }++ ]++ }++}nnew file mode 100644@@@ -0,0 +1,17 @@++{++ "results": {++ "mmlu": {++ "alias": "mmlu",++ "acc,none": 0.6052631578947368,++ "acc_stderr,none": 0.0044,++ "acc_by_task": { "abstract_algebra": 0.25, "anatomy": 0.5 }++ }++ },++ "configs": {++ "mmlu": { "model": "hf", "model_args": { "model_name": "meta-llama/Llama-3-8B" }, "batch_size": 1, "device": "cpu" }++ },++ "versions": { "mmlu": 1 },++ "n-shot": { "mmlu": 0 },++ "higher_is_better": { "acc,none": true },++ "config": { "model": "hf", "batch_size": 1 }++}nnew file mode 100644@@@ -0,0 +1,37 @@++{++ "version": 3,++ "timestamp": "2026-10-03T12:00:00Z",++ "results": [++ {++ "id": "case-1",++ "testIdx": 0,++ "promptIdx": 0,++ "success": true,++ "score": 1,++ "provider": { "id": "openai:gpt-4o" },++ "response": { "output": "42", "tokenUsage": { "total": 120 }, "cost": 0.003 },++ "gradingResult": { "pass": true, "score": 1, "reason": "matches" }++ },++ {++ "id": "case-2",++ "testIdx": 1,++ "promptIdx": 0,++ "success": false,++ "score": 0,++ "provider": { "id": "openai:gpt-4o" },++ "response": { "output": "41", "tokenUsage": { "total": 130 }, "cost": 0.004 },++ "gradingResult": { "pass": false, "score": 0, "reason": "wrong" }++ },++ {++ "id": "case-3",++ "testIdx": 2,++ "promptIdx": 0,++ "success": true,++ "score": 1,++ "provider": { "id": "anthropic:claude-sonnet-4" },++ "response": { "output": "42", "tokenUsage": { "total": 90 }, "cost": 0.002 },++ "gradingResult": { "pass": true, "score": 1 }++ }++ ],++ "stats": { "successes": 2, "failures": 1, "errors": 0 }++}nnew file mode 100644@@@ -0,0 +1,380 @@++'use strict';++++/**++ * Importing results people bring themselves (B12): the four tools, the limits, the privacy, the account data++ * and the mirror's own rules.++ */++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { boot, check, done } = require('./helpers/boot');++const { uncovered } = require('openvibe-sdk/account-data');++const accountDataLib = require('../server/identity/account-data');++const importers = require('../server/bench/importers');++const limits = require('../server/bench/importers/limits');++const { matchSuite } = require('../server/bench/suites');++const { createMirror } = require('../server/bench/mirror');++++const FIX = path.join(__dirname, 'fixtures', 'imports');++const read = (name) => fs.readFileSync(path.join(FIX, name));++++/** A zip with the parts stored (not deflated), so the reader is tested on a real archive. */++function zipOf(parts) {++ const chunks = [];++ const central = [];++ let offset = 0;++ for (const part of parts) {++ const name = Buffer.from(part.name, 'utf8');++ const data = Buffer.isBuffer(part.data) ? part.data : Buffer.from(part.data, 'utf8');++ const local = Buffer.alloc(30);++ local.writeUInt32LE(0x04034b50, 0);++ local.writeUInt16LE(20, 4);++ local.writeUInt16LE(0, 6);++ local.writeUInt16LE(0, 8);++ local.writeUInt32LE(0, 10);++ local.writeUInt32LE(data.length, 18);++ local.writeUInt32LE(data.length, 22);++ local.writeUInt16LE(name.length, 26);++ local.writeUInt16LE(0, 28);++ chunks.push(local, name, data);++ const entry = Buffer.alloc(46);++ entry.writeUInt32LE(0x02014b50, 0);++ entry.writeUInt16LE(20, 4);++ entry.writeUInt16LE(20, 6);++ entry.writeUInt16LE(0, 8);++ entry.writeUInt16LE(0, 10);++ entry.writeUInt32LE(0, 12);++ entry.writeUInt32LE(data.length, 20);++ entry.writeUInt32LE(data.length, 24);++ entry.writeUInt16LE(name.length, 28);++ entry.writeUInt16LE(0, 30);++ entry.writeUInt16LE(0, 32);++ entry.writeUInt16LE(0, 34);++ entry.writeUInt16LE(0, 36);++ entry.writeUInt32LE(0, 38);++ entry.writeUInt32LE(offset, 42);++ central.push(entry, name);++ offset += local.length + name.length + data.length;++ }++ const centralBuffer = Buffer.concat(central);++ const end = Buffer.alloc(22);++ end.writeUInt32LE(0x06054b50, 0);++ end.writeUInt16LE(parts.length, 8);++ end.writeUInt16LE(parts.length, 10);++ end.writeUInt32LE(centralBuffer.length, 12);++ end.writeUInt32LE(offset, 16);++ return Buffer.concat([...chunks, centralBuffer, end]);++}++++++/**++ * A zip that lies: each part really inflates to 'size' bytes, but the directory says it is a kilobyte.++ * This is the bomb the cap has to catch: the memory is bounded by what is really unpacked, not by the claim.++ */++function bombZip(parts) {++ const zlib = require('zlib');++ const chunks = [];++ const central = [];++ let offset = 0;++ for (const part of parts) {++ const name = Buffer.from(part.name, 'utf8');++ const raw = Buffer.alloc(part.size, 0x61);++ const deflated = zlib.deflateRawSync(raw, { level: 9 });++ const local = Buffer.alloc(30);++ local.writeUInt32LE(0x04034b50, 0);++ local.writeUInt16LE(20, 4);++ local.writeUInt16LE(0, 6);++ local.writeUInt16LE(8, 8);++ local.writeUInt32LE(0, 10);++ local.writeUInt32LE(0, 14);++ local.writeUInt32LE(deflated.length, 18);++ local.writeUInt32LE(part.declared, 22);++ local.writeUInt16LE(name.length, 26);++ local.writeUInt16LE(0, 28);++ chunks.push(local, name, deflated);++ const entry = Buffer.alloc(46);++ entry.writeUInt32LE(0x02014b50, 0);++ entry.writeUInt16LE(20, 4);++ entry.writeUInt16LE(20, 6);++ entry.writeUInt16LE(0, 8);++ entry.writeUInt16LE(8, 10);++ entry.writeUInt32LE(0, 12);++ entry.writeUInt32LE(0, 16);++ entry.writeUInt32LE(deflated.length, 20);++ entry.writeUInt32LE(part.declared, 24);++ entry.writeUInt16LE(name.length, 28);++ entry.writeUInt16LE(0, 30);++ entry.writeUInt16LE(0, 32);++ entry.writeUInt16LE(0, 34);++ entry.writeUInt16LE(0, 36);++ entry.writeUInt32LE(0, 38);++ entry.writeUInt32LE(offset, 42);++ central.push(entry, name);++ offset += local.length + name.length + deflated.length;++ }++ const centralBuffer = Buffer.concat(central);++ const end = Buffer.alloc(22);++ end.writeUInt32LE(0x06054b50, 0);++ end.writeUInt16LE(parts.length, 8);++ end.writeUInt16LE(parts.length, 10);++ end.writeUInt32LE(centralBuffer.length, 12);++ end.writeUInt32LE(offset, 16);++ return Buffer.concat([...chunks, centralBuffer, end]);++}++++/** A multipart body with one file part, the way a browser sends it. */++function multipart(filename, bytes) {++ const boundary = '----b12fixture';++ const head = Buffer.from(++ '--' + boundary + '\r\n'++ + 'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n'++ + 'Content-Type: application/octet-stream\r\n\r\n', 'utf8');++ const tail = Buffer.from('\r\n--' + boundary + '--\r\n', 'utf8');++ return { body: Buffer.concat([head, bytes, tail]), type: 'multipart/form-data; boundary=' + boundary };++}++++(async () => {++ // ── the parsers, on the published shapes ─────────────────────────────────────────────────────────────────++ await check('the Inspect log is read: the model, its score and its tasks', () => {++ const out = importers.parse(read('inspect-log.json'), { filename: 'log.json' });++ assert.strictEqual(out.tool, 'inspect');++ assert.strictEqual(out.toolVersion, '0.3.286');++ assert.strictEqual(out.models.length, 1);++ assert.strictEqual(out.models[0].model, 'openai/gpt-4o');++ assert.strictEqual(out.models[0].provider, 'openai');++ assert.ok(Math.abs(out.models[0].score - 66.6666) < 0.01, 'the score is out of 100: ' + out.models[0].score);++ assert.strictEqual(out.models[0].tasks.length, 3);++ assert.strictEqual(out.models[0].cost_usd, 0.42);++ assert.strictEqual(out.suite.name, 'mmlu');++ });++++ await check('an .eval zip is opened in memory', () => {++ const zip = zipOf([{ name: 'logs/2026-10-01.json', data: read('inspect-log.json') }]);++ const out = importers.parse(zip, { filename: 'run.eval' });++ assert.strictEqual(out.tool, 'inspect');++ assert.strictEqual(out.models[0].model, 'openai/gpt-4o');++ });++++ await check('the Harbor results are read: one model, the reward of every trial', () => {++ const out = importers.parse(read('harbor-results.json'), { filename: 'results.json' });++ assert.strictEqual(out.tool, 'harbor');++ assert.strictEqual(out.models.length, 1);++ assert.strictEqual(out.models[0].model, 'claude-sonnet-4');++ assert.strictEqual(out.models[0].provider, 'anthropic');++ assert.strictEqual(out.models[0].tasks.length, 2);++ assert.strictEqual(out.models[0].tasks[0].score, 100);++ assert.strictEqual(out.models[0].tasks[1].score, 0);++ });++++ await check('the lm-evaluation-harness results are read: the model and its subtasks', () => {++ const out = importers.parse(read('lm-eval-results.json'), { filename: 'results.json' });++ assert.strictEqual(out.tool, 'lm-eval');++ assert.strictEqual(out.models[0].model, 'meta-llama/Llama-3-8B');++ assert.strictEqual(out.models[0].tasks.length, 2);++ assert.strictEqual(out.suite.name, 'mmlu');++ });++++ await check('the promptfoo results are read: one run per provider', () => {++ const out = importers.parse(read('promptfoo-output.json'), { filename: 'output.json' });++ assert.strictEqual(out.tool, 'promptfoo');++ assert.strictEqual(out.models.length, 2);++ const gpt = out.models.find((m) => m.model === 'openai:gpt-4o');++ assert.ok(gpt, 'the openai run is there');++ assert.strictEqual(gpt.tasks.length, 2);++ assert.strictEqual(gpt.cost_usd, 0.007);++ });++++ // ── the limits ─────────────────────────────────────────────────────────────────────────────────────────++ await check('a file over 20 MB is refused', () => {++ assert.throws(() => importers.parse(Buffer.alloc(20 * 1024 * 1024 + 1)), /20 MB|too large/i);++ });++++ await check('a zip that claims more than it holds is refused', () => {++ const zip = zipOf([{ name: 'logs/x.json', data: read('inspect-log.json') }]);++ // Say the entry unpacks to 200 MB: a bomb, and it is refused before anything is inflated.++ const centralAt = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));++ zip.writeUInt32LE(200 * 1024 * 1024, centralAt + 24);++ assert.throws(() => limits.readZip(zip), /100 MB|unpacks/i);++ });++++++ await check('a zip that lies about its size is refused at the real cap, and the memory stays bounded', () => {++ const rssBefore = process.memoryUsage().rss;++ const zip = bombZip([++ { name: 'logs/one.json', size: 60 * 1024 * 1024, declared: 1024 },++ { name: 'logs/two.json', size: 60 * 1024 * 1024, declared: 1024 },++ ]);++ let caught = null;++ try {++ limits.jsonPartsOfZip(zip);++ } catch (err) {++ caught = err;++ }++ assert.ok(caught, 'the bomb is refused');++ assert.strictEqual(caught.code, 'bench.import_unpacks_too_much');++ assert.strictEqual(caught.status, 413);++ assert.ok(caught.inflated <= limits.MAX_DECOMPRESSED_BYTES,++ 'the bytes really unpacked stay under the cap: ' + caught.inflated);++ const grew = process.memoryUsage().rss - rssBefore;++ assert.ok(grew < 250 * 1024 * 1024, 'the memory stayed bounded: ' + Math.round(grew / 1048576) + ' MB');++ });++++ await check('the operator token is compared in constant time', () => {++ const { tokenMatches } = require('../server/http/mirror');++ assert.ok(tokenMatches('secret', 'secret'));++ assert.ok(!tokenMatches('secrat', 'secret'));++ assert.ok(!tokenMatches('secret', 'secre'));++ assert.ok(!tokenMatches('', 'secret'));++ assert.ok(!tokenMatches('secret', ''));++ assert.ok(!tokenMatches(null, 'secret'));++ });++++ await check('a JSON that nests too deeply is refused', () => {++ const deep = '['.repeat(80) + ']'.repeat(80);++ assert.throws(() => limits.parseJson(deep), /deeply/i);++ });++++ await check('a file that is none of ours is refused', () => {++ assert.throws(() => importers.parse(Buffer.from('{"hello":"world"}')), /do not know/i);++ });++++ // ── the suites ───────────────────────────────────────────────────────────────────────────────────────++ await check('a known suite is matched by its name', () => {++ assert.ok(matchSuite('MMLU'));++ assert.strictEqual(matchSuite('MMLU').key, 'mmlu');++ assert.ok(matchSuite('gsm8k'));++ assert.ok(matchSuite('HumanEval'));++ assert.strictEqual(matchSuite('my own prompts'), null);++ });++++ const t = await boot();++ const alice = t.network.addUser('alice');++ const bob = t.network.addUser('bob');++++ await check('the upload form is there and asks for a sign-in', async () => {++ const out = await t.get('/imports');++ assert.strictEqual(out.status, 200, out.text.slice(0, 200));++ assert.ok(/Import a results file/.test(out.text), 'the page says what it is');++ assert.ok(/Sign in/.test(out.text), 'it asks for a sign-in');++ });++++ let importId = null;++ let runId = null;++ let benchmarkId = null;++++ await check('an upload becomes a preview, and then submitted runs', async () => {++ const up = multipart('inspect-log.json', read('inspect-log.json'));++ const posted = await t.get('/imports', { as: alice, method: 'POST', headers: { 'content-type': up.type }, raw: up.body });++ assert.strictEqual(posted.status, 303, posted.text.slice(0, 300));++ importId = String(posted.headers.get('location') || '').split('/').pop();++ assert.ok(importId, 'the upload has an id');++++ const preview = await t.get('/imports/' + importId, { as: alice });++ assert.strictEqual(preview.status, 200, preview.text.slice(0, 200));++ assert.ok(/1 model/.test(preview.text), 'the preview says what it holds: ' + preview.text.slice(0, 200));++ assert.ok(/from inspect 0\.3\.286/.test(preview.text), 'and which tool it came from');++++ const done = await t.get('/imports/' + importId + '/confirm', { as: alice, method: 'POST' });++ assert.strictEqual(done.status, 303, done.text.slice(0, 300));++ benchmarkId = String(done.headers.get('location') || '').split('/').pop();++++ const runs = await t.ctx.registry.listRuns({ benchmark: benchmarkId });++ assert.strictEqual(runs.length, 1);++ runId = runs[0].id;++ assert.strictEqual(runs[0].provenance, 'submitted', 'an imported run is never verified');++ assert.strictEqual(runs[0].source.kind, 'import');++ assert.strictEqual(runs[0].source.tool, 'inspect');++ assert.strictEqual(runs[0].source.tool_version, '0.3.286');++ const full = await t.ctx.registry.getRun(runId);++ assert.strictEqual(full.tasks.length, 3, 'the per-task scores came with it');++ assert.strictEqual(runs[0].harness.name, 'inspect-ai');++ });++++ await check('the run page says where it came from', async () => {++ const page = await t.get('/runs/' + runId, { as: alice });++ assert.strictEqual(page.status, 200, page.text.slice(0, 200));++ assert.ok(/Imported from inspect 0\.3\.286/.test(page.text), 'the label is there');++ });++++ await check('the file stays private to its owner', async () => {++ const mine = await t.get('/imports/' + importId + '/file', { as: alice });++ assert.strictEqual(mine.status, 200);++ assert.ok(mine.buffer.length > 0);++ const theirs = await t.get('/imports/' + importId + '/file', { as: bob });++ assert.strictEqual(theirs.status, 404, 'another person may not read the file');++ const other = await t.get('/imports/' + importId, { as: bob });++ assert.strictEqual(other.status, 404, 'nor see the preview');++ });++++ await check('the uploads are in the account data, and the guard is happy', async () => {++ const row = accountDataLib.TABLES.find((x) => x.table === 'bench_imports');++ assert.ok(row, 'bench_imports is in the account data');++ assert.strictEqual(row.subject, 'owner');++ assert.strictEqual(row.erase, 'delete');++ const missing = await uncovered(t.ctx.s.db, {++ tables: [...accountDataLib.TABLES, ...accountDataLib.GUARDED],++ exempt: {++ account_data_events: 'the ledger of the requests themselves',++ bench_ratings: 'the ratings are an aggregate over the models',++ },++ });++ assert.deepStrictEqual(missing, [], 'a table with a person column is neither covered nor exempted: ' + missing.join(', '));++ });++++ // ── the mirror ───────────────────────────────────────────────────────────────────────────────────────++ await check('the mirror reads a source under its licence, and never lands the same row twice', async () => {++ const licence = ' Apache License\n Version 2.0, January 2004\n';++ const table = [{++ title: 'MMLU',++ header: [++ { value: 'Model', metadata: {} },++ { value: 'EM', metadata: { metric: 'EM', run_group: 'MMLU' } },++ ],++ rows: [++ [{ value: 'Phi-3 (14B)' }, { value: 0.675 }],++ [{ value: 'Llama-3 (8B)' }, { value: 0.51 }],++ ],++ }];++ const fetchImpl = async (url) => {++ const text = String(url);++ if (text.includes('/LICENSE')) return { ok: true, status: 200, text: async () => licence, json: async () => ({}) };++ if (text.includes('/groups/')) return { ok: true, status: 200, text: async () => JSON.stringify(table), json: async () => table };++ return { ok: false, status: 404, text: async () => '', json: async () => ({}) };++ };++ const mirror = createMirror({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} } });++ const looked = await mirror.preview('helm', { group: 'mmlu', release: 'v1.13.0' });++ assert.ok(looked.ok, 'the source cleared: ' + JSON.stringify(looked).slice(0, 200));++ assert.strictEqual(looked.licence, 'Apache-2.0');++ assert.strictEqual(looked.count, 2);++++ const applied = await mirror.apply('helm', { group: 'mmlu', release: 'v1.13.0' });++ assert.ok(applied.ok);++ assert.strictEqual(applied.written.length, 2);++ const runs = await t.ctx.registry.listRuns({ benchmark: applied.written[0].benchmark });++ assert.strictEqual(runs.length, 2, 'one mirrored run per model');++ const mirrored = runs.find((r) => r.model === 'Phi-3 (14B)');++ assert.ok(mirrored, 'the mirrored run is there');++ assert.strictEqual(mirrored.provenance, 'submitted', 'a mirrored run is never verified');++ assert.strictEqual(mirrored.source.kind, 'mirror');++ assert.strictEqual(mirrored.source.licence, 'Apache-2.0');++ assert.ok(mirrored.source.source_url, 'it says where it came from');++++ const again = await mirror.apply('helm', { group: 'mmlu', release: 'v1.13.0' });++ assert.strictEqual(again.written.length, 0, 'the same model, version and source do not land twice');++ assert.ok(again.skipped.some((x) => x.why === 'already mirrored'));++ });++++ await check('a source whose licence cannot be read is not imported', async () => {++ const fetchImpl = async () => ({ ok: false, status: 401, text: async () => '', json: async () => ({}) });++ const mirror = createMirror({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} } });++ const looked = await mirror.preview('helm', { group: 'mmlu' });++ assert.ok(!looked.ok, 'it does not clear');++ assert.ok(/licence/.test(looked.reason), looked.reason);++ });++++ await t.close();++ done();++})().catch((err) => {++ process.stderr.write('The test file failed: ' + ((err && err.stack) || err) + '\n');++ process.exit(1);++});
new file mode 100644
new file mode 100644
@@ -0,0 +1,68 @@
@@ -0,0 +1,68 @@
+-- phase: expand
+-- B12: imported results. An upload is the owner's own file (private until they share it), the runs it becomes are
+-- submitted (never verified) and carry the provenance of where they came from: a tool, or a mirror's source.
+CREATE TABLE bench_imports (
+ id text COLLATE "C" PRIMARY KEY, -- bim_ + ULID
+ owner text COLLATE "C" NOT NULL,
+ tool text COLLATE "C" NOT NULL, -- inspect | harbor | lm-eval | promptfoo
+ tool_version text COLLATE "C" NOT NULL DEFAULT '',
+ filename text COLLATE "C" NOT NULL DEFAULT '', -- the name the person gave it; never a path
+ sha256 text COLLATE "C" NOT NULL,
+ size integer NOT NULL DEFAULT 0,
+ status text COLLATE "C" NOT NULL DEFAULT 'preview', -- preview | imported | failed
+ summary jsonb NOT NULL DEFAULT '{}'::jsonb,
+ benchmark_id text COLLATE "C" REFERENCES bench_benchmarks (id) ON DELETE SET NULL,
+ run_ids jsonb NOT NULL DEFAULT '[]'::jsonb,
+ created_by text COLLATE "C" NOT NULL,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT bench_imports_id CHECK (id ~ '^bim_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_imports_tool CHECK (tool IN ('inspect', 'harbor', 'lm-eval', 'promptfoo')),
+ CONSTRAINT bench_imports_status CHECK (status IN ('preview', 'imported', 'failed')),
+ CONSTRAINT bench_imports_sha256 CHECK (sha256 ~ '^[0-9a-f]{64}$'),
+ CONSTRAINT bench_imports_size CHECK (size >= 0)
+);
+CREATE INDEX bench_imports_by_owner ON bench_imports (owner, created_at DESC);
+
+-- The uploaded file itself. It hangs off the import, so erasing the person erases the file with it.
+CREATE TABLE bench_import_files (
+ import_id text COLLATE "C" PRIMARY KEY REFERENCES bench_imports (id) ON DELETE CASCADE,
+ content bytea NOT NULL,
+ size integer NOT NULL DEFAULT 0,
+ CONSTRAINT bench_import_files_size CHECK (size >= 0)
+);
+
+-- Where a run came from: an upload (tool and its version) or a mirror (source url, licence, the day it was read).
+-- It sits beside the run, not inside the published bench.run document, which the contract closes to extras.
+ALTER TABLE bench_runs ADD COLUMN source jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+-- The mirror's ledger: one row per source read, with the licence it was read under.
+CREATE TABLE bench_mirror_reads (
+ id text COLLATE "C" PRIMARY KEY, -- bmr_ + ULID
+ source_key text COLLATE "C" NOT NULL,
+ source_url text COLLATE "C" NOT NULL,
+ licence text COLLATE "C" NOT NULL,
+ licence_url text COLLATE "C" NOT NULL DEFAULT '',
+ attribution text COLLATE "C" NOT NULL DEFAULT '',
+ read_at timestamptz NOT NULL DEFAULT now(),
+ rows integer NOT NULL DEFAULT 0,
+ CONSTRAINT bench_mirror_reads_id CHECK (id ~ '^bmr_[0-9A-HJKMNP-TV-Z]{26}$')
+);
+CREATE UNIQUE INDEX bench_mirror_reads_source ON bench_mirror_reads (source_key, source_url);
+
+-- One mirrored result per model, benchmark version and source: the same one never lands twice.
+CREATE TABLE bench_mirror_runs (
+ id text COLLATE "C" PRIMARY KEY, -- bmr_ + ULID
+ source_key text COLLATE "C" NOT NULL,
+ source_url text COLLATE "C" NOT NULL,
+ licence text COLLATE "C" NOT NULL,
+ licence_url text COLLATE "C" NOT NULL DEFAULT '',
+ read_at timestamptz NOT NULL DEFAULT now(),
+ benchmark_id text COLLATE "C" NOT NULL REFERENCES bench_benchmarks (id) ON DELETE CASCADE,
+ benchmark_version integer NOT NULL,
+ model text COLLATE "C" NOT NULL,
+ run_id text COLLATE "C" REFERENCES bench_runs (id) ON DELETE SET NULL,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT bench_mirror_runs_id CHECK (id ~ '^bmr_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_mirror_runs_version CHECK (benchmark_version >= 1)
+);
+CREATE UNIQUE INDEX bench_mirror_runs_dedupe ON bench_mirror_runs (source_key, benchmark_id, benchmark_version, model);
@@ -108,6 +108,12 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
@@ -108,6 +108,12 @@ html, body { margin: 0; background: var(--bg-primary, #0a0f1c); color: var(--tex
.br-prov.is-verified { background: color-mix(in srgb, var(--success, #22c55e) 15%, transparent); color: var(--success, #22c55e); }
.br-prov.is-verified { background: color-mix(in srgb, var(--success, #22c55e) 15%, transparent); color: var(--success, #22c55e); }
.br-prov.is-verified::before { content: ""; width: 6px; height: 6px; border-radius: 50%; background: currentColor; }
.br-prov.is-verified::before { content: ""; width: 6px; height: 6px; border-radius: 50%; background: currentColor; }
.br-prov.is-submitted { border: 1px solid var(--border, #1f2d47); color: var(--text-muted, #7386a3); }
.br-prov.is-submitted { border: 1px solid var(--border, #1f2d47); color: var(--text-muted, #7386a3); }
+/* A source url is long: it breaks inside the line, so the page never scrolls sideways */
+.br-source, .br-source a { overflow-wrap: anywhere; word-break: break-word; }
+
+/* Imported (someone's own file) and mirrored (a published result): both are never verified */
+.br-prov.is-imported { border: 1px dashed var(--warning, #f59e0b); color: var(--warning, #f59e0b); }
+.br-prov.is-mirrored { border: 1px dotted var(--info, #38bdf8); color: var(--info, #38bdf8); }
/* Rank-by tabs: links, so every metric has its own URL */
/* Rank-by tabs: links, so every metric has its own URL */
.br-tabs { display: flex; flex-wrap: wrap; gap: 4px; padding: 3px; border-radius: 999px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }
.br-tabs { display: flex; flex-wrap: wrap; gap: 4px; padding: 3px; border-radius: 999px; background: var(--bg-card, #0f1729); border: 1px solid var(--border, #1f2d47); }
new file mode 100644
new file mode 100644
@@ -0,0 +1,66 @@
@@ -0,0 +1,66 @@
+'use strict';
+
+/**
+ * Mirror published results into Bench (B12).
+ *
+ * node scripts/mirror-import.js # what HELM would bring (writes nothing)
+ * node scripts/mirror-import.js --source helm --group mmlu --release v1.13.0
+ * node scripts/mirror-import.js --apply # write it
+ *
+ * A source is read only when its licence clears, and every row that lands says where it came from, under which
+ * licence and the day it was read. The same model, benchmark version and source never land twice.
+ *
+ * Run it as the service user, with the service's own environment (never printed):
+ *
+ * sudo systemd-run --wait --pipe --uid=ubuntu --gid=ubuntu \
+ * --property=EnvironmentFile=/etc/openvibe/bench.env \
+ * --working-directory=/opt/openvibe.bench /usr/bin/env node scripts/mirror-import.js
+ */
+const { load } = require('../server/config');
+const { openStore } = require('../server/db');
+const { createRegistry } = require('../server/bench/registry');
+const { createMirror } = require('../server/bench/mirror');
+
+const argv = process.argv.slice(2);
+const flag = (name) => argv.includes('--' + name);
+const value = (name, fallback) => {
+ const at = argv.indexOf('--' + name);
+ return at >= 0 && argv[at + 1] ? argv[at + 1] : fallback;
+};
+
+(async () => {
+ const config = load();
+ const s = await openStore(config, { log: console });
+ const registry = createRegistry(s, { log: console });
+ const mirror = createMirror({ s, registry, fetchImpl: globalThis.fetch, log: console });
+ const source = value('source', 'helm');
+ const options = {
+ suite: value('suite', undefined),
+ release: value('release', undefined),
+ group: value('group', undefined),
+ limit: value('limit', undefined),
+ };
+ const out = flag('apply')
+ ? await mirror.apply(source, options)
+ : await mirror.preview(source, options);
+ if (!out.ok) {
+ process.stdout.write(JSON.stringify(out, null, 2) + '\n');
+ process.exitCode = 1;
+ return;
+ }
+ if (flag('apply')) {
+ process.stdout.write(JSON.stringify({
+ source: out.source, licence: out.licence, licence_url: out.licence_url, read_at: out.read_at,
+ written: out.written.length, skipped: out.skipped.length,
+ }, null, 2) + '\n');
+ return;
+ }
+ const sample = (out.rows || []).slice(0, 10).map((r) => ({ benchmark: r.benchmark, model: r.model, score: Math.round(r.score * 100) / 100, source_url: r.source_url }));
+ process.stdout.write(JSON.stringify({
+ source: out.source, title: out.title, url: out.url, licence: out.licence, licence_url: out.licence_url,
+ licence_quote: out.licence_quote, attribution: out.attribution, rows: out.count, sample,
+ }, null, 2) + '\n');
+})().catch((err) => {
+ process.stderr.write('The mirror failed: ' + ((err && err.message) || err) + '\n');
+ process.exitCode = 1;
+});
@@ -75,6 +75,9 @@ async function createApp(opts = {}) {
@@ -75,6 +75,9 @@ async function createApp(opts = {}) {
const bench = opts.bench || createStore(s, { config, log, random: opts.random });
const bench = opts.bench || createStore(s, { config, log, random: opts.random });
const registry = opts.registry || createRegistry(s, { log });
const registry = opts.registry || createRegistry(s, { log });
+ // B12: bringing your own results, and the operator's mirror.
+ const imports = require('./bench/imports').createImports({ s, registry, log });
+ const mirror = require('./bench/mirror').createMirror({ s, registry, fetchImpl, log });
const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
const publisher = createPublisher({ config, s, fetchImpl, log });
const publisher = createPublisher({ config, s, fetchImpl, log });
@@ -91,6 +94,7 @@ async function createApp(opts = {}) {
@@ -91,6 +94,7 @@ async function createApp(opts = {}) {
const launcher = createLauncher({ config, registry, fetchImpl: fetchImpl || globalThis.fetch, log });
const launcher = createLauncher({ config, registry, fetchImpl: fetchImpl || globalThis.fetch, log });
const ctx = {
const ctx = {
config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
+ imports, mirror,
random: opts.random || Math.random,
random: opts.random || Math.random,
};
};
@@ -183,6 +187,7 @@ async function createApp(opts = {}) {
@@ -183,6 +187,7 @@ async function createApp(opts = {}) {
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/showcase', createShowcaseRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1/gallery', createGalleryRoutes(ctx));
app.use('/api/v1', createPickRoutes(ctx));
app.use('/api/v1', createPickRoutes(ctx));
+ app.use(require('./http/mirror').createMirrorRoutes(ctx));
app.use(mcpRouter(ctx));
app.use(mcpRouter(ctx));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
app.use('/api', (req, res) => contracts.http.sendProblem(res, 404, 'route.not_found', { detail: 'No such API route', ctx: req.ov }));
@@ -204,6 +209,7 @@ async function createApp(opts = {}) {
@@ -204,6 +209,7 @@ async function createApp(opts = {}) {
app.use(createWatchPages(ctx));
app.use(createWatchPages(ctx));
app.use('/battle', createBattlePages(ctx));
app.use('/battle', createBattlePages(ctx));
app.use('/ratings', createRatingsPages(ctx));
app.use('/ratings', createRatingsPages(ctx));
+ app.use(require('./http/imports').createImportRoutes(ctx));
app.use(createPageRoutes(ctx));
app.use(createPageRoutes(ctx));
app.use((req, res) => send(res, 404, {
app.use((req, res) => send(res, 404, {
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
viewer: req.viewer, config, path: req.originalUrl, title: 'Not found',
new file mode 100644
new file mode 100644
@@ -0,0 +1,93 @@
@@ -0,0 +1,93 @@
+'use strict';
+
+/**
+ * Harbor (harbor-framework/harbor) results: the results.json of a job.
+ *
+ * A Harbor job result (src/harbor/models/job/result.py) is { id, started_at, finished_at, n_total_trials, stats,
+ * trial_results }. Every trial carries the agent that ran it (agent_info.model_info.name is the model) and the
+ * verifier's reward (verifier_result.reward), so one model here is one agent over a set of tasks.
+ */
+const { parseJson, asText } = require('./limits');
+
+const TOOL = 'harbor';
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+function trialsOf(doc) {
+ if (Array.isArray(doc)) return doc;
+ if (doc && Array.isArray(doc.trial_results)) return doc.trial_results;
+ if (doc && Array.isArray(doc.trials)) return doc.trials;
+ return [];
+}
+
+function agentOf(trial) {
+ const agent = (trial && trial.agent_info) || (trial && trial.agent) || {};
+ const model = (agent.model_info && agent.model_info.name) || agent.model || trial.model || '';
+ const provider = (agent.model_info && agent.model_info.provider) || agent.provider || 'harbor';
+ return { model: String(model || '').trim(), provider: String(provider || 'harbor').trim(), agent: agent };
+}
+
+function rewardOf(trial) {
+ const verifier = trial && (trial.verifier_result || trial.verifier || trial.rewards || {});
+ const value = num(verifier && (verifier.reward != null ? verifier.reward : verifier.score != null ? verifier.score : verifier.value));
+ return value == null ? null : value;
+}
+
+function parse(bytes) {
+ const doc = parseJson(asText(bytes), { what: 'the results' });
+ const trials = trialsOf(doc);
+ if (!trials.length) throw new Error('That file holds no trials.');
+ const stats = (doc && doc.stats) || {};
+ const byModel = new Map();
+ for (const trial of trials) {
+ const { model, provider, agent } = agentOf(trial);
+ const key = model + '|' + provider;
+ if (!byModel.has(key)) {
+ byModel.set(key, {
+ model: model || 'unknown',
+ provider: provider || 'harbor',
+ score: null,
+ cost_usd: 0,
+ hasCost: false,
+ tasks: [],
+ agentName: String(agent.name || ''),
+ agentVersion: String(agent.version || ''),
+ });
+ }
+ const row = byModel.get(key);
+ const reward = rewardOf(trial);
+ const taskId = String((trial && (trial.task_id || trial.task_name || trial.name)) || ('trial-' + row.tasks.length));
+ row.tasks.push({ task_id: taskId, score: reward == null ? null : reward * 100 });
+ const cost = num(trial && (trial.cost_usd != null ? trial.cost_usd : trial.cost));
+ if (cost != null) { row.cost_usd += cost; row.hasCost = true; }
+ if (reward != null) row.score = row.score == null ? reward * 100 : (row.score + reward * 100) / 2;
+ }
+ const models = [...byModel.values()].map((row) => ({
+ model: row.model,
+ provider: row.provider,
+ score: row.score,
+ cost_usd: row.hasCost ? row.cost_usd : null,
+ tasks: row.tasks,
+ }));
+ const totalCost = num(stats.cost_usd != null ? stats.cost_usd : stats.total_cost);
+ if (totalCost != null && models.length === 1) models[0].cost_usd = totalCost;
+ const date = doc.started_at || (doc.stats && doc.stats.started_at) || doc.created_at || null;
+ const version = String((doc && (doc.harbor_version || doc.version)) || (stats.harness_version) || '').trim();
+ const agentVersion = models.length && models[0] ? '' : '';
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'harbor', version: version || 'unknown' },
+ date: date ? new Date(date).toISOString() : null,
+ suite: null,
+ models,
+ };
+}
+
+function looksLike(doc) {
+ if (!doc || typeof doc !== 'object') return false;
+ if (Array.isArray(doc.trial_results)) return true;
+ if (doc.n_total_trials != null && doc.stats) return true;
+ return false;
+}
+
+module.exports = { TOOL, parse, looksLike };
new file mode 100644
new file mode 100644
@@ -0,0 +1,74 @@
@@ -0,0 +1,74 @@
+'use strict';
+
+/**
+ * The four tools whose results a person can bring to Bench (B12): Inspect AI, Harbor, the EleutherAI
+ * lm-evaluation-harness and promptfoo. The file says which one it is; when it does not, the shape of the
+ * document says it.
+ */
+const { ImportError, MAX_UPLOAD_BYTES } = require('./limits');
+const inspectTool = require('./inspect');
+const harborTool = require('./harbor');
+const lmEvalTool = require('./lm-eval');
+const promptfooTool = require('./promptfoo');
+
+const TOOLS = {
+ inspect: inspectTool,
+ harbor: harborTool,
+ 'lm-eval': lmEvalTool,
+ promptfoo: promptfooTool,
+};
+
+const TOOL_LABELS = {
+ inspect: 'Inspect AI',
+ harbor: 'Harbor',
+ 'lm-eval': 'lm-evaluation-harness',
+ promptfoo: 'promptfoo',
+};
+
+/** The tool that wrote this file, or null when it is none of ours. */
+function detect(doc, filename = '') {
+ const name = String(filename || '').toLowerCase();
+ if (name.endsWith('.eval')) return 'inspect';
+ if (inspectTool.looksLike(doc)) return 'inspect';
+ if (harborTool.looksLike(doc)) return 'harbor';
+ if (lmEvalTool.looksLike(doc)) return 'lm-eval';
+ if (promptfooTool.looksLike(doc)) return 'promptfoo';
+ return null;
+}
+
+/**
+ * Read a results file: check its size, find the tool, read it.
+ * The tool's own parser throws when the file is not its shape; that error is passed on as a plain message.
+ */
+function parse(bytes, { filename = '', tool = null } = {}) {
+ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');
+ if (!buf.length) throw new ImportError(422, 'bench.import_empty', 'That file is empty.');
+ if (buf.length > MAX_UPLOAD_BYTES) {
+ throw new ImportError(413, 'bench.import_too_large', 'That file is larger than 20 MB.');
+ }
+ const wanted = tool ? String(tool) : null;
+ if (wanted && !TOOLS[wanted]) {
+ throw new ImportError(422, 'bench.import_unknown_tool', 'I do not know that tool.');
+ }
+ if (wanted) {
+ const out = TOOLS[wanted].parse(buf);
+ out.filename = String(filename || '');
+ return out;
+ }
+ // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others.
+ const errors = [];
+ for (const key of ['inspect', 'harbor', 'lm-eval', 'promptfoo']) {
+ try {
+ const out = TOOLS[key].parse(buf);
+ out.filename = String(filename || '');
+ return out;
+ } catch (err) {
+ if (err instanceof ImportError) throw err; // a size or a shape problem is the answer
+ errors.push(key + ': ' + ((err && err.message) || 'unknown'));
+ }
+ }
+ throw new ImportError(422, 'bench.import_unknown_format',
+ 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, an lm-evaluation-harness results.json or a promptfoo output.json.');
+}
+
+module.exports = { TOOLS, TOOL_LABELS, detect, parse, ImportError, MAX_UPLOAD_BYTES };
new file mode 100644
new file mode 100644
@@ -0,0 +1,139 @@
@@ -0,0 +1,139 @@
+'use strict';
+
+/**
+ * Inspect AI (inspect_ai) results: the .eval log (a zip of the log directory) or its JSON log.
+ *
+ * The JSON log of inspect_ai 0.3.x is an EvalLog: eval (the task, the model, the samples), results (the scores of
+ * the scorers), stats (the model usage, with the cost) and reductions (the per-sample scores). Nothing in it is
+ * executed: it is read as JSON and nothing else.
+ */
+const { parseJson, asText, jsonPartsOfZip } = require('./limits');
+
+const TOOL = 'inspect';
+
+function isZip(buffer) {
+ return buffer.length > 4 && buffer[0] === 0x50 && buffer[1] === 0x4b && (buffer[2] === 0x03 || buffer[2] === 0x05 || buffer[2] === 0x07);
+}
+
+/** The log inside the file: a zip (an .eval) or the JSON log itself. */
+function readLog(bytes) {
+ if (isZip(bytes)) {
+ const parts = jsonPartsOfZip(bytes);
+ for (const part of parts) {
+ const doc = parseJson(asText(part.data), { what: 'the log' });
+ if (doc && typeof doc === 'object' && doc.eval) return doc;
+ }
+ const doc = parseJson(asText(parts[0].data), { what: 'the log' });
+ return doc;
+ }
+ return parseJson(asText(bytes), { what: 'the log' });
+}
+
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+/** The overall score: the first scorer's accuracy, or the first metric that is not an error bar. */
+function overallScore(results) {
+ const scores = results && Array.isArray(results.scores) ? results.scores : [];
+ for (const score of scores) {
+ const metrics = score && score.metrics ? score.metrics : {};
+ if (metrics.accuracy && num(metrics.accuracy.value) != null) return num(metrics.accuracy.value);
+ }
+ for (const score of scores) {
+ const metrics = score && score.metrics ? score.metrics : {};
+ for (const [name, metric] of Object.entries(metrics)) {
+ if (/stderr|error|nan/i.test(name)) continue;
+ if (num(metric && metric.value) != null) return num(metric.value);
+ }
+ }
+ return null;
+}
+
+/** The per-sample scores: the reductions of the log, or the samples themselves. */
+function taskScores(doc) {
+ const out = [];
+ const reductions = doc && doc.reductions ? doc.reductions : null;
+ const samples = reductions && Array.isArray(reductions.samples) ? reductions.samples : null;
+ if (samples) {
+ for (const row of samples) {
+ if (!row || typeof row !== 'object') continue;
+ const value = num(row.value);
+ if (value == null) continue;
+ out.push({ task_id: String(row.sample_id != null ? row.sample_id : row.id), score: value * 100 });
+ }
+ if (out.length) return out;
+ }
+ const list = doc && Array.isArray(doc.samples) ? doc.samples : [];
+ for (const sample of list) {
+ if (!sample || typeof sample !== 'object') continue;
+ const scores = sample.scores && typeof sample.scores === 'object' ? sample.scores : {};
+ let value = null;
+ for (const entry of Object.values(scores)) {
+ const inner = entry && typeof entry === 'object' ? entry : {};
+ if (num(inner.value) != null) { value = num(inner.value); break; }
+ if (num(inner.accuracy) != null) { value = num(inner.accuracy); break; }
+ }
+ if (value == null) continue;
+ out.push({ task_id: String(sample.id != null ? sample.id : sample.sample_id), score: value * 100 });
+ }
+ return out;
+}
+
+/** The cost the log carries: the model usage of the run. */
+function costOf(doc) {
+ const stats = doc && doc.stats ? doc.stats : {};
+ const usage = stats.model_usage && typeof stats.model_usage === 'object' ? stats.model_usage : {};
+ let total = 0;
+ let found = false;
+ const add = (v) => {
+ const n = num(v);
+ if (n != null) { total += n; found = true; }
+ };
+ for (const entry of Object.values(usage)) {
+ if (entry == null) continue;
+ if (typeof entry === 'number') { add(entry); continue; }
+ if (typeof entry === 'object') {
+ add(entry.cost);
+ add(entry.total_cost);
+ }
+ }
+ if (!found) {
+ add(stats.total_cost);
+ add(stats.cost);
+ }
+ return found ? total : null;
+}
+
+function parse(bytes) {
+ const doc = readLog(bytes);
+ if (!looksLike(doc)) throw new Error('That file is not an Inspect AI log.');
+ const evalSpec = doc && doc.eval ? doc.eval : {};
+ const model = String(evalSpec.model || '').trim();
+ const provider = model.includes('/')
+ ? model.split('/')[0]
+ : String(evalSpec.model_api || evalSpec.provider || 'inspect');
+ const version = String(doc.version || evalSpec.eval_version || '').trim();
+ const score = overallScore(doc.results);
+ const tasks = taskScores(doc);
+ const stats = doc.stats && typeof doc.stats === 'object' ? doc.stats : {};
+ const date = stats.started_at || evalSpec.created || doc.created_at || null;
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'inspect-ai', version: version || 'unknown' },
+ date: date ? new Date(date).toISOString() : null,
+ suite: evalSpec.task ? { name: String(evalSpec.task), version: evalSpec.task_version == null ? null : String(evalSpec.task_version) } : null,
+ models: [{
+ model: model || 'unknown',
+ provider: provider || 'inspect',
+ score: score == null ? null : score * 100,
+ cost_usd: costOf(doc),
+ tasks,
+ }],
+ };
+}
+
+function looksLike(doc) {
+ return Boolean(doc && typeof doc === 'object' && doc.eval && typeof doc.eval === 'object' && (doc.eval.model || doc.eval.task));
+}
+
+module.exports = { TOOL, parse, looksLike, readLog, isZip };
new file mode 100644
new file mode 100644
@@ -0,0 +1,199 @@
@@ -0,0 +1,199 @@
+'use strict';
+
+/**
+ * What an uploaded file may be (B12). The rules are the same for every tool:
+ *
+ * 20 MB the file as it arrives
+ * 100 MB everything it unpacks to (a zip that says more is refused, so a bomb never fills the disk)
+ * 64 how deep a JSON document may nest
+ *
+ * Nothing in the file is ever run. A name in the file is never used as a path: the entries of a zip are read in
+ * memory, and only their bytes matter.
+ */
+const zlib = require('zlib');
+
+const MAX_UPLOAD_BYTES = 20 * 1024 * 1024;
+const MAX_DECOMPRESSED_BYTES = 100 * 1024 * 1024;
+const MAX_JSON_DEPTH = 64;
+const MAX_ZIP_ENTRIES = 2000;
+const MAX_COMPRESSION_RATIO = 200;
+
+class ImportError extends Error {
+ constructor(status, code, detail) {
+ super(detail);
+ this.name = 'ImportError';
+ this.status = status;
+ this.code = code;
+ this.detail = detail;
+ }
+}
+
+/** How deep a parsed document nests, counted with an explicit stack so a deep file cannot blow the call stack. */
+function depthOf(value) {
+ let max = 0;
+ const stack = [[value, 1]];
+ while (stack.length) {
+ const [node, depth] = stack.pop();
+ if (depth > max) max = depth;
+ if (node && typeof node === 'object') {
+ for (const child of Object.values(node)) {
+ if (child && typeof child === 'object') stack.push([child, depth + 1]);
+ }
+ }
+ if (stack.length > 200000) return max + 1; // a document this wide is refused on its size alone
+ }
+ return max;
+}
+
+/** JSON with a size and a depth limit: a file that is not JSON, or nests too deep, is refused, never guessed at. */
+function parseJson(text, { maxDepth = MAX_JSON_DEPTH, what = 'file' } = {}) {
+ const body = String(text == null ? '' : text);
+ if (!body.trim()) throw new ImportError(422, 'bench.import_empty', 'That ' + what + ' is empty.');
+ let value;
+ try {
+ value = JSON.parse(body);
+ } catch (err) {
+ throw new ImportError(422, 'bench.import_not_json', 'That ' + what + ' is not JSON I can read.');
+ }
+ if (depthOf(value) > maxDepth) {
+ throw new ImportError(422, 'bench.import_too_deep', 'That ' + what + ' nests too deeply for me to read.');
+ }
+ return value;
+}
+
+/** The bytes as text, refusing anything that is not text (a binary file is not a results file). */
+function asText(bytes, { what = 'file' } = {}) {
+ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');
+ // A NUL byte or a lone high-byte run means binary: refuse rather than mangle it.
+ let nul = 0;
+ for (let i = 0; i < buf.length && nul < 2; i += 1) if (buf[i] === 0) nul += 1;
+ if (nul >= 2) throw new ImportError(422, 'bench.import_not_text', 'That ' + what + ' is not a text file.');
+ return buf.toString('utf8');
+}
+
+// ── zip ─────────────────────────────────────────────────────────────────────────────────────────────────────────
+
+const EOCD = 0x06054b50;
+const CENTRAL = 0x02014b50;
+const LOCAL = 0x04034b50;
+
+/**
+ * The entries of a zip, in memory. Only what a results file needs: the names and the bytes. A zip that claims
+ * more than it may unpack to, or that hides a thousand parts, is refused before anything is inflated.
+ */
+function readZip(buffer) {
+ const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer || '');
+ // The end-of-central-directory record sits at the end, behind a comment of up to 64 KB.
+ let eocd = -1;
+ const from = Math.max(0, buf.length - (22 + 0xffff));
+ for (let i = buf.length - 22; i >= from; i -= 1) {
+ if (buf.readUInt32LE(i) === EOCD) { eocd = i; break; }
+ }
+ if (eocd < 0) throw new ImportError(422, 'bench.import_not_zip', 'That file is not a zip I can read.');
+ const count = buf.readUInt16LE(eocd + 10);
+ const size = buf.readUInt32LE(eocd + 12);
+ const offset = buf.readUInt32LE(eocd + 16);
+ if (count > MAX_ZIP_ENTRIES) throw new ImportError(413, 'bench.import_too_many_parts', 'That zip holds too many parts.');
+ if (size > buf.length) throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');
+ const entries = [];
+ let at = offset;
+ let total = 0;
+ for (let n = 0; n < count; n += 1) {
+ if (at + 46 > buf.length || buf.readUInt32LE(at) !== CENTRAL) {
+ throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');
+ }
+ const method = buf.readUInt16LE(at + 10);
+ const compressed = buf.readUInt32LE(at + 20);
+ const uncompressed = buf.readUInt32LE(at + 24);
+ const nameLen = buf.readUInt16LE(at + 28);
+ const extraLen = buf.readUInt16LE(at + 30);
+ const commentLen = buf.readUInt16LE(at + 32);
+ const localOffset = buf.readUInt32LE(at + 42);
+ const name = buf.subarray(at + 46, at + 46 + nameLen).toString('utf8');
+ at += 46 + nameLen + extraLen + commentLen;
+ if (uncompressed > MAX_DECOMPRESSED_BYTES) {
+ throw new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');
+ }
+ if (compressed > 0 && uncompressed / compressed > MAX_COMPRESSION_RATIO) {
+ throw new ImportError(413, 'bench.import_zip_bomb', 'That zip packs far too much into too little; I will not open it.');
+ }
+ total += uncompressed;
+ if (total > MAX_DECOMPRESSED_BYTES) {
+ throw new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');
+ }
+ entries.push({ name, method, compressed, uncompressed, localOffset });
+ }
+ return entries;
+}
+
+/**
+ * The bytes of one entry of a zip, inflated in memory.
+ *
+ * The cap is the real one: the inflater is given the room that is left in the whole zip, so a part that lies
+ * about its size cannot fill the memory. A zip that says it holds a kilobyte and really holds sixty megabytes is
+ * cut off at the budget, not believed.
+ */
+function readZipEntry(buffer, entry, { maxOutput = MAX_DECOMPRESSED_BYTES, used = 0 } = {}) {
+ const buf = buffer;
+ const at = entry.localOffset;
+ if (at + 30 > buf.length || buf.readUInt32LE(at) !== LOCAL) {
+ throw new ImportError(422, 'bench.import_bad_zip', 'That zip is damaged.');
+ }
+ const nameLen = buf.readUInt16LE(at + 26);
+ const extraLen = buf.readUInt16LE(at + 28);
+ const start = at + 30 + nameLen + extraLen;
+ const raw = buf.subarray(start, start + entry.compressed);
+ if (entry.method === 0) {
+ const out = Buffer.from(raw);
+ if (out.length > maxOutput) throw unpackedTooMuch(used + out.length);
+ return out;
+ }
+ if (entry.method !== 8) throw new ImportError(422, 'bench.import_bad_zip', 'That zip uses a compression I do not know.');
+ let out;
+ try {
+ out = zlib.inflateRawSync(raw, { maxOutputLength: maxOutput });
+ } catch (err) {
+ // The inflater stops at the budget: that is the cap working, not a broken file.
+ if (err && (err.code === 'ERR_BUFFER_TOO_LARGE' || /maxOutputLength/i.test(String(err.message)))) {
+ throw unpackedTooMuch(used + maxOutput);
+ }
+ throw new ImportError(422, 'bench.import_bad_zip', 'That zip could not be unpacked.');
+ }
+ if (out.length > maxOutput) throw unpackedTooMuch(used + out.length);
+ return out;
+}
+
+/** The refusal, with the bytes that were really unpacked: the cap is on the memory, not on the claim. */
+function unpackedTooMuch(inflated) {
+ const err = new ImportError(413, 'bench.import_unpacks_too_much', 'That zip unpacks to more than 100 MB.');
+ err.inflated = Number(inflated) || 0;
+ return err;
+}
+
+/**
+ * Every JSON part of a zip, biggest first: the results log of an .eval file is the one that matters.
+ *
+ * The running total is of the bytes really inflated, and each part is only given the room that is left. When the
+ * room runs out the import is refused, so a zip of many parts cannot unpack to more than the cap in all.
+ */
+function jsonPartsOfZip(buffer) {
+ const entries = readZip(buffer);
+ const out = [];
+ let used = 0;
+ for (const entry of entries) {
+ if (!/\.json$/i.test(entry.name)) continue;
+ const budget = MAX_DECOMPRESSED_BYTES - used;
+ if (budget <= 0) throw unpackedTooMuch(used);
+ const data = readZipEntry(buffer, entry, { maxOutput: budget, used });
+ used += data.length;
+ out.push({ name: entry.name, data });
+ }
+ if (!out.length) throw new ImportError(422, 'bench.import_no_json', 'That zip holds no JSON file I can read.');
+ out.sort((a, b) => b.data.length - a.data.length);
+ return out;
+}
+
+module.exports = {
+ MAX_UPLOAD_BYTES, MAX_DECOMPRESSED_BYTES, MAX_JSON_DEPTH, MAX_ZIP_ENTRIES, MAX_COMPRESSION_RATIO,
+ ImportError, depthOf, parseJson, asText, readZip, readZipEntry, jsonPartsOfZip,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,134 @@
@@ -0,0 +1,134 @@
+'use strict';
+
+/**
+ * EleutherAI lm-evaluation-harness results: the results.json that lm_eval writes.
+ *
+ * The file is { results: { <task>: { "acc,none": 0.61, "acc_stderr,none": 0.006, ... } }, configs: { <task>: {...} },
+ * versions: { <task>: 0 }, "n-shot": {...}, higher_is_better: {...} }. One config is one model, so one model here is
+ * one run; every task of the file is a task of that run.
+ */
+const { parseJson, asText } = require('./limits');
+
+const TOOL = 'lm-eval';
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+const METRIC_ORDER = ['acc,none', 'acc_norm,none', 'exact_match,none', 'normalized_accuracy,none', 'f1,none', 'acc', 'acc_norm', 'exact_match'];
+const BACKENDS = {
+ hf: 'huggingface', 'huggingface': 'huggingface', openai: 'openai', 'openai-chat-completions': 'openai',
+ anthropic: 'anthropic', 'anthropic-chat': 'anthropic', google: 'google', 'gemini': 'google', mistral: 'mistral',
+ local: 'local', 'local-completions': 'local', dummy: 'dummy',
+};
+
+/** The metric of a result that means "how good was it", skipping the error bars and the losses. */
+function metricOf(entry, higherIsBetter) {
+ if (!entry || typeof entry !== 'object') return null;
+ for (const key of METRIC_ORDER) {
+ if (num(entry[key]) != null) return { metric: key, value: num(entry[key]) };
+ }
+ for (const [key, value] of Object.entries(entry)) {
+ if (/stderr|stderr|nan|perplexity|word_perplexity|byte_perplexity|bits_per_byte/i.test(key)) continue;
+ const n = num(value);
+ if (n == null) continue;
+ if (higherIsBetter && higherIsBetter[key] === false) continue;
+ return { metric: key, value: n };
+ }
+ return null;
+}
+
+/** The subtasks of a task, when the harness wrote them down (MMLU's 57 subjects). */
+function subtasksOf(results, task, entry) {
+ const out = [];
+ if (entry && entry.acc_by_task && typeof entry.acc_by_task === 'object') {
+ for (const [name, value] of Object.entries(entry.acc_by_task)) {
+ const n = num(value);
+ if (n != null) out.push({ task_id: task + '/' + name, score: n * 100 });
+ }
+ if (out.length) return out;
+ }
+ const grouped = results[task + '_subtasks'] || results[task + '_groups'];
+ if (grouped && typeof grouped === 'object') {
+ for (const [name, sub] of Object.entries(grouped)) {
+ const m = metricOf(sub, null);
+ if (m) out.push({ task_id: task + '/' + name, score: m.value * 100 });
+ }
+ if (out.length) return out;
+ }
+ return out;
+}
+
+function modelOf(config) {
+ const args = (config && config.model_args) || {};
+ const name = args.model_name || args.pretrained || args.model || args.name || (config && config.model) || '';
+ const backend = (config && config.model) || '';
+ return { name: String(name || '').trim(), backend: String(backend || '').trim() };
+}
+
+function providerOf(name, backend) {
+ const key = String(backend || '').toLowerCase();
+ if (BACKENDS[key]) return BACKENDS[key];
+ if (name.includes('/')) return name.split('/')[0];
+ return key || 'lm-eval';
+}
+
+function parse(bytes) {
+ const doc = parseJson(asText(bytes), { what: 'the results' });
+ const results = doc && doc.results && typeof doc.results === 'object' && !Array.isArray(doc.results) ? doc.results : null;
+ if (!results) throw new Error('That file has no results.');
+ // The harness's own markers: its results file always carries the configs and the versions of the tasks.
+ if (!doc.configs && !doc.versions && !doc['n-shot'] && !doc.higher_is_better) {
+ throw new Error('That file is not an lm-evaluation-harness results file.');
+ }
+ const configs = doc.configs && typeof doc.configs === 'object' ? doc.configs : {};
+ const versions = doc.versions && typeof doc.versions === 'object' ? doc.versions : {};
+ const higher = doc.higher_is_better && typeof doc.higher_is_better === 'object' ? doc.higher_is_better : {};
+
+ const tasks = [];
+ let suiteName = null;
+ let suiteVersion = null;
+ for (const [task, entry] of Object.entries(results)) {
+ if (task.endsWith('_subtasks') || task.endsWith('_groups')) continue;
+ const subtasks = subtasksOf(results, task, entry);
+ if (subtasks.length) {
+ for (const t of subtasks) tasks.push(t);
+ } else {
+ const m = metricOf(entry, higher);
+ if (m) tasks.push({ task_id: task, score: m.value * 100 });
+ }
+ if (!suiteName) { suiteName = task; suiteVersion = versions[task] == null ? null : String(versions[task]); }
+ }
+ if (Object.keys(results).filter((t) => !t.endsWith('_subtasks') && !t.endsWith('_groups')).length > 1) {
+ suiteName = null; // a file with many tasks is a mixed file: it becomes its own private benchmark
+ }
+ if (!tasks.length) throw new Error('That file holds no scores I can read.');
+
+ const config = configs[suiteName] || configs[Object.keys(configs)[0]] || doc.config || {};
+ const { name, backend } = modelOf(config);
+ const provider = providerOf(name, backend);
+ const model = name || backend || 'unknown';
+ const overall = tasks.reduce((n, t) => n + t.score, 0) / tasks.length;
+ const version = String(doc.harness_version || doc.lm_eval_version || doc.version || '').trim();
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'lm-evaluation-harness', version: version || 'unknown' },
+ date: doc.created_at || doc.date || null,
+ suite: suiteName ? { name: suiteName, version: suiteVersion } : null,
+ models: [{
+ model,
+ provider,
+ score: overall,
+ cost_usd: num(doc.cost_usd),
+ tasks,
+ }],
+ };
+}
+
+function looksLike(doc) {
+ if (!doc || typeof doc !== 'object') return false;
+ if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) return false;
+ if (doc.configs || doc.versions || doc['n-shot'] || doc.higher_is_better) return true;
+ const keys = Object.keys(doc.results);
+ return keys.length > 0 && keys.every((k) => doc.results[k] && typeof doc.results[k] === 'object');
+}
+
+module.exports = { TOOL, parse, looksLike };
new file mode 100644
new file mode 100644
@@ -0,0 +1,137 @@
@@ -0,0 +1,137 @@
+'use strict';
+
+/**
+ * promptfoo results: the JSON that "promptfoo eval -o output.json" writes.
+ *
+ * The file is { version, timestamp, results, stats }. The results are either the rows of the run (one row per test
+ * case, each with the provider that answered and the score it got) or the newer shape, where results.outputs holds
+ * one entry per provider with its metrics. Both are read; each row becomes a task.
+ */
+const { parseJson, asText } = require('./limits');
+
+const TOOL = 'promptfoo';
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+function providerIdOf(row) {
+ const p = row && (row.provider != null ? row.provider : row.providerId != null ? row.providerId : row.target);
+ if (p == null) return 'promptfoo';
+ if (typeof p === 'string') return p;
+ if (typeof p === 'object') return String(p.id || p.label || p.name || 'promptfoo');
+ return 'promptfoo';
+}
+
+/** "openai:gpt-4o" -> provider "openai", model "openai:gpt-4o". */
+function splitProvider(id) {
+ const at = id.indexOf(':');
+ if (at <= 0) return { provider: 'promptfoo', model: id };
+ return { provider: id.slice(0, at), model: id };
+}
+
+function scoreOf(row) {
+ const grading = row && row.gradingResult;
+ if (grading && num(grading.score) != null) return num(grading.score);
+ if (num(row && row.score) != null) return num(row.score);
+ if (row && typeof row.success === 'boolean') return row.success ? 1 : 0;
+ return null;
+}
+
+function costOf(row) {
+ const response = (row && row.response) || {};
+ const c = num(response.cost != null ? response.cost : row && row.cost);
+ return c;
+}
+
+function tokensOf(row) {
+ const response = (row && row.response) || {};
+ const usage = response.tokenUsage || response.tokens || null;
+ if (!usage || typeof usage !== 'object') return null;
+ return num(usage.total != null ? usage.total : usage.total_tokens);
+}
+
+function rowId(row, index) {
+ if (row && row.id != null) return String(row.id);
+ if (row && row.testIdx != null) return 'test-' + row.testIdx;
+ if (row && row.testCase && row.testCase.vars) return 'vars-' + index;
+ return 'case-' + index;
+}
+
+function parse(bytes) {
+ const doc = parseJson(asText(bytes), { what: 'the results' });
+ const results = doc && doc.results != null ? doc.results : null;
+ if (!results) throw new Error('That file has no results.');
+
+ const rows = [];
+ if (Array.isArray(results)) {
+ rows.push(...results);
+ } else if (Array.isArray(results.outputs)) {
+ // The newer shape: one entry per provider, with the test results inside it.
+ results.outputs.forEach((output, index) => {
+ const cases = Array.isArray(output.tests) ? output.tests : Array.isArray(output.testCases) ? output.testCases : [];
+ if (cases.length) {
+ for (const c of cases) rows.push({ ...c, provider: output.provider, id: (c && c.id) || 'case-' + index });
+ } else {
+ const named = output.metrics && output.metrics.namedScores ? output.metrics.namedScores : {};
+ const score = num(output.score) != null ? num(output.score)
+ : (Object.values(named).map(num).find((v) => v != null) != null ? Object.values(named).map(num).find((v) => v != null) : null);
+ rows.push({
+ id: output.id != null ? String(output.id) : 'output-' + index,
+ provider: output.provider,
+ score: score == null ? null : score,
+ gradingResult: null,
+ });
+ }
+ });
+ } else {
+ throw new Error('That file has no results I can read.');
+ }
+ if (!rows.length) throw new Error('That file holds no results I can read.');
+
+ const byProvider = new Map();
+ for (let i = 0; i < rows.length; i += 1) {
+ const row = rows[i];
+ const id = providerIdOf(row);
+ const { provider, model } = splitProvider(id);
+ if (!byProvider.has(id)) {
+ byProvider.set(id, { model, provider, score: null, cost_usd: 0, hasCost: false, tokens: 0, hasTokens: false, tasks: [] });
+ }
+ const entry = byProvider.get(id);
+ const score = scoreOf(row);
+ entry.tasks.push({ task_id: rowId(row, i), score: score == null ? null : score * 100 });
+ const cost = costOf(row);
+ if (cost != null) { entry.cost_usd += cost; entry.hasCost = true; }
+ const tokens = tokensOf(row);
+ if (tokens != null) { entry.tokens += tokens; entry.hasTokens = true; }
+ }
+
+ const models = [...byProvider.values()].map((entry) => {
+ const scored = entry.tasks.filter((t) => t.score != null);
+ return {
+ model: entry.model,
+ provider: entry.provider,
+ score: scored.length ? scored.reduce((n, t) => n + t.score, 0) / scored.length : null,
+ cost_usd: entry.hasCost ? entry.cost_usd : null,
+ tokens: entry.hasTokens ? entry.tokens : null,
+ tasks: entry.tasks,
+ };
+ });
+ const version = String(doc.promptfoo_version || doc.tool_version || doc.promptfooVersion || '').trim();
+ const date = doc.timestamp || doc.created_at || null;
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'promptfoo', version: version || 'unknown' },
+ date: date ? new Date(date).toISOString() : null,
+ suite: null,
+ models,
+ };
+}
+
+function looksLike(doc) {
+ if (!doc || typeof doc !== 'object') return false;
+ if (Array.isArray(doc.results)) return true;
+ if (doc.results && Array.isArray(doc.results.outputs)) return true;
+ if (doc.results && typeof doc.results === 'object' && (doc.results.stats || doc.results.prompts)) return true;
+ return false;
+}
+
+module.exports = { TOOL, parse, looksLike };
new file mode 100644
new file mode 100644
@@ -0,0 +1,269 @@
@@ -0,0 +1,269 @@
+'use strict';
+
+/**
+ * Importing results a person already has (B12).
+ *
+ * A person brings a file from Inspect AI, Harbor, the lm-evaluation-harness or promptfoo. The file becomes:
+ *
+ * a benchmark matched to a suite Bench knows (MMLU, GSM8K, HumanEval, ...) when the file names one, and a
+ * private benchmark of their own otherwise
+ * runs one per model in the file, always submitted (never verified: nobody re-ran them), with the
+ * per-task scores the file carries, the harness and its version, the date and the cost
+ *
+ * The uploaded file stays the owner's own: private until they share the benchmark it became.
+ */
+const crypto = require('crypto');
+const { ImportError, MAX_UPLOAD_BYTES } = require('./importers/limits');
+const importers = require('./importers');
+const { matchSuite, suiteTag, IMPORT_TAG } = require('./suites');
+const { validateBenchmark } = require('./validate');
+
+const MAX_MODELS = 200;
+const MAX_TASKS_PER_MODEL = 10000;
+
+const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');
+const num = (v, d = 0) => (v == null || v === '' || Number.isNaN(Number(v)) ? d : Number(v));
+
+/** The name of the file, without any path: a name is never a path here. */
+function safeFilename(name) {
+ const raw = String(name == null ? '' : name);
+ const base = raw.split(/[\\/]/).pop() || '';
+ return base.replace(/[\u0000-\u001f]/g, '').trim().slice(0, 160) || 'results';
+}
+
+function createImports({ s, registry, log = console }) {
+ const db = s.db;
+
+ const wireImport = (row) => {
+ if (!row) return null;
+ const summary = typeof row.summary === 'string' ? JSON.parse(row.summary) : (row.summary || {});
+ return {
+ id: row.id,
+ owner: row.owner,
+ tool: row.tool,
+ tool_version: row.tool_version,
+ filename: row.filename,
+ sha256: row.sha256,
+ size: num(row.size),
+ status: row.status,
+ summary,
+ benchmark_id: row.benchmark_id || null,
+ run_ids: Array.isArray(row.run_ids) ? row.run_ids : [],
+ created_at: row.created_at instanceof Date ? row.created_at.toISOString() : row.created_at,
+ };
+ };
+
+ /** Read the file, work out which tool wrote it, and keep it (and what it says) for the preview. */
+ async function store({ owner, subject, filename, bytes, tool = null }) {
+ const subjectId = String(subject || owner || '');
+ if (!subjectId) throw new ImportError(401, 'bench.import_signed_in', 'Sign in to bring your own results.');
+ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || '');
+ if (!buf.length) throw new ImportError(422, 'bench.import_empty', 'That file is empty.');
+ if (buf.length > MAX_UPLOAD_BYTES) {
+ throw new ImportError(413, 'bench.import_too_large', 'That file is larger than 20 MB.');
+ }
+ let parsed;
+ try {
+ parsed = importers.parse(buf, { filename: safeFilename(filename), tool });
+ } catch (err) {
+ if (err instanceof ImportError) throw err;
+ throw new ImportError(422, 'bench.import_unreadable', (err && err.message) || 'I could not read that file.');
+ }
+ const models = (parsed.models || []).slice(0, MAX_MODELS);
+ for (const model of models) {
+ if ((model.tasks || []).length > MAX_TASKS_PER_MODEL) {
+ throw new ImportError(413, 'bench.import_too_many_tasks', 'That file holds more tasks than I can import.');
+ }
+ }
+ const id = s.newId('bim');
+ const name = safeFilename(filename);
+ const summary = {
+ tool: parsed.tool,
+ tool_version: parsed.toolVersion,
+ harness: parsed.harness,
+ date: parsed.date,
+ suite: parsed.suite || null,
+ models: models.map((m) => ({
+ model: m.model,
+ provider: m.provider,
+ score: m.score == null ? null : num(m.score),
+ cost_usd: m.cost_usd == null ? null : num(m.cost_usd),
+ tokens: m.tokens == null ? null : num(m.tokens),
+ tasks: (m.tasks || []).map((t) => ({ task_id: String(t.task_id), score: t.score == null ? null : num(t.score) })),
+ })),
+ };
+ await db.exec(
+ `INSERT INTO bench_imports (id, owner, tool, tool_version, filename, sha256, size, status, summary, created_by, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, 'preview', $8::jsonb, $9, $10::timestamptz)`,
+ [id, subjectId, parsed.tool, String(parsed.toolVersion || ''), name, sha256(buf), buf.length,
+ JSON.stringify(summary), subjectId, s.iso()],
+ );
+ await db.exec('INSERT INTO bench_import_files (import_id, content, size) VALUES ($1, $2, $3)', [id, buf, buf.length]);
+ return get(id);
+ }
+
+ async function get(id) {
+ return wireImport(await db.maybe('SELECT * FROM bench_imports WHERE id = $1', [String(id || '')]));
+ }
+
+ /** The imports of a person, newest first. */
+ async function listFor(owner, { limit = 20 } = {}) {
+ const rows = await db.many(
+ 'SELECT * FROM bench_imports WHERE owner = $1 ORDER BY created_at DESC LIMIT $2',
+ [String(owner || ''), Math.max(1, Math.min(100, num(limit, 20)))],
+ );
+ return rows.map(wireImport);
+ }
+
+ /** The task ids of an import: every task any model in the file scored. */
+ function taskIdsOf(summary) {
+ const ids = [];
+ for (const model of summary.models || []) {
+ for (const task of model.tasks || []) {
+ if (!ids.includes(task.task_id)) ids.push(task.task_id);
+ }
+ }
+ return ids;
+ }
+
+ /** The benchmark an import becomes: the one of a suite Bench knows, or a new private one. */
+ async function benchmarkFor(summary, { subject }) {
+ const suite = summary.suite ? matchSuite(summary.suite.name) : null;
+ const tag = suite ? suiteTag(suite.key) : null;
+ if (tag) {
+ const list = await registry.listBenchmarks({ tag, owner: subject, limit: 50 });
+ const found = list.find((b) => Array.isArray(b.tags) && b.tags.includes(tag));
+ if (found) return { benchmark: found, suite, matched: true };
+ }
+ const title = suite
+ ? suite.title
+ : (summary.suite && summary.suite.name) || (importers.TOOL_LABELS[summary.tool] + ' results');
+ const toolLabel = importers.TOOL_LABELS[summary.tool] || summary.tool;
+ const made = await registry.createBenchmark({
+ owner: subject,
+ title,
+ description: 'Imported from ' + toolLabel + ' ' + (summary.tool_version || '') + '.',
+ kind: 'text.answer',
+ tags: [IMPORT_TAG].concat(tag ? [tag] : []),
+ visibility: 'private',
+ subject,
+ });
+ if (!made.ok) throw new ImportError(made.status || 422, made.code || 'bench.import_failed', made.message || 'The benchmark could not be made.');
+ return { benchmark: made.benchmark, suite, matched: false };
+ }
+
+ /** The version the runs hang on: one that has the tasks, or a new one frozen with them. */
+ async function versionFor(benchmark, summary, { subject, toolLabel }) {
+ const taskIds = taskIdsOf(summary);
+ const latest = await registry.latestVersion(benchmark.id);
+ if (latest && latest.definition) {
+ const known = new Set((latest.definition.tasks || []).map((t) => String(t.id)));
+ if (taskIds.length && taskIds.every((id) => known.has(id))) return latest.version;
+ }
+ const title = benchmark.title;
+ const definition = {
+ title,
+ description: benchmark.description || ('Imported from ' + toolLabel + '.'),
+ kind: benchmark.kind,
+ tags: benchmark.tags || [],
+ visibility: 'unlisted',
+ tasks: taskIds.map((id) => ({
+ id,
+ prompt: 'Imported task ' + id + '. The file carried the score, not the prompt.',
+ holdout: false,
+ })),
+ judges: [],
+ scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: {},
+ matrix: {},
+ };
+ const frozen = await registry.freezeVersion(benchmark.id, { definition, subject, validate: validateBenchmark });
+ if (!frozen.ok) throw new ImportError(frozen.status || 422, frozen.code || 'bench.import_failed', frozen.message || 'The benchmark could not be frozen.');
+ return frozen.version.version;
+ }
+
+ /** Turn the upload into runs: one per model, always submitted. */
+ async function commit(id, { subject }) {
+ const row = await get(id);
+ if (!row) throw new ImportError(404, 'bench.import_not_found', 'No such upload.');
+ if (String(row.owner) !== String(subject)) throw new ImportError(404, 'bench.import_not_found', 'No such upload.');
+ if (row.status === 'imported') {
+ const benchmark = row.benchmark_id ? await registry.getBenchmark(row.benchmark_id) : null;
+ return { ok: true, benchmark, runs: row.run_ids.map((rid) => ({ id: rid })), already: true };
+ }
+ const summary = row.summary || {};
+ const toolLabel = importers.TOOL_LABELS[summary.tool] || summary.tool;
+ const { benchmark } = await benchmarkFor(summary, { subject });
+ const version = await versionFor(benchmark, summary, { subject, toolLabel });
+ const known = new Set(taskIdsOf(summary));
+ const runIds = [];
+ for (const model of summary.models || []) {
+ const tasks = (model.tasks || [])
+ .filter((t) => known.has(t.task_id))
+ .map((t) => ({ task_id: t.task_id, score: t.score == null ? null : num(t.score), cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }));
+ if (!tasks.length) continue;
+ const published = await registry.publishRun({
+ benchmark: benchmark.id + '@' + version,
+ model: String(model.model || 'unknown').slice(0, 160),
+ provider: String(model.provider || 'unknown').slice(0, 160),
+ route: {},
+ harness: summary.harness || { name: toolLabel, version: summary.tool_version || 'unknown' },
+ settings: {},
+ tasks,
+ score: model.score == null ? null : num(model.score),
+ cost_usd: num(model.cost_usd),
+ ms: 0,
+ tokens: num(model.tokens),
+ source: {
+ kind: 'import',
+ tool: summary.tool,
+ tool_version: summary.tool_version || '',
+ import_id: row.id,
+ filename: row.filename,
+ date: summary.date || null,
+ },
+ }, { provenance: 'submitted', subject });
+ if (!published.ok) {
+ log.warn('[OpenVibe.Bench] an imported run was refused:', published.code || '', published.message || '');
+ continue;
+ }
+ runIds.push(published.run.id);
+ }
+ await db.exec(
+ "UPDATE bench_imports SET status = 'imported', benchmark_id = $1, run_ids = $2::jsonb WHERE id = $3",
+ [benchmark.id, JSON.stringify(runIds), row.id],
+ );
+ return { ok: true, benchmark, runs: runIds.map((rid) => ({ id: rid })), import: await get(row.id) };
+ }
+
+ /**
+ * The uploaded file itself. The owner may always read it. Once the benchmark it became is public, anyone may;
+ * an unlisted benchmark shares it with the people its link reaches.
+ */
+ async function file(id, { subject = null, shareToken = null } = {}) {
+ const row = await db.maybe('SELECT * FROM bench_imports WHERE id = $1', [String(id || '')]);
+ if (!row) return null;
+ const owner = subject != null && String(row.owner) === String(subject);
+ if (!owner) {
+ const benchmark = row.benchmark_id ? await registry.getBenchmark(row.benchmark_id) : null;
+ let allowed = Boolean(benchmark && benchmark.visibility === 'public');
+ if (!allowed && benchmark && benchmark.visibility === 'unlisted' && benchmark.shared && shareToken) {
+ const shared = await registry.findShared(String(shareToken));
+ allowed = Boolean(shared && shared.id === benchmark.id);
+ }
+ if (!allowed) return null;
+ }
+ const content = await db.maybe('SELECT content, size FROM bench_import_files WHERE import_id = $1', [row.id]);
+ if (!content) return null;
+ return {
+ import: wireImport(row),
+ content: content.content,
+ filename: row.filename || 'results',
+ size: num(content.size),
+ };
+ }
+
+ return { store, get, listFor, commit, file, taskIdsOf };
+}
+
+module.exports = { createImports, safeFilename, MAX_MODELS, MAX_TASKS_PER_MODEL };
new file mode 100644
new file mode 100644
@@ -0,0 +1,290 @@
@@ -0,0 +1,290 @@
+'use strict';
+
+/**
+ * Mirroring standard results (B12).
+ *
+ * A mirror is an operator's job: it reads a published leaderboard from an open source and writes it into Bench
+ * with its provenance. Two rules hold it together:
+ *
+ * the licence a source is read only when its licence allows the reuse, and the licence (with its text and
+ * where it was read) travels with every row it brings
+ * the label a mirrored run is never verified: it is submitted, and it says where it came from
+ *
+ * The same model, benchmark version and source never land twice: bench_mirror_runs holds one row for each, with a
+ * unique index over the three.
+ */
+const { matchSuite, suiteTag, IMPORT_TAG } = require('./suites');
+
+const MIRROR_TAG = 'mirrored';
+
+const SOURCES = {
+ helm: {
+ key: 'helm',
+ title: 'HELM',
+ base: 'https://storage.googleapis.com/crfm-helm-public',
+ page: 'https://www.helm.ai/',
+ licence: 'Apache-2.0',
+ licence_url: 'https://github.com/stanford-crfm/helm/blob/main/LICENSE',
+ attribution: 'HELM (Stanford Center for Research on Foundation Models), stanford-crfm/helm',
+ },
+ 'open-llm-leaderboard': {
+ key: 'open-llm-leaderboard',
+ title: 'Open LLM Leaderboard',
+ base: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',
+ page: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',
+ licence: '',
+ licence_url: 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard',
+ attribution: 'Open LLM Leaderboard (Hugging Face)',
+ },
+};
+
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+/**
+ * The licence of a source, read at the moment of the import. A source whose licence cannot be read does not
+ * clear, and nothing of it is imported.
+ */
+async function checkLicence(source, { fetchImpl = globalThis.fetch } = {}) {
+ if (source.key === 'helm') {
+ const url = 'https://raw.githubusercontent.com/stanford-crfm/helm/main/LICENSE';
+ const res = await fetchImpl(url);
+ if (!res.ok) return { cleared: false, reason: 'the licence file could not be read (' + res.status + ')' };
+ const text = await res.text();
+ const line = text.split('\n').map((l) => l.trim()).find((l) => /Apache License/i.test(l)) || '';
+ if (!line) return { cleared: false, reason: 'the licence file names no licence I recognise' };
+ return { cleared: true, licence: 'Apache-2.0', licence_url: url, quote: line };
+ }
+ if (source.key === 'open-llm-leaderboard') {
+ const url = 'https://huggingface.co/datasets/open-llm-leaderboard/open_llm_leaderboard/raw/main/README.md';
+ const res = await fetchImpl(url);
+ if (!res.ok) return { cleared: false, reason: 'the dataset card could not be read (' + res.status + ')' };
+ const text = await res.text();
+ const line = text.split('\n').map((l) => l.trim()).find((l) => /^license:\s*/i.test(l)) || '';
+ if (!line) return { cleared: false, reason: 'the dataset card states no licence' };
+ const licence = line.replace(/^license:\s*/i, '').replace(/^["']|["']$/g, '').trim();
+ return { cleared: true, licence, licence_url: url, quote: line };
+ }
+ return { cleared: false, reason: 'no licence is known for this source' };
+}
+
+/** The rows of a HELM group table: one per model, with the score of the group. */
+function helmRows(table, { group, release }) {
+ const out = [];
+ const tables = Array.isArray(table) ? table : [table];
+ for (const one of tables) {
+ if (!one || !Array.isArray(one.header) || !Array.isArray(one.rows)) continue;
+ const header = one.header;
+ const wanted = ['EM', 'Accuracy', 'exact_match', 'Normalized accuracy', 'acc'];
+ let index = -1;
+ for (const name of wanted) {
+ const at = header.findIndex((h) => h && h.metadata && String(h.metadata.metric || '') === name);
+ if (at > 0) { index = at; break; }
+ }
+ if (index < 0) continue;
+ for (const row of one.rows) {
+ if (!Array.isArray(row) || !row.length) continue;
+ const model = row[0] && row[0].value != null ? String(row[0].value) : '';
+ const score = num(row[index] && row[index].value);
+ if (!model || score == null) continue;
+ const group_ = (one.header[index] && one.header[index].metadata && one.header[index].metadata.run_group) || group;
+ out.push({
+ benchmark: group_,
+ benchmark_version: release,
+ model,
+ score: score * 100,
+ task_id: String(group_).toLowerCase().replace(/[^a-z0-9]+/g, '_'),
+ source_url: sourceUrlHelm(group, release),
+ });
+ }
+ }
+ return out;
+}
+
+function sourceUrlHelm(group, release) {
+ return 'https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/' + release + '/groups/' + group + '.json';
+}
+
+/** Read a HELM group: the leaderboard table of one scenario group of one release. */
+async function loadHelm(source, { fetchImpl = globalThis.fetch, suite = 'lite', release = 'v1.13.0', group = 'mmlu' } = {}) {
+ const url = source.base + '/' + suite + '/benchmark_output/releases/' + release + '/groups/' + group + '.json';
+ const res = await fetchImpl(url);
+ if (!res.ok) throw new Error('the source answered ' + res.status + ' for ' + url);
+ const table = await res.json();
+ return { url, rows: helmRows(table, { group, release }) };
+}
+
+/** Read the Open LLM Leaderboard dataset: one row per model and task. */
+async function loadOpenLlmLeaderboard(source, { fetchImpl = globalThis.fetch, limit = 200 } = {}) {
+ const rows = [];
+ const configsUrl = 'https://datasets-server.huggingface.co/splits?dataset=' + encodeURIComponent('open-llm-leaderboard/open_llm_leaderboard');
+ const res = await fetchImpl(configsUrl);
+ if (!res.ok) throw new Error('the source answered ' + res.status + ' for the dataset splits');
+ const splits = await res.json();
+ const configs = (splits.splits || []).slice(0, 6);
+ for (const config of configs) {
+ const url = 'https://datasets-server.huggingface.co/rows?dataset='
+ + encodeURIComponent('open-llm-leaderboard/open_llm_leaderboard')
+ + '&config=' + encodeURIComponent(config.config) + '&split=' + encodeURIComponent(config.split)
+ + '&offset=0&length=' + Math.max(1, Math.min(100, limit));
+ const r2 = await fetchImpl(url);
+ if (!r2.ok) continue;
+ const body = await r2.json();
+ for (const item of body.rows || []) {
+ const row = item.row || {};
+ const model = row.model_name || row.model || row.Model || row.model_full_name;
+ const score = num(row.accuracy != null ? row.accuracy : row.acc != null ? row.acc : row.score);
+ if (!model || score == null) continue;
+ rows.push({
+ benchmark: config.config,
+ benchmark_version: 'leaderboard',
+ model: String(model),
+ score: score * 100,
+ task_id: String(config.config),
+ source_url: source.page,
+ });
+ }
+ }
+ return { url: source.page, rows };
+}
+
+/**
+ * The mirror: read the sources, work out what they would bring, and (only when asked) write it.
+ * Every mirrored run is submitted and carries the source, its licence and the day it was read.
+ */
+function createMirror({ s, registry, fetchImpl = globalThis.fetch, log = console } = {}) {
+ const db = s.db;
+
+ /** What a source would bring, without writing anything. */
+ async function preview(sourceKey, options = {}) {
+ const source = SOURCES[sourceKey];
+ if (!source) return { ok: false, error: 'No such source.' };
+ const licence = await checkLicence(source, { fetchImpl });
+ if (!licence.cleared) {
+ return { ok: false, source: source.key, blocked: true, reason: 'the licence does not clear: ' + licence.reason };
+ }
+ let loaded;
+ try {
+ loaded = source.key === 'helm'
+ ? await loadHelm(source, { fetchImpl, ...options })
+ : await loadOpenLlmLeaderboard(source, { fetchImpl, ...options });
+ } catch (err) {
+ return { ok: false, source: source.key, blocked: true, reason: (err && err.message) || 'the source could not be read' };
+ }
+ return {
+ ok: true,
+ source: source.key,
+ title: source.title,
+ url: loaded.url,
+ licence: licence.licence,
+ licence_url: licence.licence_url,
+ licence_quote: licence.quote,
+ attribution: source.attribution,
+ rows: loaded.rows,
+ count: loaded.rows.length,
+ };
+ }
+
+ /** The benchmark a mirrored group becomes: one per source and group, public, so anybody can read it. */
+ async function benchmarkFor(row, { source, licence }) {
+ const suite = matchSuite(row.benchmark);
+ const tags = [MIRROR_TAG, IMPORT_TAG].concat(suite ? [suiteTag(suite.key)] : []);
+ const tag = 'mirror:' + source.key + ':' + String(row.benchmark).toLowerCase().replace(/[^a-z0-9]+/g, '_');
+ const list = await registry.listBenchmarks({ tag, limit: 20 });
+ const found = list.find((b) => Array.isArray(b.tags) && b.tags.includes(tag));
+ if (found) return found;
+ const made = await registry.createBenchmark({
+ owner: 'openvibe',
+ title: (suite ? suite.title : row.benchmark) + ' (' + source.title + ')',
+ description: 'Mirrored from ' + source.title + '. Licence: ' + licence.licence + '. ' + source.attribution + '.',
+ kind: 'text.answer',
+ tags: tags.concat([tag]),
+ visibility: 'public',
+ subject: 'openvibe',
+ });
+ if (!made.ok) throw new Error(made.message || 'the benchmark could not be made');
+ return made.benchmark;
+ }
+
+ /** Write the rows: one mirrored run per model, benchmark version and source; the same one never lands twice. */
+ async function apply(sourceKey, options = {}) {
+ const looked = await preview(sourceKey, options);
+ if (!looked.ok) return looked;
+ const source = SOURCES[sourceKey];
+ const readAt = s.iso();
+ const written = [];
+ const skipped = [];
+ for (const row of looked.rows) {
+ const benchmark = await benchmarkFor(row, { source, licence: looked.licence });
+ const definition = {
+ title: benchmark.title,
+ description: benchmark.description,
+ kind: benchmark.kind,
+ tags: benchmark.tags || [],
+ visibility: 'public',
+ tasks: [{ id: row.task_id, prompt: 'Mirrored result: ' + row.benchmark + '.', holdout: false }],
+ judges: [],
+ scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: {},
+ matrix: {},
+ };
+ const latest = await registry.latestVersion(benchmark.id);
+ const known = latest && latest.definition
+ ? new Set((latest.definition.tasks || []).map((t) => String(t.id)))
+ : new Set();
+ let version;
+ if (latest && known.has(row.task_id)) {
+ version = latest.version;
+ } else {
+ const frozen = await registry.freezeVersion(benchmark.id, { definition, subject: 'openvibe' });
+ if (!frozen.ok) { skipped.push({ model: row.model, why: frozen.message }); continue; }
+ version = frozen.version.version;
+ }
+ const prior = await db.maybe(
+ 'SELECT id FROM bench_mirror_runs WHERE source_key = $1 AND benchmark_id = $2 AND benchmark_version = $3 AND model = $4',
+ [source.key, benchmark.id, version, row.model],
+ );
+ if (prior) { skipped.push({ model: row.model, why: 'already mirrored' }); continue; }
+ const published = await registry.publishRun({
+ benchmark: benchmark.id + '@' + version,
+ model: String(row.model).slice(0, 160),
+ provider: source.key,
+ route: {},
+ harness: { name: source.title, version: String(row.benchmark_version || 'published') },
+ settings: {},
+ tasks: [{ task_id: row.task_id, score: row.score, cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }],
+ score: row.score,
+ cost_usd: 0,
+ ms: 0,
+ tokens: 0,
+ source: {
+ kind: 'mirror',
+ source_key: source.key,
+ source_url: row.source_url,
+ licence: looked.licence,
+ licence_url: looked.licence_url,
+ attribution: looked.attribution,
+ read_at: readAt,
+ },
+ }, { provenance: 'submitted', subject: 'openvibe' });
+ if (!published.ok) { skipped.push({ model: row.model, why: published.message }); continue; }
+ await db.exec(
+ `INSERT INTO bench_mirror_runs (id, source_key, source_url, licence, licence_url, read_at, benchmark_id, benchmark_version, model, run_id)
+ VALUES ($1, $2, $3, $4, $5, $6::timestamptz, $7, $8, $9, $10)`,
+ [s.newId('bmr'), source.key, row.source_url, looked.licence, looked.licence_url, readAt,
+ benchmark.id, version, row.model, published.run.id],
+ );
+ written.push({ model: row.model, benchmark: benchmark.id, run: published.run.id, score: row.score });
+ }
+ await db.exec(
+ `INSERT INTO bench_mirror_reads (id, source_key, source_url, licence, licence_url, attribution, read_at, rows)
+ VALUES ($1, $2, $3, $4, $5, $6, $7::timestamptz, $8)
+ ON CONFLICT (source_key, source_url) DO UPDATE SET read_at = EXCLUDED.read_at, rows = EXCLUDED.rows`,
+ [s.newId('bmr'), source.key, looked.url, looked.licence, looked.licence_url, looked.attribution, readAt, written.length],
+ );
+ return { ok: true, source: source.key, licence: looked.licence, licence_url: looked.licence_url, written, skipped, read_at: readAt };
+ }
+
+ return { preview, apply, SOURCES, checkLicence, MIRROR_TAG };
+}
+
+module.exports = { createMirror, SOURCES, checkLicence, MIRROR_TAG };
@@ -137,6 +137,9 @@ function wireRun(row, tasks = null) {
@@ -137,6 +137,9 @@ function wireRun(row, tasks = null) {
ms: num(row.ms),
ms: num(row.ms),
tokens: num(row.tokens),
tokens: num(row.tokens),
provenance: row.provenance,
provenance: row.provenance,
+ // Where the run came from: an upload, or a mirror's source with its licence. It is Bench's own field:
+ // the published bench.run document closes its shape to the contract, so this sits beside it.
+ source: json(row.source, {}),
session: row.session || null,
session: row.session || null,
created_by: row.created_by,
created_by: row.created_by,
created_at: iso(row.created_at),
created_at: iso(row.created_at),
@@ -520,11 +523,12 @@ function createRegistry(s, { log = console } = {}) {
@@ -520,11 +523,12 @@ function createRegistry(s, { log = console } = {}) {
await db.tx(async () => {
await db.tx(async () => {
await db.exec(
await db.exec(
`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, route, provider, harness, settings, executor,
`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, route, provider, harness, settings, executor,
- hardware, score, cost_usd, ms, tokens, provenance, session, created_by, created_at)
- VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15, $16, $17, $18::timestamptz)`,
+ hardware, score, cost_usd, ms, tokens, provenance, source, session, created_by, created_at)
+ VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15, $16::jsonb, $17, $18, $19::timestamptz)`,
[id, benchmarkId, version, model, JSON.stringify(doc.route || {}), provider,
[id, benchmarkId, version, model, JSON.stringify(doc.route || {}), provider,
JSON.stringify(doc.harness || {}), JSON.stringify(doc.settings || {}), JSON.stringify(executor),
JSON.stringify(doc.harness || {}), JSON.stringify(doc.settings || {}), JSON.stringify(executor),
doc.hardware == null ? null : JSON.stringify(doc.hardware), score, cost, ms, tokens, provenance,
doc.hardware == null ? null : JSON.stringify(doc.hardware), score, cost, ms, tokens, provenance,
+ JSON.stringify(doc.source || {}),
doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],
doc.session == null ? null : String(doc.session).slice(0, 120), String(subject || 'unknown'), s.iso()],
);
);
for (const t of tasks) {
for (const t of tasks) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,56 @@
@@ -0,0 +1,56 @@
+'use strict';
+
+/**
+ * The suites Bench knows by name (B12). A results file that names one of them is matched to the benchmark of that
+ * name in the registry; anything else becomes a new private benchmark of the person's own.
+ *
+ * A suite is a name, a version and the aliases the tools use for it. The alias list is what makes "mmlu",
+ * "MMLU" and "massively multitask language understanding" the same suite.
+ */
+const SUITES = [
+ { key: 'mmlu', title: 'MMLU', aliases: ['mmlu', 'massively multitask language understanding', 'massive multitask language understanding'] },
+ { key: 'mmlu_pro', title: 'MMLU-Pro', aliases: ['mmlu_pro', 'mmlu-pro', 'mmlu pro'] },
+ { key: 'gsm8k', title: 'GSM8K', aliases: ['gsm8k', 'gsm-8k', 'grade school math 8k'] },
+ { key: 'humaneval', title: 'HumanEval', aliases: ['humaneval', 'human_eval', 'human-eval'] },
+ { key: 'hellaswag', title: 'HellaSwag', aliases: ['hellaswag'] },
+ { key: 'arc_challenge', title: 'ARC-Challenge', aliases: ['arc_challenge', 'arc-challenge', 'arc challenge'] },
+ { key: 'arc_easy', title: 'ARC-Easy', aliases: ['arc_easy', 'arc-easy'] },
+ { key: 'truthful_qa', title: 'TruthfulQA', aliases: ['truthful_qa', 'truthful-qa', 'truthfulqa'] },
+ { key: 'winogrande', title: 'WinoGrande', aliases: ['winogrande', 'winogrande_xl'] },
+ { key: 'hendrycks_math', title: 'Hendrycks MATH', aliases: ['hendrycks_math', 'math', 'hendrycks-math'] },
+ { key: 'gpqa', title: 'GPQA', aliases: ['gpqa', 'gpqa_main', 'gpqa_diamond'] },
+ { key: 'bigbench', title: 'BIG-bench', aliases: ['bigbench', 'big_bench', 'big-bench'] },
+ { key: 'tau', title: 'tau-bench', aliases: ['tau', 'tau_bench', 'tau-bench', 'taubench'] },
+ { key: 'swe_bench', title: 'SWE-bench', aliases: ['swe_bench', 'swe-bench', 'swebench'] },
+ { key: 'bfcl', title: 'Berkeley Function Calling Leaderboard', aliases: ['bfcl', 'bfcl_v3', 'function_calling'] },
+ { key: 'mt_bench', title: 'MT-Bench', aliases: ['mt_bench', 'mt-bench', 'mtbench'] },
+ { key: 'hle', title: "Humanity's Last Exam", aliases: ['hle', 'humanitys_last_exam'] },
+];
+
+/** "ARC-Challenge" -> "arcchallenge": the name without its punctuation. */
+function normalize(name) {
+ return String(name == null ? '' : name).toLowerCase().replace(/[^a-z0-9]+/g, '');
+}
+
+/** The suite a name means, or null. */
+function matchSuite(name) {
+ const key = normalize(name);
+ if (!key) return null;
+ for (const suite of SUITES) {
+ if (normalize(suite.key) === key) return suite;
+ for (const alias of suite.aliases) {
+ if (normalize(alias) === key) return suite;
+ }
+ }
+ return null;
+}
+
+/** The tag a benchmark of that suite carries, so a later import finds it again. */
+function suiteTag(key) {
+ return 'suite:' + String(key || '').toLowerCase();
+}
+
+/** The tag every imported benchmark carries. */
+const IMPORT_TAG = 'imported';
+
+module.exports = { SUITES, normalize, matchSuite, suiteTag, IMPORT_TAG };
new file mode 100644
new file mode 100644
@@ -0,0 +1,114 @@
@@ -0,0 +1,114 @@
+'use strict';
+
+/**
+ * Bringing your own results (B12): the upload form, the preview and the file itself.
+ *
+ * The upload is a plain form post, same-origin (CSRF), signed in, and capped at 20 MB by the body parser. The
+ * file is the person's own: the page that shows it is theirs, and the file stays private until they share the
+ * benchmark it became.
+ */
+const express = require('express');
+const { asyncRouter } = require('./router');
+const { sameOrigin } = require('./same-origin');
+const { parseMultipart } = require('./multipart');
+const { raw } = require('../render/html');
+const { send } = require('../render/layout');
+const views = require('../views/imports');
+const { ImportError } = require('../bench/importers/limits');
+
+function createImportRoutes(ctx) {
+ const { config } = ctx;
+ const r = asyncRouter();
+ const subjectOf = (req) => (req.viewer && req.viewer.kind === 'user' ? (req.viewer.subject || req.viewer.id || null) : null);
+ const page = (req, res, o, status = 200) => send(res, status, { viewer: req.viewer, config, path: req.originalUrl, ...o });
+
+ const upload = async (req, res) => {
+ const subject = subjectOf(req);
+ const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');
+ const imports = subject ? await ctx.imports.listFor(subject, { limit: 10 }) : [];
+ return page(req, res, {
+ index: false,
+ title: 'Import a results file',
+ description: 'Bring the results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo, and see the runs it holds before they are imported.',
+ body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, imports, error: req.query.error ? String(req.query.error).slice(0, 200) : null }))),
+ });
+ };
+ r.get('/imports', upload);
+
+ r.post('/imports', express.raw({ type: 'multipart/form-data', limit: '21mb' }), ctx.principal.middleware, async (req, res) => {
+ const subject = subjectOf(req);
+ const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');
+ const fail = (status, error) => page(req, res, {
+ index: false, title: 'Import a results file',
+ description: 'Bring a results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo.',
+ body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, error }))),
+ }, status);
+ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
+ if (!subject) return fail(401, 'Sign in to bring your own results.');
+ const parts = parseMultipart(req.body, req.get('content-type'));
+ if (!parts) return fail(422, 'That upload did not arrive as a form. Try again.');
+ const file = parts.parts.find((p) => p.name === 'file' && p.data && p.data.length);
+ if (!file) return fail(422, 'Pick a results file to read.');
+ const toolPart = parts.parts.find((p) => p.name === 'tool');
+ const tool = toolPart ? String(toolPart.data.toString('utf8')).trim() : '';
+ try {
+ const imp = await ctx.imports.store({
+ owner: subject, subject, filename: file.filename || 'results', bytes: file.data, tool: tool || null,
+ });
+ return res.redirect(303, '/imports/' + encodeURIComponent(imp.id));
+ } catch (err) {
+ const detail = err instanceof ImportError ? err.detail : 'That file could not be read.';
+ return fail(err instanceof ImportError ? err.status : 422, detail);
+ }
+ });
+
+ /** The preview: what the file says, before anything is written. */
+ r.get('/imports/:id', async (req, res) => {
+ const subject = subjectOf(req);
+ const imp = await ctx.imports.get(String(req.params.id || ''));
+ if (!imp || String(imp.owner) !== String(subject)) {
+ return page(req, res, { index: false, title: 'No such upload', body: raw(String(require('../views').notFound())) }, 404);
+ }
+ return page(req, res, {
+ index: false,
+ title: 'Import preview',
+ description: 'What this results file holds, before it is imported.',
+ body: raw(String(views.previewPage({ imp }))),
+ });
+ });
+
+ r.post('/imports/:id/confirm', ctx.principal.middleware, async (req, res) => {
+ const subject = subjectOf(req);
+ if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
+ const imp = await ctx.imports.get(String(req.params.id || ''));
+ if (!imp || String(imp.owner) !== String(subject)) {
+ return page(req, res, { index: false, title: 'No such upload', body: raw(String(require('../views').notFound())) }, 404);
+ }
+ try {
+ const done = await ctx.imports.commit(imp.id, { subject });
+ if (done.benchmark) return res.redirect(303, '/benchmarks/' + encodeURIComponent(done.benchmark.id));
+ return page(req, res, { index: false, title: 'Import preview', body: raw(String(views.previewPage({ imp, error: 'Nothing could be imported from that file.' }))) }, 422);
+ } catch (err) {
+ const detail = err instanceof ImportError ? err.detail : 'The import did not go through.';
+ return page(req, res, { index: false, title: 'Import preview', body: raw(String(views.previewPage({ imp, error: detail }))) }, 422);
+ }
+ });
+
+ /** The uploaded file: the owner's own, and anyone else's once the benchmark it became is shared. */
+ r.get('/imports/:id/file', async (req, res) => {
+ const subject = subjectOf(req);
+ const shareToken = req.query.share ? String(req.query.share) : null;
+ const found = await ctx.imports.file(String(req.params.id || ''), { subject, shareToken });
+ if (!found) {
+ return page(req, res, { index: false, title: 'No such file', body: raw(String(require('../views').notFound())) }, 404);
+ }
+ res.setHeader('Content-Type', 'application/octet-stream');
+ res.setHeader('Content-Disposition', 'attachment; filename="' + String(found.filename).replace(/[^A-Za-z0-9._-]+/g, '_') + '"');
+ res.setHeader('Cache-Control', 'private, max-age=0, no-store');
+ return res.send(found.content);
+ });
+
+ return r;
+}
+
+module.exports = { createImportRoutes };
new file mode 100644
new file mode 100644
@@ -0,0 +1,53 @@
@@ -0,0 +1,53 @@
+'use strict';
+
+/**
+ * The operator's mirror route (B12). It is not a public API: it needs the operator's own token, and it reads
+ * the sources' licences before it writes anything. A dry run (the default) says what it would import and from
+ * where, and writes nothing.
+ */
+const crypto = require('crypto');
+const { asyncRouter } = require('./router');
+
+function operatorToken(req) {
+ const h = String(req.get('authorization') || '');
+ return /^Bearer\s+\S/i.test(h) ? h.replace(/^Bearer\s+/i, '').trim() : null;
+}
+
+/**
+ * The operator's token, compared in constant time: the digests of both are compared with timingSafeEqual, so
+ * the answer does not depend on how many characters matched.
+ */
+function tokenMatches(given, expected) {
+ if (!given || !expected) return false;
+ const a = crypto.createHash('sha256').update(String(given), 'utf8').digest();
+ const b = crypto.createHash('sha256').update(String(expected), 'utf8').digest();
+ return a.length === b.length && crypto.timingSafeEqual(a, b);
+}
+
+function createMirrorRoutes(ctx) {
+ const r = asyncRouter();
+ const token = process.env.BENCH_ADMIN_TOKEN || '';
+
+ r.post('/api/v1/admin/mirror', async (req, res) => {
+ if (!token) return res.status(404).type('text').send('No such route.');
+ const given = operatorToken(req);
+ if (!tokenMatches(given, token)) return res.status(403).type('text').send('The operator token is required.');
+ const body = req.body && typeof req.body === 'object' ? req.body : {};
+ const source = String(body.source || 'helm');
+ const options = {
+ suite: body.suite ? String(body.suite) : undefined,
+ release: body.release ? String(body.release) : undefined,
+ group: body.group ? String(body.group) : undefined,
+ limit: body.limit ? Number(body.limit) : undefined,
+ };
+ const apply = body.apply === true || body.apply === 'true';
+ const out = apply
+ ? await ctx.mirror.apply(source, options)
+ : await ctx.mirror.preview(source, options);
+ return res.json(out);
+ });
+
+ return r;
+}
+
+module.exports = { createMirrorRoutes, tokenMatches };
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+'use strict';
+
+/**
+ * A multipart/form-data body, read in memory (B12). Express has no body parser for it and a results file needs
+ * one: the upload form posts one file and nothing else.
+ *
+ * The body is already capped by the caller (express.raw with a limit), so this only has to find the parts.
+ * A part's name is used as a field name; a part's filename is only ever shown, never used as a path.
+ */
+function parseMultipart(buffer, contentType) {
+ const type = String(contentType || '');
+ const at = type.toLowerCase().indexOf('multipart/form-data');
+ if (at < 0) return null;
+ const match = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(type.slice(at));
+ if (!match) return null;
+ const boundary = (match[1] || match[2] || '').trim();
+ if (!boundary || boundary.length > 200) return null;
+ const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer || '');
+ const delimiter = Buffer.from('--' + boundary);
+ const parts = [];
+ let start = buf.indexOf(delimiter);
+ if (start < 0) return null;
+ start += delimiter.length;
+ for (let guard = 0; guard < 50; guard += 1) {
+ if (start + 2 <= buf.length && buf[start] === 0x2d && buf[start + 1] === 0x2d) break; // the closing --
+ // The part's headers run to the first blank line.
+ const headEnd = buf.indexOf('\r\n\r\n', start, 'latin1');
+ if (headEnd < 0) break;
+ const head = buf.subarray(start, headEnd).toString('latin1');
+ const bodyStart = headEnd + 4;
+ const next = buf.indexOf(delimiter, bodyStart);
+ if (next < 0) break;
+ const bodyEnd = Math.max(bodyStart, next - 2); // the CRLF before the boundary belongs to the delimiter
+ const name = /name="([^"]*)"/i.exec(head);
+ const filename = /filename="([^"]*)"/i.exec(head);
+ parts.push({
+ name: name ? name[1] : '',
+ filename: filename ? filename[1] : null,
+ contentType: (/content-type:\s*([^\r\n]+)/i.exec(head) || [])[1] || null,
+ data: buf.subarray(bodyStart, bodyEnd),
+ });
+ start = next + delimiter.length;
+ }
+ return { parts };
+}
+
+module.exports = { parseMultipart };
@@ -35,6 +35,8 @@ const TABLES = [
@@ -35,6 +35,8 @@ const TABLES = [
// The runs they submitted and the benchmarks they watch: theirs, so they go.
// The runs they submitted and the benchmarks they watch: theirs, so they go.
{ table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },
{ table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },
{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
{ table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
+ // The results files they brought (B12): theirs, so they go with them. The bytes hang off the row and go with it.
+ { table: 'bench_imports', subject: 'owner', file: 'bench_imports.json', erase: 'delete' },
];
];
// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
@@ -59,6 +61,11 @@ async function extraExport(db, subject) {
@@ -59,6 +61,11 @@ async function extraExport(db, subject) {
AND (visibility = 'private' OR tags @> '["your-prompts"]'::jsonb))
AND (visibility = 'private' OR tags @> '["your-prompts"]'::jsonb))
ORDER BY created_at DESC LIMIT 5000`, [subject]);
ORDER BY created_at DESC LIMIT 5000`, [subject]);
if (runs.length) files.push({ name: 'benchmark_runs.json', content: runs });
if (runs.length) files.push({ name: 'benchmark_runs.json', content: runs });
+ // The files they uploaded: the bytes are theirs too, so the export carries them.
+ const uploads = await db.many(`SELECT f.import_id, i.filename, i.sha256, i.size, encode(f.content, 'base64') AS content_base64
+ FROM bench_import_files f JOIN bench_imports i ON i.id = f.import_id
+ WHERE i.owner = $1 OR i.created_by = $1 ORDER BY i.created_at DESC LIMIT 5000`, [subject]);
+ if (uploads.length) files.push({ name: 'import_files.json', content: uploads });
return files;
return files;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,68 @@
@@ -0,0 +1,68 @@
+'use strict';
+
+/**
+ * The pages of an import (B12): the upload form and the preview. Both are plain forms: they work without
+ * JavaScript, they are narrow enough for a phone, and they say plainly what will happen.
+ */
+const { html, raw } = require('../render/html');
+
+const TOOLS = [
+ { key: 'inspect', label: 'Inspect AI', hint: 'an .eval log (a zip) or its JSON log' },
+ { key: 'harbor', label: 'Harbor', hint: 'the results.json of a Harbor job' },
+ { key: 'lm-eval', label: 'lm-evaluation-harness', hint: 'the results.json of a run' },
+ { key: 'promptfoo', label: 'promptfoo', hint: 'the output.json of promptfoo eval -o' },
+];
+
+const fmtDate = (v) => {
+ if (!v) return '';
+ const d = new Date(v);
+ return Number.isNaN(d.getTime()) ? '' : d.toISOString().slice(0, 10);
+};
+
+function uploadPage({ signedIn = false, loginUrl = '/auth/login', error = null, imports = [] } = {}) {
+ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><span>Bring your results</span></nav>
+<header class="bm-page-h"><p class="bm-kicker">Bring your results</p><h1>Import a results file</h1><p class="bm-lede">Run your evals with the tools you already use, then bring the file here. Every run it holds becomes a submitted run: Bench never calls them verified, because nobody re-ran them.</p></header>`;
+ if (!signedIn) {
+ return html`<div class="bm-wrap br bm-narrow" id="br-import">${head}<div class="br-note" role="alert">Sign in first: the file is your own, and it stays yours until you share it.</div><p><a class="bm-btn" href="${loginUrl}">Sign in with OpenVibe</a></p></div>`;
+ }
+ const rows = imports.map((i) => html`<li class="br-import-row"><a href="/imports/${encodeURIComponent(i.id)}">${i.filename || 'results'}</a><span class="br-fine">${i.tool} ${i.tool_version} · ${fmtDate(i.created_at)} · ${i.status === 'imported' ? 'imported' : 'waiting for you'}</span></li>`);
+ return html`<div class="bm-wrap br bm-narrow" id="br-import">${head}
+${error ? html`<div class="br-note is-bad" role="alert">${error}</div>` : ''}
+<form method="post" action="/imports" enctype="multipart/form-data" class="bm-form">
+ <p class="bm-field"><label for="br-file">The results file</label>
+ <input id="br-file" name="file" type="file" accept=".json,.eval,application/json,application/zip" required></p>
+ <p class="bm-field"><label for="br-tool">Which tool wrote it</label>
+ <select id="br-tool" name="tool">
+ <option value="">Let me see (I will work it out)</option>
+ ${TOOLS.map((t) => html`<option value="${t.key}">${t.label} — ${t.hint}</option>`)}
+ </select></p>
+ <p class="br-fine">Up to 20 MB. A zip is opened in memory and never unpacked onto the disk, and nothing inside it is ever run.</p>
+ <p><button class="bm-btn" type="submit">Read the file</button></p>
+</form>
+${rows.length ? html`<section class="bm-section"><h2>Your uploads</h2><ul class="br-imports">${rows}</ul></section>` : ''}
+</div>`;
+}
+
+function previewPage({ imp, benchmark = null, error = null } = {}) {
+ const summary = imp.summary || {};
+ const models = summary.models || [];
+ const taskCount = (imp.summary && imp.summary.models || []).reduce((n, m) => n + ((m.tasks || []).length), 0);
+ const suite = summary.suite ? summary.suite.name : null;
+ const head = html`<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/imports">Bring your results</a><span aria-hidden="true">/</span><span>Preview</span></nav>
+<header class="bm-page-h"><p class="bm-kicker">Preview</p><h1>${models.length} model${models.length === 1 ? '' : 's'}, ${taskCount} task${taskCount === 1 ? '' : 's'}, from ${imp.tool} ${imp.tool_version}</h1><p class="bm-lede">Import? Every run below is submitted, not verified, and says which tool it came from.</p></header>`;
+ const list = models.map((m) => html`<li><b>${m.model}</b><span class="br-fine">${m.score == null ? '' : ' · score ' + Math.round(m.score)} · ${(m.tasks || []).length} task${(m.tasks || []).length === 1 ? '' : 's'}${m.cost_usd ? ' · $' + m.cost_usd : ''}</span></li>`);
+ return html`<div class="bm-wrap br bm-narrow" id="br-import-preview">${head}
+${error ? html`<div class="br-note is-bad" role="alert">${error}</div>` : ''}
+<ul class="br-plain">${list}</ul>
+<dl class="br-env"><div><dt>File</dt><dd>${imp.filename}</dd></div><div><dt>Tool</dt><dd>${imp.tool} ${imp.tool_version}</dd></div><div><dt>Suite</dt><dd>${suite || 'none named'}</dd></div><div><dt>Date in the file</dt><dd>${fmtDate(summary.date) || 'not stated'}</dd></div></dl>
+${suite
+ ? html`<p class="br-fine">Bench knows this suite: the runs join its benchmark.</p>`
+ : html`<p class="br-fine">This becomes a private benchmark of your own, which you can share later.</p>`}
+<form method="post" action="/imports/${encodeURIComponent(imp.id)}/confirm" class="bm-form">
+ <p><button class="bm-btn" type="submit">Import these runs</button></p>
+</form>
+<p class="br-fine"><a href="/imports">Cancel</a></p>
+</div>`;
+}
+
+module.exports = { uploadPage, previewPage, TOOLS };
@@ -56,10 +56,40 @@ const compareHref = (ids) => `/compare?runs=${ids.map(encodeURIComponent).join('
@@ -56,10 +56,40 @@ const compareHref = (ids) => `/compare?runs=${ids.map(encodeURIComponent).join('
const kindChip = (kind) => (kind ? html`<span class="br-chip br-mono">${kind}</span>` : '');
const kindChip = (kind) => (kind ? html`<span class="br-chip br-mono">${kind}</span>` : '');
-function provenance(p) {
- return p === 'verified'
+function provenance(p, source = null) {
+ const badge = p === 'verified'
? html`<span class="br-prov is-verified" title="Run by openvibe.codes, Run or a Node worker, and published by that service">Verified</span>`
? html`<span class="br-prov is-verified" title="Run by openvibe.codes, Run or a Node worker, and published by that service">Verified</span>`
: html`<span class="br-prov is-submitted" title="Published by a person; nobody re-ran it">Submitted</span>`;
: html`<span class="br-prov is-submitted" title="Published by a person; nobody re-ran it">Submitted</span>`;
+ const kind = source && typeof source === 'object' ? source.kind : null;
+ if (kind === 'import') {
+ const tool = String(source.tool || 'a tool');
+ const version = String(source.tool_version || '');
+ return html`${badge}<span class="br-prov is-imported" title="Imported from ${tool} ${version}: the file was read, never re-run">Imported from ${tool}${version ? ' ' + version : ''}</span>`;
+ }
+ if (kind === 'mirror') {
+ const url = String(source.source_url || '');
+ const licence = String(source.licence || '');
+ return html`${badge}<span class="br-prov is-mirrored" title="Mirrored from ${url}${licence ? ' (' + licence + ')' : ''}: a published result, never re-run">Mirrored</span>`;
+ }
+ return badge;
+}
+
+/** Where a run came from, in a line of its own: an upload with its file, or a mirror with its source. */
+function sourceLine(run) {
+ const source = run && run.source && typeof run.source === 'object' ? run.source : null;
+ if (!source) return '';
+ if (source.kind === 'import') {
+ const tool = String(source.tool || 'a tool');
+ const version = String(source.tool_version || '');
+ const file = source.import_id ? html` · <a href="/imports/${encodeURIComponent(source.import_id)}/file">the original file</a>` : '';
+ const said = source.date ? html` · the file says ${fmtDate(source.date)}` : '';
+ return html`<p class="br-fine">Imported from ${tool}${version ? ' ' + version : ''}${said}. The original file is the owner's, and stays private until they share it${file}.</p>`;
+ }
+ if (source.kind === 'mirror') {
+ const read = source.read_at ? html` · read ${fmtDate(source.read_at)}` : '';
+ return html`<p class="br-fine br-source">Mirrored from <a href="${String(source.source_url || '')}" rel="nofollow noopener">${String(source.source_url || 'the source')}</a>${source.licence ? ' · ' + source.licence : ''}${read}. A published result, never re-run here.</p>`;
+ }
+ return '';
}
}
/** A score out of 100 with a thin bar under it. */
/** A score out of 100 with a thin bar under it. */
@@ -201,7 +231,7 @@ ${publicEntries.length
@@ -201,7 +231,7 @@ ${publicEntries.length
function runsList(runs, { showBenchmark = false, titles = {} } = {}) {
function runsList(runs, { showBenchmark = false, titles = {} } = {}) {
if (!runs.length) return '';
if (!runs.length) return '';
- return html`<ul class="br-runs">${runs.map((r) => html`<li class="br-run"><a class="br-run-link" href="${runHref(r.id)}"><span class="br-run-main"><span class="br-run-model">${r.model}</span><span class="br-run-sub">${showBenchmark ? html`${titles[r.benchmark_id] || r.benchmark_id} · ` : ''}${harnessLabel(r.harness)} · ${fmtDate(r.created_at)}</span></span>${scoreBar(r.score)}</a>${provenance(r.provenance)}</li>`)}</ul>`;
+ return html`<ul class="br-runs">${runs.map((r) => html`<li class="br-run"><a class="br-run-link" href="${runHref(r.id)}"><span class="br-run-main"><span class="br-run-model">${r.model}</span><span class="br-run-sub">${showBenchmark ? html`${titles[r.benchmark_id] || r.benchmark_id} · ` : ''}${harnessLabel(r.harness)} · ${fmtDate(r.created_at)}</span></span>${scoreBar(r.score)}</a>${provenance(r.provenance, r.source)}</li>`)}</ul>`;
}
}
// ---- outputs -------------------------------------------------------------------------------------------------
// ---- outputs -------------------------------------------------------------------------------------------------
@@ -414,9 +444,10 @@ function runPage({ run, benchmark, definition, pct }) {
@@ -414,9 +444,10 @@ function runPage({ run, benchmark, definition, pct }) {
: html`<p class="br-fine">Where it sits shows once ${run.model} has more verified runs here.</p>`;
: html`<p class="br-fine">Where it sits shows once ${run.model} has more verified runs here.</p>`;
return html`<div class="bm-wrap br" id="br-run">
return html`<div class="bm-wrap br" id="br-run">
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${benchmarkHref(run.benchmark_id)}">${benchmark ? benchmark.title : run.benchmark_id}</a><span aria-hidden="true">/</span><span class="br-mono">${run.id}</span></nav>
<nav class="br-crumbs" aria-label="Breadcrumb"><a href="/benchmarks">Benchmarks</a><span aria-hidden="true">/</span><a href="${benchmarkHref(run.benchmark_id)}">${benchmark ? benchmark.title : run.benchmark_id}</a><span aria-hidden="true">/</span><span class="br-mono">${run.id}</span></nav>
-<header class="br-head br-run-head"><div class="br-head-main"><p class="br-head-meta">${provenance(run.provenance)}<span>${fmtDate(run.created_at)}</span></p><h1>${run.model}</h1><p class="bm-lede">on ${benchmark ? benchmark.title : run.benchmark_id}, through ${harnessLabel(run.harness)}</p></div>
+<header class="br-head br-run-head"><div class="br-head-main"><p class="br-head-meta">${provenance(run.provenance, run.source)}<span>${fmtDate(run.created_at)}</span></p><h1>${run.model}</h1><p class="bm-lede">on ${benchmark ? benchmark.title : run.benchmark_id}, through ${harnessLabel(run.harness)}</p></div>
<div class="br-head-score">${scoreBar(run.score, { big: true })}<dl class="br-totals"><div><dt>Cost</dt><dd>${fmtUsd(run.cost_usd)}</dd></div><div><dt>Time</dt><dd>${fmtMs(run.ms)}</dd></div><div><dt>Tokens</dt><dd>${fmtTokens(run.tokens)}</dd></div></dl></div></header>
<div class="br-head-score">${scoreBar(run.score, { big: true })}<dl class="br-totals"><div><dt>Cost</dt><dd>${fmtUsd(run.cost_usd)}</dd></div><div><dt>Time</dt><dd>${fmtMs(run.ms)}</dd></div><div><dt>Tokens</dt><dd>${fmtTokens(run.tokens)}</dd></div></dl></div></header>
${pctLine}
${pctLine}
+${sourceLine(run)}
<section class="bm-section" aria-labelledby="br-env-h"><h2 id="br-env-h">Environment</h2><dl class="br-env">${env.map(([k, v]) => html`<div><dt>${k}</dt><dd>${v}</dd></div>`)}</dl></section>
<section class="bm-section" aria-labelledby="br-env-h"><h2 id="br-env-h">Environment</h2><dl class="br-env">${env.map(([k, v]) => html`<div><dt>${k}</dt><dd>${v}</dd></div>`)}</dl></section>
<section class="bm-section" aria-labelledby="br-pt-h"><div class="bm-head-row"><h2 id="br-pt-h">Tasks</h2><span class="bm-count">${(run.tasks || []).length}</span></div>
<section class="bm-section" aria-labelledby="br-pt-h"><div class="bm-head-row"><h2 id="br-pt-h">Tasks</h2><span class="bm-count">${(run.tasks || []).length}</span></div>
${(run.tasks || []).length ? html`<ol class="br-ptasks">${run.tasks.map((t) => {
${(run.tasks || []).length ? html`<ol class="br-ptasks">${run.tasks.map((t) => {
@@ -438,7 +469,7 @@ function comparePage({ cmp, titles = {} }) {
@@ -438,7 +469,7 @@ function comparePage({ cmp, titles = {} }) {
const line = (label, vals, f, hi = true) => { const b = best(vals, hi); return html`<tr><th scope="row">${label}</th>${vals.map((v) => cell(v, b, f))}</tr>`; };
const line = (label, vals, f, hi = true) => { const b = best(vals, hi); return html`<tr><th scope="row">${label}</th>${vals.map((v) => cell(v, b, f))}</tr>`; };
return html`<div class="bm-wrap br" id="br-compare">
return html`<div class="bm-wrap br" id="br-compare">
<header class="bm-page-h"><p class="bm-kicker">Compare</p><h1>${runs.map((r) => r.model).join(' vs ')}</h1><p class="bm-lede">${[...new Set(runs.map((r) => titles[r.benchmark_id] || r.benchmark_id))].join(', ')}. The best value in each row is marked.</p></header>
<header class="bm-page-h"><p class="bm-kicker">Compare</p><h1>${runs.map((r) => r.model).join(' vs ')}</h1><p class="bm-lede">${[...new Set(runs.map((r) => titles[r.benchmark_id] || r.benchmark_id))].join(', ')}. The best value in each row is marked.</p></header>
-<div class="br-table-wrap"><table class="br-table br-cmp"><thead><tr><th scope="col"><span class="br-sr">Measure</span></th>${runs.map((r) => html`<th scope="col"><a href="${runHref(r.id)}">${r.model}</a><span class="br-cmp-sub">${harnessLabel(r.harness)}</span>${provenance(r.provenance)}</th>`)}</tr></thead><tbody>
+<div class="br-table-wrap"><table class="br-table br-cmp"><thead><tr><th scope="col"><span class="br-sr">Measure</span></th>${runs.map((r) => html`<th scope="col"><a href="${runHref(r.id)}">${r.model}</a><span class="br-cmp-sub">${harnessLabel(r.harness)}</span>${provenance(r.provenance, r.source)}</th>`)}</tr></thead><tbody>
${line('Score', runs.map((r) => r.score), fmtScore)}${line('Cost', runs.map((r) => r.cost_usd), fmtUsd, false)}${line('Time', runs.map((r) => r.ms), fmtMs, false)}${line('Tokens', runs.map((r) => r.tokens), fmtTokens, false)}
${line('Score', runs.map((r) => r.score), fmtScore)}${line('Cost', runs.map((r) => r.cost_usd), fmtUsd, false)}${line('Time', runs.map((r) => r.ms), fmtMs, false)}${line('Tokens', runs.map((r) => r.tokens), fmtTokens, false)}
${rows.map((row) => line(html`<span class="br-mono">${row.task_id}</span>`, row.scores, fmtScore))}
${rows.map((row) => line(html`<span class="br-mono">${row.task_id}</span>`, row.scores, fmtScore))}
</tbody></table></div>
</tbody></table></div>
new file mode 100644
new file mode 100644
@@ -0,0 +1,25 @@
@@ -0,0 +1,25 @@
+{
+ "id": "0f1c9a52-0000-4000-8000-000000000001",
+ "started_at": "2026-10-02T09:00:00Z",
+ "finished_at": "2026-10-02T09:10:00Z",
+ "n_total_trials": 2,
+ "stats": { "n_completed_trials": 2, "cost_usd": 0.31 },
+ "trial_results": [
+ {
+ "id": "trial-1",
+ "task_name": "write-a-python-parser",
+ "task_id": "task-parser",
+ "agent_info": { "name": "claude-code", "version": "1.2.3", "model_info": { "name": "claude-sonnet-4", "provider": "anthropic" } },
+ "verifier_result": { "reward": 1.0 },
+ "cost_usd": 0.2
+ },
+ {
+ "id": "trial-2",
+ "task_name": "fix-the-failing-test",
+ "task_id": "task-test",
+ "agent_info": { "name": "claude-code", "version": "1.2.3", "model_info": { "name": "claude-sonnet-4", "provider": "anthropic" } },
+ "verifier_result": { "reward": 0.0 },
+ "cost_usd": 0.11
+ }
+ ]
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
+{
+ "version": "0.3.286",
+ "status": "success",
+ "eval": {
+ "task": "mmlu",
+ "task_version": "1",
+ "dataset": { "samples": [1, 2, 3] },
+ "model": "openai/gpt-4o",
+ "model_api": "openai",
+ "created": "2026-10-01T10:00:00Z"
+ },
+ "results": {
+ "total_samples": 3,
+ "completed_samples": 3,
+ "scores": [
+ { "name": "accuracy", "scorer": "choice", "metrics": { "accuracy": { "value": 0.6666666666666666, "stderr": 0.1 } } }
+ ]
+ },
+ "stats": {
+ "started_at": "2026-10-01T10:00:00Z",
+ "completed_at": "2026-10-01T10:05:00Z",
+ "model_usage": { "openai/gpt-4o": { "input_tokens": 1200, "output_tokens": 300, "total_tokens": 1500, "cost": 0.42 } }
+ },
+ "reductions": {
+ "samples": [
+ { "sample_id": 1, "value": 1 },
+ { "sample_id": 2, "value": 0 },
+ { "sample_id": 3, "value": 1 }
+ ]
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,17 @@
@@ -0,0 +1,17 @@
+{
+ "results": {
+ "mmlu": {
+ "alias": "mmlu",
+ "acc,none": 0.6052631578947368,
+ "acc_stderr,none": 0.0044,
+ "acc_by_task": { "abstract_algebra": 0.25, "anatomy": 0.5 }
+ }
+ },
+ "configs": {
+ "mmlu": { "model": "hf", "model_args": { "model_name": "meta-llama/Llama-3-8B" }, "batch_size": 1, "device": "cpu" }
+ },
+ "versions": { "mmlu": 1 },
+ "n-shot": { "mmlu": 0 },
+ "higher_is_better": { "acc,none": true },
+ "config": { "model": "hf", "batch_size": 1 }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,37 @@
@@ -0,0 +1,37 @@
+{
+ "version": 3,
+ "timestamp": "2026-10-03T12:00:00Z",
+ "results": [
+ {
+ "id": "case-1",
+ "testIdx": 0,
+ "promptIdx": 0,
+ "success": true,
+ "score": 1,
+ "provider": { "id": "openai:gpt-4o" },
+ "response": { "output": "42", "tokenUsage": { "total": 120 }, "cost": 0.003 },
+ "gradingResult": { "pass": true, "score": 1, "reason": "matches" }
+ },
+ {
+ "id": "case-2",
+ "testIdx": 1,
+ "promptIdx": 0,
+ "success": false,
+ "score": 0,
+ "provider": { "id": "openai:gpt-4o" },
+ "response": { "output": "41", "tokenUsage": { "total": 130 }, "cost": 0.004 },
+ "gradingResult": { "pass": false, "score": 0, "reason": "wrong" }
+ },
+ {
+ "id": "case-3",
+ "testIdx": 2,
+ "promptIdx": 0,
+ "success": true,
+ "score": 1,
+ "provider": { "id": "anthropic:claude-sonnet-4" },
+ "response": { "output": "42", "tokenUsage": { "total": 90 }, "cost": 0.002 },
+ "gradingResult": { "pass": true, "score": 1 }
+ }
+ ],
+ "stats": { "successes": 2, "failures": 1, "errors": 0 }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,380 @@
@@ -0,0 +1,380 @@
+'use strict';
+
+/**
+ * Importing results people bring themselves (B12): the four tools, the limits, the privacy, the account data
+ * and the mirror's own rules.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { boot, check, done } = require('./helpers/boot');
+const { uncovered } = require('openvibe-sdk/account-data');
+const accountDataLib = require('../server/identity/account-data');
+const importers = require('../server/bench/importers');
+const limits = require('../server/bench/importers/limits');
+const { matchSuite } = require('../server/bench/suites');
+const { createMirror } = require('../server/bench/mirror');
+
+const FIX = path.join(__dirname, 'fixtures', 'imports');
+const read = (name) => fs.readFileSync(path.join(FIX, name));
+
+/** A zip with the parts stored (not deflated), so the reader is tested on a real archive. */
+function zipOf(parts) {
+ const chunks = [];
+ const central = [];
+ let offset = 0;
+ for (const part of parts) {
+ const name = Buffer.from(part.name, 'utf8');
+ const data = Buffer.isBuffer(part.data) ? part.data : Buffer.from(part.data, 'utf8');
+ const local = Buffer.alloc(30);
+ local.writeUInt32LE(0x04034b50, 0);
+ local.writeUInt16LE(20, 4);
+ local.writeUInt16LE(0, 6);
+ local.writeUInt16LE(0, 8);
+ local.writeUInt32LE(0, 10);
+ local.writeUInt32LE(data.length, 18);
+ local.writeUInt32LE(data.length, 22);
+ local.writeUInt16LE(name.length, 26);
+ local.writeUInt16LE(0, 28);
+ chunks.push(local, name, data);
+ const entry = Buffer.alloc(46);
+ entry.writeUInt32LE(0x02014b50, 0);
+ entry.writeUInt16LE(20, 4);
+ entry.writeUInt16LE(20, 6);
+ entry.writeUInt16LE(0, 8);
+ entry.writeUInt16LE(0, 10);
+ entry.writeUInt32LE(0, 12);
+ entry.writeUInt32LE(data.length, 20);
+ entry.writeUInt32LE(data.length, 24);
+ entry.writeUInt16LE(name.length, 28);
+ entry.writeUInt16LE(0, 30);
+ entry.writeUInt16LE(0, 32);
+ entry.writeUInt16LE(0, 34);
+ entry.writeUInt16LE(0, 36);
+ entry.writeUInt32LE(0, 38);
+ entry.writeUInt32LE(offset, 42);
+ central.push(entry, name);
+ offset += local.length + name.length + data.length;
+ }
+ const centralBuffer = Buffer.concat(central);
+ const end = Buffer.alloc(22);
+ end.writeUInt32LE(0x06054b50, 0);
+ end.writeUInt16LE(parts.length, 8);
+ end.writeUInt16LE(parts.length, 10);
+ end.writeUInt32LE(centralBuffer.length, 12);
+ end.writeUInt32LE(offset, 16);
+ return Buffer.concat([...chunks, centralBuffer, end]);
+}
+
+
+/**
+ * A zip that lies: each part really inflates to 'size' bytes, but the directory says it is a kilobyte.
+ * This is the bomb the cap has to catch: the memory is bounded by what is really unpacked, not by the claim.
+ */
+function bombZip(parts) {
+ const zlib = require('zlib');
+ const chunks = [];
+ const central = [];
+ let offset = 0;
+ for (const part of parts) {
+ const name = Buffer.from(part.name, 'utf8');
+ const raw = Buffer.alloc(part.size, 0x61);
+ const deflated = zlib.deflateRawSync(raw, { level: 9 });
+ const local = Buffer.alloc(30);
+ local.writeUInt32LE(0x04034b50, 0);
+ local.writeUInt16LE(20, 4);
+ local.writeUInt16LE(0, 6);
+ local.writeUInt16LE(8, 8);
+ local.writeUInt32LE(0, 10);
+ local.writeUInt32LE(0, 14);
+ local.writeUInt32LE(deflated.length, 18);
+ local.writeUInt32LE(part.declared, 22);
+ local.writeUInt16LE(name.length, 26);
+ local.writeUInt16LE(0, 28);
+ chunks.push(local, name, deflated);
+ const entry = Buffer.alloc(46);
+ entry.writeUInt32LE(0x02014b50, 0);
+ entry.writeUInt16LE(20, 4);
+ entry.writeUInt16LE(20, 6);
+ entry.writeUInt16LE(0, 8);
+ entry.writeUInt16LE(8, 10);
+ entry.writeUInt32LE(0, 12);
+ entry.writeUInt32LE(0, 16);
+ entry.writeUInt32LE(deflated.length, 20);
+ entry.writeUInt32LE(part.declared, 24);
+ entry.writeUInt16LE(name.length, 28);
+ entry.writeUInt16LE(0, 30);
+ entry.writeUInt16LE(0, 32);
+ entry.writeUInt16LE(0, 34);
+ entry.writeUInt16LE(0, 36);
+ entry.writeUInt32LE(0, 38);
+ entry.writeUInt32LE(offset, 42);
+ central.push(entry, name);
+ offset += local.length + name.length + deflated.length;
+ }
+ const centralBuffer = Buffer.concat(central);
+ const end = Buffer.alloc(22);
+ end.writeUInt32LE(0x06054b50, 0);
+ end.writeUInt16LE(parts.length, 8);
+ end.writeUInt16LE(parts.length, 10);
+ end.writeUInt32LE(centralBuffer.length, 12);
+ end.writeUInt32LE(offset, 16);
+ return Buffer.concat([...chunks, centralBuffer, end]);
+}
+
+/** A multipart body with one file part, the way a browser sends it. */
+function multipart(filename, bytes) {
+ const boundary = '----b12fixture';
+ const head = Buffer.from(
+ '--' + boundary + '\r\n'
+ + 'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n'
+ + 'Content-Type: application/octet-stream\r\n\r\n', 'utf8');
+ const tail = Buffer.from('\r\n--' + boundary + '--\r\n', 'utf8');
+ return { body: Buffer.concat([head, bytes, tail]), type: 'multipart/form-data; boundary=' + boundary };
+}
+
+(async () => {
+ // ── the parsers, on the published shapes ─────────────────────────────────────────────────────────────────
+ await check('the Inspect log is read: the model, its score and its tasks', () => {
+ const out = importers.parse(read('inspect-log.json'), { filename: 'log.json' });
+ assert.strictEqual(out.tool, 'inspect');
+ assert.strictEqual(out.toolVersion, '0.3.286');
+ assert.strictEqual(out.models.length, 1);
+ assert.strictEqual(out.models[0].model, 'openai/gpt-4o');
+ assert.strictEqual(out.models[0].provider, 'openai');
+ assert.ok(Math.abs(out.models[0].score - 66.6666) < 0.01, 'the score is out of 100: ' + out.models[0].score);
+ assert.strictEqual(out.models[0].tasks.length, 3);
+ assert.strictEqual(out.models[0].cost_usd, 0.42);
+ assert.strictEqual(out.suite.name, 'mmlu');
+ });
+
+ await check('an .eval zip is opened in memory', () => {
+ const zip = zipOf([{ name: 'logs/2026-10-01.json', data: read('inspect-log.json') }]);
+ const out = importers.parse(zip, { filename: 'run.eval' });
+ assert.strictEqual(out.tool, 'inspect');
+ assert.strictEqual(out.models[0].model, 'openai/gpt-4o');
+ });
+
+ await check('the Harbor results are read: one model, the reward of every trial', () => {
+ const out = importers.parse(read('harbor-results.json'), { filename: 'results.json' });
+ assert.strictEqual(out.tool, 'harbor');
+ assert.strictEqual(out.models.length, 1);
+ assert.strictEqual(out.models[0].model, 'claude-sonnet-4');
+ assert.strictEqual(out.models[0].provider, 'anthropic');
+ assert.strictEqual(out.models[0].tasks.length, 2);
+ assert.strictEqual(out.models[0].tasks[0].score, 100);
+ assert.strictEqual(out.models[0].tasks[1].score, 0);
+ });
+
+ await check('the lm-evaluation-harness results are read: the model and its subtasks', () => {
+ const out = importers.parse(read('lm-eval-results.json'), { filename: 'results.json' });
+ assert.strictEqual(out.tool, 'lm-eval');
+ assert.strictEqual(out.models[0].model, 'meta-llama/Llama-3-8B');
+ assert.strictEqual(out.models[0].tasks.length, 2);
+ assert.strictEqual(out.suite.name, 'mmlu');
+ });
+
+ await check('the promptfoo results are read: one run per provider', () => {
+ const out = importers.parse(read('promptfoo-output.json'), { filename: 'output.json' });
+ assert.strictEqual(out.tool, 'promptfoo');
+ assert.strictEqual(out.models.length, 2);
+ const gpt = out.models.find((m) => m.model === 'openai:gpt-4o');
+ assert.ok(gpt, 'the openai run is there');
+ assert.strictEqual(gpt.tasks.length, 2);
+ assert.strictEqual(gpt.cost_usd, 0.007);
+ });
+
+ // ── the limits ─────────────────────────────────────────────────────────────────────────────────────────
+ await check('a file over 20 MB is refused', () => {
+ assert.throws(() => importers.parse(Buffer.alloc(20 * 1024 * 1024 + 1)), /20 MB|too large/i);
+ });
+
+ await check('a zip that claims more than it holds is refused', () => {
+ const zip = zipOf([{ name: 'logs/x.json', data: read('inspect-log.json') }]);
+ // Say the entry unpacks to 200 MB: a bomb, and it is refused before anything is inflated.
+ const centralAt = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
+ zip.writeUInt32LE(200 * 1024 * 1024, centralAt + 24);
+ assert.throws(() => limits.readZip(zip), /100 MB|unpacks/i);
+ });
+
+
+ await check('a zip that lies about its size is refused at the real cap, and the memory stays bounded', () => {
+ const rssBefore = process.memoryUsage().rss;
+ const zip = bombZip([
+ { name: 'logs/one.json', size: 60 * 1024 * 1024, declared: 1024 },
+ { name: 'logs/two.json', size: 60 * 1024 * 1024, declared: 1024 },
+ ]);
+ let caught = null;
+ try {
+ limits.jsonPartsOfZip(zip);
+ } catch (err) {
+ caught = err;
+ }
+ assert.ok(caught, 'the bomb is refused');
+ assert.strictEqual(caught.code, 'bench.import_unpacks_too_much');
+ assert.strictEqual(caught.status, 413);
+ assert.ok(caught.inflated <= limits.MAX_DECOMPRESSED_BYTES,
+ 'the bytes really unpacked stay under the cap: ' + caught.inflated);
+ const grew = process.memoryUsage().rss - rssBefore;
+ assert.ok(grew < 250 * 1024 * 1024, 'the memory stayed bounded: ' + Math.round(grew / 1048576) + ' MB');
+ });
+
+ await check('the operator token is compared in constant time', () => {
+ const { tokenMatches } = require('../server/http/mirror');
+ assert.ok(tokenMatches('secret', 'secret'));
+ assert.ok(!tokenMatches('secrat', 'secret'));
+ assert.ok(!tokenMatches('secret', 'secre'));
+ assert.ok(!tokenMatches('', 'secret'));
+ assert.ok(!tokenMatches('secret', ''));
+ assert.ok(!tokenMatches(null, 'secret'));
+ });
+
+ await check('a JSON that nests too deeply is refused', () => {
+ const deep = '['.repeat(80) + ']'.repeat(80);
+ assert.throws(() => limits.parseJson(deep), /deeply/i);
+ });
+
+ await check('a file that is none of ours is refused', () => {
+ assert.throws(() => importers.parse(Buffer.from('{"hello":"world"}')), /do not know/i);
+ });
+
+ // ── the suites ───────────────────────────────────────────────────────────────────────────────────────
+ await check('a known suite is matched by its name', () => {
+ assert.ok(matchSuite('MMLU'));
+ assert.strictEqual(matchSuite('MMLU').key, 'mmlu');
+ assert.ok(matchSuite('gsm8k'));
+ assert.ok(matchSuite('HumanEval'));
+ assert.strictEqual(matchSuite('my own prompts'), null);
+ });
+
+ const t = await boot();
+ const alice = t.network.addUser('alice');
+ const bob = t.network.addUser('bob');
+
+ await check('the upload form is there and asks for a sign-in', async () => {
+ const out = await t.get('/imports');
+ assert.strictEqual(out.status, 200, out.text.slice(0, 200));
+ assert.ok(/Import a results file/.test(out.text), 'the page says what it is');
+ assert.ok(/Sign in/.test(out.text), 'it asks for a sign-in');
+ });
+
+ let importId = null;
+ let runId = null;
+ let benchmarkId = null;
+
+ await check('an upload becomes a preview, and then submitted runs', async () => {
+ const up = multipart('inspect-log.json', read('inspect-log.json'));
+ const posted = await t.get('/imports', { as: alice, method: 'POST', headers: { 'content-type': up.type }, raw: up.body });
+ assert.strictEqual(posted.status, 303, posted.text.slice(0, 300));
+ importId = String(posted.headers.get('location') || '').split('/').pop();
+ assert.ok(importId, 'the upload has an id');
+
+ const preview = await t.get('/imports/' + importId, { as: alice });
+ assert.strictEqual(preview.status, 200, preview.text.slice(0, 200));
+ assert.ok(/1 model/.test(preview.text), 'the preview says what it holds: ' + preview.text.slice(0, 200));
+ assert.ok(/from inspect 0\.3\.286/.test(preview.text), 'and which tool it came from');
+
+ const done = await t.get('/imports/' + importId + '/confirm', { as: alice, method: 'POST' });
+ assert.strictEqual(done.status, 303, done.text.slice(0, 300));
+ benchmarkId = String(done.headers.get('location') || '').split('/').pop();
+
+ const runs = await t.ctx.registry.listRuns({ benchmark: benchmarkId });
+ assert.strictEqual(runs.length, 1);
+ runId = runs[0].id;
+ assert.strictEqual(runs[0].provenance, 'submitted', 'an imported run is never verified');
+ assert.strictEqual(runs[0].source.kind, 'import');
+ assert.strictEqual(runs[0].source.tool, 'inspect');
+ assert.strictEqual(runs[0].source.tool_version, '0.3.286');
+ const full = await t.ctx.registry.getRun(runId);
+ assert.strictEqual(full.tasks.length, 3, 'the per-task scores came with it');
+ assert.strictEqual(runs[0].harness.name, 'inspect-ai');
+ });
+
+ await check('the run page says where it came from', async () => {
+ const page = await t.get('/runs/' + runId, { as: alice });
+ assert.strictEqual(page.status, 200, page.text.slice(0, 200));
+ assert.ok(/Imported from inspect 0\.3\.286/.test(page.text), 'the label is there');
+ });
+
+ await check('the file stays private to its owner', async () => {
+ const mine = await t.get('/imports/' + importId + '/file', { as: alice });
+ assert.strictEqual(mine.status, 200);
+ assert.ok(mine.buffer.length > 0);
+ const theirs = await t.get('/imports/' + importId + '/file', { as: bob });
+ assert.strictEqual(theirs.status, 404, 'another person may not read the file');
+ const other = await t.get('/imports/' + importId, { as: bob });
+ assert.strictEqual(other.status, 404, 'nor see the preview');
+ });
+
+ await check('the uploads are in the account data, and the guard is happy', async () => {
+ const row = accountDataLib.TABLES.find((x) => x.table === 'bench_imports');
+ assert.ok(row, 'bench_imports is in the account data');
+ assert.strictEqual(row.subject, 'owner');
+ assert.strictEqual(row.erase, 'delete');
+ const missing = await uncovered(t.ctx.s.db, {
+ tables: [...accountDataLib.TABLES, ...accountDataLib.GUARDED],
+ exempt: {
+ account_data_events: 'the ledger of the requests themselves',
+ bench_ratings: 'the ratings are an aggregate over the models',
+ },
+ });
+ assert.deepStrictEqual(missing, [], 'a table with a person column is neither covered nor exempted: ' + missing.join(', '));
+ });
+
+ // ── the mirror ───────────────────────────────────────────────────────────────────────────────────────
+ await check('the mirror reads a source under its licence, and never lands the same row twice', async () => {
+ const licence = ' Apache License\n Version 2.0, January 2004\n';
+ const table = [{
+ title: 'MMLU',
+ header: [
+ { value: 'Model', metadata: {} },
+ { value: 'EM', metadata: { metric: 'EM', run_group: 'MMLU' } },
+ ],
+ rows: [
+ [{ value: 'Phi-3 (14B)' }, { value: 0.675 }],
+ [{ value: 'Llama-3 (8B)' }, { value: 0.51 }],
+ ],
+ }];
+ const fetchImpl = async (url) => {
+ const text = String(url);
+ if (text.includes('/LICENSE')) return { ok: true, status: 200, text: async () => licence, json: async () => ({}) };
+ if (text.includes('/groups/')) return { ok: true, status: 200, text: async () => JSON.stringify(table), json: async () => table };
+ return { ok: false, status: 404, text: async () => '', json: async () => ({}) };
+ };
+ const mirror = createMirror({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} } });
+ const looked = await mirror.preview('helm', { group: 'mmlu', release: 'v1.13.0' });
+ assert.ok(looked.ok, 'the source cleared: ' + JSON.stringify(looked).slice(0, 200));
+ assert.strictEqual(looked.licence, 'Apache-2.0');
+ assert.strictEqual(looked.count, 2);
+
+ const applied = await mirror.apply('helm', { group: 'mmlu', release: 'v1.13.0' });
+ assert.ok(applied.ok);
+ assert.strictEqual(applied.written.length, 2);
+ const runs = await t.ctx.registry.listRuns({ benchmark: applied.written[0].benchmark });
+ assert.strictEqual(runs.length, 2, 'one mirrored run per model');
+ const mirrored = runs.find((r) => r.model === 'Phi-3 (14B)');
+ assert.ok(mirrored, 'the mirrored run is there');
+ assert.strictEqual(mirrored.provenance, 'submitted', 'a mirrored run is never verified');
+ assert.strictEqual(mirrored.source.kind, 'mirror');
+ assert.strictEqual(mirrored.source.licence, 'Apache-2.0');
+ assert.ok(mirrored.source.source_url, 'it says where it came from');
+
+ const again = await mirror.apply('helm', { group: 'mmlu', release: 'v1.13.0' });
+ assert.strictEqual(again.written.length, 0, 'the same model, version and source do not land twice');
+ assert.ok(again.skipped.some((x) => x.why === 'already mirrored'));
+ });
+
+ await check('a source whose licence cannot be read is not imported', async () => {
+ const fetchImpl = async () => ({ ok: false, status: 401, text: async () => '', json: async () => ({}) });
+ const mirror = createMirror({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} } });
+ const looked = await mirror.preview('helm', { group: 'mmlu' });
+ assert.ok(!looked.ok, 'it does not clear');
+ assert.ok(/licence/.test(looked.reason), looked.reason);
+ });
+
+ await t.close();
+ done();
+})().catch((err) => {
+ process.stderr.write('The test file failed: ' + ((err && err.stack) || err) + '\n');
+ process.exit(1);
+});