openvibers/openvibe.bench Commits
Bench B2/B7: Every Eval Ever import and export, lighteval and HELM importers (#175)
Changes
@@@ -2,6 +2,17 @@## Unreleased++- **Every Eval Ever (B2).** Bench imports the public EEE datastore (`github.com/evaleval/every_eval_ever`,++ `evaleval/EEE_datastore`) as *external* results: the model (joined by Report's rule — maker prefix and a date or++ channel suffix dropped), the harness and its version, the task and its version, few-shot, the metric, the score,++ the source link and the licence. A record whose licence does not allow redistribution is dropped and counted.++ The rows live in `bench_eee_records` and are served by `GET /api/v1/external/records` with `external: true`; they++ never enter Bench's ratings, boards or medians. A weekly, resumable, bounded job (`BENCH_EEE_SYNC`) imports them.++ Bench's own public runs are exported as EEE `evaluation_run` records at `GET /api/v1/datasets/eee` (JSONL, paged;++ per-sample scores at `/api/v1/datasets/eee/runs/:id.samples.jsonl`).++- **Two more harnesses (B7).** HF lighteval and Stanford HELM results join Inspect AI, Harbor, the++ lm-evaluation-harness and promptfoo. Every import keeps the harness name and version, each task's version, the++ few-shot and the seed on the run.- **The public pages moved to OpenVibe.Report's AI section (2026-11-08).** `/`, `/docs`, `/benchmarks`, `/models/*`, `/compare` and the other page routes answer `301` to `https://openvibe.report/ai` counterparts; the API, MCP, render origin, badges and embeds stay on bench.openvibe.services. New public reads feed Report: `GET /api/v1/runs`, `/api/v1/overview` (with every public model and its last run) and `/api/v1/models/:model/summary` (per-benchmark and per-task medians, cost, recent runs). `/api/v1/benchmarks` now carries each benchmark's runs and last run.- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:`POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, andnnew file mode 100644@@@ -0,0 +1,82 @@++-- phase: expand++-- Stage B2 (plan 2026-11-09): Every Eval Ever (EEE).++--++-- Bench imports the EEE datastore (github.com/evaleval/every_eval_ever, HF evaleval/EEE_datastore) as *external*++-- results: they sit beside Bench's own runs in their own table, with the provenance and the licence they were++-- published under, and they never enter Bench's ratings, boards or medians. The licence that does not allow++-- redistribution stops a record at the door, and the sync's ledger counts how many and why.++--++-- The exporter is the other direction: Bench's own public runs are served as EEE `evaluation_run` records from++-- /api/v1/datasets/eee (server/bench/eee.js), so nothing here stores the export.++++-- One row per (record, evaluation result, metric): the same EEE record never lands twice.++CREATE TABLE bench_eee_records (++ id text COLLATE "C" PRIMARY KEY, -- ber_ + ULID++ eee_id text COLLATE "C" NOT NULL, -- the EEE evaluation_id (record identity)++ source_name text COLLATE "C" NOT NULL DEFAULT '',++ source_type text COLLATE "C" NOT NULL, -- documentation | evaluation_run++ source_url text COLLATE "C" NOT NULL DEFAULT '',++ source_org text COLLATE "C" NOT NULL DEFAULT '',++ evaluator_relationship text COLLATE "C" NOT NULL DEFAULT 'other',++ licence text COLLATE "C" NOT NULL,++ licence_url text COLLATE "C" NOT NULL DEFAULT '',++ licence_source text COLLATE "C" NOT NULL DEFAULT '', -- record | datastore++ model text COLLATE "C" NOT NULL, -- the join name (maker prefix and date/channel suffix dropped)++ model_raw text COLLATE "C" NOT NULL DEFAULT '', -- the source's own id++ developer text COLLATE "C" NOT NULL DEFAULT '',++ harness_name text COLLATE "C" NOT NULL DEFAULT 'unknown',++ harness_version text COLLATE "C" NOT NULL DEFAULT 'unknown',++ task text COLLATE "C" NOT NULL,++ task_version text COLLATE "C" NOT NULL DEFAULT '',++ few_shot integer,++ seed text COLLATE "C" NOT NULL DEFAULT '',++ metric_id text COLLATE "C" NOT NULL DEFAULT '',++ metric_name text COLLATE "C" NOT NULL DEFAULT '',++ lower_is_better boolean,++ score numeric(14, 6) NOT NULL,++ score_min numeric(14, 6),++ score_max numeric(14, 6),++ evaluation_at timestamptz,++ retrieved_at timestamptz,++ provenance jsonb NOT NULL DEFAULT '{}'::jsonb, -- the EEE record as published (its licence, harness, link)++ created_at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT bench_eee_records_id CHECK (id ~ '^ber_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_eee_records_source_type CHECK (source_type IN ('documentation', 'evaluation_run')),++ CONSTRAINT bench_eee_records_relationship CHECK (evaluator_relationship IN ('first_party', 'third_party', 'collaborative', 'other')),++ CONSTRAINT bench_eee_records_few_shot CHECK (few_shot IS NULL OR few_shot >= 0)++);++CREATE UNIQUE INDEX bench_eee_records_dedupe ON bench_eee_records (eee_id, task, metric_id, metric_name);++CREATE INDEX bench_eee_records_by_model ON bench_eee_records (model, task);++CREATE INDEX bench_eee_records_by_task ON bench_eee_records (task, created_at DESC);++CREATE INDEX bench_eee_records_by_source ON bench_eee_records (source_type, created_at DESC);++++-- A sync run: where it got to and what it did. A run that stopped half way leaves its cursor here, and the next++-- run of the weekly job resumes from it instead of starting the datastore again.++CREATE TABLE bench_eee_syncs (++ id text COLLATE "C" PRIMARY KEY, -- bes_ + ULID++ status text COLLATE "C" NOT NULL DEFAULT 'running', -- running | finished | failed++ cursor jsonb NOT NULL DEFAULT '{}'::jsonb, -- { collection, file }++ collections integer NOT NULL DEFAULT 0,++ files_seen integer NOT NULL DEFAULT 0,++ records_read integer NOT NULL DEFAULT 0,++ records_kept integer NOT NULL DEFAULT 0,++ dropped_licence integer NOT NULL DEFAULT 0,++ dropped_invalid integer NOT NULL DEFAULT 0,++ dropped_reason jsonb NOT NULL DEFAULT '{}'::jsonb, -- { reason: count }++ default_licence text COLLATE "C" NOT NULL DEFAULT '',++ licence_url text COLLATE "C" NOT NULL DEFAULT '',++ error text COLLATE "C" NOT NULL DEFAULT '',++ started_at timestamptz NOT NULL DEFAULT now(),++ updated_at timestamptz NOT NULL DEFAULT now(),++ finished_at timestamptz,++ CONSTRAINT bench_eee_syncs_id CHECK (id ~ '^bes_[0-9A-HJKMNP-TV-Z]{26}$'),++ CONSTRAINT bench_eee_syncs_status CHECK (status IN ('running', 'finished', 'failed')),++ CONSTRAINT bench_eee_syncs_counts CHECK (files_seen >= 0 AND records_read >= 0 AND records_kept >= 0++ AND dropped_licence >= 0 AND dropped_invalid >= 0)++);++CREATE INDEX bench_eee_syncs_by_status ON bench_eee_syncs (status, started_at DESC);++++-- B7: the two harnesses added beside the four. Widening the check is the expand step.++ALTER TABLE bench_imports DROP CONSTRAINT bench_imports_tool;++ALTER TABLE bench_imports ADD CONSTRAINT bench_imports_tool++ CHECK (tool IN ('inspect', 'harbor', 'lm-eval', 'promptfoo', 'lighteval', 'helm'));@@@ -88,6 +88,9 @@ async function createApp(opts = {}) {// B12: bringing your own results, and the operator's mirror.const imports = require('./bench/imports').createImports({ s, registry, log });const mirror = require('./bench/mirror').createMirror({ s, registry, fetchImpl, log });++ // B2: the external EEE records and the exporter of Bench's own runs, plus the weekly import job.++ const eee = require('./bench/eee').createEee({ s, registry, fetchImpl, log, config });++ const eeeSync = require('./bench/eee-sync').createEeeSync({ eee, config, log });const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });const jobs = createBenchJobs({ s, bench, ai, config, log });// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.@@@ -111,7 +114,7 @@ async function createApp(opts = {}) {const ctx = {config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,anon, daily, variantEvidence, evidenceConfig, sharing,-- imports, mirror,++ imports, mirror, eee, eeeSync,random: opts.random || Math.random,};@@@ -253,6 +256,8 @@ async function createApp(opts = {}) {const SHARE_PATH = /^\/api\/v1\/(?:shared-with-me$|share\/people$|[^/]+\/[^/]+\/share(?:\/|$))/;app.use((req, res, next) => (SHARE_PATH.test(req.path) ? shareApi(req, res, next) : next()));}++ // The EEE routes come before the kind dataset route: /datasets/eee.jsonl would otherwise be read as kind 'eee'.++ app.use(require('./http/eee').createEeeRoutes(ctx));app.use('/api/v1', createApi(ctx));app.use('/api/v1', createRegistryApi(ctx));app.use('/api/v1', createVariantEvidenceApi(ctx));nnew file mode 100644@@@ -0,0 +1,46 @@++'use strict';++++/**++ * The weekly Every Eval Ever import (B2). It is the same shape as Bench's other workers: a timer, one async pass at++ * a time, and the pass itself is bounded and resumable (server/bench/eee.js `sync`). Off until the owner turns++ * BENCH_EEE_SYNC on; never under BENCH_DRILL unless BENCH_EEE_SYNC_IN_DRILL says so too.++ */++function createEeeSync({ eee, config, log = console }) {++ let timer = null;++ let running = false;++ let stopped = false;++ const settings = config.bench.eee;++++ async function tick() {++ if (running || stopped) return null;++ running = true;++ try {++ return await eee.sync({ maxFiles: settings.filesPerRun, maxRecords: settings.recordsPerRun });++ } catch (err) {++ log.warn('[OpenVibe.Bench] the EEE sync failed:', (err && err.message) || err);++ return null;++ } finally {++ running = false;++ }++ }++++ function start() {++ if (timer || stopped || !settings.enabled) return null;++ if (config.drill && !settings.enabledInDrill) return null;++ const run = () => tick().catch((err) => log.warn('[OpenVibe.Bench] EEE job:', (err && err.message) || err));++ timer = setInterval(run, settings.intervalMs);++ if (timer.unref) timer.unref();++ run();++ return timer;++ }++++ function stop() {++ stopped = true;++ if (timer) clearInterval(timer);++ timer = null;++ }++++ return { tick, start, stop, get running() { return running; } };++}++++module.exports = { createEeeSync };nnew file mode 100644@@@ -0,0 +1,664 @@++'use strict';++++/**++ * Every Eval Ever (B2): importing the EEE datastore as external results, and exporting Bench's own runs as EEE.++ *++ * Import. The EEE datastore (github.com/evaleval/every_eval_ever, HF evaleval/EEE_datastore) is a public, MIT++ * collection of eval records in one JSON shape (`eval.schema.json`). A record is read from the datastore's own JSON++ * files, its licence is checked before anything is stored, and each evaluation result becomes one `bench_eee_records`++ * row: the model (a join name with the maker prefix and a date/channel suffix dropped, the rule of Report's++ * `server/data/ai-join.js`), the harness and its version, the task and its version, few-shot, the metric, the score++ * and the source link. The rows are *external*: they never enter Bench's ratings, boards or medians.++ *++ * A weekly job walks the datastore with a cursor in `bench_eee_syncs`, so a stopped run resumes where it stopped, and++ * one run is bounded in files and records.++ *++ * Export. A public Bench run is served as an EEE `evaluation_run` record from `exportRecords` (the route is++ * /api/v1/datasets/eee), with the per-sample scores on a companion endpoint where Bench keeps them.++ */++const { ImportError } = require('./importers/limits');++++const DATASET = 'evaleval/EEE_datastore';++const REPO = 'https://github.com/evaleval/every_eval_ever';++const DATASET_URL = 'https://huggingface.co/datasets/' + DATASET;++const SCHEMA_VERSION = '0.3.0';++++const SOURCE = {++ key: 'eee',++ title: 'Every Eval Ever',++ repo: REPO,++ dataset: DATASET_URL,++ schema_version: SCHEMA_VERSION,++ // The datastore's own dataset card declares MIT; a record may override it in additional_details.++ default_licence: 'MIT',++ licence_url: DATASET_URL + '/blob/main/README.md',++};++++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++const json = (v, fallback) => {++ if (v == null) return fallback;++ if (typeof v === 'string') { try { return JSON.parse(v); } catch { return fallback; } }++ return v;++};++const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));++const details = (v) => (v && typeof v === 'object' && !Array.isArray(v) ? v : {});++++/**++ * The join rule of Report's server/data/ai-join.js: Bench names a model as its maker's API does++ * ("openai/gpt-5-nano-2025-08-07"), the catalogue names it by maker and family ("openai/gpt-5-nano"). They are the++ * same model once the maker prefix and a date or channel suffix are dropped.++ */++function bareModel(v) {++ return String(v || '').toLowerCase()++ .replace(/^[^/]+\//, '')++ .replace(/[-_@:](\d{4}-?\d{2}-?\d{2}|latest|free)$/, '');++}++++/** A licence that forbids redistribution is not imported. An empty or "unknown" one is not a licence either. */++const NON_REDISTRIBUTABLE = /all rights reserved|proprietary|no (re)?distribution|not for (re)?distribution|research only|evaluation only|internal only|confidential|closed|^unknown$|^none$|^n\/?a$/i;++function allowsRedistribution(licence) {++ const text = String(licence == null ? '' : licence).trim();++ if (!text) return false;++ return !NON_REDISTRIBUTABLE.test(text);++}++++/** The first licence a record states about itself, or the datastore's own. */++function recordLicenceOf(doc) {++ const meta = details(doc && doc.source_metadata);++ const direct = [meta.licence, meta.license, details(meta.additional_details).licence, details(meta.additional_details).license];++ for (const value of direct) {++ if (typeof value === 'string' && value.trim()) return { licence: value.trim(), source: 'record' };++ }++ const results = Array.isArray(doc && doc.evaluation_results) ? doc.evaluation_results : [];++ for (const result of results) {++ const data = details(result && result.source_data);++ const extra = details(data.additional_details);++ for (const value of [data.licence, data.license, extra.licence, extra.license]) {++ if (typeof value === 'string' && value.trim()) return { licence: value.trim(), source: 'record' };++ }++ }++ return null;++}++++function firstText(...values) {++ for (const value of values) {++ if (typeof value === 'string' && value.trim()) return value.trim();++ }++ return '';++}++++/** Few-shot: the schema has no typed field, so the sources put it in additional_details. */++function fewShotOf(result, doc) {++ const places = [++ details(result && result.source_data && result.source_data.additional_details),++ details(result && result.generation_config && result.generation_config.additional_details),++ details(result && result.generation_config && result.generation_config.generation_args),++ details(doc && doc.source_metadata && doc.source_metadata.additional_details),++ ];++ for (const place of places) {++ for (const key of ['num_fewshot', 'few_shot', 'n_shot', 'fewshot', 'num_shots', 'shots']) {++ const value = num(place[key]);++ if (value != null) return Math.max(0, Math.round(value));++ }++ }++ return null;++}++++function seedOf(result, doc) {++ const places = [++ details(result && result.generation_config && result.generation_config.generation_args),++ details(result && result.generation_config && result.generation_config.additional_details),++ details(doc && doc.source_metadata && doc.source_metadata.additional_details),++ ];++ for (const place of places) {++ for (const key of ['seed', 'random_seed']) {++ const value = place[key];++ if (value != null && String(value).trim()) return String(value).trim().slice(0, 40);++ }++ }++ return '';++}++++function taskVersionOf(result) {++ const data = details(result && result.source_data);++ const extra = details(data.additional_details);++ return firstText(extra.task_version, extra.version, extra.task_revision, extra.revision, data.version);++}++++/** The link back to where this record was reported, preferring a stated URL over the datastore's own path. */++function sourceUrlOf(doc, result, path) {++ const meta = details(doc && doc.source_metadata);++ const extra = details(meta.additional_details);++ const data = details(result && result.source_data);++ const dataExtra = details(data.additional_details);++ const candidates = [++ extra.source_url, extra.url, extra.paper_url,++ meta.source_organization_url,++ dataExtra.source_url, dataExtra.url,++ ...(Array.isArray(data.url) ? data.url : []),++ data.hf_repo ? 'https://huggingface.co/datasets/' + data.hf_repo : '',++ path ? DATASET_URL + '/blob/main/' + path : '',++ ];++ for (const value of candidates) {++ if (typeof value === 'string' && /^https?:\/\//i.test(value)) return value;++ }++ return DATASET_URL;++}++++/**++ * Read one EEE record into the rows it would store. It throws when the document is not an EEE record; a result++ * without a score is dropped (invalid), and a result whose licence forbids redistribution is dropped (licence).++ */++function normalizeRecord(doc, { fallbackLicence = SOURCE.default_licence, fallbackLicenceUrl = SOURCE.licence_url, path = '' } = {}) {++ if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {++ throw new ImportError(422, 'bench.eee_not_object', 'That file is not an EEE record.');++ }++ const resultsIn = Array.isArray(doc.evaluation_results) ? doc.evaluation_results : null;++ if (!resultsIn || !resultsIn.length) {++ throw new ImportError(422, 'bench.eee_no_results', 'That EEE record holds no evaluation results.');++ }++ const meta = details(doc.source_metadata);++ const sourceType = meta.source_type === 'evaluation_run' ? 'evaluation_run' : 'documentation';++ const relationship = ['first_party', 'third_party', 'collaborative', 'other'].includes(meta.evaluator_relationship)++ ? meta.evaluator_relationship : 'other';++ const library = details(doc.eval_library);++ const modelInfo = details(doc.model_info);++ const modelRaw = firstText(modelInfo.id, modelInfo.name);++ const model = bareModel(modelRaw) || modelRaw || 'unknown';++ const developer = firstText(modelInfo.developer, modelRaw.includes('/') ? modelRaw.split('/')[0] : '');++ const stated = recordLicenceOf(doc);++ const licence = stated ? stated.licence : fallbackLicence;++ const licenceSource = stated ? 'record' : 'datastore';++ const eeeId = firstText(doc.evaluation_id, modelRaw) || 'unknown';++++ const record = {++ eee_id: eeeId,++ source_name: firstText(meta.source_name),++ source_type: sourceType,++ source_org: firstText(meta.source_organization_name),++ evaluator_relationship: relationship,++ licence,++ licence_source: licenceSource,++ model,++ model_raw: modelRaw,++ developer,++ harness_name: firstText(library.name, meta.source_name, 'unknown'),++ harness_version: firstText(library.version, 'unknown'),++ retrieved_at: num(doc.retrieved_timestamp),++ // The provenance a reader needs, not a second copy of every evaluation result on every row.++ provenance: {++ schema_version: doc.schema_version == null ? null : String(doc.schema_version),++ evaluation_id: eeeId,++ retrieved_timestamp: doc.retrieved_timestamp == null ? null : String(doc.retrieved_timestamp),++ source_metadata: meta,++ model_info: modelInfo,++ eval_library: library,++ },++ };++++ const results = [];++ for (const result of resultsIn) {++ if (!result || typeof result !== 'object') {++ results.push({ dropped: 'invalid', reason: 'a result is not an object' });++ continue;++ }++ const score = num(result.score_details && result.score_details.score);++ if (score == null) {++ results.push({ dropped: 'invalid', reason: 'a result has no score' });++ continue;++ }++ const metric = details(result.metric_config);++ const metricId = firstText(metric.metric_id, metric.metric_name, 'score');++ const metricName = firstText(metric.metric_name, metric.metric_id, 'score');++ const task = firstText(result.evaluation_name, result.evaluation_result_id) || 'unknown';++ if (!allowsRedistribution(licence)) {++ results.push({ dropped: 'licence', reason: 'licence does not allow redistribution: ' + licence });++ continue;++ }++ results.push({++ allowed: true,++ task,++ task_version: taskVersionOf(result),++ few_shot: fewShotOf(result, doc),++ seed: seedOf(result, doc),++ metric_id: metricId,++ metric_name: metricName,++ lower_is_better: typeof metric.lower_is_better === 'boolean' ? metric.lower_is_better : null,++ score,++ score_min: num(metric.min_score),++ score_max: num(metric.max_score),++ evaluation_at: num(result.evaluation_timestamp != null ? result.evaluation_timestamp : doc.evaluation_timestamp),++ source_url: sourceUrlOf(doc, result, path),++ licence,++ });++ }++ return { record, results };++}++++/** The datastore's own licence, read from its dataset card. A card that cannot be read falls back to MIT. */++async function datastoreLicence(fetchImpl = globalThis.fetch) {++ const url = SOURCE.licence_url;++ try {++ const res = await fetchImpl(url);++ if (res && res.ok) {++ const text = await res.text();++ const line = String(text).split('\n').map((l) => l.trim()).find((l) => /^license:\s*/i.test(l));++ if (line) {++ const licence = line.replace(/^license:\s*/i, '').replace(/^["']|["']$/g, '').trim();++ if (licence) return { licence, url, read: true };++ }++ }++ } catch (err) {++ // fall through to the known card value++ }++ return { licence: SOURCE.default_licence, url, read: false };++}++++function wireExternal(row) {++ if (!row) return null;++ return {++ id: row.id,++ external: true,++ eee_id: row.eee_id,++ source_type: row.source_type,++ source: {++ name: row.source_name,++ organization: row.source_org,++ url: row.source_url,++ relationship: row.evaluator_relationship,++ },++ licence: row.licence,++ licence_url: row.licence_url,++ licence_source: row.licence_source,++ model: row.model,++ model_raw: row.model_raw,++ developer: row.developer,++ harness: { name: row.harness_name, version: row.harness_version },++ task: row.task,++ task_version: row.task_version,++ few_shot: row.few_shot == null ? null : num(row.few_shot),++ seed: row.seed || '',++ metric: { id: row.metric_id, name: row.metric_name, lower_is_better: row.lower_is_better },++ score: row.score == null ? null : num(row.score),++ score_min: row.score_min == null ? null : num(row.score_min),++ score_max: row.score_max == null ? null : num(row.score_max),++ evaluation_at: row.evaluation_at ? iso(row.evaluation_at) : null,++ retrieved_at: row.retrieved_at ? iso(row.retrieved_at) : null,++ provenance: json(row.provenance, {}),++ created_at: iso(row.created_at),++ };++}++++function createEee({ s, fetchImpl = globalThis.fetch, log = console, config = null, now = Date.now } = {}) {++ const db = s.db;++ const baseUrl = String((config && config.baseUrl) || '').replace(/\/+$/, '');++++ async function getJson(url) {++ const res = await fetchImpl(url);++ if (!res || !res.ok) throw new Error('the source answered ' + (res && res.status) + ' for ' + url);++ return res.json();++ }++++ async function storeRecord(doc, { path = '', readAt = s.iso(), fallbackLicence = SOURCE.default_licence, fallbackLicenceUrl = SOURCE.licence_url } = {}) {++ const parsed = normalizeRecord(doc, { fallbackLicence, fallbackLicenceUrl, path });++ const counts = { read: parsed.results.length, kept: 0, droppedLicence: 0, droppedInvalid: 0, reasons: {} };++ const note = (reason) => { counts.reasons[reason] = (counts.reasons[reason] || 0) + 1; };++ for (const result of parsed.results) {++ if (result.dropped === 'licence') { counts.droppedLicence += 1; note(result.reason); continue; }++ if (result.dropped) { counts.droppedInvalid += 1; note(result.reason); continue; }++ await db.exec(++ `INSERT INTO bench_eee_records++ (id, eee_id, source_name, source_type, source_url, source_org, evaluator_relationship, licence,++ licence_url, licence_source, model, model_raw, developer, harness_name, harness_version, task,++ task_version, few_shot, seed, metric_id, metric_name, lower_is_better, score, score_min,++ score_max, evaluation_at, retrieved_at, provenance, created_at)++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19,++ $20, $21, $22, $23, $24, $25, $26::timestamptz, $27::timestamptz, $28::jsonb, $29::timestamptz)++ ON CONFLICT (eee_id, task, metric_id, metric_name) DO UPDATE SET++ score = EXCLUDED.score, score_min = EXCLUDED.score_min, score_max = EXCLUDED.score_max,++ licence = EXCLUDED.licence, licence_url = EXCLUDED.licence_url, licence_source = EXCLUDED.licence_source,++ source_url = EXCLUDED.source_url, harness_version = EXCLUDED.harness_version,++ task_version = EXCLUDED.task_version, few_shot = EXCLUDED.few_shot, seed = EXCLUDED.seed,++ retrieved_at = EXCLUDED.retrieved_at, provenance = EXCLUDED.provenance`,++ [s.newId('ber'), parsed.record.eee_id, parsed.record.source_name, parsed.record.source_type,++ result.source_url, parsed.record.source_org, parsed.record.evaluator_relationship,++ parsed.record.licence, parsed.record.licence_source === 'record' ? result.source_url : fallbackLicenceUrl, parsed.record.licence_source, parsed.record.model,++ parsed.record.model_raw, parsed.record.developer, parsed.record.harness_name,++ parsed.record.harness_version, result.task, result.task_version, result.few_shot, result.seed,++ result.metric_id, result.metric_name, result.lower_is_better, result.score, result.score_min,++ result.score_max,++ result.evaluation_at == null ? null : new Date(result.evaluation_at * 1000).toISOString(),++ parsed.record.retrieved_at == null ? null : new Date(parsed.record.retrieved_at * 1000).toISOString(),++ JSON.stringify(parsed.record.provenance), readAt],++ );++ counts.kept += 1;++ }++ return counts;++ }++++ /** The external records, newest first, with `external: true` and their provenance. Never a Bench rating. */++ async function listRecords({ model = null, task = null, sourceType = null, limit = 100, offset = 0 } = {}) {++ const where = [];++ const params = [];++ const add = (clause, value) => { params.push(value); where.push(clause.replace('?', `$${params.length}`)); };++ if (model) add('model = ?', String(model));++ if (task) add('task = ?', String(task));++ if (sourceType) add('source_type = ?', String(sourceType));++ params.push(Math.max(1, Math.min(1000, Number(limit) || 100)));++ const limitAt = params.length;++ params.push(Math.max(0, Number(offset) || 0));++ const rows = await db.many(++ `SELECT * FROM bench_eee_records ${where.length ? 'WHERE ' + where.join(' AND ') : ''}++ ORDER BY created_at DESC, id LIMIT $${limitAt} OFFSET $${params.length}`,++ params,++ );++ return rows.map(wireExternal);++ }++++ async function countRecords({ model = null, task = null, sourceType = null } = {}) {++ const where = [];++ const params = [];++ const add = (clause, value) => { params.push(value); where.push(clause.replace('?', `$${params.length}`)); };++ if (model) add('model = ?', String(model));++ if (task) add('task = ?', String(task));++ if (sourceType) add('source_type = ?', String(sourceType));++ const row = await db.maybe(`SELECT COUNT(*)::int AS n FROM bench_eee_records ${where.length ? 'WHERE ' + where.join(' AND ') : ''}`, params);++ return num(row && row.n) || 0;++ }++++ async function latestSync() {++ const row = await db.maybe('SELECT * FROM bench_eee_syncs ORDER BY started_at DESC LIMIT 1');++ return row ? wireSync(row) : null;++ }++++ function wireSync(row) {++ return {++ id: row.id,++ status: row.status,++ cursor: json(row.cursor, {}),++ collections: num(row.collections) || 0,++ files_seen: num(row.files_seen) || 0,++ records_read: num(row.records_read) || 0,++ records_kept: num(row.records_kept) || 0,++ dropped_licence: num(row.dropped_licence) || 0,++ dropped_invalid: num(row.dropped_invalid) || 0,++ dropped_reason: json(row.dropped_reason, {}),++ default_licence: row.default_licence || '',++ licence_url: row.licence_url || '',++ error: row.error || '',++ started_at: iso(row.started_at),++ updated_at: iso(row.updated_at),++ finished_at: iso(row.finished_at),++ };++ }++++ async function listCollections() {++ const url = 'https://huggingface.co/api/datasets/' + DATASET + '/tree/main/data?recursive=false';++ const body = await getJson(url);++ if (!Array.isArray(body)) throw new Error('the datastore tree is not a list');++ return body++ .filter((entry) => entry && entry.type === 'directory' && entry.path)++ .map((entry) => String(entry.path).replace(/^data\//, ''))++ .sort();++ }++++ async function listFiles(collection) {++ const url = 'https://huggingface.co/api/datasets/' + DATASET + '/tree/main/data/' + encodeURIComponent(collection) + '?recursive=true&limit=1000';++ const body = await getJson(url);++ if (!Array.isArray(body)) throw new Error('the collection tree is not a list');++ return body++ .filter((entry) => entry && entry.type === 'file' && /\.json$/i.test(entry.path || ''))++ .map((entry) => String(entry.path))++ .sort();++ }++++ function rawUrl(path) {++ return DATASET_URL + '/resolve/main/' + path.split('/').map(encodeURIComponent).join('/');++ }++++ /**++ * One bounded pass of the weekly job. It resumes the newest running sync (or starts one), walks the datastore's++ * collections and JSON files from the cursor, and stops at `maxFiles` files or `maxRecords` results. The next++ * pass continues from the same cursor. Nothing is fetched when the job is disabled.++ */++ async function sync({ maxFiles = 200, maxRecords = 5000, collections = null } = {}) {++ const licence = await datastoreLicence(fetchImpl);++ let row = await db.maybe("SELECT * FROM bench_eee_syncs WHERE status = 'running' ORDER BY started_at DESC LIMIT 1");++ if (!row) {++ const id = s.newId('bes');++ row = await db.maybe(++ `INSERT INTO bench_eee_syncs (id, status, cursor, default_licence, licence_url, started_at, updated_at)++ VALUES ($1, 'running', '{}'::jsonb, $2, $3, $4::timestamptz, $4::timestamptz) RETURNING *`,++ [id, licence.licence, licence.url, s.iso()],++ );++ }++ const cursor = json(row.cursor, {});++ let collectionIndex = Number.isInteger(cursor.collection) ? cursor.collection : 0;++ let fileIndex = Number.isInteger(cursor.file) ? cursor.file : 0;++ let filesSeen = num(row.files_seen) || 0;++ let recordsRead = num(row.records_read) || 0;++ let recordsKept = num(row.records_kept) || 0;++ let droppedLicence = num(row.dropped_licence) || 0;++ let droppedInvalid = num(row.dropped_invalid) || 0;++ const reasons = { ...json(row.dropped_reason, {}) };++ let names;++ let error = '';++ let status = 'running';++ try {++ names = Array.isArray(collections) && collections.length ? collections.map(String).sort() : await listCollections();++ let inThisPass = 0;++ while (collectionIndex < names.length && inThisPass < maxFiles && recordsRead < maxRecords) {++ const paths = await listFiles(names[collectionIndex]);++ while (fileIndex < paths.length && inThisPass < maxFiles && recordsRead < maxRecords) {++ const path = paths[fileIndex];++ let doc;++ try {++ doc = await getJson(rawUrl(path));++ } catch (err) {++ // A single unreadable file is a skipped record, not the end of the pass.++ droppedInvalid += 1;++ reasons['could not read ' + path] = (reasons['could not read ' + path] || 0) + 1;++ fileIndex += 1;++ inThisPass += 1;++ filesSeen += 1;++ continue;++ }++ const out = await storeRecord(doc, { path, readAt: s.iso(), fallbackLicence: licence.licence, fallbackLicenceUrl: licence.url });++ recordsRead += out.read;++ recordsKept += out.kept;++ droppedLicence += out.droppedLicence;++ droppedInvalid += out.droppedInvalid;++ for (const [reason, count] of Object.entries(out.reasons)) reasons[reason] = (reasons[reason] || 0) + count;++ fileIndex += 1;++ inThisPass += 1;++ filesSeen += 1;++ }++ if (fileIndex >= paths.length) { collectionIndex += 1; fileIndex = 0; }++ }++ if (collectionIndex >= names.length) status = 'finished';++ } catch (err) {++ status = 'failed';++ error = String((err && err.message) || err).slice(0, 500);++ log.warn('[OpenVibe.Bench] the EEE sync stopped:', error);++ }++ const nextCursor = JSON.stringify({ collection: collectionIndex, file: fileIndex });++ const updated = await db.maybe(++ `UPDATE bench_eee_syncs SET status = $1, cursor = $2::jsonb, collections = $3, files_seen = $4,++ records_read = $5, records_kept = $6, dropped_licence = $7, dropped_invalid = $8,++ dropped_reason = $9::jsonb, default_licence = $10, licence_url = $11, error = $12,++ updated_at = $13::timestamptz, finished_at = CASE WHEN $1 = 'running' THEN NULL ELSE $13::timestamptz END++ WHERE id = $14 RETURNING *`,++ [status, nextCursor, names ? names.length : 0, filesSeen, recordsRead, recordsKept, droppedLicence,++ droppedInvalid, JSON.stringify(reasons), licence.licence, licence.url, error, s.iso(), row.id],++ );++ return wireSync(updated);++ }++++ // ── Export: Bench's own runs as EEE `evaluation_run` records ───────────────────────────────────────────────++++ /** One public Bench run as an EEE aggregate record. Top-level extras are not allowed by the schema, so the++ * Bench-specific facts live in the nested `additional_details` maps. */++ function eeeRecordOf(run, tasks) {++ const createdAt = iso(run.created_at) || s.iso();++ const retrieved = String(Math.floor((Date.parse(createdAt) || now()) / 1000));++ const settings = json(run.settings, {});++ const harness = json(run.harness, {});++ const route = json(run.route, {});++ const benchmarkRef = `${run.benchmark_id}@${run.benchmark_version}`;++ const sampleCount = Array.isArray(tasks) ? tasks.length : 0;++ const samplesUrl = baseUrl ? `${baseUrl}/api/v1/datasets/eee/runs/${run.id}.samples.jsonl` : null;++ const developer = firstText(route.maker, route.provider_name, (route.model && String(route.model).includes('/')) ? String(route.model).split('/')[0] : '');++ const generationArgs = {};++ if (settings.temperature != null) generationArgs.temperature = num(settings.temperature);++ if (settings.max_tokens != null && num(settings.max_tokens) > 0) generationArgs.max_tokens = num(settings.max_tokens);++ const result = {++ evaluation_result_id: `${run.id}:${run.benchmark_id}`,++ evaluation_name: `openvibe.bench.${run.benchmark_kind || 'run'}`,++ source_data: {++ dataset_name: String(run.benchmark_title || run.benchmark_id),++ source_type: 'other',++ additional_details: {++ benchmark_id: String(run.benchmark_id),++ benchmark_version: String(run.benchmark_version),++ benchmark_ref: benchmarkRef,++ task_count: String(sampleCount),++ ...(samplesUrl ? { samples_url: samplesUrl } : {}),++ },++ },++ metric_config: {++ metric_id: 'openvibe-bench.score',++ metric_name: 'Score',++ metric_kind: 'score',++ metric_unit: 'points',++ lower_is_better: false,++ score_type: 'continuous',++ min_score: 0,++ max_score: 100,++ },++ score_details: {++ score: num(run.score) == null ? 0 : num(run.score),++ ...(sampleCount ? { uncertainty: { num_samples: sampleCount } } : {}),++ },++ generation_config: {++ generation_args: generationArgs,++ additional_details: {++ ...(settings.seed != null ? { seed: String(settings.seed) } : {}),++ ...(settings.few_shot != null ? { few_shot: String(settings.few_shot) } : {}),++ ...(settings.effort != null ? { effort: String(settings.effort) } : {}),++ },++ },++ };++ return {++ schema_version: SCHEMA_VERSION,++ evaluation_id: `openvibe-bench/${run.id}`,++ evaluation_timestamp: createdAt,++ retrieved_timestamp: retrieved,++ source_metadata: {++ source_name: 'OpenVibe.Bench',++ source_type: 'evaluation_run',++ source_organization_name: 'OpenVibe',++ ...(baseUrl ? { source_organization_url: baseUrl } : {}),++ evaluator_relationship: 'third_party',++ additional_details: {++ run_id: String(run.id),++ provenance: String(run.provenance || ''),++ benchmark: benchmarkRef,++ provider: String(run.provider || ''),++ },++ },++ eval_library: {++ name: firstText(harness.name, 'openvibe-bench'),++ version: firstText(harness.version, 'unknown'),++ },++ model_info: {++ name: String(run.model || 'unknown'),++ id: String((route.model || run.model || 'unknown')),++ ...(developer ? { developer } : {}),++ ...(run.provider ? { inference_platform: String(run.provider) } : {}),++ additional_details: {++ deployment_type: 'externally_managed',++ model_availability: 'unknown',++ bench_model: String(run.model || ''),++ bench_provider: String(run.provider || ''),++ },++ },++ evaluation_results: [result],++ };++ }++++ /** Public Bench runs as EEE records, newest last, paged. `limit` is capped so one request stays small. */++ async function exportRecords({ offset = 0, limit = 500 } = {}) {++ const take = Math.max(1, Math.min(1000, Number(limit) || 500));++ const skip = Math.max(0, Number(offset) || 0);++ const runs = await db.many(++ `SELECT r.*, b.title AS benchmark_title, b.kind AS benchmark_kind++ FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id++ WHERE b.visibility = 'public' AND r.provenance = 'verified'++ ORDER BY r.created_at, r.id LIMIT $1 OFFSET $2`,++ [take, skip],++ );++ if (!runs.length) return [];++ const tasks = await db.many(++ 'SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id',++ [runs.map((run) => String(run.id))],++ );++ const byRun = new Map();++ for (const task of tasks) {++ if (!byRun.has(task.run_id)) byRun.set(task.run_id, []);++ byRun.get(task.run_id).push(task);++ }++ return runs.map((run) => eeeRecordOf(run, byRun.get(run.id) || []));++ }++++ async function exportCount() {++ const row = await db.maybe(++ "SELECT COUNT(*)::int AS n FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id WHERE b.visibility = 'public' AND r.provenance = 'verified'",++ );++ return num(row && row.n) || 0;++ }++++ async function exportLines(options = {}) {++ const records = await exportRecords(options);++ return records.length ? records.map((record) => JSON.stringify(record)).join('\n') + '\n' : '';++ }++++ /** The per-sample (instance-level) lines of one public run: Bench's per-task scores and where its answer is. */++ async function samplesOf(runId) {++ const run = await db.maybe(++ `SELECT r.*, b.title AS benchmark_title, b.kind AS benchmark_kind++ FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id++ WHERE r.id = $1 AND b.visibility = 'public' AND r.provenance = 'verified'`,++ [String(runId || '')],++ );++ if (!run) return null;++ const tasks = await db.many('SELECT * FROM bench_run_tasks WHERE run_id = $1 ORDER BY created_at, id', [run.id]);++ const evaluationId = `openvibe-bench/${run.id}`;++ return {++ run,++ lines: tasks.map((task) => JSON.stringify({++ evaluation_id: evaluationId,++ model_id: String(run.model || 'unknown'),++ sample_id: String(task.task_id),++ task_id: String(task.task_id),++ score: task.score == null ? null : num(task.score),++ cost_usd: num(task.cost_usd),++ ms: num(task.ms),++ tokens: num(task.tokens),++ ...(task.transcript_url ? { transcript_url: String(task.transcript_url) } : {}),++ artifacts: json(task.artifacts, []),++ created_at: iso(task.created_at),++ })),++ };++ }++++ return {++ SOURCE, sync, storeRecord, listRecords, countRecords, latestSync,++ exportRecords, exportLines, exportCount, samplesOf,++ listCollections, listFiles, normalizeRecord,++ };++}++++module.exports = {++ createEee, normalizeRecord, allowsRedistribution, recordLicenceOf, bareModel,++ datastoreLicence, wireExternal, SOURCE, DATASET, DATASET_URL, SCHEMA_VERSION,++};nnew file mode 100644@@@ -0,0 +1,96 @@++'use strict';++++/**++ * Stanford HELM results: a run's `stats.json` together with its `run_spec.json`.++ *++ * HELM writes one run directory per (scenario, method, model). `stats.json` is a list of stat objects — each has a++ * `name` ({ name, split, perturbation? }) and a `mean` — and `run_spec.json` says which model, scenario and method++ * produced them. This importer accepts the two together, either as { stats, run_spec } or as one document whose++ * top level is the run spec and whose `stats` key holds the list. One model here is one run; the run's scenario is++ * the task, the adapter's `max_train_instances` the few-shot.++ */++const { parseJson, asText } = require('./limits');++++const TOOL = 'helm';++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++const METRIC_ORDER = ['quasi_exact_match', 'exact_match', 'accuracy', 'quasi_prefix_exact_match', 'prefix_exact_match', 'f1', 'bleu', 'score'];++++function statsOf(doc) {++ if (Array.isArray(doc)) return { stats: doc, runSpec: {} };++ if (!doc || typeof doc !== 'object') return { stats: null, runSpec: {} };++ const runSpec = doc.run_spec && typeof doc.run_spec === 'object' ? doc.run_spec : doc;++ const stats = Array.isArray(doc.stats) ? doc.stats : (Array.isArray(doc) ? doc : null);++ return { stats, runSpec };++}++++/** The headline metric: the first in the preference list, from a plain split before a perturbed one. */++function metricOf(stats) {++ const usable = (stats || []).filter((s) => s && s.name && typeof s.name === 'object' && num(s.mean) != null);++ if (!usable.length) return null;++ const plain = usable.filter((s) => !s.name.perturbation);++ const splits = (list) => {++ const wanted = list.filter((s) => s.name.split === 'valid' || s.name.split === 'test' || s.name.split == null);++ return wanted.length ? wanted : list;++ };++ const pool = splits(plain.length ? plain : usable);++ for (const name of METRIC_ORDER) {++ const found = pool.find((s) => String(s.name.name) === name);++ if (found) return { metric: name, value: num(found.mean) };++ }++ const first = pool[0];++ return { metric: String(first.name.name || 'score'), value: num(first.mean) };++}++++function parse(bytes) {++ const doc = parseJson(asText(bytes), { what: 'the results' });++ const { stats, runSpec } = statsOf(doc);++ if (!Array.isArray(stats) || !stats.length) throw new Error('That file holds no HELM stats.');++ const adapter = (runSpec.adapter_spec && typeof runSpec.adapter_spec === 'object') ? runSpec.adapter_spec : {};++ const model = String(adapter.model || runSpec.model || '').trim();++ if (!model) throw new Error('That file has no run spec naming the model.');++ const metric = metricOf(stats);++ if (!metric) throw new Error('That file holds no scores I can read.');++ const scenario = runSpec.scenario_spec && typeof runSpec.scenario_spec === 'object' ? runSpec.scenario_spec : {};++ const args = (scenario.args && typeof scenario.args === 'object') ? scenario.args : {};++ const groups = Array.isArray(runSpec.groups) ? runSpec.groups : [];++ const task = String(args.dataset || groups[0] || String(runSpec.name || '').split(':')[0] || 'helm').trim();++ const deployment = String(adapter.model_deployment || '');++ const provider = deployment.includes('/') ? deployment.split('/')[0] : (model.includes('/') ? model.split('/')[0] : 'helm');++ const version = String(runSpec.helm_version || doc.helm_version || runSpec.version || '').trim();++ const score = metric.value <= 1 ? metric.value * 100 : metric.value;++ const taskVersion = String(runSpec.version || doc.version || '').trim();++ const seed = adapter.seed != null ? adapter.seed : adapter.random_seed;++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'helm', version: version || 'unknown' },++ date: runSpec.date || doc.date || null,++ suite: { name: task, version: taskVersion || null },++ models: [{++ model,++ provider: provider || 'helm',++ score,++ cost_usd: null,++ seed: seed == null ? null : String(seed),++ few_shot: num(adapter.max_train_instances),++ tasks: [{++ task_id: task,++ score,++ metric: metric.metric,++ few_shot: num(adapter.max_train_instances),++ task_version: taskVersion || null,++ higher_is_better: !/loss|perplexity|latency|runtime/i.test(metric.metric),++ }],++ }],++ };++}++++function looksLike(doc) {++ if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return false;++ if (!Array.isArray(doc.stats) || !doc.stats.length) return false;++ const first = doc.stats[0];++ return Boolean(first && first.name && typeof first.name === 'object' && ('mean' in first || 'count' in first));++}++++module.exports = { TOOL, parse, looksLike, metricOf };@@@ -1,27 +1,33 @@'use strict';/**-- * The four tools whose results a person can bring to Bench (B12): Inspect AI, Harbor, the EleutherAI-- * lm-evaluation-harness and promptfoo. The file says which one it is; when it does not, the shape of the-- * document says it.++ * The tools whose results a person can bring to Bench: Inspect AI, Harbor, HF lighteval, the EleutherAI++ * lm-evaluation-harness, Stanford HELM and promptfoo. The file says which one it is; when it does not, the shape of++ * the document says it.*/const { ImportError, MAX_UPLOAD_BYTES } = require('./limits');const inspectTool = require('./inspect');const harborTool = require('./harbor');++const lightevalTool = require('./lighteval');const lmEvalTool = require('./lm-eval');++const helmTool = require('./helm');const promptfooTool = require('./promptfoo');const TOOLS = {inspect: inspectTool,harbor: harborTool,++ lighteval: lightevalTool,'lm-eval': lmEvalTool,++ helm: helmTool,promptfoo: promptfooTool,};const TOOL_LABELS = {inspect: 'Inspect AI',harbor: 'Harbor',++ lighteval: 'HF lighteval','lm-eval': 'lm-evaluation-harness',++ helm: 'Stanford HELM',promptfoo: 'promptfoo',};@@@ -31,7 +37,9 @@ function detect(doc, filename = '') {if (name.endsWith('.eval')) return 'inspect';if (inspectTool.looksLike(doc)) return 'inspect';if (harborTool.looksLike(doc)) return 'harbor';++ if (lightevalTool.looksLike(doc)) return 'lighteval';if (lmEvalTool.looksLike(doc)) return 'lm-eval';++ if (helmTool.looksLike(doc)) return 'helm';if (promptfooTool.looksLike(doc)) return 'promptfoo';return null;}@@@ -55,9 +63,10 @@ function parse(bytes, { filename = '', tool = null } = {}) {out.filename = String(filename || '');return out;}-- // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others.++ // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others. lighteval++ // comes before lm-eval: both have `results`, and only lighteval has `config_general`/`config_tasks`.const errors = [];-- for (const key of ['inspect', 'harbor', 'lm-eval', 'promptfoo']) {++ for (const key of ['inspect', 'harbor', 'lighteval', 'lm-eval', 'helm', 'promptfoo']) {try {const out = TOOLS[key].parse(buf);out.filename = String(filename || '');@@@ -68,7 +77,7 @@ function parse(bytes, { filename = '', tool = null } = {}) {}}throw new ImportError(422, 'bench.import_unknown_format',-- 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, an lm-evaluation-harness results.json or a promptfoo output.json.');++ 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, a lighteval results file, an lm-evaluation-harness results.json, a HELM stats file or a promptfoo output.json.');}module.exports = { TOOLS, TOOL_LABELS, detect, parse, ImportError, MAX_UPLOAD_BYTES };nnew file mode 100644@@@ -0,0 +1,101 @@++'use strict';++++/**++ * Hugging Face lighteval results: the `results_<model>.json` that `lighteval` writes.++ *++ * The file is { config_general, results, versions, config_tasks }. `results` maps a task key++ * ("arc:challenge|25" — the task, then the few-shot count) to its metrics. `config_tasks` names the task, its HF++ * dataset and its metric (and whether a higher score is better). One file is one model, so one model here is one run;++ * every task key is a task of that run, with its own few-shot and task version.++ */++const { parseJson, asText } = require('./limits');++++const TOOL = 'lighteval';++const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));++++const METRIC_ORDER = ['acc', 'exact_match', 'acc_norm', 'f1', 'pass@1', 'quasi_exact_match', 'accuracy', 'em'];++++/** The metric of a task that means "how good was it", skipping its error bar. */++function metricOf(entry) {++ if (!entry || typeof entry !== 'object') return null;++ for (const key of METRIC_ORDER) {++ if (num(entry[key]) != null) return { metric: key, value: num(entry[key]) };++ }++ for (const [key, value] of Object.entries(entry)) {++ if (/_stderr$|stderr|std_err/i.test(key)) continue;++ const n = num(value);++ if (n != null) return { metric: key, value: n };++ }++ return null;++}++++/** "arc:challenge|25" -> { task: "arc:challenge", fewShot: 25 }. */++function keyOf(key) {++ const text = String(key || '');++ const parts = text.split('|');++ return { task: parts[0].trim(), fewShot: num(parts[1]) };++}++++function parse(bytes) {++ const doc = parseJson(asText(bytes), { what: 'the results' });++ const results = doc && doc.results && typeof doc.results === 'object' && !Array.isArray(doc.results) ? doc.results : null;++ if (!results) throw new Error('That file has no results.');++ const general = (doc.config_general && typeof doc.config_general === 'object') ? doc.config_general : {};++ const taskConfigs = (doc.config_tasks && typeof doc.config_tasks === 'object') ? doc.config_tasks : {};++ const versions = (doc.versions && typeof doc.versions === 'object') ? doc.versions : {};++ const model = String(general.model_name || (general.model_config && general.model_config.model_name) || doc.model_name || '').trim();++ if (!model) throw new Error('That file names no model.');++++ const tasks = [];++ for (const [key, entry] of Object.entries(results)) {++ const metric = metricOf(entry);++ if (!metric) continue;++ const { task, fewShot } = keyOf(key);++ const config = taskConfigs[key] || taskConfigs[task] || {};++ const taskVersion = versions[key] != null ? versions[key] : (versions[task] != null ? versions[task] : null);++ tasks.push({++ task_id: task || key,++ score: metric.value * 100,++ metric: metric.metric,++ few_shot: fewShot,++ task_version: taskVersion == null ? null : String(taskVersion),++ hf_repo: config.hf_repo ? String(config.hf_repo) : null,++ higher_is_better: config.metrics && Array.isArray(config.metrics) && config.metrics[0]++ ? Boolean(config.metrics[0].higher_is_better)++ : null,++ });++ }++ if (!tasks.length) throw new Error('That file holds no scores I can read.');++++ const provider = model.includes('/') ? model.split('/')[0] : 'lighteval';++ const version = String(general.lighteval_sha || general.lighteval_version || doc.version || '').trim();++ const seed = general.model_config && general.model_config.generation_parameters++ ? general.model_config.generation_parameters.seed++ : null;++ const overall = tasks.reduce((n, t) => n + t.score, 0) / tasks.length;++ const distinct = [...new Set(tasks.map((t) => t.task_id))];++ return {++ tool: TOOL,++ toolVersion: version || 'unknown',++ harness: { name: 'lighteval', version: version || 'unknown' },++ date: general.pretty_timestamp || general.date || null,++ suite: distinct.length === 1 ? { name: distinct[0], version: tasks[0].task_version } : null,++ models: [{++ model,++ provider,++ score: overall,++ cost_usd: null,++ seed: seed == null ? null : String(seed),++ few_shot: tasks.length && tasks[0].few_shot != null ? tasks[0].few_shot : null,++ tasks,++ }],++ };++}++++function looksLike(doc) {++ if (!doc || typeof doc !== 'object') return false;++ if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) return false;++ return Boolean(doc.config_general || doc.config_tasks);++}++++module.exports = { TOOL, parse, looksLike, metricOf };@@@ -89,7 +89,17 @@ function createImports({ s, registry, log = console }) {score: m.score == null ? null : num(m.score),cost_usd: m.cost_usd == null ? null : num(m.cost_usd),tokens: m.tokens == null ? null : num(m.tokens),-- tasks: (m.tasks || []).map((t) => ({ task_id: String(t.task_id), score: t.score == null ? null : num(t.score) })),++ // B7: the harness's own settings travel with the import: the run's seed and few-shot, and each++ // task's version and metric, so a result can be compared at a glance with the tool that made it.++ seed: m.seed == null ? null : String(m.seed),++ few_shot: m.few_shot == null ? null : num(m.few_shot),++ tasks: (m.tasks || []).map((t) => ({++ task_id: String(t.task_id),++ score: t.score == null ? null : num(t.score),++ task_version: t.task_version == null ? null : String(t.task_version),++ few_shot: t.few_shot == null ? null : num(t.few_shot),++ metric: t.metric == null ? null : String(t.metric),++ })),})),};await db.exec(@@@ -202,13 +212,23 @@ function createImports({ s, registry, log = console }) {.filter((t) => known.has(t.task_id)).map((t) => ({ task_id: t.task_id, score: t.score == null ? null : num(t.score), cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }));if (!tasks.length) continue;++ // B7: keep what the harness recorded beside the scores: the task versions, the few-shot and the seed.++ const taskVersions = {};++ for (const task of model.tasks || []) {++ if (task.task_version) taskVersions[String(task.task_id)] = String(task.task_version);++ }++ const settings = {++ ...(model.seed != null ? { seed: String(model.seed) } : {}),++ ...(model.few_shot != null ? { few_shot: num(model.few_shot) } : {}),++ ...(Object.keys(taskVersions).length ? { task_versions: taskVersions } : {}),++ };const published = await registry.publishRun({benchmark: benchmark.id + '@' + version,model: String(model.model || 'unknown').slice(0, 160),provider: String(model.provider || 'unknown').slice(0, 160),route: {},harness: summary.harness || { name: toolLabel, version: summary.tool_version || 'unknown' },-- settings: {},++ settings,tasks,score: model.score == null ? null : num(model.score),cost_usd: num(model.cost_usd),@@@ -136,6 +136,16 @@ function load(env = process.env) {// How often the job looks at the clock.pollMs: Math.max(30_000, int(env.BENCH_DAILY_POLL_MS, 300_000)),},++ // B2: the weekly Every Eval Ever import. It reads a public datastore (no account, no key), so it is safe++ // to run, but it is the owner's switch: off until BENCH_EEE_SYNC is on. One pass is bounded by the two++ // caps, and the cursor in bench_eee_syncs makes a stopped pass resume.++ eee: {++ enabled: bool(env.BENCH_EEE_SYNC, false),++ enabledInDrill: bool(env.BENCH_EEE_SYNC_IN_DRILL, false),++ intervalMs: Math.max(60_000, int(env.BENCH_EEE_SYNC_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),++ filesPerRun: Math.max(1, int(env.BENCH_EEE_FILES_PER_RUN, 5000)),++ recordsPerRun: Math.max(1, int(env.BENCH_EEE_RECORDS_PER_RUN, 50000)),++ },},};}@@@ -54,7 +54,7 @@ function createDiscoveryRoutes(ctx) {summary: DESCRIPTION,markdown: true,base: site,-- details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. GET /api/v1/datasets/benchmarks.jsonl is every public benchmark and its runs. POST /mcp is the Model Context Protocol endpoint.',++ details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. GET /api/v1/datasets/benchmarks.jsonl is every public benchmark and its runs. GET /api/v1/datasets/eee is Bench’s public runs as Every Eval Ever records (JSONL, paged), and GET /api/v1/external/records is the imported Every Eval Ever records with their provenance. POST /mcp is the Model Context Protocol endpoint.',sections: [{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }] },nnew file mode 100644@@@ -0,0 +1,100 @@++'use strict';++++/**++ * The EEE routes (B2).++ *++ * GET /api/v1/datasets/eee Bench's public runs as EEE `evaluation_run` records (JSONL, paged)++ * GET /api/v1/datasets/eee/runs/:id.samples.jsonl one run's per-sample scores++ * GET /api/v1/external/records the imported EEE records, each `external: true` with its provenance++ * POST /api/v1/admin/eee/sync one bounded pass of the weekly import (operator token)++ *++ * Everything here is public except the sync, which is the operator's own switch and reads the public datastore.++ */++const crypto = require('crypto');++const { asyncRouter } = require('./router');++++function operatorToken(req) {++ const h = String(req.get('authorization') || '');++ return /^Bearer\s+\S/i.test(h) ? h.replace(/^Bearer\s+/i, '').trim() : null;++}++++function tokenMatches(given, expected) {++ if (!given || !expected) return false;++ const a = crypto.createHash('sha256').update(String(given), 'utf8').digest();++ const b = crypto.createHash('sha256').update(String(expected), 'utf8').digest();++ return a.length === b.length && crypto.timingSafeEqual(a, b);++}++++function createEeeRoutes(ctx) {++ const r = asyncRouter();++ const eee = ctx.eee;++ const token = process.env.BENCH_ADMIN_TOKEN || '';++++ const paging = (req) => ({++ offset: Math.max(0, Number(req.query.offset) || 0),++ limit: Math.max(1, Math.min(1000, Number(req.query.limit) || 500)),++ });++++ const jsonl = (res, text, name) => {++ res.set('Cache-Control', 'public, max-age=60');++ res.set('Content-Disposition', `attachment; filename="${name}"`);++ return res.type('application/x-ndjson').send(text);++ };++++ r.get('/api/v1/datasets/eee', async (req, res) => {++ const { offset, limit } = paging(req);++ const [text, total] = await Promise.all([eee.exportLines({ offset, limit }), eee.exportCount()]);++ res.set('X-Total-Count', String(total));++ return jsonl(res, text, 'bench-eee.jsonl');++ });++++ r.get('/api/v1/datasets/eee.jsonl', async (req, res) => {++ const { offset, limit } = paging(req);++ const [text, total] = await Promise.all([eee.exportLines({ offset, limit }), eee.exportCount()]);++ res.set('X-Total-Count', String(total));++ return jsonl(res, text, 'bench-eee.jsonl');++ });++++ r.get('/api/v1/datasets/eee/runs/:id.samples.jsonl', async (req, res) => {++ const out = await eee.samplesOf(String(req.params.id || ''));++ if (!out) return res.status(404).type('text/plain').send('No such public run.');++ res.set('X-Total-Count', String(out.lines.length));++ return jsonl(res, out.lines.length ? out.lines.join('\n') + '\n' : '', `${out.run.id}-samples.jsonl`);++ });++++ // The external records: imported, never rated, always labelled with where they came from.++ r.get('/api/v1/external/records', async (req, res) => {++ const { offset, limit } = paging(req);++ const filters = {++ model: req.query.model ? String(req.query.model) : null,++ task: req.query.task ? String(req.query.task) : null,++ sourceType: req.query.source_type ? String(req.query.source_type) : null,++ };++ const [records, total] = await Promise.all([++ eee.listRecords({ ...filters, offset, limit }),++ eee.countRecords(filters),++ ]);++ res.set('Cache-Control', 'public, max-age=60');++ return res.json({ count: records.length, total, offset, limit, external: true, records });++ });++++ r.get('/api/v1/external/sync', async (_req, res) => {++ res.set('Cache-Control', 'public, max-age=30');++ return res.json({ sync: await eee.latestSync() });++ });++++ r.post('/api/v1/admin/eee/sync', async (req, res) => {++ if (!token) return res.status(404).type('text').send('No such route.');++ if (!tokenMatches(operatorToken(req), token)) return res.status(403).type('text').send('The operator token is required.');++ const body = req.body && typeof req.body === 'object' ? req.body : {};++ const out = await eee.sync({++ maxFiles: body.max_files ? Math.max(1, Math.min(20000, Number(body.max_files))) : undefined,++ maxRecords: body.max_records ? Math.max(1, Math.min(200000, Number(body.max_records))) : undefined,++ });++ return res.json(out);++ });++++ return r;++}++++module.exports = { createEeeRoutes, tokenMatches };@@@ -31,7 +31,7 @@ function createImportRoutes(ctx) {return page(req, res, {index: false,title: 'Import a results file',-- description: 'Bring the results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo, and see the runs it holds before they are imported.',++ description: 'Bring the results file of Inspect AI, Harbor, HF lighteval, lm-evaluation-harness, Stanford HELM or promptfoo, and see the runs it holds before they are imported.',body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, imports, error: req.query.error ? String(req.query.error).slice(0, 200) : null }))),});};@@@ -42,7 +42,7 @@ function createImportRoutes(ctx) {const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');const fail = (status, error) => page(req, res, {index: false, title: 'Import a results file',-- description: 'Bring a results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo.',++ description: 'Bring a results file of Inspect AI, Harbor, HF lighteval, lm-evaluation-harness, Stanford HELM or promptfoo.',body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, error }))),}, status);if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');@@@ -16,6 +16,7 @@ function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }() => ctx.keys.client.stop(),() => ctx.jobs.stop(),() => ctx.daily.stop(),++ () => ctx.eeeSync.stop(),() => ctx.publisher.stop(),async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); },() => ctx.s.close(),@@@ -45,6 +46,8 @@ async function start() {ctx.jobs.start();// The daily rounds (one per design kind per day). Off until BENCH_DAILY_ROUNDS is on; never under the drill.ctx.daily.start();++ // The weekly Every Eval Ever import (B2). Off until BENCH_EEE_SYNC is on; bounded and resumable.++ ctx.eeeSync.start();// Outbox publisher → OpenVibe.Events.ctx.publisher.start();@@@ -9,7 +9,9 @@ const { html, raw } = require('../render/html');const TOOLS = [{ key: 'inspect', label: 'Inspect AI', hint: 'an .eval log (a zip) or its JSON log' },{ key: 'harbor', label: 'Harbor', hint: 'the results.json of a Harbor job' },++ { key: 'lighteval', label: 'HF lighteval', hint: 'the results_<model>.json of a run' },{ key: 'lm-eval', label: 'lm-evaluation-harness', hint: 'the results.json of a run' },++ { key: 'helm', label: 'Stanford HELM', hint: 'a run’s stats.json with its run_spec.json' },{ key: 'promptfoo', label: 'promptfoo', hint: 'the output.json of promptfoo eval -o' },];@@@ -61,6 +61,8 @@ ${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&cate${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}++${ep('GET', '/api/v1/datasets/eee?offset=0&limit=500', 'Bench’s public runs as Every Eval Ever <code>evaluation_run</code> records (JSONL, paged; <code>/datasets/eee.jsonl</code> is the same). Each run’s per-sample scores are at <code>/datasets/eee/runs/:id.samples.jsonl</code>.')}++${ep('GET', '/api/v1/external/records?model=&task=&source_type=', 'The imported Every Eval Ever records: each is <code>external: true</code>, with its source, licence, harness and version, task and version, few-shot, metric and score. They never enter Bench’s ratings.')}</dl></section><section><h2>Write</h2><dl class="bm-eps">${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}nnew file mode 100644@@@ -0,0 +1,275 @@++'use strict';++++/**++ * Every Eval Ever (B2): importing the EEE datastore as external results, the licence gate, the resumable weekly++ * sync, the exporter of Bench's own runs as EEE, and the routes that serve both.++ */++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { boot, check, done } = require('./helpers/boot');++const eeeLib = require('../server/bench/eee');++const { createEeeSync } = require('../server/bench/eee-sync');++require('../server/http/eee');++++// The operator routes read the token from the environment at mount time (as server/http/mirror.js does).++process.env.BENCH_ADMIN_TOKEN = process.env.BENCH_ADMIN_TOKEN || 'op-secret';++++const FIX = path.join(__dirname, 'fixtures', 'eee');++const record = () => JSON.parse(fs.readFileSync(path.join(FIX, 'record.json'), 'utf8'));++++function definition(over = {}) {++ return {++ title: 'EEE export benchmark',++ description: 'A public benchmark for the exporter.',++ kind: 'text.answer',++ tags: ['test'],++ visibility: 'public',++ tasks: [++ { id: 't1', prompt: 'One.', files: [], expected: null, holdout: false },++ { id: 't2', prompt: 'Two.', files: [], expected: null, holdout: false },++ ],++ judges: [],++ scoring: { method: 'mean', scale: { min: 0, max: 100 } },++ limits: {},++ matrix: {},++ ...over,++ };++}++++(async () => {++ // ── the join rule and the licence gate ───────────────────────────────────────────────────────────────────++ await check('a model name drops the maker prefix and a date or channel suffix, as Report joins it', () => {++ assert.strictEqual(eeeLib.bareModel('openai/gpt-5-nano-2025-08-07'), 'gpt-5-nano');++ assert.strictEqual(eeeLib.bareModel('anthropic/claude-3-7-sonnet-20250219'), 'claude-3-7-sonnet');++ assert.strictEqual(eeeLib.bareModel('meta-llama/Llama-3.1-8B-Instruct'), 'llama-3.1-8b-instruct');++ assert.strictEqual(eeeLib.bareModel('gpt-4o-latest'), 'gpt-4o');++ assert.strictEqual(eeeLib.bareModel('openai/gpt-5-nano'), 'gpt-5-nano');++ });++++ await check('a licence that forbids redistribution is refused, a stated open one is kept', () => {++ assert.ok(eeeLib.allowsRedistribution('MIT'));++ assert.ok(eeeLib.allowsRedistribution('apache-2.0'));++ assert.ok(eeeLib.allowsRedistribution('cc-by-4.0'));++ assert.ok(eeeLib.allowsRedistribution('cc-by-nc-4.0'));++ assert.ok(!eeeLib.allowsRedistribution('All rights reserved'));++ assert.ok(!eeeLib.allowsRedistribution('proprietary'));++ assert.ok(!eeeLib.allowsRedistribution('research only'));++ assert.ok(!eeeLib.allowsRedistribution('unknown'));++ assert.ok(!eeeLib.allowsRedistribution(''));++ });++++ await check('one EEE record is read into its results, with provenance and the join name', () => {++ const doc = record();++ const parsed = eeeLib.normalizeRecord(doc, { path: 'data/IFEval/x.json' });++ assert.strictEqual(parsed.record.eee_id, doc.evaluation_id);++ assert.strictEqual(parsed.record.source_type, 'documentation');++ assert.strictEqual(parsed.record.model, 'llama-pro-8b-instruct');++ assert.strictEqual(parsed.record.model_raw, 'TencentARC/LLaMA-Pro-8B-Instruct');++ assert.strictEqual(parsed.record.developer, 'TencentARC');++ assert.strictEqual(parsed.results.length, 6);++ const ifeval = parsed.results.find((r) => r.task === 'IFEval');++ assert.ok(ifeval, 'IFEval is there');++ assert.strictEqual(ifeval.score, 0.4486);++ assert.strictEqual(ifeval.metric_name, 'score');++ assert.match(ifeval.source_url, /^https:\/\//);++ assert.strictEqual(parsed.record.licence, 'MIT', 'the datastore card licence is the fallback');++ });++++ await check('a record that states a non-redistributable licence is dropped, and the reason is recorded', () => {++ const doc = record();++ doc.source_metadata.additional_details = { license: 'All rights reserved' };++ const parsed = eeeLib.normalizeRecord(doc);++ assert.strictEqual(parsed.results.length, 6);++ assert.ok(parsed.results.every((r) => r.dropped === 'licence'), 'every result is dropped for licence');++ });++++ await check('few-shot, task version, seed and the harness are kept when the record states them', () => {++ const doc = record();++ doc.eval_library = { name: 'inspect_ai', version: '0.3.75' };++ doc.evaluation_results[0].source_data.additional_details = { task_version: '2', num_fewshot: 5 };++ doc.evaluation_results[0].generation_config = { generation_args: { temperature: 0.2, seed: 7 } };++ const parsed = eeeLib.normalizeRecord(doc);++ const first = parsed.results[0];++ assert.strictEqual(parsed.record.harness_name, 'inspect_ai');++ assert.strictEqual(parsed.record.harness_version, '0.3.75');++ assert.strictEqual(first.task_version, '2');++ assert.strictEqual(first.few_shot, 5);++ assert.strictEqual(first.seed, '7');++ });++++ await check('the weekly job starts only when the owner turns it on', async () => {++ const settings = { intervalMs: 60_000, filesPerRun: 1, recordsPerRun: 1 };++ const off = createEeeSync({++ eee: { sync: async () => ({ status: 'finished' }) },++ config: { drill: true, bench: { eee: { ...settings, enabled: true, enabledInDrill: false } } },++ log: { warn() {} },++ });++ assert.strictEqual(off.start(), null, 'never under the drill');++ const on = createEeeSync({++ eee: { sync: async () => ({ status: 'finished' }) },++ config: { drill: true, bench: { eee: { ...settings, enabled: true, enabledInDrill: true } } },++ log: { warn() {} },++ });++ assert.ok(on.start(), 'a timer when it is allowed');++ on.stop();++ });++++ const t = await boot({ env: { BENCH_ADMIN_TOKEN: 'op-secret' } });++ // A public benchmark with one verified run, for the exporter.++ const made = await t.ctx.registry.createBenchmark({++ owner: 'openvibe', title: 'EEE export benchmark', kind: 'text.answer', visibility: 'public', subject: 'openvibe',++ });++ assert.ok(made.ok, made.message);++ const frozen = await t.ctx.registry.freezeVersion(made.benchmark.id, { definition: definition(), subject: 'openvibe' });++ assert.ok(frozen.ok, frozen.message);++ const published = await t.ctx.registry.publishRun({++ benchmark: `${made.benchmark.id}@${frozen.version.version}`,++ model: 'openai/gpt-5-nano-2025-08-07', provider: 'openai',++ route: { provider: 'openai', model: 'openai/gpt-5-nano' },++ harness: { name: 'lm-evaluation-harness', version: '0.4.2' },++ settings: { temperature: 0, seed: '7', few_shot: 5 },++ tasks: [++ { task_id: 't1', score: 80, cost_usd: 0.01, ms: 1000, tokens: 100, artifacts: [] },++ { task_id: 't2', score: 60, cost_usd: 0.02, ms: 2000, tokens: 200, transcript_url: 'https://example.test/t2' },++ ],++ score: 70, cost_usd: 0.03, ms: 3000, tokens: 300,++ }, { provenance: 'verified', subject: 'service:codes' });++ assert.ok(published.ok, published.message);++++ await check('the imported record lands as external rows, deduped, with external: true', async () => {++ const eee = t.ctx.eee;++ const doc = record();++ const first = await eee.storeRecord(doc, { path: 'data/IFEval/x.json', readAt: t.ctx.s.iso() });++ assert.strictEqual(first.read, 6);++ assert.strictEqual(first.kept, 6);++ const again = await eee.storeRecord(doc, { path: 'data/IFEval/x.json', readAt: t.ctx.s.iso() });++ assert.strictEqual(again.kept, 6, 'the second store updates the same rows');++ assert.strictEqual(await eee.countRecords(), 6, 'no duplicate rows');++++ const rows = await eee.listRecords({ task: 'IFEval' });++ assert.strictEqual(rows.length, 1);++ assert.strictEqual(rows[0].external, true);++ assert.strictEqual(rows[0].licence, 'MIT');++ assert.strictEqual(rows[0].source_type, 'documentation');++ assert.strictEqual(rows[0].model, 'llama-pro-8b-instruct');++ assert.ok(rows[0].provenance.source_metadata, 'the record provenance travels with it');++ });++++ await check('the external records are served as JSON, marked external, and never rated', async () => {++ const out = await t.get('/api/v1/external/records?task=IFEval');++ assert.strictEqual(out.status, 200, out.text.slice(0, 200));++ const body = out.json();++ assert.strictEqual(body.external, true);++ assert.strictEqual(body.total, 1);++ assert.strictEqual(body.records[0].external, true);++ assert.strictEqual(body.records[0].score, 0.4486);++ // Not in the ratings: nothing external is a Bench run.++ const rated = await t.ctx.s.db.maybe("SELECT COUNT(*)::int AS n FROM bench_runs WHERE model = 'llama-pro-8b-instruct'");++ assert.strictEqual(rated.n, 0, 'an external record never becomes a Bench run');++ });++++ await check('a public Bench run is exported as an EEE evaluation_run record', async () => {++ const records = await t.ctx.eee.exportRecords({ limit: 10 });++ const mine = records.find((r) => r.evaluation_id === `openvibe-bench/${published.run.id}`);++ assert.ok(mine, 'the run is there');++ assert.strictEqual(mine.schema_version, '0.3.0');++ assert.strictEqual(mine.source_metadata.source_type, 'evaluation_run');++ assert.strictEqual(mine.source_metadata.source_name, 'OpenVibe.Bench');++ assert.strictEqual(mine.eval_library.name, 'lm-evaluation-harness');++ assert.strictEqual(mine.eval_library.version, '0.4.2');++ assert.strictEqual(mine.model_info.name, 'openai/gpt-5-nano-2025-08-07');++ assert.strictEqual(mine.evaluation_results.length, 1);++ assert.strictEqual(mine.evaluation_results[0].score_details.score, 70);++ assert.strictEqual(mine.evaluation_results[0].score_details.uncertainty.num_samples, 2);++ assert.strictEqual(mine.evaluation_results[0].metric_config.min_score, 0);++ assert.strictEqual(mine.evaluation_results[0].metric_config.max_score, 100);++ assert.ok(mine.evaluation_results[0].source_data.additional_details.samples_url, 'the per-sample URL is named');++ assert.ok(!('extra' in mine), 'no top-level key outside the EEE schema');++ // It is importable as EEE too: the round trip is the schema check a test can make without a validator.++ const back = t.ctx.eee.normalizeRecord(mine);++ assert.strictEqual(back.results.length, 1);++ assert.strictEqual(back.record.model, 'gpt-5-nano');++ });++++ await check('the EEE dataset is served as paged JSONL, and the samples as their own JSONL', async () => {++ const out = await t.get('/api/v1/datasets/eee?limit=10');++ assert.strictEqual(out.status, 200, out.text.slice(0, 200));++ assert.match(String(out.headers.get('content-type')), /ndjson/);++ const total = Number(out.headers.get('x-total-count'));++ assert.ok(total >= 1, 'there is at least the one public run');++ const lines = out.text.trim().split('\n').filter(Boolean).map((line) => JSON.parse(line));++ assert.ok(lines.some((r) => r.evaluation_id === `openvibe-bench/${published.run.id}`));++ assert.ok(lines.every((r) => r.source_metadata.source_type === 'evaluation_run'));++++ const page = await t.get('/api/v1/datasets/eee?limit=1&offset=1');++ assert.strictEqual(page.status, 200);++ assert.ok(page.text.trim().split('\n').filter(Boolean).length <= 1, 'the page is bounded');++++ // The .jsonl twin is not read as the kind "eee" (the kind dataset route is mounted after this one).++ const twin = await t.get('/api/v1/datasets/eee.jsonl?limit=1');++ assert.strictEqual(twin.status, 200, twin.text.slice(0, 200));++ assert.match(String(twin.headers.get('content-type')), /ndjson/);++++ const samples = await t.get(`/api/v1/datasets/eee/runs/${published.run.id}.samples.jsonl`);++ assert.strictEqual(samples.status, 200, samples.text.slice(0, 200));++ const rows = samples.text.trim().split('\n').filter(Boolean).map((line) => JSON.parse(line));++ assert.strictEqual(rows.length, 2);++ assert.strictEqual(rows[0].model_id, 'openai/gpt-5-nano-2025-08-07');++ assert.ok(rows.some((r) => r.transcript_url === 'https://example.test/t2'));++ });++++ await check('the sync is the operator\'s route only', async () => {++ const anon = await t.get('/api/v1/admin/eee/sync', { method: 'POST', json: {} });++ assert.strictEqual(anon.status, 403, anon.text.slice(0, 120));++ });++++ // ── the weekly sync: bounded, resumable, and the licence gate counted ───────────────────────────────────────++ await check('a bounded sync resumes from its cursor and counts what it dropped for licence', async () => {++ const good = record();++ const bad = record();++ bad.evaluation_id = 'hfopenllm_v2/TencentARC_LLaMA-Pro-8B-Instruct/1770682499.0';++ bad.source_metadata = { ...bad.source_metadata, additional_details: { license: 'All rights reserved' } };++ const files = {++ 'data/IFEval/a/good.json': good,++ 'data/IFEval/a/bad.json': bad,++ };++ const fetchImpl = async (url) => {++ const text = String(url);++ const ok = (body, type = 'json') => ({++ ok: true, status: 200,++ json: async () => (type === 'json' ? body : JSON.parse(body)),++ text: async () => (typeof body === 'string' ? body : JSON.stringify(body)),++ });++ if (text.endsWith('/README.md')) return ok('license: mit');++ if (text.includes('/tree/main/data?recursive=false')) return ok([{ type: 'directory', path: 'data/IFEval' }]);++ if (text.includes('/tree/main/data/IFEval?recursive=true')) {++ return ok(Object.keys(files).map((p) => ({ type: 'file', path: p })));++ }++ for (const [p, doc] of Object.entries(files)) if (text.endsWith('/' + p)) return ok(doc);++ return { ok: false, status: 404, json: async () => ({}), text: async () => '' };++ };++ const eee = eeeLib.createEee({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} }, config: t.ctx.config });++++ const first = await eee.sync({ maxFiles: 1, maxRecords: 100 });++ assert.strictEqual(first.status, 'running', 'one file is one pass');++ assert.strictEqual(first.files_seen, 1);++ assert.strictEqual(first.records_kept, 0, 'the first file (a/bad.json) is dropped for its licence');++ assert.strictEqual(first.dropped_licence, 6);++ assert.deepStrictEqual(first.cursor, { collection: 0, file: 1 });++++ const second = await eee.sync({ maxFiles: 1, maxRecords: 100 });++ assert.strictEqual(second.id, first.id, 'it resumes the same sync');++ assert.strictEqual(second.files_seen, 2);++ assert.strictEqual(second.dropped_licence, 6, 'the bad record is counted, not stored');++ assert.strictEqual(second.records_kept, 6, 'the good record lands');++ assert.strictEqual(second.status, 'finished');++ assert.ok(second.dropped_reason && Object.keys(second.dropped_reason).some((k) => /licence/.test(k)), 'the reason is kept');++ });++++ await t.close();++ done();++})().catch((err) => {++ process.stderr.write('The test file failed: ' + ((err && err.stack) || err) + '\n');++ process.exit(1);++});nnew file mode 100644@@@ -0,0 +1,19 @@++# Fixtures, and their licences++++These files are real, small samples from the public sources the importers read. They are test data, not part of++Bench's data.++++- `eee/record.json` — one record from Every Eval Ever (EvalEval coalition), taken verbatim from++ `every_eval_ever/tests/data/98ea850e-7019-4728-a558-8b1819ec47c2.json` in++ <https://github.com/evaleval/every_eval_ever>. The EEE datastore and its records are published under the++ dataset card's MIT licence (<https://huggingface.co/datasets/evaleval/EEE_datastore>); the repository is++ Apache-2.0.++- `imports/lighteval-results.json` — a subset of++ `tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json` from++ <https://github.com/huggingface/lighteval> (MIT). Four task entries are kept verbatim; the rest are removed.++- `imports/helm-results.json` — the real `stats.json` and `run_spec.json` of++ `commonsense-dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0` from++ <https://github.com/stanford-crfm/helm> (Apache-2.0), combined under `{ stats, run_spec }` as the importer reads++ them. They were reached through the EEE repository's test data.++- `imports/{inspect,harbor,lm-eval,promptfoo}-*.json` — the earlier B12 fixtures; their sources are named in the++ tests.nnew file mode 100644@@@ -0,0 +1,132 @@++{++ "schema_version": "0.2.0",++ "evaluation_id": "hfopenllm_v2/TencentARC_LLaMA-Pro-8B-Instruct/1770682486.623709",++ "retrieved_timestamp": "1770682486.623709",++ "source_metadata": {++ "source_name": "HF Open LLM v2",++ "source_type": "documentation",++ "source_organization_name": "Hugging Face",++ "evaluator_relationship": "third_party"++ },++ "model_info": {++ "name": "LLaMA-Pro-8B-Instruct",++ "id": "TencentARC/LLaMA-Pro-8B-Instruct",++ "developer": "TencentARC",++ "inference_platform": "unknown",++ "additional_details": {++ "precision": "bfloat16",++ "architecture": "LlamaForCausalLM",++ "params_billions": 8.357++ }++ },++ "evaluation_results": [++ {++ "evaluation_name": "IFEval",++ "source_data": {++ "dataset_name": "IFEval",++ "source_type": "hf_dataset",++ "hf_repo": "google/IFEval"++ },++ "metric_config": {++ "evaluation_description": "Accuracy on IFEval",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.4486++ }++ },++ {++ "evaluation_name": "BBH",++ "source_data": {++ "dataset_name": "BBH",++ "source_type": "hf_dataset",++ "hf_repo": "SaylorTwift/bbh"++ },++ "metric_config": {++ "evaluation_description": "Accuracy on BBH",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.4224++ }++ },++ {++ "evaluation_name": "MATH Level 5",++ "source_data": {++ "dataset_name": "MATH Level 5",++ "source_type": "hf_dataset",++ "hf_repo": "DigitalLearningGmbH/MATH-lighteval"++ },++ "metric_config": {++ "evaluation_description": "Exact Match on MATH Level 5",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.0249++ }++ },++ {++ "evaluation_name": "GPQA",++ "source_data": {++ "dataset_name": "GPQA",++ "source_type": "hf_dataset",++ "hf_repo": "Idavidrein/gpqa"++ },++ "metric_config": {++ "evaluation_description": "Accuracy on GPQA",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.2743++ }++ },++ {++ "evaluation_name": "MUSR",++ "source_data": {++ "dataset_name": "MUSR",++ "source_type": "hf_dataset",++ "hf_repo": "TAUR-Lab/MuSR"++ },++ "metric_config": {++ "evaluation_description": "Accuracy on MUSR",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.419++ }++ },++ {++ "evaluation_name": "MMLU-PRO",++ "source_data": {++ "dataset_name": "MMLU-PRO",++ "source_type": "hf_dataset",++ "hf_repo": "TIGER-Lab/MMLU-Pro"++ },++ "metric_config": {++ "evaluation_description": "Accuracy on MMLU-PRO",++ "lower_is_better": false,++ "score_type": "continuous",++ "min_score": 0.0,++ "max_score": 1.0++ },++ "score_details": {++ "score": 0.1946++ }++ }++ ]++}\\ No newline at end of filennew file mode 100644@@@ -0,0 +1,1499 @@++{++ "stats": [++ {++ "name": {++ "name": "num_references",++ "split": "valid"++ },++ "count": 1,++ "sum": 4.0,++ "sum_squared": 16.0,++ "min": 4.0,++ "max": 4.0,++ "mean": 4.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_trials",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_prompt_tokens",++ "split": "valid"++ },++ "count": 1,++ "sum": 631.4,++ "sum_squared": 398665.95999999996,++ "min": 631.4,++ "max": 631.4,++ "mean": 631.4,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_completion_tokens",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_output_tokens",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "training_co2_cost",++ "split": "valid"++ },++ "count": 0,++ "sum": 0,++ "sum_squared": 0++ },++ {++ "name": {++ "name": "training_energy_cost",++ "split": "valid"++ },++ "count": 0,++ "sum": 0,++ "sum_squared": 0++ },++ {++ "name": {++ "name": "inference_runtime",++ "split": "valid"++ },++ "count": 1,++ "sum": 14.876816749572754,++ "sum_squared": 221.31967660036844,++ "min": 14.876816749572754,++ "max": 14.876816749572754,++ "mean": 14.876816749572754,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "batch_size",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_length",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_stop",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_endoftext",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_unknown",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_instances",++ "split": "valid"++ },++ "count": 1,++ "sum": 5.0,++ "sum_squared": 25.0,++ "min": 5.0,++ "max": 5.0,++ "mean": 5.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prompt_truncated",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "max_prob",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match@5",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match@5",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match@5",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match@5",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "logprob",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_perplexity_tokens",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_bytes",++ "split": "valid"++ },++ "count": 1,++ "sum": 2.0,++ "sum_squared": 4.0,++ "min": 2.0,++ "max": 2.0,++ "mean": 2.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "perplexity",++ "split": "valid"++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "bits_per_byte",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "logprob_per_byte",++ "split": "valid"++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_references",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 4.0,++ "sum_squared": 16.0,++ "min": 4.0,++ "max": 4.0,++ "mean": 4.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_references",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 4.0,++ "sum_squared": 16.0,++ "min": 4.0,++ "max": 4.0,++ "mean": 4.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_trials",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_trials",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_prompt_tokens",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 631.4,++ "sum_squared": 398665.95999999996,++ "min": 631.4,++ "max": 631.4,++ "mean": 631.4,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_prompt_tokens",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 631.4,++ "sum_squared": 398665.95999999996,++ "min": 631.4,++ "max": 631.4,++ "mean": 631.4,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_completion_tokens",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_completion_tokens",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_output_tokens",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_output_tokens",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "inference_runtime",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 14.876816749572754,++ "sum_squared": 221.31967660036844,++ "min": 14.876816749572754,++ "max": 14.876816749572754,++ "mean": 14.876816749572754,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "inference_runtime",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 14.876816749572754,++ "sum_squared": 221.31967660036844,++ "min": 14.876816749572754,++ "max": 14.876816749572754,++ "mean": 14.876816749572754,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "batch_size",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "batch_size",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_length",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_length",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_stop",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_stop",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_endoftext",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_endoftext",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_unknown",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "finish_reason_unknown",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_instances",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 5.0,++ "sum_squared": 25.0,++ "min": 5.0,++ "max": 5.0,++ "mean": 5.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_train_instances",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 5.0,++ "sum_squared": 25.0,++ "min": 5.0,++ "max": 5.0,++ "mean": 5.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prompt_truncated",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prompt_truncated",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "max_prob",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "max_prob",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "prefix_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "quasi_prefix_exact_match@5",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.3,++ "sum_squared": 0.09,++ "min": 0.3,++ "max": 0.3,++ "mean": 0.3,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "logprob",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "logprob",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 0.0,++ "sum_squared": 0.0,++ "min": 0.0,++ "max": 0.0,++ "mean": 0.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_perplexity_tokens",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_perplexity_tokens",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 1.0,++ "sum_squared": 1.0,++ "min": 1.0,++ "max": 1.0,++ "mean": 1.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_bytes",++ "split": "valid",++ "perturbation": {++ "name": "robustness",++ "robustness": true,++ "fairness": false,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 2.0,++ "sum_squared": 4.0,++ "min": 2.0,++ "max": 2.0,++ "mean": 2.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_bytes",++ "split": "valid",++ "perturbation": {++ "name": "fairness",++ "robustness": false,++ "fairness": true,++ "computed_on": "worst"++ }++ },++ "count": 1,++ "sum": 2.0,++ "sum_squared": 4.0,++ "min": 2.0,++ "max": 2.0,++ "mean": 2.0,++ "variance": 0.0,++ "stddev": 0.0++ },++ {++ "name": {++ "name": "num_instances",++ "split": "valid"++ },++ "count": 1,++ "sum": 10.0,++ "sum_squared": 100.0,++ "min": 10.0,++ "max": 10.0,++ "mean": 10.0,++ "variance": 0.0,++ "stddev": 0.0++ }++ ],++ "run_spec": {++ "name": "commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0",++ "scenario_spec": {++ "class_name": "helm.benchmark.scenarios.commonsense_scenario.HellaSwagScenario",++ "args": {}++ },++ "adapter_spec": {++ "method": "multiple_choice_joint",++ "global_prefix": "",++ "global_suffix": "",++ "instructions": "The following are multiple choice questions (with answers) about common sense.\n",++ "input_prefix": "Question: ",++ "input_suffix": "\n",++ "reference_prefix": "A. ",++ "reference_suffix": "\n",++ "chain_of_thought_prefix": "",++ "chain_of_thought_suffix": "\n",++ "output_prefix": "Answer: ",++ "output_suffix": "\n",++ "instance_prefix": "\n",++ "substitutions": [],++ "max_train_instances": 5,++ "max_eval_instances": 10,++ "num_outputs": 5,++ "num_train_trials": 1,++ "num_trials": 1,++ "sample_train": true,++ "model_deployment": "huggingface/pythia-1b-v0",++ "model": "eleutherai/pythia-1b-v0",++ "temperature": 0.0,++ "max_tokens": 1,++ "stop_sequences": [++ "\n"++ ],++ "multi_label": false++ },++ "metric_specs": [++ {++ "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",++ "args": {++ "names": [++ "exact_match",++ "quasi_exact_match",++ "prefix_exact_match",++ "quasi_prefix_exact_match"++ ]++ }++ },++ {++ "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",++ "args": {}++ },++ {++ "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",++ "args": {}++ }++ ],++ "data_augmenter_spec": {++ "perturbation_specs": [],++ "should_augment_train_instances": false,++ "should_include_original_train": false,++ "should_skip_unchanged_train": false,++ "should_augment_eval_instances": false,++ "should_include_original_eval": false,++ "should_skip_unchanged_eval": false,++ "seeds_per_instance": 1++ },++ "groups": [++ "hellaswag"++ ]++ }++}\\ No newline at end of filennew file mode 100644@@@ -0,0 +1,238 @@++{++ "config_general": {++ "lighteval_sha": "864af4f0ca5973e42ac4def921a3387ebeeca0c1",++ "num_fewshot_seeds": 1,++ "model_config": {++ "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct",++ "generation_parameters": {++ "temperature": 0.0,++ "top_p": 0.9,++ "seed": null++ }++ },++ "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct"++ },++ "results": {++ "arc:challenge|25": {++ "acc": 0.3,++ "acc_stderr": 0.15275252316519466++ },++ "truthfulqa:mc|0": {++ "truthfulqa_mc1": 0.2,++ "truthfulqa_mc1_stderr": 0.13333333333333333,++ "truthfulqa_mc2": 0.4801027091249869,++ "truthfulqa_mc2_stderr": 0.13947336040397884++ },++ "bigbench_hard:geometric_shapes|3": {++ "acc": 0.0,++ "acc_stderr": 0.0++ },++ "agieval:logiqa-en|0": {++ "acc": 0.2,++ "acc_stderr": 0.13333333333333333++ }++ },++ "versions": {},++ "config_tasks": {++ "arc:challenge|25": {++ "name": "arc:challenge",++ "prompt_function": "arc_prompt",++ "hf_repo": "allenai/ai2_arc",++ "hf_subset": "ARC-Challenge",++ "metrics": [++ {++ "metric_name": "acc",++ "higher_is_better": true,++ "category": "LOGPROBS",++ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",++ "corpus_level_fn": "mean",++ "batched_compute": false++ }++ ],++ "hf_data_files": null,++ "solver": [++ "solve"++ ],++ "scorer": "score",++ "sample_fields": "record_to_sample",++ "sample_to_fewshot": null,++ "filter": null,++ "hf_revision": null,++ "hf_filter": null,++ "hf_avail_splits": [++ "train",++ "test"++ ],++ "evaluation_splits": [++ "test"++ ],++ "few_shots_split": null,++ "few_shots_select": "random_sampling_from_train",++ "generation_size": 1,++ "generation_grammar": null,++ "stop_sequence": [++ "\n"++ ],++ "num_samples": null,++ "original_num_docs": -1,++ "effective_num_docs": -1,++ "must_remove_duplicate_docs": false,++ "num_fewshots": 25,++ "version": 0++ },++ "truthfulqa:mc|0": {++ "name": "truthfulqa:mc",++ "prompt_function": "truthful_qa_multiple_choice_prompt",++ "hf_repo": "truthfulqa/truthful_qa",++ "hf_subset": "multiple_choice",++ "metrics": [++ {++ "metric_name": [++ "truthfulqa_mc1",++ "truthfulqa_mc2"++ ],++ "higher_is_better": {++ "truthfulqa_mc1": true,++ "truthfulqa_mc2": true++ },++ "category": "LOGPROBS",++ "sample_level_fn": "TruthfulqaMCMetrics()",++ "corpus_level_fn": {++ "truthfulqa_mc1": "mean",++ "truthfulqa_mc2": "mean"++ },++ "batched_compute": false++ }++ ],++ "hf_data_files": null,++ "solver": null,++ "scorer": null,++ "sample_fields": null,++ "sample_to_fewshot": null,++ "filter": null,++ "hf_revision": null,++ "hf_filter": null,++ "hf_avail_splits": [++ "validation"++ ],++ "evaluation_splits": [++ "validation"++ ],++ "few_shots_split": null,++ "few_shots_select": null,++ "generation_size": -1,++ "generation_grammar": null,++ "stop_sequence": [++ "\n"++ ],++ "num_samples": null,++ "original_num_docs": -1,++ "effective_num_docs": -1,++ "must_remove_duplicate_docs": false,++ "num_fewshots": 0,++ "version": 0++ },++ "bigbench_hard:geometric_shapes|3": {++ "name": "bigbench_hard:geometric_shapes",++ "prompt_function": "bbh_prompt",++ "hf_repo": "lighteval/bbh",++ "hf_subset": "geometric_shapes",++ "metrics": [++ {++ "metric_name": "acc",++ "higher_is_better": true,++ "category": "LOGPROBS",++ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",++ "corpus_level_fn": "mean",++ "batched_compute": false++ }++ ],++ "hf_data_files": null,++ "solver": [++ "solve"++ ],++ "scorer": "score",++ "sample_fields": "record_to_sample",++ "sample_to_fewshot": null,++ "filter": null,++ "hf_revision": null,++ "hf_filter": null,++ "hf_avail_splits": [++ "train"++ ],++ "evaluation_splits": [++ "train"++ ],++ "few_shots_split": null,++ "few_shots_select": null,++ "generation_size": -1,++ "generation_grammar": null,++ "stop_sequence": [++ "</s>",++ "Q=",++ "\n\n"++ ],++ "num_samples": null,++ "original_num_docs": -1,++ "effective_num_docs": -1,++ "must_remove_duplicate_docs": false,++ "num_fewshots": 3,++ "version": 0++ },++ "agieval:logiqa-en|0": {++ "name": "agieval:logiqa-en",++ "prompt_function": "agieval_prompt",++ "hf_repo": "dmayhem93/agieval-logiqa-en",++ "hf_subset": "default",++ "metrics": [++ {++ "metric_name": "acc",++ "higher_is_better": true,++ "category": "LOGPROBS",++ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",++ "corpus_level_fn": "mean",++ "batched_compute": false++ }++ ],++ "hf_data_files": null,++ "solver": [++ "solve"++ ],++ "scorer": "score",++ "sample_fields": "record_to_sample",++ "sample_to_fewshot": null,++ "filter": null,++ "hf_revision": null,++ "hf_filter": null,++ "hf_avail_splits": [++ "test"++ ],++ "evaluation_splits": [++ "test"++ ],++ "few_shots_split": null,++ "few_shots_select": "random_sampling",++ "generation_size": 1,++ "generation_grammar": null,++ "stop_sequence": [],++ "num_samples": null,++ "original_num_docs": -1,++ "effective_num_docs": -1,++ "must_remove_duplicate_docs": false,++ "num_fewshots": 0,++ "version": 0++ }++ },++ "summary_general": {++ "hashes": {++ "hash_examples": "68b0b3cbd628cac8",++ "hash_full_prompts": "6c276b4f520c3deb",++ "hash_input_tokens": "84a946d8f5709efe",++ "hash_cont_tokens": "655a877ed0bcff8a"++ },++ "truncated": 0,++ "non_truncated": 0,++ "padded": 0,++ "non_padded": 0++ }++}\\ No newline at end of file@@@ -11,6 +11,9 @@ const { boot, check, done } = require('./helpers/boot');const { uncovered } = require('openvibe-sdk/account-data');const accountDataLib = require('../server/identity/account-data');const importers = require('../server/bench/importers');++require('../server/bench/importers/helm');++require('../server/bench/importers/lighteval');++require('../server/bench/imports');const limits = require('../server/bench/importers/limits');const { matchSuite } = require('../server/bench/suites');const { createMirror } = require('../server/bench/mirror');@@@ -184,6 +187,35 @@ function multipart(filename, bytes) {assert.strictEqual(gpt.cost_usd, 0.007);});++ await check('the HF lighteval results are read: the model, every task, its few-shot and its metric', () => {++ const out = importers.parse(read('lighteval-results.json'), { filename: 'results.json' });++ assert.strictEqual(out.tool, 'lighteval');++ assert.strictEqual(out.harness.name, 'lighteval');++ assert.strictEqual(out.models.length, 1);++ assert.strictEqual(out.models[0].model, 'HuggingFaceTB/SmolLM2-1.7B-Instruct');++ assert.strictEqual(out.models[0].provider, 'HuggingFaceTB');++ assert.strictEqual(out.models[0].tasks.length, 4);++ const arc = out.models[0].tasks.find((task) => task.task_id === 'arc:challenge');++ assert.ok(arc, 'the ARC challenge task is there');++ assert.strictEqual(arc.few_shot, 25, 'the task key carries its few-shot');++ assert.strictEqual(arc.metric, 'acc');++ assert.ok(Math.abs(arc.score - 30) < 1e-9, 'the score is out of 100: ' + arc.score);++ assert.match(out.harness.version, /^[0-9a-f]{7,}$/, 'the lighteval commit is the harness version');++ });++++ await check('the Stanford HELM stats are read with their run spec: model, scenario and few-shot', () => {++ const out = importers.parse(read('helm-results.json'), { filename: 'stats.json' });++ assert.strictEqual(out.tool, 'helm');++ assert.strictEqual(out.harness.name, 'helm');++ assert.strictEqual(out.models[0].model, 'eleutherai/pythia-1b-v0');++ assert.strictEqual(out.models[0].provider, 'huggingface');++ assert.strictEqual(out.models[0].tasks.length, 1);++ assert.strictEqual(out.models[0].tasks[0].task_id, 'hellaswag');++ assert.strictEqual(out.models[0].tasks[0].metric, 'quasi_exact_match');++ assert.strictEqual(out.models[0].tasks[0].few_shot, 5, 'max_train_instances is the few-shot');++ assert.ok(Math.abs(out.models[0].score - 30) < 1e-9, 'the proportion is read out of 100: ' + out.models[0].score);++ });++// ── the limits ─────────────────────────────────────────────────────────────────────────────────────────await check('a file over 20 MB is refused', () => {assert.throws(() => importers.parse(Buffer.alloc(20 * 1024 * 1024 + 1)), /20 MB|too large/i);@@@ -297,6 +329,27 @@ function multipart(filename, bytes) {assert.strictEqual(runs[0].harness.name, 'inspect-ai');});++ await check('an import keeps the harness, task version, few-shot and seed on the run', async () => {++ const doc = JSON.parse(read('lighteval-results.json').toString('utf8'));++ doc.versions = { 'arc:challenge': 3 };++ doc.config_general.model_config.generation_parameters.seed = 42;++ const up = await t.ctx.imports.store({++ owner: alice.subject, subject: alice.subject, filename: 'lighteval-results.json',++ bytes: Buffer.from(JSON.stringify(doc)),++ });++ assert.strictEqual(up.summary.tool, 'lighteval');++ assert.strictEqual(up.summary.harness.name, 'lighteval');++ assert.strictEqual(up.summary.models[0].tasks[0].few_shot, 25);++ assert.strictEqual(up.summary.models[0].tasks[0].task_version, '3');++ const imported = await t.ctx.imports.commit(up.id, { subject: alice.subject });++ const run = await t.ctx.registry.getRun(imported.runs[0].id);++ assert.strictEqual(run.harness.name, 'lighteval', 'the harness name is on the run');++ assert.strictEqual(run.harness.version, up.summary.harness.version, 'and its version');++ assert.strictEqual(run.settings.few_shot, 25, 'the few-shot is kept');++ assert.strictEqual(run.settings.seed, '42', 'the seed is kept');++ assert.strictEqual(run.settings.task_versions['arc:challenge'], '3', 'the task version is kept');++ });++await check('the file stays private to its owner', async () => {const mine = await t.get('/imports/' + importId + '/file', { as: alice });assert.strictEqual(mine.status, 200);
@@ -2,6 +2,17 @@
@@ -2,6 +2,17 @@
## Unreleased
## Unreleased
+- **Every Eval Ever (B2).** Bench imports the public EEE datastore (`github.com/evaleval/every_eval_ever`,
+ `evaleval/EEE_datastore`) as *external* results: the model (joined by Report's rule — maker prefix and a date or
+ channel suffix dropped), the harness and its version, the task and its version, few-shot, the metric, the score,
+ the source link and the licence. A record whose licence does not allow redistribution is dropped and counted.
+ The rows live in `bench_eee_records` and are served by `GET /api/v1/external/records` with `external: true`; they
+ never enter Bench's ratings, boards or medians. A weekly, resumable, bounded job (`BENCH_EEE_SYNC`) imports them.
+ Bench's own public runs are exported as EEE `evaluation_run` records at `GET /api/v1/datasets/eee` (JSONL, paged;
+ per-sample scores at `/api/v1/datasets/eee/runs/:id.samples.jsonl`).
+- **Two more harnesses (B7).** HF lighteval and Stanford HELM results join Inspect AI, Harbor, the
+ lm-evaluation-harness and promptfoo. Every import keeps the harness name and version, each task's version, the
+ few-shot and the seed on the run.
- **The public pages moved to OpenVibe.Report's AI section (2026-11-08).** `/`, `/docs`, `/benchmarks`, `/models/*`, `/compare` and the other page routes answer `301` to `https://openvibe.report/ai` counterparts; the API, MCP, render origin, badges and embeds stay on bench.openvibe.services. New public reads feed Report: `GET /api/v1/runs`, `/api/v1/overview` (with every public model and its last run) and `/api/v1/models/:model/summary` (per-benchmark and per-task medians, cost, recent runs). `/api/v1/benchmarks` now carries each benchmark's runs and last run.
- **The public pages moved to OpenVibe.Report's AI section (2026-11-08).** `/`, `/docs`, `/benchmarks`, `/models/*`, `/compare` and the other page routes answer `301` to `https://openvibe.report/ai` counterparts; the API, MCP, render origin, badges and embeds stay on bench.openvibe.services. New public reads feed Report: `GET /api/v1/runs`, `/api/v1/overview` (with every public model and its last run) and `/api/v1/models/:model/summary` (per-benchmark and per-task medians, cost, recent runs). `/api/v1/benchmarks` now carries each benchmark's runs and last run.
- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:
- **The Exchange arena (`chat.arena`).** A new kind, and a first-party API only OpenVibe.Exchange may call:
`POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, and
`POST /api/v1/arena/rounds` builds a round from two answers the Exchange already has, and
new file mode 100644
new file mode 100644
@@ -0,0 +1,82 @@
@@ -0,0 +1,82 @@
+-- phase: expand
+-- Stage B2 (plan 2026-11-09): Every Eval Ever (EEE).
+--
+-- Bench imports the EEE datastore (github.com/evaleval/every_eval_ever, HF evaleval/EEE_datastore) as *external*
+-- results: they sit beside Bench's own runs in their own table, with the provenance and the licence they were
+-- published under, and they never enter Bench's ratings, boards or medians. The licence that does not allow
+-- redistribution stops a record at the door, and the sync's ledger counts how many and why.
+--
+-- The exporter is the other direction: Bench's own public runs are served as EEE `evaluation_run` records from
+-- /api/v1/datasets/eee (server/bench/eee.js), so nothing here stores the export.
+
+-- One row per (record, evaluation result, metric): the same EEE record never lands twice.
+CREATE TABLE bench_eee_records (
+ id text COLLATE "C" PRIMARY KEY, -- ber_ + ULID
+ eee_id text COLLATE "C" NOT NULL, -- the EEE evaluation_id (record identity)
+ source_name text COLLATE "C" NOT NULL DEFAULT '',
+ source_type text COLLATE "C" NOT NULL, -- documentation | evaluation_run
+ source_url text COLLATE "C" NOT NULL DEFAULT '',
+ source_org text COLLATE "C" NOT NULL DEFAULT '',
+ evaluator_relationship text COLLATE "C" NOT NULL DEFAULT 'other',
+ licence text COLLATE "C" NOT NULL,
+ licence_url text COLLATE "C" NOT NULL DEFAULT '',
+ licence_source text COLLATE "C" NOT NULL DEFAULT '', -- record | datastore
+ model text COLLATE "C" NOT NULL, -- the join name (maker prefix and date/channel suffix dropped)
+ model_raw text COLLATE "C" NOT NULL DEFAULT '', -- the source's own id
+ developer text COLLATE "C" NOT NULL DEFAULT '',
+ harness_name text COLLATE "C" NOT NULL DEFAULT 'unknown',
+ harness_version text COLLATE "C" NOT NULL DEFAULT 'unknown',
+ task text COLLATE "C" NOT NULL,
+ task_version text COLLATE "C" NOT NULL DEFAULT '',
+ few_shot integer,
+ seed text COLLATE "C" NOT NULL DEFAULT '',
+ metric_id text COLLATE "C" NOT NULL DEFAULT '',
+ metric_name text COLLATE "C" NOT NULL DEFAULT '',
+ lower_is_better boolean,
+ score numeric(14, 6) NOT NULL,
+ score_min numeric(14, 6),
+ score_max numeric(14, 6),
+ evaluation_at timestamptz,
+ retrieved_at timestamptz,
+ provenance jsonb NOT NULL DEFAULT '{}'::jsonb, -- the EEE record as published (its licence, harness, link)
+ created_at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT bench_eee_records_id CHECK (id ~ '^ber_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_eee_records_source_type CHECK (source_type IN ('documentation', 'evaluation_run')),
+ CONSTRAINT bench_eee_records_relationship CHECK (evaluator_relationship IN ('first_party', 'third_party', 'collaborative', 'other')),
+ CONSTRAINT bench_eee_records_few_shot CHECK (few_shot IS NULL OR few_shot >= 0)
+);
+CREATE UNIQUE INDEX bench_eee_records_dedupe ON bench_eee_records (eee_id, task, metric_id, metric_name);
+CREATE INDEX bench_eee_records_by_model ON bench_eee_records (model, task);
+CREATE INDEX bench_eee_records_by_task ON bench_eee_records (task, created_at DESC);
+CREATE INDEX bench_eee_records_by_source ON bench_eee_records (source_type, created_at DESC);
+
+-- A sync run: where it got to and what it did. A run that stopped half way leaves its cursor here, and the next
+-- run of the weekly job resumes from it instead of starting the datastore again.
+CREATE TABLE bench_eee_syncs (
+ id text COLLATE "C" PRIMARY KEY, -- bes_ + ULID
+ status text COLLATE "C" NOT NULL DEFAULT 'running', -- running | finished | failed
+ cursor jsonb NOT NULL DEFAULT '{}'::jsonb, -- { collection, file }
+ collections integer NOT NULL DEFAULT 0,
+ files_seen integer NOT NULL DEFAULT 0,
+ records_read integer NOT NULL DEFAULT 0,
+ records_kept integer NOT NULL DEFAULT 0,
+ dropped_licence integer NOT NULL DEFAULT 0,
+ dropped_invalid integer NOT NULL DEFAULT 0,
+ dropped_reason jsonb NOT NULL DEFAULT '{}'::jsonb, -- { reason: count }
+ default_licence text COLLATE "C" NOT NULL DEFAULT '',
+ licence_url text COLLATE "C" NOT NULL DEFAULT '',
+ error text COLLATE "C" NOT NULL DEFAULT '',
+ started_at timestamptz NOT NULL DEFAULT now(),
+ updated_at timestamptz NOT NULL DEFAULT now(),
+ finished_at timestamptz,
+ CONSTRAINT bench_eee_syncs_id CHECK (id ~ '^bes_[0-9A-HJKMNP-TV-Z]{26}$'),
+ CONSTRAINT bench_eee_syncs_status CHECK (status IN ('running', 'finished', 'failed')),
+ CONSTRAINT bench_eee_syncs_counts CHECK (files_seen >= 0 AND records_read >= 0 AND records_kept >= 0
+ AND dropped_licence >= 0 AND dropped_invalid >= 0)
+);
+CREATE INDEX bench_eee_syncs_by_status ON bench_eee_syncs (status, started_at DESC);
+
+-- B7: the two harnesses added beside the four. Widening the check is the expand step.
+ALTER TABLE bench_imports DROP CONSTRAINT bench_imports_tool;
+ALTER TABLE bench_imports ADD CONSTRAINT bench_imports_tool
+ CHECK (tool IN ('inspect', 'harbor', 'lm-eval', 'promptfoo', 'lighteval', 'helm'));
@@ -88,6 +88,9 @@ async function createApp(opts = {}) {
@@ -88,6 +88,9 @@ async function createApp(opts = {}) {
// B12: bringing your own results, and the operator's mirror.
// B12: bringing your own results, and the operator's mirror.
const imports = require('./bench/imports').createImports({ s, registry, log });
const imports = require('./bench/imports').createImports({ s, registry, log });
const mirror = require('./bench/mirror').createMirror({ s, registry, fetchImpl, log });
const mirror = require('./bench/mirror').createMirror({ s, registry, fetchImpl, log });
+ // B2: the external EEE records and the exporter of Bench's own runs, plus the weekly import job.
+ const eee = require('./bench/eee').createEee({ s, registry, fetchImpl, log, config });
+ const eeeSync = require('./bench/eee-sync').createEeeSync({ eee, config, log });
const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.
// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.
@@ -111,7 +114,7 @@ async function createApp(opts = {}) {
@@ -111,7 +114,7 @@ async function createApp(opts = {}) {
const ctx = {
const ctx = {
config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
config, s, keys, sso, principal, bench, registry, launcher, ai, jobs, publisher, accountData, accountSend, log, fetchImpl,
anon, daily, variantEvidence, evidenceConfig, sharing,
anon, daily, variantEvidence, evidenceConfig, sharing,
- imports, mirror,
+ imports, mirror, eee, eeeSync,
random: opts.random || Math.random,
random: opts.random || Math.random,
};
};
@@ -253,6 +256,8 @@ async function createApp(opts = {}) {
@@ -253,6 +256,8 @@ async function createApp(opts = {}) {
const SHARE_PATH = /^\/api\/v1\/(?:shared-with-me$|share\/people$|[^/]+\/[^/]+\/share(?:\/|$))/;
const SHARE_PATH = /^\/api\/v1\/(?:shared-with-me$|share\/people$|[^/]+\/[^/]+\/share(?:\/|$))/;
app.use((req, res, next) => (SHARE_PATH.test(req.path) ? shareApi(req, res, next) : next()));
app.use((req, res, next) => (SHARE_PATH.test(req.path) ? shareApi(req, res, next) : next()));
}
}
+ // The EEE routes come before the kind dataset route: /datasets/eee.jsonl would otherwise be read as kind 'eee'.
+ app.use(require('./http/eee').createEeeRoutes(ctx));
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createRegistryApi(ctx));
app.use('/api/v1', createVariantEvidenceApi(ctx));
app.use('/api/v1', createVariantEvidenceApi(ctx));
new file mode 100644
new file mode 100644
@@ -0,0 +1,46 @@
@@ -0,0 +1,46 @@
+'use strict';
+
+/**
+ * The weekly Every Eval Ever import (B2). It is the same shape as Bench's other workers: a timer, one async pass at
+ * a time, and the pass itself is bounded and resumable (server/bench/eee.js `sync`). Off until the owner turns
+ * BENCH_EEE_SYNC on; never under BENCH_DRILL unless BENCH_EEE_SYNC_IN_DRILL says so too.
+ */
+function createEeeSync({ eee, config, log = console }) {
+ let timer = null;
+ let running = false;
+ let stopped = false;
+ const settings = config.bench.eee;
+
+ async function tick() {
+ if (running || stopped) return null;
+ running = true;
+ try {
+ return await eee.sync({ maxFiles: settings.filesPerRun, maxRecords: settings.recordsPerRun });
+ } catch (err) {
+ log.warn('[OpenVibe.Bench] the EEE sync failed:', (err && err.message) || err);
+ return null;
+ } finally {
+ running = false;
+ }
+ }
+
+ function start() {
+ if (timer || stopped || !settings.enabled) return null;
+ if (config.drill && !settings.enabledInDrill) return null;
+ const run = () => tick().catch((err) => log.warn('[OpenVibe.Bench] EEE job:', (err && err.message) || err));
+ timer = setInterval(run, settings.intervalMs);
+ if (timer.unref) timer.unref();
+ run();
+ return timer;
+ }
+
+ function stop() {
+ stopped = true;
+ if (timer) clearInterval(timer);
+ timer = null;
+ }
+
+ return { tick, start, stop, get running() { return running; } };
+}
+
+module.exports = { createEeeSync };
new file mode 100644
new file mode 100644
@@ -0,0 +1,664 @@
@@ -0,0 +1,664 @@
+'use strict';
+
+/**
+ * Every Eval Ever (B2): importing the EEE datastore as external results, and exporting Bench's own runs as EEE.
+ *
+ * Import. The EEE datastore (github.com/evaleval/every_eval_ever, HF evaleval/EEE_datastore) is a public, MIT
+ * collection of eval records in one JSON shape (`eval.schema.json`). A record is read from the datastore's own JSON
+ * files, its licence is checked before anything is stored, and each evaluation result becomes one `bench_eee_records`
+ * row: the model (a join name with the maker prefix and a date/channel suffix dropped, the rule of Report's
+ * `server/data/ai-join.js`), the harness and its version, the task and its version, few-shot, the metric, the score
+ * and the source link. The rows are *external*: they never enter Bench's ratings, boards or medians.
+ *
+ * A weekly job walks the datastore with a cursor in `bench_eee_syncs`, so a stopped run resumes where it stopped, and
+ * one run is bounded in files and records.
+ *
+ * Export. A public Bench run is served as an EEE `evaluation_run` record from `exportRecords` (the route is
+ * /api/v1/datasets/eee), with the per-sample scores on a companion endpoint where Bench keeps them.
+ */
+const { ImportError } = require('./importers/limits');
+
+const DATASET = 'evaleval/EEE_datastore';
+const REPO = 'https://github.com/evaleval/every_eval_ever';
+const DATASET_URL = 'https://huggingface.co/datasets/' + DATASET;
+const SCHEMA_VERSION = '0.3.0';
+
+const SOURCE = {
+ key: 'eee',
+ title: 'Every Eval Ever',
+ repo: REPO,
+ dataset: DATASET_URL,
+ schema_version: SCHEMA_VERSION,
+ // The datastore's own dataset card declares MIT; a record may override it in additional_details.
+ default_licence: 'MIT',
+ licence_url: DATASET_URL + '/blob/main/README.md',
+};
+
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+const json = (v, fallback) => {
+ if (v == null) return fallback;
+ if (typeof v === 'string') { try { return JSON.parse(v); } catch { return fallback; } }
+ return v;
+};
+const iso = (v) => (v == null ? null : (v instanceof Date ? v.toISOString() : String(v)));
+const details = (v) => (v && typeof v === 'object' && !Array.isArray(v) ? v : {});
+
+/**
+ * The join rule of Report's server/data/ai-join.js: Bench names a model as its maker's API does
+ * ("openai/gpt-5-nano-2025-08-07"), the catalogue names it by maker and family ("openai/gpt-5-nano"). They are the
+ * same model once the maker prefix and a date or channel suffix are dropped.
+ */
+function bareModel(v) {
+ return String(v || '').toLowerCase()
+ .replace(/^[^/]+\//, '')
+ .replace(/[-_@:](\d{4}-?\d{2}-?\d{2}|latest|free)$/, '');
+}
+
+/** A licence that forbids redistribution is not imported. An empty or "unknown" one is not a licence either. */
+const NON_REDISTRIBUTABLE = /all rights reserved|proprietary|no (re)?distribution|not for (re)?distribution|research only|evaluation only|internal only|confidential|closed|^unknown$|^none$|^n\/?a$/i;
+function allowsRedistribution(licence) {
+ const text = String(licence == null ? '' : licence).trim();
+ if (!text) return false;
+ return !NON_REDISTRIBUTABLE.test(text);
+}
+
+/** The first licence a record states about itself, or the datastore's own. */
+function recordLicenceOf(doc) {
+ const meta = details(doc && doc.source_metadata);
+ const direct = [meta.licence, meta.license, details(meta.additional_details).licence, details(meta.additional_details).license];
+ for (const value of direct) {
+ if (typeof value === 'string' && value.trim()) return { licence: value.trim(), source: 'record' };
+ }
+ const results = Array.isArray(doc && doc.evaluation_results) ? doc.evaluation_results : [];
+ for (const result of results) {
+ const data = details(result && result.source_data);
+ const extra = details(data.additional_details);
+ for (const value of [data.licence, data.license, extra.licence, extra.license]) {
+ if (typeof value === 'string' && value.trim()) return { licence: value.trim(), source: 'record' };
+ }
+ }
+ return null;
+}
+
+function firstText(...values) {
+ for (const value of values) {
+ if (typeof value === 'string' && value.trim()) return value.trim();
+ }
+ return '';
+}
+
+/** Few-shot: the schema has no typed field, so the sources put it in additional_details. */
+function fewShotOf(result, doc) {
+ const places = [
+ details(result && result.source_data && result.source_data.additional_details),
+ details(result && result.generation_config && result.generation_config.additional_details),
+ details(result && result.generation_config && result.generation_config.generation_args),
+ details(doc && doc.source_metadata && doc.source_metadata.additional_details),
+ ];
+ for (const place of places) {
+ for (const key of ['num_fewshot', 'few_shot', 'n_shot', 'fewshot', 'num_shots', 'shots']) {
+ const value = num(place[key]);
+ if (value != null) return Math.max(0, Math.round(value));
+ }
+ }
+ return null;
+}
+
+function seedOf(result, doc) {
+ const places = [
+ details(result && result.generation_config && result.generation_config.generation_args),
+ details(result && result.generation_config && result.generation_config.additional_details),
+ details(doc && doc.source_metadata && doc.source_metadata.additional_details),
+ ];
+ for (const place of places) {
+ for (const key of ['seed', 'random_seed']) {
+ const value = place[key];
+ if (value != null && String(value).trim()) return String(value).trim().slice(0, 40);
+ }
+ }
+ return '';
+}
+
+function taskVersionOf(result) {
+ const data = details(result && result.source_data);
+ const extra = details(data.additional_details);
+ return firstText(extra.task_version, extra.version, extra.task_revision, extra.revision, data.version);
+}
+
+/** The link back to where this record was reported, preferring a stated URL over the datastore's own path. */
+function sourceUrlOf(doc, result, path) {
+ const meta = details(doc && doc.source_metadata);
+ const extra = details(meta.additional_details);
+ const data = details(result && result.source_data);
+ const dataExtra = details(data.additional_details);
+ const candidates = [
+ extra.source_url, extra.url, extra.paper_url,
+ meta.source_organization_url,
+ dataExtra.source_url, dataExtra.url,
+ ...(Array.isArray(data.url) ? data.url : []),
+ data.hf_repo ? 'https://huggingface.co/datasets/' + data.hf_repo : '',
+ path ? DATASET_URL + '/blob/main/' + path : '',
+ ];
+ for (const value of candidates) {
+ if (typeof value === 'string' && /^https?:\/\//i.test(value)) return value;
+ }
+ return DATASET_URL;
+}
+
+/**
+ * Read one EEE record into the rows it would store. It throws when the document is not an EEE record; a result
+ * without a score is dropped (invalid), and a result whose licence forbids redistribution is dropped (licence).
+ */
+function normalizeRecord(doc, { fallbackLicence = SOURCE.default_licence, fallbackLicenceUrl = SOURCE.licence_url, path = '' } = {}) {
+ if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {
+ throw new ImportError(422, 'bench.eee_not_object', 'That file is not an EEE record.');
+ }
+ const resultsIn = Array.isArray(doc.evaluation_results) ? doc.evaluation_results : null;
+ if (!resultsIn || !resultsIn.length) {
+ throw new ImportError(422, 'bench.eee_no_results', 'That EEE record holds no evaluation results.');
+ }
+ const meta = details(doc.source_metadata);
+ const sourceType = meta.source_type === 'evaluation_run' ? 'evaluation_run' : 'documentation';
+ const relationship = ['first_party', 'third_party', 'collaborative', 'other'].includes(meta.evaluator_relationship)
+ ? meta.evaluator_relationship : 'other';
+ const library = details(doc.eval_library);
+ const modelInfo = details(doc.model_info);
+ const modelRaw = firstText(modelInfo.id, modelInfo.name);
+ const model = bareModel(modelRaw) || modelRaw || 'unknown';
+ const developer = firstText(modelInfo.developer, modelRaw.includes('/') ? modelRaw.split('/')[0] : '');
+ const stated = recordLicenceOf(doc);
+ const licence = stated ? stated.licence : fallbackLicence;
+ const licenceSource = stated ? 'record' : 'datastore';
+ const eeeId = firstText(doc.evaluation_id, modelRaw) || 'unknown';
+
+ const record = {
+ eee_id: eeeId,
+ source_name: firstText(meta.source_name),
+ source_type: sourceType,
+ source_org: firstText(meta.source_organization_name),
+ evaluator_relationship: relationship,
+ licence,
+ licence_source: licenceSource,
+ model,
+ model_raw: modelRaw,
+ developer,
+ harness_name: firstText(library.name, meta.source_name, 'unknown'),
+ harness_version: firstText(library.version, 'unknown'),
+ retrieved_at: num(doc.retrieved_timestamp),
+ // The provenance a reader needs, not a second copy of every evaluation result on every row.
+ provenance: {
+ schema_version: doc.schema_version == null ? null : String(doc.schema_version),
+ evaluation_id: eeeId,
+ retrieved_timestamp: doc.retrieved_timestamp == null ? null : String(doc.retrieved_timestamp),
+ source_metadata: meta,
+ model_info: modelInfo,
+ eval_library: library,
+ },
+ };
+
+ const results = [];
+ for (const result of resultsIn) {
+ if (!result || typeof result !== 'object') {
+ results.push({ dropped: 'invalid', reason: 'a result is not an object' });
+ continue;
+ }
+ const score = num(result.score_details && result.score_details.score);
+ if (score == null) {
+ results.push({ dropped: 'invalid', reason: 'a result has no score' });
+ continue;
+ }
+ const metric = details(result.metric_config);
+ const metricId = firstText(metric.metric_id, metric.metric_name, 'score');
+ const metricName = firstText(metric.metric_name, metric.metric_id, 'score');
+ const task = firstText(result.evaluation_name, result.evaluation_result_id) || 'unknown';
+ if (!allowsRedistribution(licence)) {
+ results.push({ dropped: 'licence', reason: 'licence does not allow redistribution: ' + licence });
+ continue;
+ }
+ results.push({
+ allowed: true,
+ task,
+ task_version: taskVersionOf(result),
+ few_shot: fewShotOf(result, doc),
+ seed: seedOf(result, doc),
+ metric_id: metricId,
+ metric_name: metricName,
+ lower_is_better: typeof metric.lower_is_better === 'boolean' ? metric.lower_is_better : null,
+ score,
+ score_min: num(metric.min_score),
+ score_max: num(metric.max_score),
+ evaluation_at: num(result.evaluation_timestamp != null ? result.evaluation_timestamp : doc.evaluation_timestamp),
+ source_url: sourceUrlOf(doc, result, path),
+ licence,
+ });
+ }
+ return { record, results };
+}
+
+/** The datastore's own licence, read from its dataset card. A card that cannot be read falls back to MIT. */
+async function datastoreLicence(fetchImpl = globalThis.fetch) {
+ const url = SOURCE.licence_url;
+ try {
+ const res = await fetchImpl(url);
+ if (res && res.ok) {
+ const text = await res.text();
+ const line = String(text).split('\n').map((l) => l.trim()).find((l) => /^license:\s*/i.test(l));
+ if (line) {
+ const licence = line.replace(/^license:\s*/i, '').replace(/^["']|["']$/g, '').trim();
+ if (licence) return { licence, url, read: true };
+ }
+ }
+ } catch (err) {
+ // fall through to the known card value
+ }
+ return { licence: SOURCE.default_licence, url, read: false };
+}
+
+function wireExternal(row) {
+ if (!row) return null;
+ return {
+ id: row.id,
+ external: true,
+ eee_id: row.eee_id,
+ source_type: row.source_type,
+ source: {
+ name: row.source_name,
+ organization: row.source_org,
+ url: row.source_url,
+ relationship: row.evaluator_relationship,
+ },
+ licence: row.licence,
+ licence_url: row.licence_url,
+ licence_source: row.licence_source,
+ model: row.model,
+ model_raw: row.model_raw,
+ developer: row.developer,
+ harness: { name: row.harness_name, version: row.harness_version },
+ task: row.task,
+ task_version: row.task_version,
+ few_shot: row.few_shot == null ? null : num(row.few_shot),
+ seed: row.seed || '',
+ metric: { id: row.metric_id, name: row.metric_name, lower_is_better: row.lower_is_better },
+ score: row.score == null ? null : num(row.score),
+ score_min: row.score_min == null ? null : num(row.score_min),
+ score_max: row.score_max == null ? null : num(row.score_max),
+ evaluation_at: row.evaluation_at ? iso(row.evaluation_at) : null,
+ retrieved_at: row.retrieved_at ? iso(row.retrieved_at) : null,
+ provenance: json(row.provenance, {}),
+ created_at: iso(row.created_at),
+ };
+}
+
+function createEee({ s, fetchImpl = globalThis.fetch, log = console, config = null, now = Date.now } = {}) {
+ const db = s.db;
+ const baseUrl = String((config && config.baseUrl) || '').replace(/\/+$/, '');
+
+ async function getJson(url) {
+ const res = await fetchImpl(url);
+ if (!res || !res.ok) throw new Error('the source answered ' + (res && res.status) + ' for ' + url);
+ return res.json();
+ }
+
+ async function storeRecord(doc, { path = '', readAt = s.iso(), fallbackLicence = SOURCE.default_licence, fallbackLicenceUrl = SOURCE.licence_url } = {}) {
+ const parsed = normalizeRecord(doc, { fallbackLicence, fallbackLicenceUrl, path });
+ const counts = { read: parsed.results.length, kept: 0, droppedLicence: 0, droppedInvalid: 0, reasons: {} };
+ const note = (reason) => { counts.reasons[reason] = (counts.reasons[reason] || 0) + 1; };
+ for (const result of parsed.results) {
+ if (result.dropped === 'licence') { counts.droppedLicence += 1; note(result.reason); continue; }
+ if (result.dropped) { counts.droppedInvalid += 1; note(result.reason); continue; }
+ await db.exec(
+ `INSERT INTO bench_eee_records
+ (id, eee_id, source_name, source_type, source_url, source_org, evaluator_relationship, licence,
+ licence_url, licence_source, model, model_raw, developer, harness_name, harness_version, task,
+ task_version, few_shot, seed, metric_id, metric_name, lower_is_better, score, score_min,
+ score_max, evaluation_at, retrieved_at, provenance, created_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19,
+ $20, $21, $22, $23, $24, $25, $26::timestamptz, $27::timestamptz, $28::jsonb, $29::timestamptz)
+ ON CONFLICT (eee_id, task, metric_id, metric_name) DO UPDATE SET
+ score = EXCLUDED.score, score_min = EXCLUDED.score_min, score_max = EXCLUDED.score_max,
+ licence = EXCLUDED.licence, licence_url = EXCLUDED.licence_url, licence_source = EXCLUDED.licence_source,
+ source_url = EXCLUDED.source_url, harness_version = EXCLUDED.harness_version,
+ task_version = EXCLUDED.task_version, few_shot = EXCLUDED.few_shot, seed = EXCLUDED.seed,
+ retrieved_at = EXCLUDED.retrieved_at, provenance = EXCLUDED.provenance`,
+ [s.newId('ber'), parsed.record.eee_id, parsed.record.source_name, parsed.record.source_type,
+ result.source_url, parsed.record.source_org, parsed.record.evaluator_relationship,
+ parsed.record.licence, parsed.record.licence_source === 'record' ? result.source_url : fallbackLicenceUrl, parsed.record.licence_source, parsed.record.model,
+ parsed.record.model_raw, parsed.record.developer, parsed.record.harness_name,
+ parsed.record.harness_version, result.task, result.task_version, result.few_shot, result.seed,
+ result.metric_id, result.metric_name, result.lower_is_better, result.score, result.score_min,
+ result.score_max,
+ result.evaluation_at == null ? null : new Date(result.evaluation_at * 1000).toISOString(),
+ parsed.record.retrieved_at == null ? null : new Date(parsed.record.retrieved_at * 1000).toISOString(),
+ JSON.stringify(parsed.record.provenance), readAt],
+ );
+ counts.kept += 1;
+ }
+ return counts;
+ }
+
+ /** The external records, newest first, with `external: true` and their provenance. Never a Bench rating. */
+ async function listRecords({ model = null, task = null, sourceType = null, limit = 100, offset = 0 } = {}) {
+ const where = [];
+ const params = [];
+ const add = (clause, value) => { params.push(value); where.push(clause.replace('?', `$${params.length}`)); };
+ if (model) add('model = ?', String(model));
+ if (task) add('task = ?', String(task));
+ if (sourceType) add('source_type = ?', String(sourceType));
+ params.push(Math.max(1, Math.min(1000, Number(limit) || 100)));
+ const limitAt = params.length;
+ params.push(Math.max(0, Number(offset) || 0));
+ const rows = await db.many(
+ `SELECT * FROM bench_eee_records ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
+ ORDER BY created_at DESC, id LIMIT $${limitAt} OFFSET $${params.length}`,
+ params,
+ );
+ return rows.map(wireExternal);
+ }
+
+ async function countRecords({ model = null, task = null, sourceType = null } = {}) {
+ const where = [];
+ const params = [];
+ const add = (clause, value) => { params.push(value); where.push(clause.replace('?', `$${params.length}`)); };
+ if (model) add('model = ?', String(model));
+ if (task) add('task = ?', String(task));
+ if (sourceType) add('source_type = ?', String(sourceType));
+ const row = await db.maybe(`SELECT COUNT(*)::int AS n FROM bench_eee_records ${where.length ? 'WHERE ' + where.join(' AND ') : ''}`, params);
+ return num(row && row.n) || 0;
+ }
+
+ async function latestSync() {
+ const row = await db.maybe('SELECT * FROM bench_eee_syncs ORDER BY started_at DESC LIMIT 1');
+ return row ? wireSync(row) : null;
+ }
+
+ function wireSync(row) {
+ return {
+ id: row.id,
+ status: row.status,
+ cursor: json(row.cursor, {}),
+ collections: num(row.collections) || 0,
+ files_seen: num(row.files_seen) || 0,
+ records_read: num(row.records_read) || 0,
+ records_kept: num(row.records_kept) || 0,
+ dropped_licence: num(row.dropped_licence) || 0,
+ dropped_invalid: num(row.dropped_invalid) || 0,
+ dropped_reason: json(row.dropped_reason, {}),
+ default_licence: row.default_licence || '',
+ licence_url: row.licence_url || '',
+ error: row.error || '',
+ started_at: iso(row.started_at),
+ updated_at: iso(row.updated_at),
+ finished_at: iso(row.finished_at),
+ };
+ }
+
+ async function listCollections() {
+ const url = 'https://huggingface.co/api/datasets/' + DATASET + '/tree/main/data?recursive=false';
+ const body = await getJson(url);
+ if (!Array.isArray(body)) throw new Error('the datastore tree is not a list');
+ return body
+ .filter((entry) => entry && entry.type === 'directory' && entry.path)
+ .map((entry) => String(entry.path).replace(/^data\//, ''))
+ .sort();
+ }
+
+ async function listFiles(collection) {
+ const url = 'https://huggingface.co/api/datasets/' + DATASET + '/tree/main/data/' + encodeURIComponent(collection) + '?recursive=true&limit=1000';
+ const body = await getJson(url);
+ if (!Array.isArray(body)) throw new Error('the collection tree is not a list');
+ return body
+ .filter((entry) => entry && entry.type === 'file' && /\.json$/i.test(entry.path || ''))
+ .map((entry) => String(entry.path))
+ .sort();
+ }
+
+ function rawUrl(path) {
+ return DATASET_URL + '/resolve/main/' + path.split('/').map(encodeURIComponent).join('/');
+ }
+
+ /**
+ * One bounded pass of the weekly job. It resumes the newest running sync (or starts one), walks the datastore's
+ * collections and JSON files from the cursor, and stops at `maxFiles` files or `maxRecords` results. The next
+ * pass continues from the same cursor. Nothing is fetched when the job is disabled.
+ */
+ async function sync({ maxFiles = 200, maxRecords = 5000, collections = null } = {}) {
+ const licence = await datastoreLicence(fetchImpl);
+ let row = await db.maybe("SELECT * FROM bench_eee_syncs WHERE status = 'running' ORDER BY started_at DESC LIMIT 1");
+ if (!row) {
+ const id = s.newId('bes');
+ row = await db.maybe(
+ `INSERT INTO bench_eee_syncs (id, status, cursor, default_licence, licence_url, started_at, updated_at)
+ VALUES ($1, 'running', '{}'::jsonb, $2, $3, $4::timestamptz, $4::timestamptz) RETURNING *`,
+ [id, licence.licence, licence.url, s.iso()],
+ );
+ }
+ const cursor = json(row.cursor, {});
+ let collectionIndex = Number.isInteger(cursor.collection) ? cursor.collection : 0;
+ let fileIndex = Number.isInteger(cursor.file) ? cursor.file : 0;
+ let filesSeen = num(row.files_seen) || 0;
+ let recordsRead = num(row.records_read) || 0;
+ let recordsKept = num(row.records_kept) || 0;
+ let droppedLicence = num(row.dropped_licence) || 0;
+ let droppedInvalid = num(row.dropped_invalid) || 0;
+ const reasons = { ...json(row.dropped_reason, {}) };
+ let names;
+ let error = '';
+ let status = 'running';
+ try {
+ names = Array.isArray(collections) && collections.length ? collections.map(String).sort() : await listCollections();
+ let inThisPass = 0;
+ while (collectionIndex < names.length && inThisPass < maxFiles && recordsRead < maxRecords) {
+ const paths = await listFiles(names[collectionIndex]);
+ while (fileIndex < paths.length && inThisPass < maxFiles && recordsRead < maxRecords) {
+ const path = paths[fileIndex];
+ let doc;
+ try {
+ doc = await getJson(rawUrl(path));
+ } catch (err) {
+ // A single unreadable file is a skipped record, not the end of the pass.
+ droppedInvalid += 1;
+ reasons['could not read ' + path] = (reasons['could not read ' + path] || 0) + 1;
+ fileIndex += 1;
+ inThisPass += 1;
+ filesSeen += 1;
+ continue;
+ }
+ const out = await storeRecord(doc, { path, readAt: s.iso(), fallbackLicence: licence.licence, fallbackLicenceUrl: licence.url });
+ recordsRead += out.read;
+ recordsKept += out.kept;
+ droppedLicence += out.droppedLicence;
+ droppedInvalid += out.droppedInvalid;
+ for (const [reason, count] of Object.entries(out.reasons)) reasons[reason] = (reasons[reason] || 0) + count;
+ fileIndex += 1;
+ inThisPass += 1;
+ filesSeen += 1;
+ }
+ if (fileIndex >= paths.length) { collectionIndex += 1; fileIndex = 0; }
+ }
+ if (collectionIndex >= names.length) status = 'finished';
+ } catch (err) {
+ status = 'failed';
+ error = String((err && err.message) || err).slice(0, 500);
+ log.warn('[OpenVibe.Bench] the EEE sync stopped:', error);
+ }
+ const nextCursor = JSON.stringify({ collection: collectionIndex, file: fileIndex });
+ const updated = await db.maybe(
+ `UPDATE bench_eee_syncs SET status = $1, cursor = $2::jsonb, collections = $3, files_seen = $4,
+ records_read = $5, records_kept = $6, dropped_licence = $7, dropped_invalid = $8,
+ dropped_reason = $9::jsonb, default_licence = $10, licence_url = $11, error = $12,
+ updated_at = $13::timestamptz, finished_at = CASE WHEN $1 = 'running' THEN NULL ELSE $13::timestamptz END
+ WHERE id = $14 RETURNING *`,
+ [status, nextCursor, names ? names.length : 0, filesSeen, recordsRead, recordsKept, droppedLicence,
+ droppedInvalid, JSON.stringify(reasons), licence.licence, licence.url, error, s.iso(), row.id],
+ );
+ return wireSync(updated);
+ }
+
+ // ── Export: Bench's own runs as EEE `evaluation_run` records ───────────────────────────────────────────────
+
+ /** One public Bench run as an EEE aggregate record. Top-level extras are not allowed by the schema, so the
+ * Bench-specific facts live in the nested `additional_details` maps. */
+ function eeeRecordOf(run, tasks) {
+ const createdAt = iso(run.created_at) || s.iso();
+ const retrieved = String(Math.floor((Date.parse(createdAt) || now()) / 1000));
+ const settings = json(run.settings, {});
+ const harness = json(run.harness, {});
+ const route = json(run.route, {});
+ const benchmarkRef = `${run.benchmark_id}@${run.benchmark_version}`;
+ const sampleCount = Array.isArray(tasks) ? tasks.length : 0;
+ const samplesUrl = baseUrl ? `${baseUrl}/api/v1/datasets/eee/runs/${run.id}.samples.jsonl` : null;
+ const developer = firstText(route.maker, route.provider_name, (route.model && String(route.model).includes('/')) ? String(route.model).split('/')[0] : '');
+ const generationArgs = {};
+ if (settings.temperature != null) generationArgs.temperature = num(settings.temperature);
+ if (settings.max_tokens != null && num(settings.max_tokens) > 0) generationArgs.max_tokens = num(settings.max_tokens);
+ const result = {
+ evaluation_result_id: `${run.id}:${run.benchmark_id}`,
+ evaluation_name: `openvibe.bench.${run.benchmark_kind || 'run'}`,
+ source_data: {
+ dataset_name: String(run.benchmark_title || run.benchmark_id),
+ source_type: 'other',
+ additional_details: {
+ benchmark_id: String(run.benchmark_id),
+ benchmark_version: String(run.benchmark_version),
+ benchmark_ref: benchmarkRef,
+ task_count: String(sampleCount),
+ ...(samplesUrl ? { samples_url: samplesUrl } : {}),
+ },
+ },
+ metric_config: {
+ metric_id: 'openvibe-bench.score',
+ metric_name: 'Score',
+ metric_kind: 'score',
+ metric_unit: 'points',
+ lower_is_better: false,
+ score_type: 'continuous',
+ min_score: 0,
+ max_score: 100,
+ },
+ score_details: {
+ score: num(run.score) == null ? 0 : num(run.score),
+ ...(sampleCount ? { uncertainty: { num_samples: sampleCount } } : {}),
+ },
+ generation_config: {
+ generation_args: generationArgs,
+ additional_details: {
+ ...(settings.seed != null ? { seed: String(settings.seed) } : {}),
+ ...(settings.few_shot != null ? { few_shot: String(settings.few_shot) } : {}),
+ ...(settings.effort != null ? { effort: String(settings.effort) } : {}),
+ },
+ },
+ };
+ return {
+ schema_version: SCHEMA_VERSION,
+ evaluation_id: `openvibe-bench/${run.id}`,
+ evaluation_timestamp: createdAt,
+ retrieved_timestamp: retrieved,
+ source_metadata: {
+ source_name: 'OpenVibe.Bench',
+ source_type: 'evaluation_run',
+ source_organization_name: 'OpenVibe',
+ ...(baseUrl ? { source_organization_url: baseUrl } : {}),
+ evaluator_relationship: 'third_party',
+ additional_details: {
+ run_id: String(run.id),
+ provenance: String(run.provenance || ''),
+ benchmark: benchmarkRef,
+ provider: String(run.provider || ''),
+ },
+ },
+ eval_library: {
+ name: firstText(harness.name, 'openvibe-bench'),
+ version: firstText(harness.version, 'unknown'),
+ },
+ model_info: {
+ name: String(run.model || 'unknown'),
+ id: String((route.model || run.model || 'unknown')),
+ ...(developer ? { developer } : {}),
+ ...(run.provider ? { inference_platform: String(run.provider) } : {}),
+ additional_details: {
+ deployment_type: 'externally_managed',
+ model_availability: 'unknown',
+ bench_model: String(run.model || ''),
+ bench_provider: String(run.provider || ''),
+ },
+ },
+ evaluation_results: [result],
+ };
+ }
+
+ /** Public Bench runs as EEE records, newest last, paged. `limit` is capped so one request stays small. */
+ async function exportRecords({ offset = 0, limit = 500 } = {}) {
+ const take = Math.max(1, Math.min(1000, Number(limit) || 500));
+ const skip = Math.max(0, Number(offset) || 0);
+ const runs = await db.many(
+ `SELECT r.*, b.title AS benchmark_title, b.kind AS benchmark_kind
+ FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id
+ WHERE b.visibility = 'public' AND r.provenance = 'verified'
+ ORDER BY r.created_at, r.id LIMIT $1 OFFSET $2`,
+ [take, skip],
+ );
+ if (!runs.length) return [];
+ const tasks = await db.many(
+ 'SELECT * FROM bench_run_tasks WHERE run_id = ANY($1::text[]) ORDER BY created_at, id',
+ [runs.map((run) => String(run.id))],
+ );
+ const byRun = new Map();
+ for (const task of tasks) {
+ if (!byRun.has(task.run_id)) byRun.set(task.run_id, []);
+ byRun.get(task.run_id).push(task);
+ }
+ return runs.map((run) => eeeRecordOf(run, byRun.get(run.id) || []));
+ }
+
+ async function exportCount() {
+ const row = await db.maybe(
+ "SELECT COUNT(*)::int AS n FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id WHERE b.visibility = 'public' AND r.provenance = 'verified'",
+ );
+ return num(row && row.n) || 0;
+ }
+
+ async function exportLines(options = {}) {
+ const records = await exportRecords(options);
+ return records.length ? records.map((record) => JSON.stringify(record)).join('\n') + '\n' : '';
+ }
+
+ /** The per-sample (instance-level) lines of one public run: Bench's per-task scores and where its answer is. */
+ async function samplesOf(runId) {
+ const run = await db.maybe(
+ `SELECT r.*, b.title AS benchmark_title, b.kind AS benchmark_kind
+ FROM bench_runs r JOIN bench_benchmarks b ON b.id = r.benchmark_id
+ WHERE r.id = $1 AND b.visibility = 'public' AND r.provenance = 'verified'`,
+ [String(runId || '')],
+ );
+ if (!run) return null;
+ const tasks = await db.many('SELECT * FROM bench_run_tasks WHERE run_id = $1 ORDER BY created_at, id', [run.id]);
+ const evaluationId = `openvibe-bench/${run.id}`;
+ return {
+ run,
+ lines: tasks.map((task) => JSON.stringify({
+ evaluation_id: evaluationId,
+ model_id: String(run.model || 'unknown'),
+ sample_id: String(task.task_id),
+ task_id: String(task.task_id),
+ score: task.score == null ? null : num(task.score),
+ cost_usd: num(task.cost_usd),
+ ms: num(task.ms),
+ tokens: num(task.tokens),
+ ...(task.transcript_url ? { transcript_url: String(task.transcript_url) } : {}),
+ artifacts: json(task.artifacts, []),
+ created_at: iso(task.created_at),
+ })),
+ };
+ }
+
+ return {
+ SOURCE, sync, storeRecord, listRecords, countRecords, latestSync,
+ exportRecords, exportLines, exportCount, samplesOf,
+ listCollections, listFiles, normalizeRecord,
+ };
+}
+
+module.exports = {
+ createEee, normalizeRecord, allowsRedistribution, recordLicenceOf, bareModel,
+ datastoreLicence, wireExternal, SOURCE, DATASET, DATASET_URL, SCHEMA_VERSION,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,96 @@
@@ -0,0 +1,96 @@
+'use strict';
+
+/**
+ * Stanford HELM results: a run's `stats.json` together with its `run_spec.json`.
+ *
+ * HELM writes one run directory per (scenario, method, model). `stats.json` is a list of stat objects — each has a
+ * `name` ({ name, split, perturbation? }) and a `mean` — and `run_spec.json` says which model, scenario and method
+ * produced them. This importer accepts the two together, either as { stats, run_spec } or as one document whose
+ * top level is the run spec and whose `stats` key holds the list. One model here is one run; the run's scenario is
+ * the task, the adapter's `max_train_instances` the few-shot.
+ */
+const { parseJson, asText } = require('./limits');
+
+const TOOL = 'helm';
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+const METRIC_ORDER = ['quasi_exact_match', 'exact_match', 'accuracy', 'quasi_prefix_exact_match', 'prefix_exact_match', 'f1', 'bleu', 'score'];
+
+function statsOf(doc) {
+ if (Array.isArray(doc)) return { stats: doc, runSpec: {} };
+ if (!doc || typeof doc !== 'object') return { stats: null, runSpec: {} };
+ const runSpec = doc.run_spec && typeof doc.run_spec === 'object' ? doc.run_spec : doc;
+ const stats = Array.isArray(doc.stats) ? doc.stats : (Array.isArray(doc) ? doc : null);
+ return { stats, runSpec };
+}
+
+/** The headline metric: the first in the preference list, from a plain split before a perturbed one. */
+function metricOf(stats) {
+ const usable = (stats || []).filter((s) => s && s.name && typeof s.name === 'object' && num(s.mean) != null);
+ if (!usable.length) return null;
+ const plain = usable.filter((s) => !s.name.perturbation);
+ const splits = (list) => {
+ const wanted = list.filter((s) => s.name.split === 'valid' || s.name.split === 'test' || s.name.split == null);
+ return wanted.length ? wanted : list;
+ };
+ const pool = splits(plain.length ? plain : usable);
+ for (const name of METRIC_ORDER) {
+ const found = pool.find((s) => String(s.name.name) === name);
+ if (found) return { metric: name, value: num(found.mean) };
+ }
+ const first = pool[0];
+ return { metric: String(first.name.name || 'score'), value: num(first.mean) };
+}
+
+function parse(bytes) {
+ const doc = parseJson(asText(bytes), { what: 'the results' });
+ const { stats, runSpec } = statsOf(doc);
+ if (!Array.isArray(stats) || !stats.length) throw new Error('That file holds no HELM stats.');
+ const adapter = (runSpec.adapter_spec && typeof runSpec.adapter_spec === 'object') ? runSpec.adapter_spec : {};
+ const model = String(adapter.model || runSpec.model || '').trim();
+ if (!model) throw new Error('That file has no run spec naming the model.');
+ const metric = metricOf(stats);
+ if (!metric) throw new Error('That file holds no scores I can read.');
+ const scenario = runSpec.scenario_spec && typeof runSpec.scenario_spec === 'object' ? runSpec.scenario_spec : {};
+ const args = (scenario.args && typeof scenario.args === 'object') ? scenario.args : {};
+ const groups = Array.isArray(runSpec.groups) ? runSpec.groups : [];
+ const task = String(args.dataset || groups[0] || String(runSpec.name || '').split(':')[0] || 'helm').trim();
+ const deployment = String(adapter.model_deployment || '');
+ const provider = deployment.includes('/') ? deployment.split('/')[0] : (model.includes('/') ? model.split('/')[0] : 'helm');
+ const version = String(runSpec.helm_version || doc.helm_version || runSpec.version || '').trim();
+ const score = metric.value <= 1 ? metric.value * 100 : metric.value;
+ const taskVersion = String(runSpec.version || doc.version || '').trim();
+ const seed = adapter.seed != null ? adapter.seed : adapter.random_seed;
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'helm', version: version || 'unknown' },
+ date: runSpec.date || doc.date || null,
+ suite: { name: task, version: taskVersion || null },
+ models: [{
+ model,
+ provider: provider || 'helm',
+ score,
+ cost_usd: null,
+ seed: seed == null ? null : String(seed),
+ few_shot: num(adapter.max_train_instances),
+ tasks: [{
+ task_id: task,
+ score,
+ metric: metric.metric,
+ few_shot: num(adapter.max_train_instances),
+ task_version: taskVersion || null,
+ higher_is_better: !/loss|perplexity|latency|runtime/i.test(metric.metric),
+ }],
+ }],
+ };
+}
+
+function looksLike(doc) {
+ if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return false;
+ if (!Array.isArray(doc.stats) || !doc.stats.length) return false;
+ const first = doc.stats[0];
+ return Boolean(first && first.name && typeof first.name === 'object' && ('mean' in first || 'count' in first));
+}
+
+module.exports = { TOOL, parse, looksLike, metricOf };
@@ -1,27 +1,33 @@
@@ -1,27 +1,33 @@
'use strict';
'use strict';
/**
/**
- * The four tools whose results a person can bring to Bench (B12): Inspect AI, Harbor, the EleutherAI
- * lm-evaluation-harness and promptfoo. The file says which one it is; when it does not, the shape of the
- * document says it.
+ * The tools whose results a person can bring to Bench: Inspect AI, Harbor, HF lighteval, the EleutherAI
+ * lm-evaluation-harness, Stanford HELM and promptfoo. The file says which one it is; when it does not, the shape of
+ * the document says it.
*/
*/
const { ImportError, MAX_UPLOAD_BYTES } = require('./limits');
const { ImportError, MAX_UPLOAD_BYTES } = require('./limits');
const inspectTool = require('./inspect');
const inspectTool = require('./inspect');
const harborTool = require('./harbor');
const harborTool = require('./harbor');
+const lightevalTool = require('./lighteval');
const lmEvalTool = require('./lm-eval');
const lmEvalTool = require('./lm-eval');
+const helmTool = require('./helm');
const promptfooTool = require('./promptfoo');
const promptfooTool = require('./promptfoo');
const TOOLS = {
const TOOLS = {
inspect: inspectTool,
inspect: inspectTool,
harbor: harborTool,
harbor: harborTool,
+ lighteval: lightevalTool,
'lm-eval': lmEvalTool,
'lm-eval': lmEvalTool,
+ helm: helmTool,
promptfoo: promptfooTool,
promptfoo: promptfooTool,
};
};
const TOOL_LABELS = {
const TOOL_LABELS = {
inspect: 'Inspect AI',
inspect: 'Inspect AI',
harbor: 'Harbor',
harbor: 'Harbor',
+ lighteval: 'HF lighteval',
'lm-eval': 'lm-evaluation-harness',
'lm-eval': 'lm-evaluation-harness',
+ helm: 'Stanford HELM',
promptfoo: 'promptfoo',
promptfoo: 'promptfoo',
};
};
@@ -31,7 +37,9 @@ function detect(doc, filename = '') {
@@ -31,7 +37,9 @@ function detect(doc, filename = '') {
if (name.endsWith('.eval')) return 'inspect';
if (name.endsWith('.eval')) return 'inspect';
if (inspectTool.looksLike(doc)) return 'inspect';
if (inspectTool.looksLike(doc)) return 'inspect';
if (harborTool.looksLike(doc)) return 'harbor';
if (harborTool.looksLike(doc)) return 'harbor';
+ if (lightevalTool.looksLike(doc)) return 'lighteval';
if (lmEvalTool.looksLike(doc)) return 'lm-eval';
if (lmEvalTool.looksLike(doc)) return 'lm-eval';
+ if (helmTool.looksLike(doc)) return 'helm';
if (promptfooTool.looksLike(doc)) return 'promptfoo';
if (promptfooTool.looksLike(doc)) return 'promptfoo';
return null;
return null;
}
}
@@ -55,9 +63,10 @@ function parse(bytes, { filename = '', tool = null } = {}) {
@@ -55,9 +63,10 @@ function parse(bytes, { filename = '', tool = null } = {}) {
out.filename = String(filename || '');
out.filename = String(filename || '');
return out;
return out;
}
}
- // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others.
+ // No tool named: look at the shape. Inspect's log is read first (it may be a zip), then the others. lighteval
+ // comes before lm-eval: both have `results`, and only lighteval has `config_general`/`config_tasks`.
const errors = [];
const errors = [];
- for (const key of ['inspect', 'harbor', 'lm-eval', 'promptfoo']) {
+ for (const key of ['inspect', 'harbor', 'lighteval', 'lm-eval', 'helm', 'promptfoo']) {
try {
try {
const out = TOOLS[key].parse(buf);
const out = TOOLS[key].parse(buf);
out.filename = String(filename || '');
out.filename = String(filename || '');
@@ -68,7 +77,7 @@ function parse(bytes, { filename = '', tool = null } = {}) {
@@ -68,7 +77,7 @@ function parse(bytes, { filename = '', tool = null } = {}) {
}
}
}
}
throw new ImportError(422, 'bench.import_unknown_format',
throw new ImportError(422, 'bench.import_unknown_format',
- 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, an lm-evaluation-harness results.json or a promptfoo output.json.');
+ 'I do not know that file. It should be an Inspect AI log, a Harbor results.json, a lighteval results file, an lm-evaluation-harness results.json, a HELM stats file or a promptfoo output.json.');
}
}
module.exports = { TOOLS, TOOL_LABELS, detect, parse, ImportError, MAX_UPLOAD_BYTES };
module.exports = { TOOLS, TOOL_LABELS, detect, parse, ImportError, MAX_UPLOAD_BYTES };
new file mode 100644
new file mode 100644
@@ -0,0 +1,101 @@
@@ -0,0 +1,101 @@
+'use strict';
+
+/**
+ * Hugging Face lighteval results: the `results_<model>.json` that `lighteval` writes.
+ *
+ * The file is { config_general, results, versions, config_tasks }. `results` maps a task key
+ * ("arc:challenge|25" — the task, then the few-shot count) to its metrics. `config_tasks` names the task, its HF
+ * dataset and its metric (and whether a higher score is better). One file is one model, so one model here is one run;
+ * every task key is a task of that run, with its own few-shot and task version.
+ */
+const { parseJson, asText } = require('./limits');
+
+const TOOL = 'lighteval';
+const num = (v) => (v == null || v === '' || Number.isNaN(Number(v)) ? null : Number(v));
+
+const METRIC_ORDER = ['acc', 'exact_match', 'acc_norm', 'f1', 'pass@1', 'quasi_exact_match', 'accuracy', 'em'];
+
+/** The metric of a task that means "how good was it", skipping its error bar. */
+function metricOf(entry) {
+ if (!entry || typeof entry !== 'object') return null;
+ for (const key of METRIC_ORDER) {
+ if (num(entry[key]) != null) return { metric: key, value: num(entry[key]) };
+ }
+ for (const [key, value] of Object.entries(entry)) {
+ if (/_stderr$|stderr|std_err/i.test(key)) continue;
+ const n = num(value);
+ if (n != null) return { metric: key, value: n };
+ }
+ return null;
+}
+
+/** "arc:challenge|25" -> { task: "arc:challenge", fewShot: 25 }. */
+function keyOf(key) {
+ const text = String(key || '');
+ const parts = text.split('|');
+ return { task: parts[0].trim(), fewShot: num(parts[1]) };
+}
+
+function parse(bytes) {
+ const doc = parseJson(asText(bytes), { what: 'the results' });
+ const results = doc && doc.results && typeof doc.results === 'object' && !Array.isArray(doc.results) ? doc.results : null;
+ if (!results) throw new Error('That file has no results.');
+ const general = (doc.config_general && typeof doc.config_general === 'object') ? doc.config_general : {};
+ const taskConfigs = (doc.config_tasks && typeof doc.config_tasks === 'object') ? doc.config_tasks : {};
+ const versions = (doc.versions && typeof doc.versions === 'object') ? doc.versions : {};
+ const model = String(general.model_name || (general.model_config && general.model_config.model_name) || doc.model_name || '').trim();
+ if (!model) throw new Error('That file names no model.');
+
+ const tasks = [];
+ for (const [key, entry] of Object.entries(results)) {
+ const metric = metricOf(entry);
+ if (!metric) continue;
+ const { task, fewShot } = keyOf(key);
+ const config = taskConfigs[key] || taskConfigs[task] || {};
+ const taskVersion = versions[key] != null ? versions[key] : (versions[task] != null ? versions[task] : null);
+ tasks.push({
+ task_id: task || key,
+ score: metric.value * 100,
+ metric: metric.metric,
+ few_shot: fewShot,
+ task_version: taskVersion == null ? null : String(taskVersion),
+ hf_repo: config.hf_repo ? String(config.hf_repo) : null,
+ higher_is_better: config.metrics && Array.isArray(config.metrics) && config.metrics[0]
+ ? Boolean(config.metrics[0].higher_is_better)
+ : null,
+ });
+ }
+ if (!tasks.length) throw new Error('That file holds no scores I can read.');
+
+ const provider = model.includes('/') ? model.split('/')[0] : 'lighteval';
+ const version = String(general.lighteval_sha || general.lighteval_version || doc.version || '').trim();
+ const seed = general.model_config && general.model_config.generation_parameters
+ ? general.model_config.generation_parameters.seed
+ : null;
+ const overall = tasks.reduce((n, t) => n + t.score, 0) / tasks.length;
+ const distinct = [...new Set(tasks.map((t) => t.task_id))];
+ return {
+ tool: TOOL,
+ toolVersion: version || 'unknown',
+ harness: { name: 'lighteval', version: version || 'unknown' },
+ date: general.pretty_timestamp || general.date || null,
+ suite: distinct.length === 1 ? { name: distinct[0], version: tasks[0].task_version } : null,
+ models: [{
+ model,
+ provider,
+ score: overall,
+ cost_usd: null,
+ seed: seed == null ? null : String(seed),
+ few_shot: tasks.length && tasks[0].few_shot != null ? tasks[0].few_shot : null,
+ tasks,
+ }],
+ };
+}
+
+function looksLike(doc) {
+ if (!doc || typeof doc !== 'object') return false;
+ if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) return false;
+ return Boolean(doc.config_general || doc.config_tasks);
+}
+
+module.exports = { TOOL, parse, looksLike, metricOf };
@@ -89,7 +89,17 @@ function createImports({ s, registry, log = console }) {
@@ -89,7 +89,17 @@ function createImports({ s, registry, log = console }) {
score: m.score == null ? null : num(m.score),
score: m.score == null ? null : num(m.score),
cost_usd: m.cost_usd == null ? null : num(m.cost_usd),
cost_usd: m.cost_usd == null ? null : num(m.cost_usd),
tokens: m.tokens == null ? null : num(m.tokens),
tokens: m.tokens == null ? null : num(m.tokens),
- tasks: (m.tasks || []).map((t) => ({ task_id: String(t.task_id), score: t.score == null ? null : num(t.score) })),
+ // B7: the harness's own settings travel with the import: the run's seed and few-shot, and each
+ // task's version and metric, so a result can be compared at a glance with the tool that made it.
+ seed: m.seed == null ? null : String(m.seed),
+ few_shot: m.few_shot == null ? null : num(m.few_shot),
+ tasks: (m.tasks || []).map((t) => ({
+ task_id: String(t.task_id),
+ score: t.score == null ? null : num(t.score),
+ task_version: t.task_version == null ? null : String(t.task_version),
+ few_shot: t.few_shot == null ? null : num(t.few_shot),
+ metric: t.metric == null ? null : String(t.metric),
+ })),
})),
})),
};
};
await db.exec(
await db.exec(
@@ -202,13 +212,23 @@ function createImports({ s, registry, log = console }) {
@@ -202,13 +212,23 @@ function createImports({ s, registry, log = console }) {
.filter((t) => known.has(t.task_id))
.filter((t) => known.has(t.task_id))
.map((t) => ({ task_id: t.task_id, score: t.score == null ? null : num(t.score), cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }));
.map((t) => ({ task_id: t.task_id, score: t.score == null ? null : num(t.score), cost_usd: 0, ms: 0, tokens: 0, artifacts: [] }));
if (!tasks.length) continue;
if (!tasks.length) continue;
+ // B7: keep what the harness recorded beside the scores: the task versions, the few-shot and the seed.
+ const taskVersions = {};
+ for (const task of model.tasks || []) {
+ if (task.task_version) taskVersions[String(task.task_id)] = String(task.task_version);
+ }
+ const settings = {
+ ...(model.seed != null ? { seed: String(model.seed) } : {}),
+ ...(model.few_shot != null ? { few_shot: num(model.few_shot) } : {}),
+ ...(Object.keys(taskVersions).length ? { task_versions: taskVersions } : {}),
+ };
const published = await registry.publishRun({
const published = await registry.publishRun({
benchmark: benchmark.id + '@' + version,
benchmark: benchmark.id + '@' + version,
model: String(model.model || 'unknown').slice(0, 160),
model: String(model.model || 'unknown').slice(0, 160),
provider: String(model.provider || 'unknown').slice(0, 160),
provider: String(model.provider || 'unknown').slice(0, 160),
route: {},
route: {},
harness: summary.harness || { name: toolLabel, version: summary.tool_version || 'unknown' },
harness: summary.harness || { name: toolLabel, version: summary.tool_version || 'unknown' },
- settings: {},
+ settings,
tasks,
tasks,
score: model.score == null ? null : num(model.score),
score: model.score == null ? null : num(model.score),
cost_usd: num(model.cost_usd),
cost_usd: num(model.cost_usd),
@@ -136,6 +136,16 @@ function load(env = process.env) {
@@ -136,6 +136,16 @@ function load(env = process.env) {
// How often the job looks at the clock.
// How often the job looks at the clock.
pollMs: Math.max(30_000, int(env.BENCH_DAILY_POLL_MS, 300_000)),
pollMs: Math.max(30_000, int(env.BENCH_DAILY_POLL_MS, 300_000)),
},
},
+ // B2: the weekly Every Eval Ever import. It reads a public datastore (no account, no key), so it is safe
+ // to run, but it is the owner's switch: off until BENCH_EEE_SYNC is on. One pass is bounded by the two
+ // caps, and the cursor in bench_eee_syncs makes a stopped pass resume.
+ eee: {
+ enabled: bool(env.BENCH_EEE_SYNC, false),
+ enabledInDrill: bool(env.BENCH_EEE_SYNC_IN_DRILL, false),
+ intervalMs: Math.max(60_000, int(env.BENCH_EEE_SYNC_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
+ filesPerRun: Math.max(1, int(env.BENCH_EEE_FILES_PER_RUN, 5000)),
+ recordsPerRun: Math.max(1, int(env.BENCH_EEE_RECORDS_PER_RUN, 50000)),
+ },
},
},
};
};
}
}
@@ -54,7 +54,7 @@ function createDiscoveryRoutes(ctx) {
@@ -54,7 +54,7 @@ function createDiscoveryRoutes(ctx) {
summary: DESCRIPTION,
summary: DESCRIPTION,
markdown: true,
markdown: true,
base: site,
base: site,
- details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. GET /api/v1/datasets/benchmarks.jsonl is every public benchmark and its runs. POST /mcp is the Model Context Protocol endpoint.',
+ details: 'GET /api/v1/kinds lists the benchmark kinds. GET /api/v1/boards/:kind?type=elo|pareto|cost renders a board as JSON. GET /api/v1/datasets/:kind.jsonl is the open dataset without voter ids. GET /api/v1/datasets/benchmarks.jsonl is every public benchmark and its runs. GET /api/v1/datasets/eee is Bench’s public runs as Every Eval Ever records (JSONL, paged), and GET /api/v1/external/records is the imported Every Eval Ever records with their provenance. POST /mcp is the Model Context Protocol endpoint.',
sections: [
sections: [
{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },
{ title: 'Start here', links: [{ title: SITE_NAME, url: abs('/') }, { title: 'The API', url: abs('/docs') }] },
{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }] },
{ title: 'Machine-readable', links: [{ title: 'Kinds (JSON)', url: abs('/api/v1/kinds') }] },
new file mode 100644
new file mode 100644
@@ -0,0 +1,100 @@
@@ -0,0 +1,100 @@
+'use strict';
+
+/**
+ * The EEE routes (B2).
+ *
+ * GET /api/v1/datasets/eee Bench's public runs as EEE `evaluation_run` records (JSONL, paged)
+ * GET /api/v1/datasets/eee/runs/:id.samples.jsonl one run's per-sample scores
+ * GET /api/v1/external/records the imported EEE records, each `external: true` with its provenance
+ * POST /api/v1/admin/eee/sync one bounded pass of the weekly import (operator token)
+ *
+ * Everything here is public except the sync, which is the operator's own switch and reads the public datastore.
+ */
+const crypto = require('crypto');
+const { asyncRouter } = require('./router');
+
+function operatorToken(req) {
+ const h = String(req.get('authorization') || '');
+ return /^Bearer\s+\S/i.test(h) ? h.replace(/^Bearer\s+/i, '').trim() : null;
+}
+
+function tokenMatches(given, expected) {
+ if (!given || !expected) return false;
+ const a = crypto.createHash('sha256').update(String(given), 'utf8').digest();
+ const b = crypto.createHash('sha256').update(String(expected), 'utf8').digest();
+ return a.length === b.length && crypto.timingSafeEqual(a, b);
+}
+
+function createEeeRoutes(ctx) {
+ const r = asyncRouter();
+ const eee = ctx.eee;
+ const token = process.env.BENCH_ADMIN_TOKEN || '';
+
+ const paging = (req) => ({
+ offset: Math.max(0, Number(req.query.offset) || 0),
+ limit: Math.max(1, Math.min(1000, Number(req.query.limit) || 500)),
+ });
+
+ const jsonl = (res, text, name) => {
+ res.set('Cache-Control', 'public, max-age=60');
+ res.set('Content-Disposition', `attachment; filename="${name}"`);
+ return res.type('application/x-ndjson').send(text);
+ };
+
+ r.get('/api/v1/datasets/eee', async (req, res) => {
+ const { offset, limit } = paging(req);
+ const [text, total] = await Promise.all([eee.exportLines({ offset, limit }), eee.exportCount()]);
+ res.set('X-Total-Count', String(total));
+ return jsonl(res, text, 'bench-eee.jsonl');
+ });
+
+ r.get('/api/v1/datasets/eee.jsonl', async (req, res) => {
+ const { offset, limit } = paging(req);
+ const [text, total] = await Promise.all([eee.exportLines({ offset, limit }), eee.exportCount()]);
+ res.set('X-Total-Count', String(total));
+ return jsonl(res, text, 'bench-eee.jsonl');
+ });
+
+ r.get('/api/v1/datasets/eee/runs/:id.samples.jsonl', async (req, res) => {
+ const out = await eee.samplesOf(String(req.params.id || ''));
+ if (!out) return res.status(404).type('text/plain').send('No such public run.');
+ res.set('X-Total-Count', String(out.lines.length));
+ return jsonl(res, out.lines.length ? out.lines.join('\n') + '\n' : '', `${out.run.id}-samples.jsonl`);
+ });
+
+ // The external records: imported, never rated, always labelled with where they came from.
+ r.get('/api/v1/external/records', async (req, res) => {
+ const { offset, limit } = paging(req);
+ const filters = {
+ model: req.query.model ? String(req.query.model) : null,
+ task: req.query.task ? String(req.query.task) : null,
+ sourceType: req.query.source_type ? String(req.query.source_type) : null,
+ };
+ const [records, total] = await Promise.all([
+ eee.listRecords({ ...filters, offset, limit }),
+ eee.countRecords(filters),
+ ]);
+ res.set('Cache-Control', 'public, max-age=60');
+ return res.json({ count: records.length, total, offset, limit, external: true, records });
+ });
+
+ r.get('/api/v1/external/sync', async (_req, res) => {
+ res.set('Cache-Control', 'public, max-age=30');
+ return res.json({ sync: await eee.latestSync() });
+ });
+
+ r.post('/api/v1/admin/eee/sync', async (req, res) => {
+ if (!token) return res.status(404).type('text').send('No such route.');
+ if (!tokenMatches(operatorToken(req), token)) return res.status(403).type('text').send('The operator token is required.');
+ const body = req.body && typeof req.body === 'object' ? req.body : {};
+ const out = await eee.sync({
+ maxFiles: body.max_files ? Math.max(1, Math.min(20000, Number(body.max_files))) : undefined,
+ maxRecords: body.max_records ? Math.max(1, Math.min(200000, Number(body.max_records))) : undefined,
+ });
+ return res.json(out);
+ });
+
+ return r;
+}
+
+module.exports = { createEeeRoutes, tokenMatches };
@@ -31,7 +31,7 @@ function createImportRoutes(ctx) {
@@ -31,7 +31,7 @@ function createImportRoutes(ctx) {
return page(req, res, {
return page(req, res, {
index: false,
index: false,
title: 'Import a results file',
title: 'Import a results file',
- description: 'Bring the results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo, and see the runs it holds before they are imported.',
+ description: 'Bring the results file of Inspect AI, Harbor, HF lighteval, lm-evaluation-harness, Stanford HELM or promptfoo, and see the runs it holds before they are imported.',
body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, imports, error: req.query.error ? String(req.query.error).slice(0, 200) : null }))),
body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, imports, error: req.query.error ? String(req.query.error).slice(0, 200) : null }))),
});
});
};
};
@@ -42,7 +42,7 @@ function createImportRoutes(ctx) {
@@ -42,7 +42,7 @@ function createImportRoutes(ctx) {
const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');
const loginUrl = '/auth/login?next=' + encodeURIComponent('/imports');
const fail = (status, error) => page(req, res, {
const fail = (status, error) => page(req, res, {
index: false, title: 'Import a results file',
index: false, title: 'Import a results file',
- description: 'Bring a results file of Inspect AI, Harbor, lm-evaluation-harness or promptfoo.',
+ description: 'Bring a results file of Inspect AI, Harbor, HF lighteval, lm-evaluation-harness, Stanford HELM or promptfoo.',
body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, error }))),
body: raw(String(views.uploadPage({ signedIn: Boolean(subject), loginUrl, error }))),
}, status);
}, status);
if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
if (!sameOrigin(req, config)) return res.status(403).type('text').send('This form only works from Bench itself.');
@@ -16,6 +16,7 @@ function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }
@@ -16,6 +16,7 @@ function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }
() => ctx.keys.client.stop(),
() => ctx.keys.client.stop(),
() => ctx.jobs.stop(),
() => ctx.jobs.stop(),
() => ctx.daily.stop(),
() => ctx.daily.stop(),
+ () => ctx.eeeSync.stop(),
() => ctx.publisher.stop(),
() => ctx.publisher.stop(),
async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); },
async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); },
() => ctx.s.close(),
() => ctx.s.close(),
@@ -45,6 +46,8 @@ async function start() {
@@ -45,6 +46,8 @@ async function start() {
ctx.jobs.start();
ctx.jobs.start();
// The daily rounds (one per design kind per day). Off until BENCH_DAILY_ROUNDS is on; never under the drill.
// The daily rounds (one per design kind per day). Off until BENCH_DAILY_ROUNDS is on; never under the drill.
ctx.daily.start();
ctx.daily.start();
+ // The weekly Every Eval Ever import (B2). Off until BENCH_EEE_SYNC is on; bounded and resumable.
+ ctx.eeeSync.start();
// Outbox publisher → OpenVibe.Events.
// Outbox publisher → OpenVibe.Events.
ctx.publisher.start();
ctx.publisher.start();
@@ -9,7 +9,9 @@ const { html, raw } = require('../render/html');
@@ -9,7 +9,9 @@ const { html, raw } = require('../render/html');
const TOOLS = [
const TOOLS = [
{ key: 'inspect', label: 'Inspect AI', hint: 'an .eval log (a zip) or its JSON log' },
{ key: 'inspect', label: 'Inspect AI', hint: 'an .eval log (a zip) or its JSON log' },
{ key: 'harbor', label: 'Harbor', hint: 'the results.json of a Harbor job' },
{ key: 'harbor', label: 'Harbor', hint: 'the results.json of a Harbor job' },
+ { key: 'lighteval', label: 'HF lighteval', hint: 'the results_<model>.json of a run' },
{ key: 'lm-eval', label: 'lm-evaluation-harness', hint: 'the results.json of a run' },
{ key: 'lm-eval', label: 'lm-evaluation-harness', hint: 'the results.json of a run' },
+ { key: 'helm', label: 'Stanford HELM', hint: 'a run’s stats.json with its run_spec.json' },
{ key: 'promptfoo', label: 'promptfoo', hint: 'the output.json of promptfoo eval -o' },
{ key: 'promptfoo', label: 'promptfoo', hint: 'the output.json of promptfoo eval -o' },
];
];
@@ -61,6 +61,8 @@ ${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&cate
@@ -61,6 +61,8 @@ ${ep('GET', '/api/v1/boards/:kindId?type=elo|pareto|cost&voters=people|anon&cate
${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}
${ep('GET', '/api/v1/models/:model', 'One model across every kind.')}
${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}
${ep('GET', '/api/v1/datasets/:kindId.jsonl', 'The open dataset (prompts, entries, judgements without voter ids, ratings).')}
${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}
${ep('GET', '/api/v1/datasets/benchmarks.jsonl', 'Every public benchmark, its visible tasks and every run, with no person id.')}
+${ep('GET', '/api/v1/datasets/eee?offset=0&limit=500', 'Bench’s public runs as Every Eval Ever <code>evaluation_run</code> records (JSONL, paged; <code>/datasets/eee.jsonl</code> is the same). Each run’s per-sample scores are at <code>/datasets/eee/runs/:id.samples.jsonl</code>.')}
+${ep('GET', '/api/v1/external/records?model=&task=&source_type=', 'The imported Every Eval Ever records: each is <code>external: true</code>, with its source, licence, harness and version, task and version, few-shot, metric and score. They never enter Bench’s ratings.')}
</dl></section>
</dl></section>
<section><h2>Write</h2><dl class="bm-eps">
<section><h2>Write</h2><dl class="bm-eps">
${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}
${ep('POST', '/api/v1/kinds/:kindId/tasks', '<code>{ prompt, task_class?, public? }</code>. A person or a service with bench.task.create.')}
new file mode 100644
new file mode 100644
@@ -0,0 +1,275 @@
@@ -0,0 +1,275 @@
+'use strict';
+
+/**
+ * Every Eval Ever (B2): importing the EEE datastore as external results, the licence gate, the resumable weekly
+ * sync, the exporter of Bench's own runs as EEE, and the routes that serve both.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { boot, check, done } = require('./helpers/boot');
+const eeeLib = require('../server/bench/eee');
+const { createEeeSync } = require('../server/bench/eee-sync');
+require('../server/http/eee');
+
+// The operator routes read the token from the environment at mount time (as server/http/mirror.js does).
+process.env.BENCH_ADMIN_TOKEN = process.env.BENCH_ADMIN_TOKEN || 'op-secret';
+
+const FIX = path.join(__dirname, 'fixtures', 'eee');
+const record = () => JSON.parse(fs.readFileSync(path.join(FIX, 'record.json'), 'utf8'));
+
+function definition(over = {}) {
+ return {
+ title: 'EEE export benchmark',
+ description: 'A public benchmark for the exporter.',
+ kind: 'text.answer',
+ tags: ['test'],
+ visibility: 'public',
+ tasks: [
+ { id: 't1', prompt: 'One.', files: [], expected: null, holdout: false },
+ { id: 't2', prompt: 'Two.', files: [], expected: null, holdout: false },
+ ],
+ judges: [],
+ scoring: { method: 'mean', scale: { min: 0, max: 100 } },
+ limits: {},
+ matrix: {},
+ ...over,
+ };
+}
+
+(async () => {
+ // ── the join rule and the licence gate ───────────────────────────────────────────────────────────────────
+ await check('a model name drops the maker prefix and a date or channel suffix, as Report joins it', () => {
+ assert.strictEqual(eeeLib.bareModel('openai/gpt-5-nano-2025-08-07'), 'gpt-5-nano');
+ assert.strictEqual(eeeLib.bareModel('anthropic/claude-3-7-sonnet-20250219'), 'claude-3-7-sonnet');
+ assert.strictEqual(eeeLib.bareModel('meta-llama/Llama-3.1-8B-Instruct'), 'llama-3.1-8b-instruct');
+ assert.strictEqual(eeeLib.bareModel('gpt-4o-latest'), 'gpt-4o');
+ assert.strictEqual(eeeLib.bareModel('openai/gpt-5-nano'), 'gpt-5-nano');
+ });
+
+ await check('a licence that forbids redistribution is refused, a stated open one is kept', () => {
+ assert.ok(eeeLib.allowsRedistribution('MIT'));
+ assert.ok(eeeLib.allowsRedistribution('apache-2.0'));
+ assert.ok(eeeLib.allowsRedistribution('cc-by-4.0'));
+ assert.ok(eeeLib.allowsRedistribution('cc-by-nc-4.0'));
+ assert.ok(!eeeLib.allowsRedistribution('All rights reserved'));
+ assert.ok(!eeeLib.allowsRedistribution('proprietary'));
+ assert.ok(!eeeLib.allowsRedistribution('research only'));
+ assert.ok(!eeeLib.allowsRedistribution('unknown'));
+ assert.ok(!eeeLib.allowsRedistribution(''));
+ });
+
+ await check('one EEE record is read into its results, with provenance and the join name', () => {
+ const doc = record();
+ const parsed = eeeLib.normalizeRecord(doc, { path: 'data/IFEval/x.json' });
+ assert.strictEqual(parsed.record.eee_id, doc.evaluation_id);
+ assert.strictEqual(parsed.record.source_type, 'documentation');
+ assert.strictEqual(parsed.record.model, 'llama-pro-8b-instruct');
+ assert.strictEqual(parsed.record.model_raw, 'TencentARC/LLaMA-Pro-8B-Instruct');
+ assert.strictEqual(parsed.record.developer, 'TencentARC');
+ assert.strictEqual(parsed.results.length, 6);
+ const ifeval = parsed.results.find((r) => r.task === 'IFEval');
+ assert.ok(ifeval, 'IFEval is there');
+ assert.strictEqual(ifeval.score, 0.4486);
+ assert.strictEqual(ifeval.metric_name, 'score');
+ assert.match(ifeval.source_url, /^https:\/\//);
+ assert.strictEqual(parsed.record.licence, 'MIT', 'the datastore card licence is the fallback');
+ });
+
+ await check('a record that states a non-redistributable licence is dropped, and the reason is recorded', () => {
+ const doc = record();
+ doc.source_metadata.additional_details = { license: 'All rights reserved' };
+ const parsed = eeeLib.normalizeRecord(doc);
+ assert.strictEqual(parsed.results.length, 6);
+ assert.ok(parsed.results.every((r) => r.dropped === 'licence'), 'every result is dropped for licence');
+ });
+
+ await check('few-shot, task version, seed and the harness are kept when the record states them', () => {
+ const doc = record();
+ doc.eval_library = { name: 'inspect_ai', version: '0.3.75' };
+ doc.evaluation_results[0].source_data.additional_details = { task_version: '2', num_fewshot: 5 };
+ doc.evaluation_results[0].generation_config = { generation_args: { temperature: 0.2, seed: 7 } };
+ const parsed = eeeLib.normalizeRecord(doc);
+ const first = parsed.results[0];
+ assert.strictEqual(parsed.record.harness_name, 'inspect_ai');
+ assert.strictEqual(parsed.record.harness_version, '0.3.75');
+ assert.strictEqual(first.task_version, '2');
+ assert.strictEqual(first.few_shot, 5);
+ assert.strictEqual(first.seed, '7');
+ });
+
+ await check('the weekly job starts only when the owner turns it on', async () => {
+ const settings = { intervalMs: 60_000, filesPerRun: 1, recordsPerRun: 1 };
+ const off = createEeeSync({
+ eee: { sync: async () => ({ status: 'finished' }) },
+ config: { drill: true, bench: { eee: { ...settings, enabled: true, enabledInDrill: false } } },
+ log: { warn() {} },
+ });
+ assert.strictEqual(off.start(), null, 'never under the drill');
+ const on = createEeeSync({
+ eee: { sync: async () => ({ status: 'finished' }) },
+ config: { drill: true, bench: { eee: { ...settings, enabled: true, enabledInDrill: true } } },
+ log: { warn() {} },
+ });
+ assert.ok(on.start(), 'a timer when it is allowed');
+ on.stop();
+ });
+
+ const t = await boot({ env: { BENCH_ADMIN_TOKEN: 'op-secret' } });
+ // A public benchmark with one verified run, for the exporter.
+ const made = await t.ctx.registry.createBenchmark({
+ owner: 'openvibe', title: 'EEE export benchmark', kind: 'text.answer', visibility: 'public', subject: 'openvibe',
+ });
+ assert.ok(made.ok, made.message);
+ const frozen = await t.ctx.registry.freezeVersion(made.benchmark.id, { definition: definition(), subject: 'openvibe' });
+ assert.ok(frozen.ok, frozen.message);
+ const published = await t.ctx.registry.publishRun({
+ benchmark: `${made.benchmark.id}@${frozen.version.version}`,
+ model: 'openai/gpt-5-nano-2025-08-07', provider: 'openai',
+ route: { provider: 'openai', model: 'openai/gpt-5-nano' },
+ harness: { name: 'lm-evaluation-harness', version: '0.4.2' },
+ settings: { temperature: 0, seed: '7', few_shot: 5 },
+ tasks: [
+ { task_id: 't1', score: 80, cost_usd: 0.01, ms: 1000, tokens: 100, artifacts: [] },
+ { task_id: 't2', score: 60, cost_usd: 0.02, ms: 2000, tokens: 200, transcript_url: 'https://example.test/t2' },
+ ],
+ score: 70, cost_usd: 0.03, ms: 3000, tokens: 300,
+ }, { provenance: 'verified', subject: 'service:codes' });
+ assert.ok(published.ok, published.message);
+
+ await check('the imported record lands as external rows, deduped, with external: true', async () => {
+ const eee = t.ctx.eee;
+ const doc = record();
+ const first = await eee.storeRecord(doc, { path: 'data/IFEval/x.json', readAt: t.ctx.s.iso() });
+ assert.strictEqual(first.read, 6);
+ assert.strictEqual(first.kept, 6);
+ const again = await eee.storeRecord(doc, { path: 'data/IFEval/x.json', readAt: t.ctx.s.iso() });
+ assert.strictEqual(again.kept, 6, 'the second store updates the same rows');
+ assert.strictEqual(await eee.countRecords(), 6, 'no duplicate rows');
+
+ const rows = await eee.listRecords({ task: 'IFEval' });
+ assert.strictEqual(rows.length, 1);
+ assert.strictEqual(rows[0].external, true);
+ assert.strictEqual(rows[0].licence, 'MIT');
+ assert.strictEqual(rows[0].source_type, 'documentation');
+ assert.strictEqual(rows[0].model, 'llama-pro-8b-instruct');
+ assert.ok(rows[0].provenance.source_metadata, 'the record provenance travels with it');
+ });
+
+ await check('the external records are served as JSON, marked external, and never rated', async () => {
+ const out = await t.get('/api/v1/external/records?task=IFEval');
+ assert.strictEqual(out.status, 200, out.text.slice(0, 200));
+ const body = out.json();
+ assert.strictEqual(body.external, true);
+ assert.strictEqual(body.total, 1);
+ assert.strictEqual(body.records[0].external, true);
+ assert.strictEqual(body.records[0].score, 0.4486);
+ // Not in the ratings: nothing external is a Bench run.
+ const rated = await t.ctx.s.db.maybe("SELECT COUNT(*)::int AS n FROM bench_runs WHERE model = 'llama-pro-8b-instruct'");
+ assert.strictEqual(rated.n, 0, 'an external record never becomes a Bench run');
+ });
+
+ await check('a public Bench run is exported as an EEE evaluation_run record', async () => {
+ const records = await t.ctx.eee.exportRecords({ limit: 10 });
+ const mine = records.find((r) => r.evaluation_id === `openvibe-bench/${published.run.id}`);
+ assert.ok(mine, 'the run is there');
+ assert.strictEqual(mine.schema_version, '0.3.0');
+ assert.strictEqual(mine.source_metadata.source_type, 'evaluation_run');
+ assert.strictEqual(mine.source_metadata.source_name, 'OpenVibe.Bench');
+ assert.strictEqual(mine.eval_library.name, 'lm-evaluation-harness');
+ assert.strictEqual(mine.eval_library.version, '0.4.2');
+ assert.strictEqual(mine.model_info.name, 'openai/gpt-5-nano-2025-08-07');
+ assert.strictEqual(mine.evaluation_results.length, 1);
+ assert.strictEqual(mine.evaluation_results[0].score_details.score, 70);
+ assert.strictEqual(mine.evaluation_results[0].score_details.uncertainty.num_samples, 2);
+ assert.strictEqual(mine.evaluation_results[0].metric_config.min_score, 0);
+ assert.strictEqual(mine.evaluation_results[0].metric_config.max_score, 100);
+ assert.ok(mine.evaluation_results[0].source_data.additional_details.samples_url, 'the per-sample URL is named');
+ assert.ok(!('extra' in mine), 'no top-level key outside the EEE schema');
+ // It is importable as EEE too: the round trip is the schema check a test can make without a validator.
+ const back = t.ctx.eee.normalizeRecord(mine);
+ assert.strictEqual(back.results.length, 1);
+ assert.strictEqual(back.record.model, 'gpt-5-nano');
+ });
+
+ await check('the EEE dataset is served as paged JSONL, and the samples as their own JSONL', async () => {
+ const out = await t.get('/api/v1/datasets/eee?limit=10');
+ assert.strictEqual(out.status, 200, out.text.slice(0, 200));
+ assert.match(String(out.headers.get('content-type')), /ndjson/);
+ const total = Number(out.headers.get('x-total-count'));
+ assert.ok(total >= 1, 'there is at least the one public run');
+ const lines = out.text.trim().split('\n').filter(Boolean).map((line) => JSON.parse(line));
+ assert.ok(lines.some((r) => r.evaluation_id === `openvibe-bench/${published.run.id}`));
+ assert.ok(lines.every((r) => r.source_metadata.source_type === 'evaluation_run'));
+
+ const page = await t.get('/api/v1/datasets/eee?limit=1&offset=1');
+ assert.strictEqual(page.status, 200);
+ assert.ok(page.text.trim().split('\n').filter(Boolean).length <= 1, 'the page is bounded');
+
+ // The .jsonl twin is not read as the kind "eee" (the kind dataset route is mounted after this one).
+ const twin = await t.get('/api/v1/datasets/eee.jsonl?limit=1');
+ assert.strictEqual(twin.status, 200, twin.text.slice(0, 200));
+ assert.match(String(twin.headers.get('content-type')), /ndjson/);
+
+ const samples = await t.get(`/api/v1/datasets/eee/runs/${published.run.id}.samples.jsonl`);
+ assert.strictEqual(samples.status, 200, samples.text.slice(0, 200));
+ const rows = samples.text.trim().split('\n').filter(Boolean).map((line) => JSON.parse(line));
+ assert.strictEqual(rows.length, 2);
+ assert.strictEqual(rows[0].model_id, 'openai/gpt-5-nano-2025-08-07');
+ assert.ok(rows.some((r) => r.transcript_url === 'https://example.test/t2'));
+ });
+
+ await check('the sync is the operator\'s route only', async () => {
+ const anon = await t.get('/api/v1/admin/eee/sync', { method: 'POST', json: {} });
+ assert.strictEqual(anon.status, 403, anon.text.slice(0, 120));
+ });
+
+ // ── the weekly sync: bounded, resumable, and the licence gate counted ───────────────────────────────────────
+ await check('a bounded sync resumes from its cursor and counts what it dropped for licence', async () => {
+ const good = record();
+ const bad = record();
+ bad.evaluation_id = 'hfopenllm_v2/TencentARC_LLaMA-Pro-8B-Instruct/1770682499.0';
+ bad.source_metadata = { ...bad.source_metadata, additional_details: { license: 'All rights reserved' } };
+ const files = {
+ 'data/IFEval/a/good.json': good,
+ 'data/IFEval/a/bad.json': bad,
+ };
+ const fetchImpl = async (url) => {
+ const text = String(url);
+ const ok = (body, type = 'json') => ({
+ ok: true, status: 200,
+ json: async () => (type === 'json' ? body : JSON.parse(body)),
+ text: async () => (typeof body === 'string' ? body : JSON.stringify(body)),
+ });
+ if (text.endsWith('/README.md')) return ok('license: mit');
+ if (text.includes('/tree/main/data?recursive=false')) return ok([{ type: 'directory', path: 'data/IFEval' }]);
+ if (text.includes('/tree/main/data/IFEval?recursive=true')) {
+ return ok(Object.keys(files).map((p) => ({ type: 'file', path: p })));
+ }
+ for (const [p, doc] of Object.entries(files)) if (text.endsWith('/' + p)) return ok(doc);
+ return { ok: false, status: 404, json: async () => ({}), text: async () => '' };
+ };
+ const eee = eeeLib.createEee({ s: t.ctx.s, registry: t.ctx.registry, fetchImpl, log: { warn() {} }, config: t.ctx.config });
+
+ const first = await eee.sync({ maxFiles: 1, maxRecords: 100 });
+ assert.strictEqual(first.status, 'running', 'one file is one pass');
+ assert.strictEqual(first.files_seen, 1);
+ assert.strictEqual(first.records_kept, 0, 'the first file (a/bad.json) is dropped for its licence');
+ assert.strictEqual(first.dropped_licence, 6);
+ assert.deepStrictEqual(first.cursor, { collection: 0, file: 1 });
+
+ const second = await eee.sync({ maxFiles: 1, maxRecords: 100 });
+ assert.strictEqual(second.id, first.id, 'it resumes the same sync');
+ assert.strictEqual(second.files_seen, 2);
+ assert.strictEqual(second.dropped_licence, 6, 'the bad record is counted, not stored');
+ assert.strictEqual(second.records_kept, 6, 'the good record lands');
+ assert.strictEqual(second.status, 'finished');
+ assert.ok(second.dropped_reason && Object.keys(second.dropped_reason).some((k) => /licence/.test(k)), 'the reason is kept');
+ });
+
+ await t.close();
+ done();
+})().catch((err) => {
+ process.stderr.write('The test file failed: ' + ((err && err.stack) || err) + '\n');
+ process.exit(1);
+});
new file mode 100644
new file mode 100644
@@ -0,0 +1,19 @@
@@ -0,0 +1,19 @@
+# Fixtures, and their licences
+
+These files are real, small samples from the public sources the importers read. They are test data, not part of
+Bench's data.
+
+- `eee/record.json` — one record from Every Eval Ever (EvalEval coalition), taken verbatim from
+ `every_eval_ever/tests/data/98ea850e-7019-4728-a558-8b1819ec47c2.json` in
+ <https://github.com/evaleval/every_eval_ever>. The EEE datastore and its records are published under the
+ dataset card's MIT licence (<https://huggingface.co/datasets/evaleval/EEE_datastore>); the repository is
+ Apache-2.0.
+- `imports/lighteval-results.json` — a subset of
+ `tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json` from
+ <https://github.com/huggingface/lighteval> (MIT). Four task entries are kept verbatim; the rest are removed.
+- `imports/helm-results.json` — the real `stats.json` and `run_spec.json` of
+ `commonsense-dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0` from
+ <https://github.com/stanford-crfm/helm> (Apache-2.0), combined under `{ stats, run_spec }` as the importer reads
+ them. They were reached through the EEE repository's test data.
+- `imports/{inspect,harbor,lm-eval,promptfoo}-*.json` — the earlier B12 fixtures; their sources are named in the
+ tests.
new file mode 100644
new file mode 100644
@@ -0,0 +1,132 @@
@@ -0,0 +1,132 @@
+{
+ "schema_version": "0.2.0",
+ "evaluation_id": "hfopenllm_v2/TencentARC_LLaMA-Pro-8B-Instruct/1770682486.623709",
+ "retrieved_timestamp": "1770682486.623709",
+ "source_metadata": {
+ "source_name": "HF Open LLM v2",
+ "source_type": "documentation",
+ "source_organization_name": "Hugging Face",
+ "evaluator_relationship": "third_party"
+ },
+ "model_info": {
+ "name": "LLaMA-Pro-8B-Instruct",
+ "id": "TencentARC/LLaMA-Pro-8B-Instruct",
+ "developer": "TencentARC",
+ "inference_platform": "unknown",
+ "additional_details": {
+ "precision": "bfloat16",
+ "architecture": "LlamaForCausalLM",
+ "params_billions": 8.357
+ }
+ },
+ "evaluation_results": [
+ {
+ "evaluation_name": "IFEval",
+ "source_data": {
+ "dataset_name": "IFEval",
+ "source_type": "hf_dataset",
+ "hf_repo": "google/IFEval"
+ },
+ "metric_config": {
+ "evaluation_description": "Accuracy on IFEval",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.4486
+ }
+ },
+ {
+ "evaluation_name": "BBH",
+ "source_data": {
+ "dataset_name": "BBH",
+ "source_type": "hf_dataset",
+ "hf_repo": "SaylorTwift/bbh"
+ },
+ "metric_config": {
+ "evaluation_description": "Accuracy on BBH",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.4224
+ }
+ },
+ {
+ "evaluation_name": "MATH Level 5",
+ "source_data": {
+ "dataset_name": "MATH Level 5",
+ "source_type": "hf_dataset",
+ "hf_repo": "DigitalLearningGmbH/MATH-lighteval"
+ },
+ "metric_config": {
+ "evaluation_description": "Exact Match on MATH Level 5",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.0249
+ }
+ },
+ {
+ "evaluation_name": "GPQA",
+ "source_data": {
+ "dataset_name": "GPQA",
+ "source_type": "hf_dataset",
+ "hf_repo": "Idavidrein/gpqa"
+ },
+ "metric_config": {
+ "evaluation_description": "Accuracy on GPQA",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.2743
+ }
+ },
+ {
+ "evaluation_name": "MUSR",
+ "source_data": {
+ "dataset_name": "MUSR",
+ "source_type": "hf_dataset",
+ "hf_repo": "TAUR-Lab/MuSR"
+ },
+ "metric_config": {
+ "evaluation_description": "Accuracy on MUSR",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.419
+ }
+ },
+ {
+ "evaluation_name": "MMLU-PRO",
+ "source_data": {
+ "dataset_name": "MMLU-PRO",
+ "source_type": "hf_dataset",
+ "hf_repo": "TIGER-Lab/MMLU-Pro"
+ },
+ "metric_config": {
+ "evaluation_description": "Accuracy on MMLU-PRO",
+ "lower_is_better": false,
+ "score_type": "continuous",
+ "min_score": 0.0,
+ "max_score": 1.0
+ },
+ "score_details": {
+ "score": 0.1946
+ }
+ }
+ ]
+}
\ No newline at end of file
\ No newline at end of file
new file mode 100644
new file mode 100644
@@ -0,0 +1,1499 @@
@@ -0,0 +1,1499 @@
+{
+ "stats": [
+ {
+ "name": {
+ "name": "num_references",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 4.0,
+ "sum_squared": 16.0,
+ "min": 4.0,
+ "max": 4.0,
+ "mean": 4.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_trials",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_prompt_tokens",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 631.4,
+ "sum_squared": 398665.95999999996,
+ "min": 631.4,
+ "max": 631.4,
+ "mean": 631.4,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_completion_tokens",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_output_tokens",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "training_co2_cost",
+ "split": "valid"
+ },
+ "count": 0,
+ "sum": 0,
+ "sum_squared": 0
+ },
+ {
+ "name": {
+ "name": "training_energy_cost",
+ "split": "valid"
+ },
+ "count": 0,
+ "sum": 0,
+ "sum_squared": 0
+ },
+ {
+ "name": {
+ "name": "inference_runtime",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 14.876816749572754,
+ "sum_squared": 221.31967660036844,
+ "min": 14.876816749572754,
+ "max": 14.876816749572754,
+ "mean": 14.876816749572754,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "batch_size",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_length",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_stop",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_endoftext",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_unknown",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_instances",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 5.0,
+ "sum_squared": 25.0,
+ "min": 5.0,
+ "max": 5.0,
+ "mean": 5.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prompt_truncated",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "max_prob",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match@5",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match@5",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match@5",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match@5",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "logprob",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_perplexity_tokens",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_bytes",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 2.0,
+ "sum_squared": 4.0,
+ "min": 2.0,
+ "max": 2.0,
+ "mean": 2.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "perplexity",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "bits_per_byte",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "logprob_per_byte",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_references",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 4.0,
+ "sum_squared": 16.0,
+ "min": 4.0,
+ "max": 4.0,
+ "mean": 4.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_references",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 4.0,
+ "sum_squared": 16.0,
+ "min": 4.0,
+ "max": 4.0,
+ "mean": 4.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_trials",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_trials",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_prompt_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 631.4,
+ "sum_squared": 398665.95999999996,
+ "min": 631.4,
+ "max": 631.4,
+ "mean": 631.4,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_prompt_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 631.4,
+ "sum_squared": 398665.95999999996,
+ "min": 631.4,
+ "max": 631.4,
+ "mean": 631.4,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_completion_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_completion_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_output_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_output_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "inference_runtime",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 14.876816749572754,
+ "sum_squared": 221.31967660036844,
+ "min": 14.876816749572754,
+ "max": 14.876816749572754,
+ "mean": 14.876816749572754,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "inference_runtime",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 14.876816749572754,
+ "sum_squared": 221.31967660036844,
+ "min": 14.876816749572754,
+ "max": 14.876816749572754,
+ "mean": 14.876816749572754,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "batch_size",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "batch_size",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_length",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_length",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_stop",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_stop",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_endoftext",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_endoftext",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_unknown",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "finish_reason_unknown",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_instances",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 5.0,
+ "sum_squared": 25.0,
+ "min": 5.0,
+ "max": 5.0,
+ "mean": 5.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_train_instances",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 5.0,
+ "sum_squared": 25.0,
+ "min": 5.0,
+ "max": 5.0,
+ "mean": 5.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prompt_truncated",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prompt_truncated",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "max_prob",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "max_prob",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "prefix_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "quasi_prefix_exact_match@5",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.3,
+ "sum_squared": 0.09,
+ "min": 0.3,
+ "max": 0.3,
+ "mean": 0.3,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "logprob",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "logprob",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 0.0,
+ "sum_squared": 0.0,
+ "min": 0.0,
+ "max": 0.0,
+ "mean": 0.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_perplexity_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_perplexity_tokens",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 1.0,
+ "sum_squared": 1.0,
+ "min": 1.0,
+ "max": 1.0,
+ "mean": 1.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_bytes",
+ "split": "valid",
+ "perturbation": {
+ "name": "robustness",
+ "robustness": true,
+ "fairness": false,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 2.0,
+ "sum_squared": 4.0,
+ "min": 2.0,
+ "max": 2.0,
+ "mean": 2.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_bytes",
+ "split": "valid",
+ "perturbation": {
+ "name": "fairness",
+ "robustness": false,
+ "fairness": true,
+ "computed_on": "worst"
+ }
+ },
+ "count": 1,
+ "sum": 2.0,
+ "sum_squared": 4.0,
+ "min": 2.0,
+ "max": 2.0,
+ "mean": 2.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ },
+ {
+ "name": {
+ "name": "num_instances",
+ "split": "valid"
+ },
+ "count": 1,
+ "sum": 10.0,
+ "sum_squared": 100.0,
+ "min": 10.0,
+ "max": 10.0,
+ "mean": 10.0,
+ "variance": 0.0,
+ "stddev": 0.0
+ }
+ ],
+ "run_spec": {
+ "name": "commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0",
+ "scenario_spec": {
+ "class_name": "helm.benchmark.scenarios.commonsense_scenario.HellaSwagScenario",
+ "args": {}
+ },
+ "adapter_spec": {
+ "method": "multiple_choice_joint",
+ "global_prefix": "",
+ "global_suffix": "",
+ "instructions": "The following are multiple choice questions (with answers) about common sense.\n",
+ "input_prefix": "Question: ",
+ "input_suffix": "\n",
+ "reference_prefix": "A. ",
+ "reference_suffix": "\n",
+ "chain_of_thought_prefix": "",
+ "chain_of_thought_suffix": "\n",
+ "output_prefix": "Answer: ",
+ "output_suffix": "\n",
+ "instance_prefix": "\n",
+ "substitutions": [],
+ "max_train_instances": 5,
+ "max_eval_instances": 10,
+ "num_outputs": 5,
+ "num_train_trials": 1,
+ "num_trials": 1,
+ "sample_train": true,
+ "model_deployment": "huggingface/pythia-1b-v0",
+ "model": "eleutherai/pythia-1b-v0",
+ "temperature": 0.0,
+ "max_tokens": 1,
+ "stop_sequences": [
+ "\n"
+ ],
+ "multi_label": false
+ },
+ "metric_specs": [
+ {
+ "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
+ "args": {
+ "names": [
+ "exact_match",
+ "quasi_exact_match",
+ "prefix_exact_match",
+ "quasi_prefix_exact_match"
+ ]
+ }
+ },
+ {
+ "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
+ "args": {}
+ },
+ {
+ "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
+ "args": {}
+ }
+ ],
+ "data_augmenter_spec": {
+ "perturbation_specs": [],
+ "should_augment_train_instances": false,
+ "should_include_original_train": false,
+ "should_skip_unchanged_train": false,
+ "should_augment_eval_instances": false,
+ "should_include_original_eval": false,
+ "should_skip_unchanged_eval": false,
+ "seeds_per_instance": 1
+ },
+ "groups": [
+ "hellaswag"
+ ]
+ }
+}
\ No newline at end of file
\ No newline at end of file
new file mode 100644
new file mode 100644
@@ -0,0 +1,238 @@
@@ -0,0 +1,238 @@
+{
+ "config_general": {
+ "lighteval_sha": "864af4f0ca5973e42ac4def921a3387ebeeca0c1",
+ "num_fewshot_seeds": 1,
+ "model_config": {
+ "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+ "generation_parameters": {
+ "temperature": 0.0,
+ "top_p": 0.9,
+ "seed": null
+ }
+ },
+ "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+ },
+ "results": {
+ "arc:challenge|25": {
+ "acc": 0.3,
+ "acc_stderr": 0.15275252316519466
+ },
+ "truthfulqa:mc|0": {
+ "truthfulqa_mc1": 0.2,
+ "truthfulqa_mc1_stderr": 0.13333333333333333,
+ "truthfulqa_mc2": 0.4801027091249869,
+ "truthfulqa_mc2_stderr": 0.13947336040397884
+ },
+ "bigbench_hard:geometric_shapes|3": {
+ "acc": 0.0,
+ "acc_stderr": 0.0
+ },
+ "agieval:logiqa-en|0": {
+ "acc": 0.2,
+ "acc_stderr": 0.13333333333333333
+ }
+ },
+ "versions": {},
+ "config_tasks": {
+ "arc:challenge|25": {
+ "name": "arc:challenge",
+ "prompt_function": "arc_prompt",
+ "hf_repo": "allenai/ai2_arc",
+ "hf_subset": "ARC-Challenge",
+ "metrics": [
+ {
+ "metric_name": "acc",
+ "higher_is_better": true,
+ "category": "LOGPROBS",
+ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",
+ "corpus_level_fn": "mean",
+ "batched_compute": false
+ }
+ ],
+ "hf_data_files": null,
+ "solver": [
+ "solve"
+ ],
+ "scorer": "score",
+ "sample_fields": "record_to_sample",
+ "sample_to_fewshot": null,
+ "filter": null,
+ "hf_revision": null,
+ "hf_filter": null,
+ "hf_avail_splits": [
+ "train",
+ "test"
+ ],
+ "evaluation_splits": [
+ "test"
+ ],
+ "few_shots_split": null,
+ "few_shots_select": "random_sampling_from_train",
+ "generation_size": 1,
+ "generation_grammar": null,
+ "stop_sequence": [
+ "\n"
+ ],
+ "num_samples": null,
+ "original_num_docs": -1,
+ "effective_num_docs": -1,
+ "must_remove_duplicate_docs": false,
+ "num_fewshots": 25,
+ "version": 0
+ },
+ "truthfulqa:mc|0": {
+ "name": "truthfulqa:mc",
+ "prompt_function": "truthful_qa_multiple_choice_prompt",
+ "hf_repo": "truthfulqa/truthful_qa",
+ "hf_subset": "multiple_choice",
+ "metrics": [
+ {
+ "metric_name": [
+ "truthfulqa_mc1",
+ "truthfulqa_mc2"
+ ],
+ "higher_is_better": {
+ "truthfulqa_mc1": true,
+ "truthfulqa_mc2": true
+ },
+ "category": "LOGPROBS",
+ "sample_level_fn": "TruthfulqaMCMetrics()",
+ "corpus_level_fn": {
+ "truthfulqa_mc1": "mean",
+ "truthfulqa_mc2": "mean"
+ },
+ "batched_compute": false
+ }
+ ],
+ "hf_data_files": null,
+ "solver": null,
+ "scorer": null,
+ "sample_fields": null,
+ "sample_to_fewshot": null,
+ "filter": null,
+ "hf_revision": null,
+ "hf_filter": null,
+ "hf_avail_splits": [
+ "validation"
+ ],
+ "evaluation_splits": [
+ "validation"
+ ],
+ "few_shots_split": null,
+ "few_shots_select": null,
+ "generation_size": -1,
+ "generation_grammar": null,
+ "stop_sequence": [
+ "\n"
+ ],
+ "num_samples": null,
+ "original_num_docs": -1,
+ "effective_num_docs": -1,
+ "must_remove_duplicate_docs": false,
+ "num_fewshots": 0,
+ "version": 0
+ },
+ "bigbench_hard:geometric_shapes|3": {
+ "name": "bigbench_hard:geometric_shapes",
+ "prompt_function": "bbh_prompt",
+ "hf_repo": "lighteval/bbh",
+ "hf_subset": "geometric_shapes",
+ "metrics": [
+ {
+ "metric_name": "acc",
+ "higher_is_better": true,
+ "category": "LOGPROBS",
+ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",
+ "corpus_level_fn": "mean",
+ "batched_compute": false
+ }
+ ],
+ "hf_data_files": null,
+ "solver": [
+ "solve"
+ ],
+ "scorer": "score",
+ "sample_fields": "record_to_sample",
+ "sample_to_fewshot": null,
+ "filter": null,
+ "hf_revision": null,
+ "hf_filter": null,
+ "hf_avail_splits": [
+ "train"
+ ],
+ "evaluation_splits": [
+ "train"
+ ],
+ "few_shots_split": null,
+ "few_shots_select": null,
+ "generation_size": -1,
+ "generation_grammar": null,
+ "stop_sequence": [
+ "</s>",
+ "Q=",
+ "\n\n"
+ ],
+ "num_samples": null,
+ "original_num_docs": -1,
+ "effective_num_docs": -1,
+ "must_remove_duplicate_docs": false,
+ "num_fewshots": 3,
+ "version": 0
+ },
+ "agieval:logiqa-en|0": {
+ "name": "agieval:logiqa-en",
+ "prompt_function": "agieval_prompt",
+ "hf_repo": "dmayhem93/agieval-logiqa-en",
+ "hf_subset": "default",
+ "metrics": [
+ {
+ "metric_name": "acc",
+ "higher_is_better": true,
+ "category": "LOGPROBS",
+ "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)",
+ "corpus_level_fn": "mean",
+ "batched_compute": false
+ }
+ ],
+ "hf_data_files": null,
+ "solver": [
+ "solve"
+ ],
+ "scorer": "score",
+ "sample_fields": "record_to_sample",
+ "sample_to_fewshot": null,
+ "filter": null,
+ "hf_revision": null,
+ "hf_filter": null,
+ "hf_avail_splits": [
+ "test"
+ ],
+ "evaluation_splits": [
+ "test"
+ ],
+ "few_shots_split": null,
+ "few_shots_select": "random_sampling",
+ "generation_size": 1,
+ "generation_grammar": null,
+ "stop_sequence": [],
+ "num_samples": null,
+ "original_num_docs": -1,
+ "effective_num_docs": -1,
+ "must_remove_duplicate_docs": false,
+ "num_fewshots": 0,
+ "version": 0
+ }
+ },
+ "summary_general": {
+ "hashes": {
+ "hash_examples": "68b0b3cbd628cac8",
+ "hash_full_prompts": "6c276b4f520c3deb",
+ "hash_input_tokens": "84a946d8f5709efe",
+ "hash_cont_tokens": "655a877ed0bcff8a"
+ },
+ "truncated": 0,
+ "non_truncated": 0,
+ "padded": 0,
+ "non_padded": 0
+ }
+}
\ No newline at end of file
\ No newline at end of file
@@ -11,6 +11,9 @@ const { boot, check, done } = require('./helpers/boot');
@@ -11,6 +11,9 @@ const { boot, check, done } = require('./helpers/boot');
const { uncovered } = require('openvibe-sdk/account-data');
const { uncovered } = require('openvibe-sdk/account-data');
const accountDataLib = require('../server/identity/account-data');
const accountDataLib = require('../server/identity/account-data');
const importers = require('../server/bench/importers');
const importers = require('../server/bench/importers');
+require('../server/bench/importers/helm');
+require('../server/bench/importers/lighteval');
+require('../server/bench/imports');
const limits = require('../server/bench/importers/limits');
const limits = require('../server/bench/importers/limits');
const { matchSuite } = require('../server/bench/suites');
const { matchSuite } = require('../server/bench/suites');
const { createMirror } = require('../server/bench/mirror');
const { createMirror } = require('../server/bench/mirror');
@@ -184,6 +187,35 @@ function multipart(filename, bytes) {
@@ -184,6 +187,35 @@ function multipart(filename, bytes) {
assert.strictEqual(gpt.cost_usd, 0.007);
assert.strictEqual(gpt.cost_usd, 0.007);
});
});
+ await check('the HF lighteval results are read: the model, every task, its few-shot and its metric', () => {
+ const out = importers.parse(read('lighteval-results.json'), { filename: 'results.json' });
+ assert.strictEqual(out.tool, 'lighteval');
+ assert.strictEqual(out.harness.name, 'lighteval');
+ assert.strictEqual(out.models.length, 1);
+ assert.strictEqual(out.models[0].model, 'HuggingFaceTB/SmolLM2-1.7B-Instruct');
+ assert.strictEqual(out.models[0].provider, 'HuggingFaceTB');
+ assert.strictEqual(out.models[0].tasks.length, 4);
+ const arc = out.models[0].tasks.find((task) => task.task_id === 'arc:challenge');
+ assert.ok(arc, 'the ARC challenge task is there');
+ assert.strictEqual(arc.few_shot, 25, 'the task key carries its few-shot');
+ assert.strictEqual(arc.metric, 'acc');
+ assert.ok(Math.abs(arc.score - 30) < 1e-9, 'the score is out of 100: ' + arc.score);
+ assert.match(out.harness.version, /^[0-9a-f]{7,}$/, 'the lighteval commit is the harness version');
+ });
+
+ await check('the Stanford HELM stats are read with their run spec: model, scenario and few-shot', () => {
+ const out = importers.parse(read('helm-results.json'), { filename: 'stats.json' });
+ assert.strictEqual(out.tool, 'helm');
+ assert.strictEqual(out.harness.name, 'helm');
+ assert.strictEqual(out.models[0].model, 'eleutherai/pythia-1b-v0');
+ assert.strictEqual(out.models[0].provider, 'huggingface');
+ assert.strictEqual(out.models[0].tasks.length, 1);
+ assert.strictEqual(out.models[0].tasks[0].task_id, 'hellaswag');
+ assert.strictEqual(out.models[0].tasks[0].metric, 'quasi_exact_match');
+ assert.strictEqual(out.models[0].tasks[0].few_shot, 5, 'max_train_instances is the few-shot');
+ assert.ok(Math.abs(out.models[0].score - 30) < 1e-9, 'the proportion is read out of 100: ' + out.models[0].score);
+ });
+
// ── the limits ─────────────────────────────────────────────────────────────────────────────────────────
// ── the limits ─────────────────────────────────────────────────────────────────────────────────────────
await check('a file over 20 MB is refused', () => {
await check('a file over 20 MB is refused', () => {
assert.throws(() => importers.parse(Buffer.alloc(20 * 1024 * 1024 + 1)), /20 MB|too large/i);
assert.throws(() => importers.parse(Buffer.alloc(20 * 1024 * 1024 + 1)), /20 MB|too large/i);
@@ -297,6 +329,27 @@ function multipart(filename, bytes) {
@@ -297,6 +329,27 @@ function multipart(filename, bytes) {
assert.strictEqual(runs[0].harness.name, 'inspect-ai');
assert.strictEqual(runs[0].harness.name, 'inspect-ai');
});
});
+ await check('an import keeps the harness, task version, few-shot and seed on the run', async () => {
+ const doc = JSON.parse(read('lighteval-results.json').toString('utf8'));
+ doc.versions = { 'arc:challenge': 3 };
+ doc.config_general.model_config.generation_parameters.seed = 42;
+ const up = await t.ctx.imports.store({
+ owner: alice.subject, subject: alice.subject, filename: 'lighteval-results.json',
+ bytes: Buffer.from(JSON.stringify(doc)),
+ });
+ assert.strictEqual(up.summary.tool, 'lighteval');
+ assert.strictEqual(up.summary.harness.name, 'lighteval');
+ assert.strictEqual(up.summary.models[0].tasks[0].few_shot, 25);
+ assert.strictEqual(up.summary.models[0].tasks[0].task_version, '3');
+ const imported = await t.ctx.imports.commit(up.id, { subject: alice.subject });
+ const run = await t.ctx.registry.getRun(imported.runs[0].id);
+ assert.strictEqual(run.harness.name, 'lighteval', 'the harness name is on the run');
+ assert.strictEqual(run.harness.version, up.summary.harness.version, 'and its version');
+ assert.strictEqual(run.settings.few_shot, 25, 'the few-shot is kept');
+ assert.strictEqual(run.settings.seed, '42', 'the seed is kept');
+ assert.strictEqual(run.settings.task_versions['arc:challenge'], '3', 'the task version is kept');
+ });
+
await check('the file stays private to its owner', async () => {
await check('the file stays private to its owner', async () => {
const mine = await t.get('/imports/' + importId + '/file', { as: alice });
const mine = await t.get('/imports/' + importId + '/file', { as: alice });
assert.strictEqual(mine.status, 200);
assert.strictEqual(mine.status, 200);