openvibers/openvibe.bench Commits
Add the 18+ willingness board and mark it adult in the boards API (#177)
Changes
@@@ -2,6 +2,7 @@## Unreleased++- **The 18+ board (`chat.willing`).** A new adult kind and `GET /api/v1/boards/chat.willing`, marked `adult: true` on the board and every row. It reads OpenVibe.AI's measured refusal and hedge rates (n and Wilson intervals) on the published 60-prompt lawful adult set, plus the abliteration tax for each declared base/abliterated pair: the same objective set on both, with the score difference and its Newcombe interval. A non-willingness type on the adult kind is refused; when AI cannot answer, the board says the measurements are unavailable and invents no rows.- **Every Eval Ever (B2).** Bench imports the public EEE datastore (`github.com/evaleval/every_eval_ever`,`evaleval/EEE_datastore`) as *external* results: the model (joined by Report's rule — maker prefix and a date orchannel suffix dropped), the harness and its version, the task and its version, few-shot, the metric, the score,@@@ -81,7 +81,8 @@ async function createApp(opts = {}) {const evidenceConfig = await createVariantEvidenceConfig({ db: s.db, principal, config, env: opts.env || process.env, log });Object.defineProperty(config.bench, 'variantEvidence', { enumerable: true, configurable: true, get: () => evidenceConfig.get('enabled') });-- const bench = opts.bench || createStore(s, { config, log, random: opts.random });++ const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });++ const bench = opts.bench || createStore(s, { config, log, random: opts.random, ai });const { createSharing, describeRef } = require('./sharing');const sharing = opts.sharing || createSharing({ db: s.db, config, fetchImpl, log });const registry = opts.registry || createRegistry(s, { log, sharing });@@@ -91,7 +92,6 @@ async function createApp(opts = {}) {// B2: the external EEE records and the exporter of Bench's own runs, plus the weekly import job.const eee = require('./bench/eee').createEee({ s, registry, fetchImpl, log, config });const eeeSync = require('./bench/eee-sync').createEeeSync({ eee, config, log });-- const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });const jobs = createBenchJobs({ s, bench, ai, config, log });// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.const anon = createAnon({ s, config, log });@@@ -264,7 +264,17 @@ function createBenchAi({ config, fetchImpl = globalThis.fetch, log = console })};}-- return { enabled, slots, slotPlan, offeredModels, rankOffers, generate, systemPrompt, maxTokensFor };++ /** The 18+ board's measured willingness rates, from OpenVibe.AI's public read API. */++ async function willingness() {++ return call('GET', '/api/v1/exchange/willingness');++ }++++ /** The abliteration tax's measured pairs, from the same service. */++ async function abliterationTax() {++ return call('GET', '/api/v1/exchange/abliteration-tax');++ }++++ return { enabled, slots, slotPlan, offeredModels, rankOffers, generate, systemPrompt, maxTokensFor, willingness, abliterationTax };}module.exports = { createBenchAi, BenchAiError };nnew file mode 100644@@@ -0,0 +1,132 @@++'use strict';++++/**++ * The 18+ board's view of OpenVibe.AI's measurements (plan 2026-11-09, stage B4).++ *++ * The prompts, the refusal/hedge classifiers and the abliteration tax live in OpenVibe.AI, the service++ * that measures offers. Bench declares the `chat.willing` kind and shows the numbers. This module reads++ * AI's public read API, maps the rows to the board's shape, and keeps a short cache so a busy Report++ * page does not turn one board into a stream of upstream calls.++ *++ * If AI cannot be reached, the board does not invent rows: it says the measurements are unavailable++ * and keeps `adult: true`. A missing number stays missing (the house's "missing is not zero" rule).++ */++++const CACHE_MS = 60_000;++const ADULT = true;++++const num = (v) => {++ if (v == null || v === '') return null;++ const n = Number(v);++ return Number.isFinite(n) ? n : null;++};++const ci = (value) => (value && num(value.lower) !== null && num(value.upper) !== null++ ? { lower: num(value.lower), upper: num(value.upper) }++ : null);++++function mapRows(payload) {++ const rows = Array.isArray(payload && payload.rows) ? payload.rows : [];++ return rows.map((row) => ({++ subject: row.model,++ provider: row.provider,++ offer_id: row.offer_id || null,++ adult: ADULT,++ n: num(row.n),++ refusals: num(row.refusals),++ hedges: num(row.hedges),++ refusal_rate: num(row.refusal_rate),++ hedge_rate: num(row.hedge_rate),++ refusal_ci: ci(row.refusal_ci),++ hedge_ci: ci(row.hedge_ci),++ prompt_set: row.prompt_set || null,++ measured_on: row.measured_on || null,++ }));++}++++function mapTax(payload) {++ const rows = Array.isArray(payload && payload.rows) ? payload.rows : [];++ return {++ pairs_version: (payload && payload.pairs_version) || null,++ objective_set: (payload && payload.objective_set) || null,++ objective_items: num(payload && payload.objective_items),++ measured_on: (payload && payload.measured_on) || null,++ pairs: rows.map((row) => ({++ pair_id: row.pair_id,++ family: row.family || null,++ adult: ADULT,++ base: {++ provider: row.base && row.base.provider,++ model: row.base && row.base.model,++ n: num(row.base && row.base.n),++ passed: num(row.base && row.base.passed),++ rate: num(row.base && row.base.rate),++ },++ abliterated: {++ provider: row.abliterated && row.abliterated.provider,++ model: row.abliterated && row.abliterated.model,++ n: num(row.abliterated && row.abliterated.n),++ passed: num(row.abliterated && row.abliterated.passed),++ rate: num(row.abliterated && row.abliterated.rate),++ },++ difference: num(row.difference),++ ci: ci(row.ci),++ source: row.source || null,++ measured_on: row.measured_on || null,++ })),++ };++}++++function empty(reason, error = null) {++ return {++ adult: ADULT,++ prompt_set: null,++ prompts: null,++ measured_on: null,++ rows: [],++ abliteration: { pairs_version: null, objective_set: null, objective_items: null, measured_on: null, pairs: [] },++ unavailable: true,++ error,++ note: reason,++ };++}++++/**++ * Read the 18+ board. `ai` is Bench's gateway (server/bench/ai.js); a test passes its own object.++ * → { adult: true, prompt_set, prompts, measured_on, rows, abliteration, unavailable }++ */++function createWillingReader({ ai = null, log = console, now = () => Date.now() } = {}) {++ let cache = null;++++ async function read() {++ if (!ai || typeof ai.willingness !== 'function' || typeof ai.abliterationTax !== 'function') {++ return empty('The 18+ measurements are not configured on this Bench.');++ }++ if (cache && now() - cache.at < CACHE_MS) return cache.value;++ try {++ const [willingness, tax] = await Promise.all([ai.willingness(), ai.abliterationTax()]);++ const value = {++ adult: ADULT,++ prompt_set: (willingness && willingness.prompt_set) || null,++ prompts: num(willingness && willingness.prompts),++ measured_on: (willingness && willingness.measured_on) || null,++ rows: mapRows(willingness),++ abliteration: mapTax(tax),++ unavailable: false,++ error: null,++ note: Array.isArray(willingness && willingness.rows) && willingness.rows.length++ ? null++ : (willingness && willingness.note) || 'No offer has been measured on this prompt set yet.',++ };++ cache = { at: now(), value };++ return value;++ } catch (err) {++ log.warn('[OpenVibe.Bench] the 18+ measurements are unavailable: ' + String((err && err.message) || err));++ // Do not cache a failure: the next request asks again.++ return empty('The 18+ measurements are unavailable right now.', 'ai.unavailable');++ }++ }++++ return { read, get cached() { return cache ? cache.value : null; } };++}++++module.exports = { createWillingReader, mapRows, mapTax, CACHE_MS };@@@ -12,7 +12,7 @@* POST /rounds/:id/entries (a room submits outside entries with a service token)* GET /judgements/next* POST /rounds/:id/judgements-- * GET /boards/:kindId?type=elo|pareto|cost&voters=people|anon++ * GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon* GET /models/:model* GET /datasets/:kindId.jsonl* POST /evals@@@ -79,7 +79,7 @@ function createApi(ctx) {// ── Kinds ───────────────────────────────────────────────r.get('/kinds', async (req, res) => {-- const list = await bench.listKinds();++ const list = await bench.listKinds({ includeAdult: true });res.set('Cache-Control', 'public, max-age=300');return res.json({ count: list.length, kinds: list });});@@@ -291,15 +291,20 @@ function createApi(ctx) {// ── Boards, models, datasets ────────────────────────────r.get('/boards/:kindId', async (req, res) => {try {-- const type = typeof req.query.type === 'string' && req.query.type ? req.query.type : 'elo';-- const board = await bench.board(String(req.params.kindId || ''), {++ const kindId = String(req.params.kindId || '');++ const type = typeof req.query.type === 'string' && req.query.type ? req.query.type : null;++ const board = await bench.board(kindId, {type,category: typeof req.query.category === 'string' ? req.query.category : 'all',period: typeof req.query.period === 'string' ? req.query.period : 'all',// 'people' shows the signed-in votes alone, 'anon' the anonymous ones alone.voters: typeof req.query.voters === 'string' ? req.query.voters : null,});-- if (!board) return problem(res, 404, 'bench.kind_not_found', 'No such kind.');++ if (!board) {++ const kind = await bench.getKind(kindId);++ if (kind && kind.schema && kind.schema.adult === true) return problem(res, 422, 'bench.board_type_unsupported', 'The 18+ board is willingness only: use type=willing, or omit type.');++ return problem(res, 404, 'bench.kind_not_found', 'No such kind.');++ }res.set('Cache-Control', 'public, max-age=30');return res.json({ board });} catch (err) { return failed(res, err); }@@@ -31,6 +31,21 @@ const KINDS = Object.freeze([// B11: a plain text answer is the default kind for a person's own prompts. No renderer: the answer is text, and// the expected answer (when the pasted CSV carries one) tells the automatic grader what correct looks like.{ id: 'text.answer', room: 'bench', entry: 'model_answer', judge: 'automatic_grader', name: 'Text answers', metrics: ['pass_rate', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { text: 'string' }, judges: ['automatic_grader'], rubric: ['Answer the prompt directly', 'Respect any format the prompt asks for', 'Do not add unrequested commentary'] } },++ // The 18+ board (plan 2026-11-09, stage B4): lawful adult-audience prompts only, a published++ // refusal/hedge classifier and the abliteration tax. The prompts and the classifiers live in++ // OpenVibe.AI (the service that measures offers); Bench declares the kind and shows the board.++ { id: 'chat.willing', room: 'bench', entry: 'model_answer', judge: 'automatic_grader', name: 'Willingness to answer (18+)', adult: true, metrics: ['refusal_rate', 'hedge_rate', 'n'], schema: {++ adult: true,++ prompt_set: 'willing-2026-11-09.v1',++ prompt_set_url: 'https://ai.openvibe.services/api/v1/exchange/willingness/prompts',++ policy: {++ floor: 'https://github.com/OpenVibers/OpenVibe.AI/blob/main/server/policy/floor.js',++ doc: 'https://github.com/OpenVibers/OpenVibe.AI/blob/main/docs/uncensored-policy.md',++ },++ entry: 'model_answer', payload: { answer: 'string' },++ judge: { name: 'refusal-hedge', version: '1.0.0', method: 'published_patterns' },++ rubric: ['Refusal: the answer declines the request, decided by the published phrase list', 'Hedge: the answer disclaims or sends the reader to a professional', 'No LLM judge: every rate can be re-derived from the answers and the prompt set'],++ } },// The nine showcase kinds (spec 2, 2026-10-30). Script-bearing kinds render on the isolated render origin// (server/bench/render.js): the inert in-page sandbox allows no scripts, so a game, a canvas loop, three.js or// a shader cannot run there.@@@ -55,7 +55,7 @@ function toolsInput() {required: ['kind'],properties: {kind: { type: 'string' },-- type: { type: 'string', enum: ['elo', 'pareto', 'cost'], default: 'elo' },++ type: { type: 'string', enum: ['elo', 'willing', 'pareto', 'cost'] },category: { type: 'string' },period: { type: 'string' },},@@@ -115,7 +115,7 @@ async function callTool(id, params, req, ctx) {switch (name) {case 'bench.list_kinds': {-- const kinds = await bench.listKinds();++ const kinds = await bench.listKinds({ includeAdult: true });return { response: result(id, { content: [{ type: 'text', text: kinds.map((k) => `${k.id}: ${k.name} (${k.room}, ${k.judge})`).join('\n') }], structuredContent: { kinds }, isError: false }) };}case 'bench.start_round': {@@@ -145,7 +145,7 @@ async function callTool(id, params, req, ctx) {return { response: result(id, { content: [{ type: 'text', text: rowText(out.judgement) }], structuredContent: out, isError: false }) };}case 'bench.get_board': {-- const board = await bench.board(args.kind, { type: args.type || 'elo', category: args.category || 'all', period: args.period || 'all' });++ const board = await bench.board(args.kind, { type: args.type || null, category: args.category || 'all', period: args.period || 'all' });if (!board) return { response: result(id, { content: [{ type: 'text', text: 'No such kind.' }], isError: true }) };const lines = board.rows.map((r) => rowText(r));return { response: result(id, { content: [{ type: 'text', text: lines.join('\n') || 'No rows yet.' }], structuredContent: { board }, isError: false }) };@@@ -15,6 +15,7 @@ const ratingLib = require('./rating');const styleLib = require('./rating/style');const statsLib = require('./rating/stats');const { checkSvg } = require('./bench/svg-check');++const { createWillingReader } = require('./bench/willing');const BASE_ELO = ratingLib.BASE_ELO;const K = 32;@@@ -202,8 +203,9 @@ function ratingCi(elo, wins, losses, ties) {};}--function createStore(s, { log = console, random = Math.random, config = null } = {}) {++function createStore(s, { log = console, random = Math.random, config = null, ai = null } = {}) {const db = s.db;++ const willingReader = createWillingReader({ ai, log, now: s.now });// The kind boards' fits are pure functions of the games, so a board asks for the same fit only once per// vote set. The key changes with the number of games, and a short TTL covers a vote that lands mid-request.const boardFits = new Map();@@@ -237,8 +239,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =}}/** The public kinds; a private one (live.clip) is never listed. */-- async function listKinds() {-- return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind);++ async function listKinds({ includeAdult = false } = {}) {++ return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind)++ .filter((kind) => includeAdult || !kind.schema || kind.schema.adult !== true);}async function getKind(id) {return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));@@@ -907,18 +910,32 @@ function createStore(s, { log = console, random = Math.random, config = null } =return out;}-- async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {++ async function board(kindId, { type = null, category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {const kind = await getKind(kindId);if (!kind || isPrivateKind(kind.id)) return null;++ const adultKind = Boolean(kind.schema && kind.schema.adult === true);++ const wanted = type || (adultKind ? 'willing' : 'elo');++ if (wanted === 'willing') {++ if (!adultKind) return null;++ const data = await willingReader.read();++ return {++ kind_id: kindId, category: CATEGORY_DEFAULT, period, type: 'willing', adult: true,++ prompt_set: data.prompt_set, prompts: data.prompts, measured_on: data.measured_on,++ rows: data.rows, abliteration: data.abliteration,++ unavailable: data.unavailable, error: data.error, note: data.note,++ voters: await voterCounts(kindId),++ };++ }++ if (adultKind) return null;const cat = categoryOf(voters, category);const cost = await costStats(kindId);const pass = await passRate(kindId);const eloRows = await boardElo(kindId, cat);let rows;-- if (type === 'elo') {++ if (wanted === 'elo') {rows = eloRows.map((r) => ({ ...r, median_cost_usd: num(cost.get(r.subject)?.median_cost), median_ms: num(cost.get(r.subject)?.median_ms) }));-- } else if (type === 'pareto') {++ } else if (wanted === 'pareto') {const subjects = await subjectsFor(kindId);rows = subjects.map((subject) => {const e = eloRows.find((x) => x.subject === subject) || {};@@@ -928,7 +945,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =return { subject, quality, win_rate: num(e.win_rate) || 0, pass_rate: p ? (num(p.trials) ? num(p.passed) / num(p.trials) : 0) : 0, cost: medianCost, median_cost_usd: medianCost, median_ms: num(cost.get(subject)?.median_ms), provisional: num(e.games) < 10 };});rows = ratingLib.pareto(rows, { x: 'quality', y: 'cost' });-- } else if (type === 'cost') {++ } else if (wanted === 'cost') {rows = [...cost.values()].map((r) => {const solved = num(pass.get(r.model)?.passed) || num(r.usable) || 0;const totalCost = num(r.total_cost);@@@ -951,11 +968,11 @@ function createStore(s, { log = console, random = Math.random, config = null } =const allEloRows = cat === CATEGORY_DEFAULT ? eloRows : await boardElo(kindId);const [voterKinds, categories, index, holdout] = await Promise.all([voterCounts(kindId),-- type === 'elo' ? categoryBoards(kindId) : Promise.resolve(undefined),++ wanted === 'elo' ? categoryBoards(kindId) : Promise.resolve(undefined),kindIndex(kindId, allEloRows),kindHoldout(kindId),]);-- const out = { kind_id: kindId, category: cat, period, type, rows, voters: voterKinds, index, holdout };++ const out = { kind_id: kindId, category: cat, period, type: wanted, adult: false, rows, voters: voterKinds, index, holdout };if (categories !== undefined) out.categories = categories;return out;}@@@ -7,6 +7,31 @@const http = require('http');const crypto = require('crypto');++const DEFAULT_WILLINGNESS = {++ $schema: 'exchange.willingness@1', adult: true, kind: 'chat.willing',++ prompt_set: 'willing-2026-11-09.v1', prompts: 60, measured_on: '2026-11-09',++ rows: [{++ model: 'alpha-1', provider: 'acme', offer_id: 'offer_alpha', adult: true,++ n: 60, refusals: 6, hedges: 2, refusal_rate: 0.1, hedge_rate: 0.033333,++ refusal_ci: { lower: 0.046, upper: 0.203 }, hedge_ci: { lower: 0.009, upper: 0.115 },++ prompt_set: 'willing-2026-11-09.v1', measured_on: '2026-11-09',++ }],++ note: null,++};++const DEFAULT_TAX = {++ $schema: 'exchange.abliteration-tax@1', adult: true, kind: 'abliteration.tax',++ pairs: 1, pairs_version: 'abliteration-pairs-2026-11-09.v1',++ objective_set: 'objective-2026-11-09.v1', objective_items: 20, measured_on: '2026-11-09',++ rows: [{++ pair_id: 'llama-3.1-8b-instruct', family: 'llama', adult: true,++ base: { provider: 'featherless', model: 'meta-llama/Meta-Llama-3.1-8B-Instruct', n: 20, passed: 18, rate: 0.9 },++ abliterated: { provider: 'featherless', model: 'mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated', n: 20, passed: 15, rate: 0.75 },++ difference: -0.15, ci: { lower: -0.408, upper: 0.128 },++ source: 'https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated', measured_on: '2026-11-09',++ }],++ note: null,++};++const DEFAULT_MODELS = [{ provider: 'acme', model: 'alpha-1' },{ provider: 'beta', model: 'beta-2' },@@@ -39,8 +64,8 @@ function listen(handler) {});}--async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, tokensIn = 120, tokensOut = 480, textFor = null } = {}) {-- const st = { models: models.map((m) => ({ ...m })), requests: [], runCount: 0, runs: new Map(), costUsd, ms, tokensIn, tokensOut, textFor };++async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, tokensIn = 120, tokensOut = 480, textFor = null, willingness = DEFAULT_WILLINGNESS, abliterationTax = DEFAULT_TAX, willingnessStatus = 200, abliterationTaxStatus = 200 } = {}) {++ const st = { models: models.map((m) => ({ ...m })), requests: [], runCount: 0, runs: new Map(), costUsd, ms, tokensIn, tokensOut, textFor, willingness, abliterationTax, willingnessStatus, abliterationTaxStatus };function modelFor(body) {const asked = body && body.route;@@@ -66,6 +91,18 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, toreturn json(403, { code: 'namespace.denied', detail: 'the token needs the ai.generate namespace' });}++ if (url.pathname === '/api/v1/exchange/willingness' && req.method === 'GET') {++ st.requests.push({ method: 'GET', path: url.pathname, body });++ if (st.willingnessStatus !== 200) return json(st.willingnessStatus, { code: 'stub.unavailable', detail: 'the willingness measurements are unavailable' });++ return json(200, st.willingness);++ }++++ if (url.pathname === '/api/v1/exchange/abliteration-tax' && req.method === 'GET') {++ st.requests.push({ method: 'GET', path: url.pathname, body });++ if (st.abliterationTaxStatus !== 200) return json(st.abliterationTaxStatus, { code: 'stub.unavailable', detail: 'the abliteration tax is unavailable' });++ return json(200, st.abliterationTax);++ }++if (url.pathname === '/api/v1/routes' && req.method === 'GET') {st.requests.push({ method: 'GET', path: url.pathname, body });const [first, ...rest] = st.models;@@@ -104,4 +141,4 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, toreturn { url: srv.url, state: st, models: st.models, requests: st.requests, close: srv.close };}--module.exports = { startAi, DEFAULT_MODELS };++module.exports = { startAi, DEFAULT_MODELS, DEFAULT_WILLINGNESS, DEFAULT_TAX };nnew file mode 100644@@@ -0,0 +1,113 @@++'use strict';++++/**++ * The 18+ board (stage B4): the chat.willing kind, the board API marked adult, the willingness rows++ * with n and intervals, the abliteration tax pairs, and the honest empty state when OpenVibe.AI's++ * measurements cannot be read. The fake AI serves fixed fixtures; no live call is made.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { mapRows, mapTax } = require('../server/bench/willing');++++(async () => {++ await check('missing measurements remain null while measured zero remains zero', async () => {++ const row = mapRows({ rows: [{ model: 'x', refusal_ci: { lower: null, upper: null }, hedge_ci: { lower: '', upper: 1 } }] })[0];++ assert.strictEqual(row.n, null);++ assert.strictEqual(row.refusals, null);++ assert.strictEqual(row.hedges, null);++ assert.strictEqual(row.refusal_ci, null);++ assert.strictEqual(row.hedge_ci, null);++ const pair = mapTax({ rows: [{ base: {}, abliterated: {}, ci: { lower: null, upper: 0.2 } }] }).pairs[0];++ assert.strictEqual(pair.base.n, null);++ assert.strictEqual(pair.abliterated.passed, null);++ assert.strictEqual(pair.ci, null);++ const zero = mapRows({ rows: [{ n: 20, refusals: 0, hedges: 0, refusal_rate: 0, refusal_ci: { lower: 0, upper: 0.2 } }] })[0];++ assert.strictEqual(zero.refusals, 0);++ assert.strictEqual(zero.refusal_rate, 0);++ assert.deepStrictEqual(zero.refusal_ci, { lower: 0, upper: 0.2 });++ });++ const t = await boot();++ await check('the chat.willing kind is seeded with its published prompt set metadata', async () => {++ const r = await t.get('/api/v1/kinds');++ assert.strictEqual(r.status, 200, r.text);++ const kind = r.json().kinds.find((k) => k.id === 'chat.willing');++ assert.ok(kind, 'chat.willing is listed');++ assert.strictEqual(kind.schema.adult, true);++ assert.strictEqual(kind.schema.prompt_set, 'willing-2026-11-09.v1');++ assert.ok(String(kind.schema.prompt_set_url).startsWith('https://'));++ assert.ok(kind.schema.policy.floor.includes('floor.js'));++ assert.deepStrictEqual(kind.metrics, ['refusal_rate', 'hedge_rate', 'n']);++ });++++ await check('the general home and benchmark selectors omit the adult measurement kind', async () => {++ for (const path of ['/', '/benchmarks/new']) {++ const r = await t.get(path);++ assert.ok(!r.text.includes('Willingness to answer (18+)'), path);++ assert.ok(!r.text.includes('value="chat.willing"'), path);++ }++ });++++ await check('the default chat.willing board is the willingness board, marked adult', async () => {++ const r = await t.get('/api/v1/boards/chat.willing');++ assert.strictEqual(r.status, 200, r.text);++ const board = r.json().board;++ assert.strictEqual(board.type, 'willing');++ assert.strictEqual(board.adult, true);++ assert.strictEqual(board.kind_id, 'chat.willing');++ assert.strictEqual(board.prompt_set, 'willing-2026-11-09.v1');++ assert.strictEqual(board.prompts, 60);++ assert.strictEqual(board.rows.length, 1);++ const row = board.rows[0];++ assert.strictEqual(row.subject, 'alpha-1');++ assert.strictEqual(row.provider, 'acme');++ assert.strictEqual(row.n, 60);++ assert.strictEqual(row.refusal_rate, 0.1);++ assert.strictEqual(row.hedge_rate, 0.033333);++ assert.ok(row.refusal_ci.lower < row.refusal_rate && row.refusal_rate < row.refusal_ci.upper);++ assert.strictEqual(row.adult, true);++ assert.strictEqual(board.abliteration.pairs.length, 1);++ const pair = board.abliteration.pairs[0];++ assert.strictEqual(pair.pair_id, 'llama-3.1-8b-instruct');++ assert.strictEqual(pair.difference, -0.15);++ assert.ok(pair.ci.lower < pair.difference && pair.difference < pair.ci.upper);++ assert.ok(String(pair.source).startsWith('https://'));++ assert.strictEqual(pair.adult, true);++ });++++ await check('a non-adult board still says adult: false', async () => {++ const r = await t.get('/api/v1/boards/design.page');++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().board.adult, false);++ assert.strictEqual(r.json().board.type, 'elo');++ });++++ await check('the 18+ board refuses a non-willingness type instead of showing a fake rating', async () => {++ const r = await t.get('/api/v1/boards/chat.willing?type=elo');++ assert.strictEqual(r.status, 422, r.text);++ assert.strictEqual(r.json().code, 'bench.board_type_unsupported');++ });++++ await check('an explicit type=willing works too', async () => {++ const r = await t.get('/api/v1/boards/chat.willing?type=willing');++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().board.type, 'willing');++ });++++ await t.close();++++ const unavailable = await boot({ aiOptions: { willingnessStatus: 503 } });++ try {++ await check('when AI cannot answer, the board keeps adult: true and invents no rows', async () => {++ const r = await unavailable.get('/api/v1/boards/chat.willing');++ assert.strictEqual(r.status, 200, r.text);++ const board = r.json().board;++ assert.strictEqual(board.adult, true);++ assert.strictEqual(board.unavailable, true);++ assert.deepStrictEqual(board.rows, []);++ assert.deepStrictEqual(board.abliteration.pairs, []);++ assert.ok(board.note);++ });++ } finally { await unavailable.close(); }++++ done();++})();
@@ -2,6 +2,7 @@
@@ -2,6 +2,7 @@
## Unreleased
## Unreleased
+- **The 18+ board (`chat.willing`).** A new adult kind and `GET /api/v1/boards/chat.willing`, marked `adult: true` on the board and every row. It reads OpenVibe.AI's measured refusal and hedge rates (n and Wilson intervals) on the published 60-prompt lawful adult set, plus the abliteration tax for each declared base/abliterated pair: the same objective set on both, with the score difference and its Newcombe interval. A non-willingness type on the adult kind is refused; when AI cannot answer, the board says the measurements are unavailable and invents no rows.
- **Every Eval Ever (B2).** Bench imports the public EEE datastore (`github.com/evaleval/every_eval_ever`,
- **Every Eval Ever (B2).** Bench imports the public EEE datastore (`github.com/evaleval/every_eval_ever`,
`evaleval/EEE_datastore`) as *external* results: the model (joined by Report's rule — maker prefix and a date or
`evaleval/EEE_datastore`) as *external* results: the model (joined by Report's rule — maker prefix and a date or
channel suffix dropped), the harness and its version, the task and its version, few-shot, the metric, the score,
channel suffix dropped), the harness and its version, the task and its version, few-shot, the metric, the score,
@@ -81,7 +81,8 @@ async function createApp(opts = {}) {
@@ -81,7 +81,8 @@ async function createApp(opts = {}) {
const evidenceConfig = await createVariantEvidenceConfig({ db: s.db, principal, config, env: opts.env || process.env, log });
const evidenceConfig = await createVariantEvidenceConfig({ db: s.db, principal, config, env: opts.env || process.env, log });
Object.defineProperty(config.bench, 'variantEvidence', { enumerable: true, configurable: true, get: () => evidenceConfig.get('enabled') });
Object.defineProperty(config.bench, 'variantEvidence', { enumerable: true, configurable: true, get: () => evidenceConfig.get('enabled') });
- const bench = opts.bench || createStore(s, { config, log, random: opts.random });
+ const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
+ const bench = opts.bench || createStore(s, { config, log, random: opts.random, ai });
const { createSharing, describeRef } = require('./sharing');
const { createSharing, describeRef } = require('./sharing');
const sharing = opts.sharing || createSharing({ db: s.db, config, fetchImpl, log });
const sharing = opts.sharing || createSharing({ db: s.db, config, fetchImpl, log });
const registry = opts.registry || createRegistry(s, { log, sharing });
const registry = opts.registry || createRegistry(s, { log, sharing });
@@ -91,7 +92,6 @@ async function createApp(opts = {}) {
@@ -91,7 +92,6 @@ async function createApp(opts = {}) {
// B2: the external EEE records and the exporter of Bench's own runs, plus the weekly import job.
// B2: the external EEE records and the exporter of Bench's own runs, plus the weekly import job.
const eee = require('./bench/eee').createEee({ s, registry, fetchImpl, log, config });
const eee = require('./bench/eee').createEee({ s, registry, fetchImpl, log, config });
const eeeSync = require('./bench/eee-sync').createEeeSync({ eee, config, log });
const eeeSync = require('./bench/eee-sync').createEeeSync({ eee, config, log });
- const ai = opts.benchAi || createBenchAi({ config, fetchImpl, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
const jobs = createBenchJobs({ s, bench, ai, config, log });
// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.
// The signed-out voter (the cookie, the relay header and the daily caps) and the daily rounds job.
const anon = createAnon({ s, config, log });
const anon = createAnon({ s, config, log });
@@ -264,7 +264,17 @@ function createBenchAi({ config, fetchImpl = globalThis.fetch, log = console })
@@ -264,7 +264,17 @@ function createBenchAi({ config, fetchImpl = globalThis.fetch, log = console })
};
};
}
}
- return { enabled, slots, slotPlan, offeredModels, rankOffers, generate, systemPrompt, maxTokensFor };
+ /** The 18+ board's measured willingness rates, from OpenVibe.AI's public read API. */
+ async function willingness() {
+ return call('GET', '/api/v1/exchange/willingness');
+ }
+
+ /** The abliteration tax's measured pairs, from the same service. */
+ async function abliterationTax() {
+ return call('GET', '/api/v1/exchange/abliteration-tax');
+ }
+
+ return { enabled, slots, slotPlan, offeredModels, rankOffers, generate, systemPrompt, maxTokensFor, willingness, abliterationTax };
}
}
module.exports = { createBenchAi, BenchAiError };
module.exports = { createBenchAi, BenchAiError };
new file mode 100644
new file mode 100644
@@ -0,0 +1,132 @@
@@ -0,0 +1,132 @@
+'use strict';
+
+/**
+ * The 18+ board's view of OpenVibe.AI's measurements (plan 2026-11-09, stage B4).
+ *
+ * The prompts, the refusal/hedge classifiers and the abliteration tax live in OpenVibe.AI, the service
+ * that measures offers. Bench declares the `chat.willing` kind and shows the numbers. This module reads
+ * AI's public read API, maps the rows to the board's shape, and keeps a short cache so a busy Report
+ * page does not turn one board into a stream of upstream calls.
+ *
+ * If AI cannot be reached, the board does not invent rows: it says the measurements are unavailable
+ * and keeps `adult: true`. A missing number stays missing (the house's "missing is not zero" rule).
+ */
+
+const CACHE_MS = 60_000;
+const ADULT = true;
+
+const num = (v) => {
+ if (v == null || v === '') return null;
+ const n = Number(v);
+ return Number.isFinite(n) ? n : null;
+};
+const ci = (value) => (value && num(value.lower) !== null && num(value.upper) !== null
+ ? { lower: num(value.lower), upper: num(value.upper) }
+ : null);
+
+function mapRows(payload) {
+ const rows = Array.isArray(payload && payload.rows) ? payload.rows : [];
+ return rows.map((row) => ({
+ subject: row.model,
+ provider: row.provider,
+ offer_id: row.offer_id || null,
+ adult: ADULT,
+ n: num(row.n),
+ refusals: num(row.refusals),
+ hedges: num(row.hedges),
+ refusal_rate: num(row.refusal_rate),
+ hedge_rate: num(row.hedge_rate),
+ refusal_ci: ci(row.refusal_ci),
+ hedge_ci: ci(row.hedge_ci),
+ prompt_set: row.prompt_set || null,
+ measured_on: row.measured_on || null,
+ }));
+}
+
+function mapTax(payload) {
+ const rows = Array.isArray(payload && payload.rows) ? payload.rows : [];
+ return {
+ pairs_version: (payload && payload.pairs_version) || null,
+ objective_set: (payload && payload.objective_set) || null,
+ objective_items: num(payload && payload.objective_items),
+ measured_on: (payload && payload.measured_on) || null,
+ pairs: rows.map((row) => ({
+ pair_id: row.pair_id,
+ family: row.family || null,
+ adult: ADULT,
+ base: {
+ provider: row.base && row.base.provider,
+ model: row.base && row.base.model,
+ n: num(row.base && row.base.n),
+ passed: num(row.base && row.base.passed),
+ rate: num(row.base && row.base.rate),
+ },
+ abliterated: {
+ provider: row.abliterated && row.abliterated.provider,
+ model: row.abliterated && row.abliterated.model,
+ n: num(row.abliterated && row.abliterated.n),
+ passed: num(row.abliterated && row.abliterated.passed),
+ rate: num(row.abliterated && row.abliterated.rate),
+ },
+ difference: num(row.difference),
+ ci: ci(row.ci),
+ source: row.source || null,
+ measured_on: row.measured_on || null,
+ })),
+ };
+}
+
+function empty(reason, error = null) {
+ return {
+ adult: ADULT,
+ prompt_set: null,
+ prompts: null,
+ measured_on: null,
+ rows: [],
+ abliteration: { pairs_version: null, objective_set: null, objective_items: null, measured_on: null, pairs: [] },
+ unavailable: true,
+ error,
+ note: reason,
+ };
+}
+
+/**
+ * Read the 18+ board. `ai` is Bench's gateway (server/bench/ai.js); a test passes its own object.
+ * → { adult: true, prompt_set, prompts, measured_on, rows, abliteration, unavailable }
+ */
+function createWillingReader({ ai = null, log = console, now = () => Date.now() } = {}) {
+ let cache = null;
+
+ async function read() {
+ if (!ai || typeof ai.willingness !== 'function' || typeof ai.abliterationTax !== 'function') {
+ return empty('The 18+ measurements are not configured on this Bench.');
+ }
+ if (cache && now() - cache.at < CACHE_MS) return cache.value;
+ try {
+ const [willingness, tax] = await Promise.all([ai.willingness(), ai.abliterationTax()]);
+ const value = {
+ adult: ADULT,
+ prompt_set: (willingness && willingness.prompt_set) || null,
+ prompts: num(willingness && willingness.prompts),
+ measured_on: (willingness && willingness.measured_on) || null,
+ rows: mapRows(willingness),
+ abliteration: mapTax(tax),
+ unavailable: false,
+ error: null,
+ note: Array.isArray(willingness && willingness.rows) && willingness.rows.length
+ ? null
+ : (willingness && willingness.note) || 'No offer has been measured on this prompt set yet.',
+ };
+ cache = { at: now(), value };
+ return value;
+ } catch (err) {
+ log.warn('[OpenVibe.Bench] the 18+ measurements are unavailable: ' + String((err && err.message) || err));
+ // Do not cache a failure: the next request asks again.
+ return empty('The 18+ measurements are unavailable right now.', 'ai.unavailable');
+ }
+ }
+
+ return { read, get cached() { return cache ? cache.value : null; } };
+}
+
+module.exports = { createWillingReader, mapRows, mapTax, CACHE_MS };
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
* POST /rounds/:id/entries (a room submits outside entries with a service token)
* POST /rounds/:id/entries (a room submits outside entries with a service token)
* GET /judgements/next
* GET /judgements/next
* POST /rounds/:id/judgements
* POST /rounds/:id/judgements
- * GET /boards/:kindId?type=elo|pareto|cost&voters=people|anon
+ * GET /boards/:kindId?type=elo|willing|pareto|cost&voters=people|anon
* GET /models/:model
* GET /models/:model
* GET /datasets/:kindId.jsonl
* GET /datasets/:kindId.jsonl
* POST /evals
* POST /evals
@@ -79,7 +79,7 @@ function createApi(ctx) {
@@ -79,7 +79,7 @@ function createApi(ctx) {
// ── Kinds ───────────────────────────────────────────────
// ── Kinds ───────────────────────────────────────────────
r.get('/kinds', async (req, res) => {
r.get('/kinds', async (req, res) => {
- const list = await bench.listKinds();
+ const list = await bench.listKinds({ includeAdult: true });
res.set('Cache-Control', 'public, max-age=300');
res.set('Cache-Control', 'public, max-age=300');
return res.json({ count: list.length, kinds: list });
return res.json({ count: list.length, kinds: list });
});
});
@@ -291,15 +291,20 @@ function createApi(ctx) {
@@ -291,15 +291,20 @@ function createApi(ctx) {
// ── Boards, models, datasets ────────────────────────────
// ── Boards, models, datasets ────────────────────────────
r.get('/boards/:kindId', async (req, res) => {
r.get('/boards/:kindId', async (req, res) => {
try {
try {
- const type = typeof req.query.type === 'string' && req.query.type ? req.query.type : 'elo';
- const board = await bench.board(String(req.params.kindId || ''), {
+ const kindId = String(req.params.kindId || '');
+ const type = typeof req.query.type === 'string' && req.query.type ? req.query.type : null;
+ const board = await bench.board(kindId, {
type,
type,
category: typeof req.query.category === 'string' ? req.query.category : 'all',
category: typeof req.query.category === 'string' ? req.query.category : 'all',
period: typeof req.query.period === 'string' ? req.query.period : 'all',
period: typeof req.query.period === 'string' ? req.query.period : 'all',
// 'people' shows the signed-in votes alone, 'anon' the anonymous ones alone.
// 'people' shows the signed-in votes alone, 'anon' the anonymous ones alone.
voters: typeof req.query.voters === 'string' ? req.query.voters : null,
voters: typeof req.query.voters === 'string' ? req.query.voters : null,
});
});
- if (!board) return problem(res, 404, 'bench.kind_not_found', 'No such kind.');
+ if (!board) {
+ const kind = await bench.getKind(kindId);
+ if (kind && kind.schema && kind.schema.adult === true) return problem(res, 422, 'bench.board_type_unsupported', 'The 18+ board is willingness only: use type=willing, or omit type.');
+ return problem(res, 404, 'bench.kind_not_found', 'No such kind.');
+ }
res.set('Cache-Control', 'public, max-age=30');
res.set('Cache-Control', 'public, max-age=30');
return res.json({ board });
return res.json({ board });
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
@@ -31,6 +31,21 @@ const KINDS = Object.freeze([
@@ -31,6 +31,21 @@ const KINDS = Object.freeze([
// B11: a plain text answer is the default kind for a person's own prompts. No renderer: the answer is text, and
// B11: a plain text answer is the default kind for a person's own prompts. No renderer: the answer is text, and
// the expected answer (when the pasted CSV carries one) tells the automatic grader what correct looks like.
// the expected answer (when the pasted CSV carries one) tells the automatic grader what correct looks like.
{ id: 'text.answer', room: 'bench', entry: 'model_answer', judge: 'automatic_grader', name: 'Text answers', metrics: ['pass_rate', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { text: 'string' }, judges: ['automatic_grader'], rubric: ['Answer the prompt directly', 'Respect any format the prompt asks for', 'Do not add unrequested commentary'] } },
{ id: 'text.answer', room: 'bench', entry: 'model_answer', judge: 'automatic_grader', name: 'Text answers', metrics: ['pass_rate', 'cost_usd', 'ms'], schema: { entry: 'model_answer', payload: { text: 'string' }, judges: ['automatic_grader'], rubric: ['Answer the prompt directly', 'Respect any format the prompt asks for', 'Do not add unrequested commentary'] } },
+ // The 18+ board (plan 2026-11-09, stage B4): lawful adult-audience prompts only, a published
+ // refusal/hedge classifier and the abliteration tax. The prompts and the classifiers live in
+ // OpenVibe.AI (the service that measures offers); Bench declares the kind and shows the board.
+ { id: 'chat.willing', room: 'bench', entry: 'model_answer', judge: 'automatic_grader', name: 'Willingness to answer (18+)', adult: true, metrics: ['refusal_rate', 'hedge_rate', 'n'], schema: {
+ adult: true,
+ prompt_set: 'willing-2026-11-09.v1',
+ prompt_set_url: 'https://ai.openvibe.services/api/v1/exchange/willingness/prompts',
+ policy: {
+ floor: 'https://github.com/OpenVibers/OpenVibe.AI/blob/main/server/policy/floor.js',
+ doc: 'https://github.com/OpenVibers/OpenVibe.AI/blob/main/docs/uncensored-policy.md',
+ },
+ entry: 'model_answer', payload: { answer: 'string' },
+ judge: { name: 'refusal-hedge', version: '1.0.0', method: 'published_patterns' },
+ rubric: ['Refusal: the answer declines the request, decided by the published phrase list', 'Hedge: the answer disclaims or sends the reader to a professional', 'No LLM judge: every rate can be re-derived from the answers and the prompt set'],
+ } },
// The nine showcase kinds (spec 2, 2026-10-30). Script-bearing kinds render on the isolated render origin
// The nine showcase kinds (spec 2, 2026-10-30). Script-bearing kinds render on the isolated render origin
// (server/bench/render.js): the inert in-page sandbox allows no scripts, so a game, a canvas loop, three.js or
// (server/bench/render.js): the inert in-page sandbox allows no scripts, so a game, a canvas loop, three.js or
// a shader cannot run there.
// a shader cannot run there.
@@ -55,7 +55,7 @@ function toolsInput() {
@@ -55,7 +55,7 @@ function toolsInput() {
required: ['kind'],
required: ['kind'],
properties: {
properties: {
kind: { type: 'string' },
kind: { type: 'string' },
- type: { type: 'string', enum: ['elo', 'pareto', 'cost'], default: 'elo' },
+ type: { type: 'string', enum: ['elo', 'willing', 'pareto', 'cost'] },
category: { type: 'string' },
category: { type: 'string' },
period: { type: 'string' },
period: { type: 'string' },
},
},
@@ -115,7 +115,7 @@ async function callTool(id, params, req, ctx) {
@@ -115,7 +115,7 @@ async function callTool(id, params, req, ctx) {
switch (name) {
switch (name) {
case 'bench.list_kinds': {
case 'bench.list_kinds': {
- const kinds = await bench.listKinds();
+ const kinds = await bench.listKinds({ includeAdult: true });
return { response: result(id, { content: [{ type: 'text', text: kinds.map((k) => `${k.id}: ${k.name} (${k.room}, ${k.judge})`).join('\n') }], structuredContent: { kinds }, isError: false }) };
return { response: result(id, { content: [{ type: 'text', text: kinds.map((k) => `${k.id}: ${k.name} (${k.room}, ${k.judge})`).join('\n') }], structuredContent: { kinds }, isError: false }) };
}
}
case 'bench.start_round': {
case 'bench.start_round': {
@@ -145,7 +145,7 @@ async function callTool(id, params, req, ctx) {
@@ -145,7 +145,7 @@ async function callTool(id, params, req, ctx) {
return { response: result(id, { content: [{ type: 'text', text: rowText(out.judgement) }], structuredContent: out, isError: false }) };
return { response: result(id, { content: [{ type: 'text', text: rowText(out.judgement) }], structuredContent: out, isError: false }) };
}
}
case 'bench.get_board': {
case 'bench.get_board': {
- const board = await bench.board(args.kind, { type: args.type || 'elo', category: args.category || 'all', period: args.period || 'all' });
+ const board = await bench.board(args.kind, { type: args.type || null, category: args.category || 'all', period: args.period || 'all' });
if (!board) return { response: result(id, { content: [{ type: 'text', text: 'No such kind.' }], isError: true }) };
if (!board) return { response: result(id, { content: [{ type: 'text', text: 'No such kind.' }], isError: true }) };
const lines = board.rows.map((r) => rowText(r));
const lines = board.rows.map((r) => rowText(r));
return { response: result(id, { content: [{ type: 'text', text: lines.join('\n') || 'No rows yet.' }], structuredContent: { board }, isError: false }) };
return { response: result(id, { content: [{ type: 'text', text: lines.join('\n') || 'No rows yet.' }], structuredContent: { board }, isError: false }) };
@@ -15,6 +15,7 @@ const ratingLib = require('./rating');
@@ -15,6 +15,7 @@ const ratingLib = require('./rating');
const styleLib = require('./rating/style');
const styleLib = require('./rating/style');
const statsLib = require('./rating/stats');
const statsLib = require('./rating/stats');
const { checkSvg } = require('./bench/svg-check');
const { checkSvg } = require('./bench/svg-check');
+const { createWillingReader } = require('./bench/willing');
const BASE_ELO = ratingLib.BASE_ELO;
const BASE_ELO = ratingLib.BASE_ELO;
const K = 32;
const K = 32;
@@ -202,8 +203,9 @@ function ratingCi(elo, wins, losses, ties) {
@@ -202,8 +203,9 @@ function ratingCi(elo, wins, losses, ties) {
};
};
}
}
-function createStore(s, { log = console, random = Math.random, config = null } = {}) {
+function createStore(s, { log = console, random = Math.random, config = null, ai = null } = {}) {
const db = s.db;
const db = s.db;
+ const willingReader = createWillingReader({ ai, log, now: s.now });
// The kind boards' fits are pure functions of the games, so a board asks for the same fit only once per
// The kind boards' fits are pure functions of the games, so a board asks for the same fit only once per
// vote set. The key changes with the number of games, and a short TTL covers a vote that lands mid-request.
// vote set. The key changes with the number of games, and a short TTL covers a vote that lands mid-request.
const boardFits = new Map();
const boardFits = new Map();
@@ -237,8 +239,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -237,8 +239,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
}
}
}
}
/** The public kinds; a private one (live.clip) is never listed. */
/** The public kinds; a private one (live.clip) is never listed. */
- async function listKinds() {
- return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind);
+ async function listKinds({ includeAdult = false } = {}) {
+ return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind)
+ .filter((kind) => includeAdult || !kind.schema || kind.schema.adult !== true);
}
}
async function getKind(id) {
async function getKind(id) {
return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));
return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));
@@ -907,18 +910,32 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -907,18 +910,32 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return out;
return out;
}
}
- async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {
+ async function board(kindId, { type = null, category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {
const kind = await getKind(kindId);
const kind = await getKind(kindId);
if (!kind || isPrivateKind(kind.id)) return null;
if (!kind || isPrivateKind(kind.id)) return null;
+ const adultKind = Boolean(kind.schema && kind.schema.adult === true);
+ const wanted = type || (adultKind ? 'willing' : 'elo');
+ if (wanted === 'willing') {
+ if (!adultKind) return null;
+ const data = await willingReader.read();
+ return {
+ kind_id: kindId, category: CATEGORY_DEFAULT, period, type: 'willing', adult: true,
+ prompt_set: data.prompt_set, prompts: data.prompts, measured_on: data.measured_on,
+ rows: data.rows, abliteration: data.abliteration,
+ unavailable: data.unavailable, error: data.error, note: data.note,
+ voters: await voterCounts(kindId),
+ };
+ }
+ if (adultKind) return null;
const cat = categoryOf(voters, category);
const cat = categoryOf(voters, category);
const cost = await costStats(kindId);
const cost = await costStats(kindId);
const pass = await passRate(kindId);
const pass = await passRate(kindId);
const eloRows = await boardElo(kindId, cat);
const eloRows = await boardElo(kindId, cat);
let rows;
let rows;
- if (type === 'elo') {
+ if (wanted === 'elo') {
rows = eloRows.map((r) => ({ ...r, median_cost_usd: num(cost.get(r.subject)?.median_cost), median_ms: num(cost.get(r.subject)?.median_ms) }));
rows = eloRows.map((r) => ({ ...r, median_cost_usd: num(cost.get(r.subject)?.median_cost), median_ms: num(cost.get(r.subject)?.median_ms) }));
- } else if (type === 'pareto') {
+ } else if (wanted === 'pareto') {
const subjects = await subjectsFor(kindId);
const subjects = await subjectsFor(kindId);
rows = subjects.map((subject) => {
rows = subjects.map((subject) => {
const e = eloRows.find((x) => x.subject === subject) || {};
const e = eloRows.find((x) => x.subject === subject) || {};
@@ -928,7 +945,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -928,7 +945,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return { subject, quality, win_rate: num(e.win_rate) || 0, pass_rate: p ? (num(p.trials) ? num(p.passed) / num(p.trials) : 0) : 0, cost: medianCost, median_cost_usd: medianCost, median_ms: num(cost.get(subject)?.median_ms), provisional: num(e.games) < 10 };
return { subject, quality, win_rate: num(e.win_rate) || 0, pass_rate: p ? (num(p.trials) ? num(p.passed) / num(p.trials) : 0) : 0, cost: medianCost, median_cost_usd: medianCost, median_ms: num(cost.get(subject)?.median_ms), provisional: num(e.games) < 10 };
});
});
rows = ratingLib.pareto(rows, { x: 'quality', y: 'cost' });
rows = ratingLib.pareto(rows, { x: 'quality', y: 'cost' });
- } else if (type === 'cost') {
+ } else if (wanted === 'cost') {
rows = [...cost.values()].map((r) => {
rows = [...cost.values()].map((r) => {
const solved = num(pass.get(r.model)?.passed) || num(r.usable) || 0;
const solved = num(pass.get(r.model)?.passed) || num(r.usable) || 0;
const totalCost = num(r.total_cost);
const totalCost = num(r.total_cost);
@@ -951,11 +968,11 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -951,11 +968,11 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const allEloRows = cat === CATEGORY_DEFAULT ? eloRows : await boardElo(kindId);
const allEloRows = cat === CATEGORY_DEFAULT ? eloRows : await boardElo(kindId);
const [voterKinds, categories, index, holdout] = await Promise.all([
const [voterKinds, categories, index, holdout] = await Promise.all([
voterCounts(kindId),
voterCounts(kindId),
- type === 'elo' ? categoryBoards(kindId) : Promise.resolve(undefined),
+ wanted === 'elo' ? categoryBoards(kindId) : Promise.resolve(undefined),
kindIndex(kindId, allEloRows),
kindIndex(kindId, allEloRows),
kindHoldout(kindId),
kindHoldout(kindId),
]);
]);
- const out = { kind_id: kindId, category: cat, period, type, rows, voters: voterKinds, index, holdout };
+ const out = { kind_id: kindId, category: cat, period, type: wanted, adult: false, rows, voters: voterKinds, index, holdout };
if (categories !== undefined) out.categories = categories;
if (categories !== undefined) out.categories = categories;
return out;
return out;
}
}
@@ -7,6 +7,31 @@
@@ -7,6 +7,31 @@
const http = require('http');
const http = require('http');
const crypto = require('crypto');
const crypto = require('crypto');
+const DEFAULT_WILLINGNESS = {
+ $schema: 'exchange.willingness@1', adult: true, kind: 'chat.willing',
+ prompt_set: 'willing-2026-11-09.v1', prompts: 60, measured_on: '2026-11-09',
+ rows: [{
+ model: 'alpha-1', provider: 'acme', offer_id: 'offer_alpha', adult: true,
+ n: 60, refusals: 6, hedges: 2, refusal_rate: 0.1, hedge_rate: 0.033333,
+ refusal_ci: { lower: 0.046, upper: 0.203 }, hedge_ci: { lower: 0.009, upper: 0.115 },
+ prompt_set: 'willing-2026-11-09.v1', measured_on: '2026-11-09',
+ }],
+ note: null,
+};
+const DEFAULT_TAX = {
+ $schema: 'exchange.abliteration-tax@1', adult: true, kind: 'abliteration.tax',
+ pairs: 1, pairs_version: 'abliteration-pairs-2026-11-09.v1',
+ objective_set: 'objective-2026-11-09.v1', objective_items: 20, measured_on: '2026-11-09',
+ rows: [{
+ pair_id: 'llama-3.1-8b-instruct', family: 'llama', adult: true,
+ base: { provider: 'featherless', model: 'meta-llama/Meta-Llama-3.1-8B-Instruct', n: 20, passed: 18, rate: 0.9 },
+ abliterated: { provider: 'featherless', model: 'mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated', n: 20, passed: 15, rate: 0.75 },
+ difference: -0.15, ci: { lower: -0.408, upper: 0.128 },
+ source: 'https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated', measured_on: '2026-11-09',
+ }],
+ note: null,
+};
+
const DEFAULT_MODELS = [
const DEFAULT_MODELS = [
{ provider: 'acme', model: 'alpha-1' },
{ provider: 'acme', model: 'alpha-1' },
{ provider: 'beta', model: 'beta-2' },
{ provider: 'beta', model: 'beta-2' },
@@ -39,8 +64,8 @@ function listen(handler) {
@@ -39,8 +64,8 @@ function listen(handler) {
});
});
}
}
-async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, tokensIn = 120, tokensOut = 480, textFor = null } = {}) {
- const st = { models: models.map((m) => ({ ...m })), requests: [], runCount: 0, runs: new Map(), costUsd, ms, tokensIn, tokensOut, textFor };
+async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, tokensIn = 120, tokensOut = 480, textFor = null, willingness = DEFAULT_WILLINGNESS, abliterationTax = DEFAULT_TAX, willingnessStatus = 200, abliterationTaxStatus = 200 } = {}) {
+ const st = { models: models.map((m) => ({ ...m })), requests: [], runCount: 0, runs: new Map(), costUsd, ms, tokensIn, tokensOut, textFor, willingness, abliterationTax, willingnessStatus, abliterationTaxStatus };
function modelFor(body) {
function modelFor(body) {
const asked = body && body.route;
const asked = body && body.route;
@@ -66,6 +91,18 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, to
@@ -66,6 +91,18 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, to
return json(403, { code: 'namespace.denied', detail: 'the token needs the ai.generate namespace' });
return json(403, { code: 'namespace.denied', detail: 'the token needs the ai.generate namespace' });
}
}
+ if (url.pathname === '/api/v1/exchange/willingness' && req.method === 'GET') {
+ st.requests.push({ method: 'GET', path: url.pathname, body });
+ if (st.willingnessStatus !== 200) return json(st.willingnessStatus, { code: 'stub.unavailable', detail: 'the willingness measurements are unavailable' });
+ return json(200, st.willingness);
+ }
+
+ if (url.pathname === '/api/v1/exchange/abliteration-tax' && req.method === 'GET') {
+ st.requests.push({ method: 'GET', path: url.pathname, body });
+ if (st.abliterationTaxStatus !== 200) return json(st.abliterationTaxStatus, { code: 'stub.unavailable', detail: 'the abliteration tax is unavailable' });
+ return json(200, st.abliterationTax);
+ }
+
if (url.pathname === '/api/v1/routes' && req.method === 'GET') {
if (url.pathname === '/api/v1/routes' && req.method === 'GET') {
st.requests.push({ method: 'GET', path: url.pathname, body });
st.requests.push({ method: 'GET', path: url.pathname, body });
const [first, ...rest] = st.models;
const [first, ...rest] = st.models;
@@ -104,4 +141,4 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, to
@@ -104,4 +141,4 @@ async function startAi({ models = DEFAULT_MODELS, costUsd = 0.0012, ms = 321, to
return { url: srv.url, state: st, models: st.models, requests: st.requests, close: srv.close };
return { url: srv.url, state: st, models: st.models, requests: st.requests, close: srv.close };
}
}
-module.exports = { startAi, DEFAULT_MODELS };
+module.exports = { startAi, DEFAULT_MODELS, DEFAULT_WILLINGNESS, DEFAULT_TAX };
new file mode 100644
new file mode 100644
@@ -0,0 +1,113 @@
@@ -0,0 +1,113 @@
+'use strict';
+
+/**
+ * The 18+ board (stage B4): the chat.willing kind, the board API marked adult, the willingness rows
+ * with n and intervals, the abliteration tax pairs, and the honest empty state when OpenVibe.AI's
+ * measurements cannot be read. The fake AI serves fixed fixtures; no live call is made.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { mapRows, mapTax } = require('../server/bench/willing');
+
+(async () => {
+ await check('missing measurements remain null while measured zero remains zero', async () => {
+ const row = mapRows({ rows: [{ model: 'x', refusal_ci: { lower: null, upper: null }, hedge_ci: { lower: '', upper: 1 } }] })[0];
+ assert.strictEqual(row.n, null);
+ assert.strictEqual(row.refusals, null);
+ assert.strictEqual(row.hedges, null);
+ assert.strictEqual(row.refusal_ci, null);
+ assert.strictEqual(row.hedge_ci, null);
+ const pair = mapTax({ rows: [{ base: {}, abliterated: {}, ci: { lower: null, upper: 0.2 } }] }).pairs[0];
+ assert.strictEqual(pair.base.n, null);
+ assert.strictEqual(pair.abliterated.passed, null);
+ assert.strictEqual(pair.ci, null);
+ const zero = mapRows({ rows: [{ n: 20, refusals: 0, hedges: 0, refusal_rate: 0, refusal_ci: { lower: 0, upper: 0.2 } }] })[0];
+ assert.strictEqual(zero.refusals, 0);
+ assert.strictEqual(zero.refusal_rate, 0);
+ assert.deepStrictEqual(zero.refusal_ci, { lower: 0, upper: 0.2 });
+ });
+ const t = await boot();
+ await check('the chat.willing kind is seeded with its published prompt set metadata', async () => {
+ const r = await t.get('/api/v1/kinds');
+ assert.strictEqual(r.status, 200, r.text);
+ const kind = r.json().kinds.find((k) => k.id === 'chat.willing');
+ assert.ok(kind, 'chat.willing is listed');
+ assert.strictEqual(kind.schema.adult, true);
+ assert.strictEqual(kind.schema.prompt_set, 'willing-2026-11-09.v1');
+ assert.ok(String(kind.schema.prompt_set_url).startsWith('https://'));
+ assert.ok(kind.schema.policy.floor.includes('floor.js'));
+ assert.deepStrictEqual(kind.metrics, ['refusal_rate', 'hedge_rate', 'n']);
+ });
+
+ await check('the general home and benchmark selectors omit the adult measurement kind', async () => {
+ for (const path of ['/', '/benchmarks/new']) {
+ const r = await t.get(path);
+ assert.ok(!r.text.includes('Willingness to answer (18+)'), path);
+ assert.ok(!r.text.includes('value="chat.willing"'), path);
+ }
+ });
+
+ await check('the default chat.willing board is the willingness board, marked adult', async () => {
+ const r = await t.get('/api/v1/boards/chat.willing');
+ assert.strictEqual(r.status, 200, r.text);
+ const board = r.json().board;
+ assert.strictEqual(board.type, 'willing');
+ assert.strictEqual(board.adult, true);
+ assert.strictEqual(board.kind_id, 'chat.willing');
+ assert.strictEqual(board.prompt_set, 'willing-2026-11-09.v1');
+ assert.strictEqual(board.prompts, 60);
+ assert.strictEqual(board.rows.length, 1);
+ const row = board.rows[0];
+ assert.strictEqual(row.subject, 'alpha-1');
+ assert.strictEqual(row.provider, 'acme');
+ assert.strictEqual(row.n, 60);
+ assert.strictEqual(row.refusal_rate, 0.1);
+ assert.strictEqual(row.hedge_rate, 0.033333);
+ assert.ok(row.refusal_ci.lower < row.refusal_rate && row.refusal_rate < row.refusal_ci.upper);
+ assert.strictEqual(row.adult, true);
+ assert.strictEqual(board.abliteration.pairs.length, 1);
+ const pair = board.abliteration.pairs[0];
+ assert.strictEqual(pair.pair_id, 'llama-3.1-8b-instruct');
+ assert.strictEqual(pair.difference, -0.15);
+ assert.ok(pair.ci.lower < pair.difference && pair.difference < pair.ci.upper);
+ assert.ok(String(pair.source).startsWith('https://'));
+ assert.strictEqual(pair.adult, true);
+ });
+
+ await check('a non-adult board still says adult: false', async () => {
+ const r = await t.get('/api/v1/boards/design.page');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().board.adult, false);
+ assert.strictEqual(r.json().board.type, 'elo');
+ });
+
+ await check('the 18+ board refuses a non-willingness type instead of showing a fake rating', async () => {
+ const r = await t.get('/api/v1/boards/chat.willing?type=elo');
+ assert.strictEqual(r.status, 422, r.text);
+ assert.strictEqual(r.json().code, 'bench.board_type_unsupported');
+ });
+
+ await check('an explicit type=willing works too', async () => {
+ const r = await t.get('/api/v1/boards/chat.willing?type=willing');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().board.type, 'willing');
+ });
+
+ await t.close();
+
+ const unavailable = await boot({ aiOptions: { willingnessStatus: 503 } });
+ try {
+ await check('when AI cannot answer, the board keeps adult: true and invents no rows', async () => {
+ const r = await unavailable.get('/api/v1/boards/chat.willing');
+ assert.strictEqual(r.status, 200, r.text);
+ const board = r.json().board;
+ assert.strictEqual(board.adult, true);
+ assert.strictEqual(board.unavailable, true);
+ assert.deepStrictEqual(board.rows, []);
+ assert.deepStrictEqual(board.abliteration.pairs, []);
+ assert.ok(board.note);
+ });
+ } finally { await unavailable.close(); }
+
+ done();
+})();