openvibers/openvibe.exchange Commits
Compare: the AI-model comparison API behind openvibe.compare (#190)
Changes
@@@ -71,6 +71,11 @@ EXCHANGE_EVENTS_URL=# optional: defaults to true in production# COOKIE_SECURE=true++# openvibe.compare: the comparison site is served by this same service behind its own name (its vhost++# deploy/nginx/openvibe.compare.conf proxies to this process). The host makes a request the compare site's.++#COMPARE_HOST=openvibe.compare++#COMPARE_BASE_URL=https://openvibe.compare++# IndexNow (openvibe-shared/indexnow): a key lets a new page or a changed sitemap be submitted for recrawl.# Unset → off; nothing is mounted and nothing is sent. The key file is served at /<key>.txt.#INDEXNOW_KEY=nnew file mode 100644@@@ -0,0 +1,27 @@++-- phase: expand++-- openvibe.compare (plan 297/300/301): a reader's remark on a comparison becomes a pending benchmark case for++-- review. The remark is plain text and public metadata only: the person is stored as their Network subject when++-- they are signed in, and never as an address or a user agent. A case is reviewed by the owner; nothing here is++-- a credential and nothing is ever written to OpenVibe.Bench by this migration.++++CREATE TABLE IF NOT EXISTS exchange_compare_remarks (++ id text COLLATE "C" PRIMARY KEY, -- excr_ + 26 Crockford base32 characters (ULID)++ kind_id text COLLATE "C" NOT NULL DEFAULT 'ai.models',++ models jsonb NOT NULL DEFAULT '[]'::jsonb, -- the model slugs the remark is about++ comparison text, -- the pair/comparison key, when the remark is on one++ field text, -- price | speed | quality | privacy | other++ remark text NOT NULL,++ proposed_prompt text, -- the benchmark prompt the reviewer would run++ source_url text,++ status text COLLATE "C" NOT NULL DEFAULT 'pending',++ subject text, -- the signed-in person's usr_…, or null++ reviewer_note text,++ reviewed_by text,++ reviewed_at timestamptz,++ created_at timestamptz NOT NULL DEFAULT now(),++ CONSTRAINT exchange_compare_remarks_status CHECK (status IN ('pending', 'accepted', 'rejected')),++ CONSTRAINT exchange_compare_remarks_field CHECK (field IS NULL OR field IN ('price', 'speed', 'quality', 'privacy', 'context', 'other')),++ CONSTRAINT exchange_compare_remarks_remark CHECK (length(remark) BETWEEN 10 AND 2000)++);++CREATE INDEX IF NOT EXISTS exchange_compare_remarks_pending ON exchange_compare_remarks (status, created_at);++CREATE INDEX IF NOT EXISTS exchange_compare_remarks_models ON exchange_compare_remarks USING gin (models);@@@ -30,6 +30,7 @@ const { createPrincipal } = require('./http/principal');const { createAi } = require('./ai');const { createNetwork } = require('./network');const { createBench } = require('./bench');++const { createCompare } = require('./compare');const { createOpenApi } = require('./openapi');const { createArenaStore } = require('./arena/store');const { createCatalogHistory } = require('./catalog-history');@@@ -46,6 +47,7 @@ const { createNoteTaker } = require('./companions/taking-notes');const { createPageRoutes } = require('./http/pages');const { createConsoleRoutes } = require('./http/console');const { createArenaRoutes } = require('./http/arena');++const { createCompareApi } = require('./http/compare');const { createCompanionRoutes } = require('./http/companions');const { createServiceReadiness } = require('./observability');const { assetVersion, send } = require('./render/layout');@@@ -84,6 +86,8 @@ async function createApp(opts = {}) {// Network, read and changed with the person's own token: their machines and their machine pools.const network = opts.network || createNetwork({ config, fetchImpl, log });const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });++ // openvibe.compare's engine (server/compare.js): the join of AI's price/speed/privacy/tier with Bench's quality.++ const compare = createCompare({ s, ai, bench, config, now: s.now, log });const analysis = opts.analysis || require('./analysis').createAnalysis({ config, ai, bench, fetchImpl: opts.aiFetchImpl || globalThis.fetch, now: s.now });const arenaStore = createArenaStore(s);const history = createCatalogHistory({ s, log });@@@ -116,7 +120,7 @@ async function createApp(opts = {}) {: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });-- const ctx = { config, s, keys, sso, principal, ai, network, bench, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };++ const ctx = { config, s, keys, sso, principal, ai, network, bench, compare, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };const app = express();app.disable('x-powered-by');@@@ -255,6 +259,10 @@ async function createApp(opts = {}) {app.use('/api/v1/analysis', rateLimit({ windowMs: 60000, limit: Number(process.env.EXCHANGE_API_RATE_LIMIT_PER_MIN) || 120, standardHeaders: true, legacyHeaders: false }), require('./http/analysis').createAnalysisRoutes(ctx));++ // ── Compare's JSON API (openvibe.compare) ───────────────++ // Mounted before Exchange v1's catch-all, which answers 410 to everything else under /api/v1.++ app.use('/api/v1/compare', createCompareApi(ctx));++// ── Exchange v1's JSON API: gone ────────────────────────app.use('/api/v1', (req, res) => {res.set('Cache-Control', 'public, max-age=86400');nnew file mode 100644@@@ -0,0 +1,682 @@++'use strict';++++/**++ * Compare — the comparison engine behind openvibe.compare (plan 296, 297, 300, 301).++ *++ * Why Exchange: Exchange already reads OpenVibe.AI's public catalogue (price, speed, privacy, tier) and++ * OpenVibe.Bench's boards (quality), and it is the service that serves the public site. Bench owns the runs but++ * has no prices; a separate service would only re-fetch the same two reads and add a deployment. So the join is++ * made here, and every field says which service it came from and when it was read.++ *++ * A comparison is data (plan 297): things (models), criteria (price, speed, quality, privacy, tier), sources, a++ * method, and results with dates. Nothing is estimated: a model with no public Bench rounds has no quality score,++ * and the row says why instead of showing a made-up number.++ */++++const QUALITY_KINDS = Object.freeze([++ { id: 'chat.arena', name: 'Exchange arena' },++ { id: 'chat.reply', name: 'Chat replies' },++ { id: 'chat.willing', name: 'Willingness to answer (18+)' },++ { id: 'vision.answer', name: 'Vision answers' },++ { id: 'codes.patch', name: 'Code patches' },++ { id: 'codes.review', name: 'Code diff review' },++ { id: 'translate.ja', name: 'Japanese translation' },++ { id: 'search.answer', name: 'Cited answers' },++]);++++const NOT_CHAT = new Set(['embedding', 'embeddings', 'stt', 'asr', 'tts', 'speech', 'image', 'images', 'rerank', 'moderation']);++const KIND = 'ai.models';++++const finite = (v) => (Number.isFinite(Number(v)) ? Number(v) : null);++const iso = (v, fallback = null) => {++ if (!v) return fallback;++ const d = new Date(v);++ return Number.isNaN(d.getTime()) ? fallback : d.toISOString();++};++const round = (v, places = 4) => (v == null ? null : Math.round(Number(v) * 10 ** places) / 10 ** places);++const pct = (a, b) => (a == null || b == null || Number(b) === 0 ? null : round(((Number(a) - Number(b)) / Math.abs(Number(b))) * 100, 2));++const norm = (s) => String(s || '').toLowerCase().replace(/^[^/]+\//, '').replace(/[^a-z0-9]+/g, '');++++/** A trailing date is a release stamp, not a version of the line: gpt-5-nano-2025-08-07 is gpt-5-nano. */++const DATE_SUFFIX = /[-_ ]20\d{2}[-_]\d{1,2}[-_]\d{1,2}$/;++++function baseNameOf(model) {++ return String(model.name || model.slug || '').replace(/@.*$/, '').replace(DATE_SUFFIX, '').trim();++}++++/**++ * Split a model name into the line it belongs to and its version numbers.++ * gpt-5-nano → line "gpt-nano", version [5]++ * gpt-5.1-nano → line "gpt-nano", version [5.1]++ * claude-opus-4-5 → line "claude-opus", version [4,5]++ * qwen3-32b → line "qwen-32b", version [3] (32b is a size, not a version)++ * A token that is only a number, or letters followed by a number (qwen3, o3), carries a version; anything else++ * (32b, mini, chat) stays in the line's name.++ */++function splitVersion(name) {++ const tokens = String(name || '').toLowerCase().split(/[^a-z0-9.]+/).filter(Boolean);++ const version = [];++ const line = [];++ for (const token of tokens) {++ if (/^\d+(?:\.\d+)*$/.test(token)) {++ version.push(...token.split('.').map(Number));++ continue;++ }++ const m = /^([a-z]+)(\d+(?:\.\d+)*)$/.exec(token);++ if (m) {++ line.push(m[1]);++ version.push(...m[2].split('.').map(Number));++ continue;++ }++ line.push(token);++ }++ return { line, version, key: line.join('-') || norm(name) };++}++++function compareVersions(a, b) {++ const x = a.version || [];++ const y = b.version || [];++ const len = Math.max(x.length, y.length);++ for (let i = 0; i < len; i++) {++ const d = (x[i] || 0) - (y[i] || 0);++ if (d) return d;++ }++ return String(a.name).localeCompare(String(b.name));++}++++function isModel(m) {++ return Boolean(m && m.slug && !NOT_CHAT.has(String(m.kind || m.type || 'chat').toLowerCase()));++}++++/** An offer's price, speed, privacy and tier, exactly as AI's catalogue says them; nothing is filled in. */++function offerOf(o, catalog) {++ const price = o && o.price && Number.isFinite(Number(o.price.in)) && Number.isFinite(Number(o.price.out))++ ? {++ in: Number(o.price.in), out: Number(o.price.out),++ cached: finite(o.price.cached), per: o.price.per || '1M tokens', currency: o.price.currency || 'USD',++ }++ : null;++ const blended = price ? round((price.in + price.out) / 2, 6) : null;++ const privacy = o && o.privacy && typeof o.privacy === 'object' ? {++ label: o.privacy.label || null, retention: o.privacy.retention == null ? null : o.privacy.retention,++ training: o.privacy.training == null ? null : o.privacy.training, region: o.privacy.region || null,++ known: Boolean(o.privacy.known),++ } : null;++ return {++ provider: o.provider || null,++ provider_name: o.provider_name || o.provider || null,++ model: o.model || null,++ id: o.id || null,++ trust: o.trust || null,++ access: o.access || null,++ // The tier is whose capacity answers (the offer's trust/access), not whether it is free: a free tier on a++ // first-party model is still first-party capacity. The free flags stay beside it, where they belong.++ tier: (() => {++ if (o.trust === 'community') return 'community';++ if (o.on_our_gpus || o.trust === 'first-party') return 'first-party';++ if (o.trust === 'authorized' || o.access === 'AUTHORIZED_WHOLESALE') return 'authorized';++ return 'third-party';++ })(),++ free: Boolean(o.free), free_tier: Boolean(o.free_tier), free_now: Boolean(o.free_now),++ on_our_gpus: Boolean(o.on_our_gpus), fixed_price: Boolean(o.fixed_price),++ status: o.status || null,++ price,++ blended,++ speed: {++ p50_ms: finite(o.speed && o.speed.p50_ms),++ p95_ms: finite(o.speed && o.speed.p95_ms),++ },++ uptime: finite(o.uptime),++ error_rate: finite(o.error_rate),++ requests_7d: finite(o.requests_7d),++ requests_today: finite(o.requests_today),++ cache_hit_share_7d: finite(o.cache_hit_share_7d),++ privacy,++ data_policy: o.data_policy || null,++ refusal_rate: finite(o.refusal_rate),++ refusal_measured_on: o.refusal_measured_on || null,++ fidelity: o.fidelity == null ? null : o.fidelity,++ measured_at: iso(o.price_checked_at) || iso(o.updated_at) || iso(catalog && catalog.generated_at),++ };++}++++function cheapestOffer(offers) {++ return offers.filter((o) => o.price).sort((a, b) => a.blended - b.blended)[0] || null;++}++++function fastestOffer(offers) {++ return offers.filter((o) => o.speed.p50_ms != null).sort((a, b) => a.speed.p50_ms - b.speed.p50_ms)[0] || null;++}++++/** The Bench board rows that name this model: an exact match, or a release date appended to the same name. */++function subjectsFor(model, rows) {++ const want = norm(model.slug || model.name);++ const tail = norm(String(model.slug || '').split('/').pop());++ return rows.filter((r) => {++ const got = norm(r.subject);++ if (!got || got.length < 5) return false;++ if (got === want || got === tail) return true;++ if (want.length >= 5 && got.startsWith(want)) return true;++ if (tail.length >= 5 && got.startsWith(tail)) return true;++ return false;++ });++}++++/**++ * Quality is OpenVibe.Bench's Elo, never a guess. A line with no games has no score: the row carries `available:++ * false` and the reason, and the page prints that instead of a number.++ */++function qualityOf(model, quality) {++ const rows = subjectsFor(model, quality.rows || []);++ if (!rows.length) {++ return {++ available: false, score: null, games: 0, provisional: true, ci: null,++ by_benchmark: [], read_at: quality.at || null,++ reason: quality.unavailable ? 'Bench is not reachable right now.' : 'No public Bench rounds for this model yet.',++ };++ }++ const byKind = new Map();++ for (const r of rows) {++ const prev = byKind.get(r.kind_id);++ if (!prev || (Number(r.games) || 0) > (Number(prev.games) || 0)) byKind.set(r.kind_id, r);++ }++ const byBenchmark = [...byKind.values()].map((r) => ({++ kind_id: r.kind_id,++ name: (QUALITY_KINDS.find((k) => k.id === r.kind_id) || {}).name || r.kind_id,++ elo: finite(r.elo), games: finite(r.games) || 0, wins: finite(r.wins), losses: finite(r.losses), ties: finite(r.ties),++ win_rate: finite(r.win_rate),++ ci_lower: finite(r.ci_lower), ci_upper: finite(r.ci_upper),++ provisional: r.provisional !== false,++ read_at: quality.at || null,++ })).sort((a, b) => b.games - a.games || a.name.localeCompare(b.name));++ const games = byBenchmark.reduce((n, r) => n + (r.games || 0), 0);++ if (!games) {++ return {++ available: false, score: null, games: 0, provisional: true, ci: null,++ by_benchmark: byBenchmark, read_at: quality.at || null,++ reason: 'Bench has rounds listed for this model, but no judged games yet.',++ };++ }++ const score = round(byBenchmark.reduce((n, r) => n + (r.elo || 0) * (r.games || 0), 0) / games, 1);++ return {++ available: true, score, games, provisional: byBenchmark.some((r) => r.provisional || r.games < 10),++ // A pooled interval across different boards would be invented; each board keeps its own interval below.++ ci: null,++ by_benchmark: byBenchmark, read_at: quality.at || null,++ reason: null,++ };++}++++function privacyOf(offers) {++ const labels = new Map();++ for (const o of offers) {++ const label = o.privacy && o.privacy.label;++ if (label) labels.set(label, (labels.get(label) || 0) + 1);++ }++ if (!labels.size) return { label: null, labels: [], known: false, reason: 'No offer states a privacy label.' };++ const sorted = [...labels.entries()].sort((a, b) => b[1] - a[1]);++ return {++ label: sorted[0][0], labels: sorted.map(([label, count]) => ({ label, offers: count })),++ known: true, reason: null,++ };++}++++function summaryOf(model, catalog, quality) {++ const offers = (model.offers || []).map((o) => offerOf(o, catalog));++ const best = cheapestOffer(offers);++ const fastest = fastestOffer(offers);++ const priced = offers.filter((o) => o.price);++ const line = splitVersion(baseNameOf(model));++ return {++ slug: model.slug,++ name: model.name || model.slug,++ family: model.family || null,++ kind: String(model.kind || model.type || 'chat').toLowerCase(),++ variant: model.variant || null,++ open_weights: Boolean(model.open_weights),++ uncensored: Boolean(model.uncensored),++ context: finite(model.context),++ max_output: finite(model.max_output),++ capabilities: model.capabilities || null,++ modalities: Array.isArray(model.modalities) ? model.modalities : [],++ line: { key: line.key, version: line.version, version_label: line.version.join('.') || null },++ offers_count: offers.length,++ priced_offers: priced.length,++ providers: [...new Set(offers.map((o) => o.provider_name).filter(Boolean))],++ requests_7d: finite(model.requests_7d),++ requests_today: finite(model.requests_today),++ price: best ? {++ in: best.price.in, out: best.price.out, cached: best.price.cached, blended: best.blended,++ per: best.price.per, currency: best.price.currency, provider: best.provider_name,++ } : { available: false, reason: offers.length ? 'No offer has a per-token price.' : 'No live offer.' },++ price_range: priced.length ? {++ min_in: Math.min(...priced.map((o) => o.price.in)), max_in: Math.max(...priced.map((o) => o.price.in)),++ min_out: Math.min(...priced.map((o) => o.price.out)), max_out: Math.max(...priced.map((o) => o.price.out)),++ } : null,++ speed: fastest ? {++ p50_ms: fastest.speed.p50_ms, p95_ms: fastest.speed.p95_ms, provider: fastest.provider_name,++ measured_at: fastest.measured_at,++ } : { available: false, reason: 'No measured speed yet.' },++ uptime: offers.length ? Math.max(...offers.map((o) => (o.uptime == null ? 0 : o.uptime))) : null,++ free: offers.some((o) => o.free || o.free_now || (o.price && o.price.in === 0 && o.price.out === 0)),++ privacy: privacyOf(offers),++ tiers: {++ 'first-party': offers.filter((o) => o.tier === 'first-party').length,++ community: offers.filter((o) => o.tier === 'community').length,++ authorized: offers.filter((o) => o.tier === 'authorized').length,++ other: offers.filter((o) => o.tier === 'third-party').length,++ free_offers: offers.filter((o) => o.free || o.free_tier || o.free_now).length,++ },++ quality: qualityOf(model, quality),++ measured_at: iso(catalog && catalog.generated_at),++ };++}++++/**++ * Read the Bench boards once: one read per board, cached by the Bench client. Quality is only ever taken from++ * these rows, and a board that cannot be read is said to be unavailable, never replaced by an estimate.++ */++async function readQuality(bench, now) {++ const reads = await Promise.all(QUALITY_KINDS.map(async (kind) => {++ try {++ const board = await bench.board({ kind: kind.id, category: 'all' });++ return { kind, board: board || { unavailable: true, rows: [] } };++ } catch (err) {++ return { kind, board: { unavailable: true, rows: [], reason: String((err && err.message) || err).slice(0, 200) } };++ }++ }));++ const rows = [];++ let unavailable = 0;++ for (const { kind, board } of reads) {++ if (board.unavailable) unavailable++;++ for (const row of board.rows || []) rows.push({ ...row, kind_id: kind.id, board_name: kind.name });++ }++ return { rows, at: new Date(now()).toISOString(), unavailable: unavailable === reads.length, unread_boards: unavailable };++}++++function qualityDelta(previous, current) {++ const oldRows = new Map((previous.quality.by_benchmark || []).map((r) => [r.kind_id, r]));++ const rows = [];++ for (const now of current.quality.by_benchmark || []) {++ const before = oldRows.get(now.kind_id);++ if (!before || before.elo == null || now.elo == null) continue;++ const overlap = before.ci_lower != null && before.ci_upper != null && now.ci_lower != null && now.ci_upper != null++ && before.ci_lower <= now.ci_upper && now.ci_lower <= before.ci_upper;++ rows.push({++ kind_id: now.kind_id, name: now.name, from: before.elo, to: now.elo,++ delta: round(now.elo - before.elo, 1), interval_overlaps: overlap,++ });++ }++ if (!rows.length) return null;++ const clear = rows.filter((r) => !r.interval_overlaps);++ return {++ from: previous.quality.score, to: current.quality.score,++ delta: previous.quality.score == null || current.quality.score == null ? null : round(current.quality.score - previous.quality.score, 1),++ meaningful: clear.length > 0,++ by_benchmark: rows,++ reason: clear.length ? null : 'Every changed score is inside its own interval, so this is not a clear quality change.',++ };++}++++function change(key, label, from, to, { better = 'lower', unit = '', note = null } = {}) {++ if (from == null || to == null) {++ return { key, label, available: false, from, to, delta: null, pct: null, direction: 'unknown', unit, note: note || 'Not measured on one of the versions.' };++ }++ const delta = round(Number(to) - Number(from), 4);++ const direction = delta === 0 ? 'same' : (better === 'lower' ? (delta < 0 ? 'better' : 'worse') : (delta > 0 ? 'better' : 'worse'));++ return {++ key, label, available: true, from, to, delta, pct: pct(to, from), direction, unit,++ note: note || null,++ };++}++++function versionDiff(previous, current) {++ const changes = [++ change('price_in', 'Price in, per 1M', previous.price && previous.price.in, current.price && current.price.in, { unit: 'USD' }),++ change('price_out', 'Price out, per 1M', previous.price && previous.price.out, current.price && current.price.out, { unit: 'USD' }),++ change('p50_ms', 'Typical response, p50', previous.speed && previous.speed.p50_ms, current.speed && current.speed.p50_ms, { unit: 'ms' }),++ change('context', 'Context', previous.context, current.context, { better: 'higher', unit: 'tokens' }),++ change('elo', 'Bench Elo', previous.quality && previous.quality.score, current.quality && current.quality.score, { better: 'higher', unit: 'Elo' }),++ ];++ const quality = qualityDelta(previous, current);++ if (quality && !quality.meaningful) {++ const elo = changes.find((c) => c.key === 'elo');++ if (elo && elo.available) elo.note = quality.reason;++ }++ const known = changes.filter((c) => c.available);++ return {++ from: previous.slug, to: current.slug,++ from_name: previous.name, to_name: current.name,++ changes,++ quality,++ summary: known.length++ ? known.filter((c) => c.direction !== 'same').map((c) => `${c.label}: ${c.direction}`).join('; ') || 'No measured change.'++ : 'No criterion is measured on both versions.',++ };++}++++function lineagesOf(summaries) {++ const groups = new Map();++ for (const s of summaries) {++ const key = s.line.key || norm(s.name);++ if (!groups.has(key)) groups.set(key, []);++ groups.get(key).push(s);++ }++ const out = [];++ for (const [key, models] of groups) {++ const sorted = [...models].sort(compareVersions);++ if (sorted.length < 2) continue;++ const rows = sorted.map((m, i) => ({++ ...m,++ changes: i ? versionDiff(sorted[i - 1], m) : null,++ }));++ out.push({++ key,++ label: sorted[0].line.key || key,++ family: sorted[0].family,++ models: rows,++ latest: rows[rows.length - 1],++ measured_at: sorted.map((m) => m.measured_at).filter(Boolean).sort().pop() || null,++ });++ }++ return out.sort((a, b) => a.label.localeCompare(b.label));++}++++const SIDES = Object.freeze([++ { key: 'price_in', label: 'Price in, per 1M', better: 'lower', unit: 'USD', get: (m) => m.price && m.price.in },++ { key: 'price_out', label: 'Price out, per 1M', better: 'lower', unit: 'USD', get: (m) => m.price && m.price.out },++ { key: 'p50_ms', label: 'Typical response, p50', better: 'lower', unit: 'ms', get: (m) => m.speed && m.speed.p50_ms },++ { key: 'p95_ms', label: 'Slow response, p95', better: 'lower', unit: 'ms', get: (m) => m.speed && m.speed.p95_ms },++ { key: 'elo', label: 'Bench Elo', better: 'higher', unit: 'Elo', get: (m) => m.quality && m.quality.score },++ { key: 'games', label: 'Judged games', better: null, unit: 'games', get: (m) => m.quality && m.quality.games },++ { key: 'context', label: 'Context', better: 'higher', unit: 'tokens', get: (m) => m.context },++ { key: 'uptime', label: 'Uptime, 7 days', better: 'higher', unit: 'share', get: (m) => m.uptime },++ { key: 'requests_7d', label: 'Requests, 7 days', better: null, unit: 'requests', get: (m) => m.requests_7d },++ { key: 'offers', label: 'Live offers', better: null, unit: 'offers', get: (m) => m.offers_count },++]);++++function sideBySide(models) {++ const criteria = SIDES.map((side) => {++ const values = models.map((m) => {++ const value = side.get(m);++ return {++ slug: m.slug, name: m.name, value,++ display: value == null ? null : value,++ available: value != null,++ reason: value == null ? (side.key === 'elo' ? (m.quality && m.quality.reason) || 'No Bench evidence.' : 'Not measured.') : null,++ };++ });++ const present = values.filter((v) => v.available);++ let winner = null;++ if (side.better && present.length > 1) {++ const best = side.better === 'lower'++ ? Math.min(...present.map((v) => Number(v.value)))++ : Math.max(...present.map((v) => Number(v.value)));++ const atBest = present.filter((v) => Number(v.value) === best);++ if (atBest.length === 1) winner = atBest[0].slug;++ }++ return { key: side.key, label: side.label, better: side.better, unit: side.unit, values, winner };++ });++++ const byKey = Object.fromEntries(criteria.map((c) => [c.key, c]));++ const winnerNames = (key) => {++ const c = byKey[key];++ if (!c || !c.winner) return null;++ const m = models.find((x) => x.slug === c.winner);++ return m ? m.name : null;++ };++ const cheapest = winnerNames('price_out');++ const fastest = winnerNames('p50_ms');++ const best = winnerNames('elo');++ const priced = models.filter((m) => m.price && m.price.out != null);++ const priceSpread = priced.length > 1 ? (() => {++ const sorted = [...priced].sort((a, b) => a.price.out - b.price.out);++ const low = sorted[0];++ const high = sorted[sorted.length - 1];++ return {++ cheapest: { slug: low.slug, name: low.name, price: low.price },++ dearest: { slug: high.slug, name: high.name, price: high.price },++ difference: round(high.price.out - low.price.out, 4),++ percent: pct(high.price.out, low.price.out),++ };++ })() : null;++++ const qualityNote = (() => {++ const scored = models.filter((m) => m.quality && m.quality.available && m.quality.by_benchmark.length);++ if (scored.length < 2) return 'Fewer than two models have public Bench rounds, so quality is not compared.';++ const all = scored.flatMap((m) => m.quality.by_benchmark.map((r) => ({ ...r, slug: m.slug })));++ const overlap = all.some((a) => all.some((b) => a.slug !== b.slug && a.kind_id === b.kind_id++ && a.ci_lower != null && a.ci_upper != null && b.ci_lower != null && b.ci_upper != null++ && a.ci_lower <= b.ci_upper && b.ci_lower <= a.ci_upper));++ return overlap ? 'At least one Bench interval overlaps between models, so a quality difference is not clear on that board.' : null;++ })();++++ return {++ kind: KIND,++ models,++ criteria,++ verdict: {++ headline: [++ cheapest ? `${cheapest} is cheapest on the output price` : null,++ fastest ? `${fastest} answers fastest at p50` : null,++ best ? `${best} has the strongest measured Bench Elo` : null,++ ].filter(Boolean).join('; ') || 'Not enough measured criteria to call a winner.',++ cheapest, fastest, best,++ quality_note: qualityNote,++ caveat: 'A winner is per criterion, never one invented overall score. Read the price against the quality row for your task.',++ },++ price_spread: priceSpread,++ sources: null,++ method: null,++ };++}++++function createCompare({ s, ai, bench, config = {}, now = () => Date.now(), log = console } = {}) {++ const remarks = {++ async create({ models, comparison = null, field = null, remark, proposedPrompt = null, sourceUrl = null, subject = null }) {++ const id = s.newId('excr');++ const row = await s.db.maybe(++ `INSERT INTO exchange_compare_remarks++ (id, kind_id, models, comparison, field, remark, proposed_prompt, source_url, status, subject, created_at)++ VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7, $8, 'pending', $9, $10::timestamptz)++ RETURNING *`,++ [id, KIND, JSON.stringify(models), comparison, field, remark, proposedPrompt, sourceUrl, subject, s.iso()],++ );++ return wireRemark(row);++ },++ async list({ status = 'pending', limit = 50 } = {}) {++ const rows = await s.db.many(++ 'SELECT * FROM exchange_compare_remarks WHERE status = $1 ORDER BY created_at DESC, id DESC LIMIT $2',++ [status, Math.max(1, Math.min(200, Number(limit) || 50))],++ );++ return rows.map(wireRemark);++ },++ async get(id) {++ return wireRemark(await s.db.maybe('SELECT * FROM exchange_compare_remarks WHERE id = $1', [String(id || '')]));++ },++ async review({ id, decision, note = null, reviewer }) {++ const row = await s.db.maybe(++ `UPDATE exchange_compare_remarks++ SET status = $1, reviewer_note = $2, reviewed_by = $3, reviewed_at = $4::timestamptz++ WHERE id = $5 AND status = 'pending'++ RETURNING *`,++ [decision, note, reviewer, s.iso(), String(id || '')],++ );++ return wireRemark(row);++ },++ };++++ let qualityCache = null;++ const QUALITY_MS = 60_000;++ async function quality() {++ if (qualityCache && now() - qualityCache.atMs < QUALITY_MS) return qualityCache.value;++ const value = await readQuality(bench, now);++ qualityCache = { atMs: now(), value };++ return value;++ }++++ async function catalogAndQuality() {++ const [catalog, q] = await Promise.all([ai.catalog(), quality()]);++ return { catalog, quality: q };++ }++++ function sources(catalog, q) {++ return [++ {++ name: 'OpenVibe.AI catalog', service: 'OpenVibe.AI', url: `${config.ai && config.ai.url ? config.ai.url : 'https://ai.openvibe.services'}/api/v1/exchange/catalog`,++ read_at: iso(catalog && catalog.generated_at) || new Date(now()).toISOString(),++ fields: ['price', 'speed', 'uptime', 'requests', 'privacy', 'tier', 'context'],++ },++ {++ name: 'OpenVibe.Bench boards', service: 'OpenVibe.Bench', url: `${config.bench && config.bench.url ? config.bench.url : 'https://bench.openvibe.services'}/api/v1/boards`,++ read_at: q.at, fields: ['quality (Elo)', 'games', 'intervals'],++ unavailable: Boolean(q.unavailable),++ },++ ];++ }++++ const method = {++ name: 'Measured, dated and not estimated',++ price: 'The cheapest live offer’s input and output price per million tokens; every offer’s own price is in the price book.',++ speed: 'The fastest live offer’s measured p50 and p95 from real requests over the last seven days; not a tokens/s figure.',++ quality: 'OpenVibe.Bench Elo from public pairwise or graded rounds, weighted by judged games. A model with no judged games has no score, and the row says why. Intervals are per board.',++ privacy: 'The privacy label and the provider’s data policy exactly as AI’s catalog records them; “not stated” is not a guess.',++ tier: 'First-party (OpenVibe’s own capacity), community (a person’s machine), authorized wholesale, free, or third-party, from the offer’s own trust and access fields.',++ versions: 'Versions are the same model line (the name without its version numbers). Each row is compared with the one before it on price, speed, context and Bench Elo. An Elo move whose intervals overlap is not called a change.',++ unavailable: 'When AI or Bench cannot be read, the field says it is unavailable and shows no figure.',++ };++++ async function models({ q = '', family = '', kind = '', sort = 'requests', limit = 100, offset = 0 } = {}) {++ const { catalog, quality: qq } = await catalogAndQuality();++ let list = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));++ if (family) list = list.filter((m) => String(m.family || '').toLowerCase() === String(family).toLowerCase());++ if (kind) list = list.filter((m) => m.kind === String(kind).toLowerCase());++ if (q) {++ const needle = String(q).toLowerCase();++ list = list.filter((m) => `${m.name} ${m.slug} ${m.family || ''} ${m.providers.join(' ')}`.toLowerCase().includes(needle));++ }++ const order = {++ requests: (a, b) => (b.requests_7d || 0) - (a.requests_7d || 0) || a.name.localeCompare(b.name),++ cheap: (a, b) => (a.price.in == null ? Infinity : a.price.in) - (b.price.in == null ? Infinity : b.price.in) || a.name.localeCompare(b.name),++ price: (a, b) => (a.price.out == null ? Infinity : a.price.out) - (b.price.out == null ? Infinity : b.price.out) || a.name.localeCompare(b.name),++ fast: (a, b) => (a.speed.p50_ms == null ? Infinity : a.speed.p50_ms) - (b.speed.p50_ms == null ? Infinity : b.speed.p50_ms) || a.name.localeCompare(b.name),++ quality: (a, b) => (b.quality.score == null ? -Infinity : b.quality.score) - (a.quality.score == null ? -Infinity : a.quality.score) || a.name.localeCompare(b.name),++ name: (a, b) => a.name.localeCompare(b.name),++ };++ list.sort(order[sort] || order.requests);++ const total = list.length;++ const from = Math.max(0, Number(offset) || 0);++ const take = Math.max(1, Math.min(250, Number(limit) || 100));++ const page = list.slice(from, from + take);++ return {++ kind: KIND,++ generated_at: new Date(now()).toISOString(),++ catalog_generated_at: iso(catalog.generated_at),++ unavailable: Boolean(catalog.unavailable),++ count: page.length, total, offset: from, limit: take,++ facets: {++ families: [...new Set(list.map((m) => m.family).filter(Boolean))].sort(),++ kinds: [...new Set(list.map((m) => m.kind).filter(Boolean))].sort(),++ providers: [...new Set(list.flatMap((m) => m.providers))].sort(),++ },++ method,++ sources: sources(catalog, qq),++ models: page,++ };++ }++++ async function model(slug) {++ const { catalog, quality: qq } = await catalogAndQuality();++ const found = (catalog.models || []).find((m) => String(m.slug).toLowerCase() === String(slug || '').toLowerCase());++ if (!found) return null;++ const summary = summaryOf(found, catalog, qq);++ const offers = (found.offers || []).map((o) => offerOf(o, catalog));++ const all = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));++ const lineage = lineagesOf(all).find((l) => l.key === summary.line.key) || null;++ const alternatives = all++ .filter((m) => m.slug !== summary.slug && m.kind === summary.kind)++ .sort((a, b) => {++ const da = a.price && a.price.in != null && summary.price.in != null ? Math.abs(a.price.in - summary.price.in) : Infinity;++ const db = b.price && b.price.in != null && summary.price.in != null ? Math.abs(b.price.in - summary.price.in) : Infinity;++ return da - db;++ })++ .slice(0, 6)++ .map((m) => ({ slug: m.slug, name: m.name, price: m.price, speed: m.speed, quality: m.quality }));++ return {++ kind: KIND,++ generated_at: new Date(now()).toISOString(),++ model: summary,++ offers,++ quality: { ...summary.quality, by_benchmark: summary.quality.by_benchmark },++ versions: lineage,++ alternatives,++ method,++ sources: sources(catalog, qq),++ };++ }++++ async function compare(refs) {++ const { catalog, quality: qq } = await catalogAndQuality();++ const wanted = (Array.isArray(refs) ? refs : []).map((x) => String(x || '').trim()).filter(Boolean);++ if (wanted.length < 2 || wanted.length > 4) {++ return { error: 'compare.selection', message: 'Choose two to four models.', status: 400 };++ }++ const all = (catalog.models || []).filter(isModel);++ const found = [];++ for (const ref of wanted) {++ const m = all.find((x) => String(x.slug).toLowerCase() === ref.toLowerCase())++ || all.find((x) => String(x.name).toLowerCase() === ref.toLowerCase())++ || all.find((x) => String(x.slug).split('/').pop().toLowerCase() === ref.toLowerCase());++ if (!m) return { error: 'compare.model_not_found', message: `No such model: ${ref}`, status: 404 };++ if (!found.some((x) => x.slug === m.slug)) found.push(m);++ }++ if (found.length < 2) return { error: 'compare.selection', message: 'Choose two different models.', status: 400 };++ const summaries = found.map((m) => summaryOf(m, catalog, qq));++ const out = sideBySide(summaries);++ return {++ ...out,++ generated_at: new Date(now()).toISOString(),++ catalog_generated_at: iso(catalog.generated_at),++ method,++ sources: sources(catalog, qq),++ };++ }++++ async function versions(key = null) {++ const { catalog, quality: qq } = await catalogAndQuality();++ const all = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));++ const list = lineagesOf(all);++ return {++ kind: `${KIND}.versions`,++ generated_at: new Date(now()).toISOString(),++ count: list.length,++ lineages: key ? list.filter((l) => l.key === String(key)) : list,++ method,++ sources: sources(catalog, qq),++ };++ }++++ return { kind: KIND, methods: { models, model, compare, versions, quality, remarks }, method, qualityKinds: QUALITY_KINDS };++}++++function wireRemark(row) {++ if (!row) return null;++ return {++ id: row.id,++ kind: row.kind_id,++ models: row.models || [],++ comparison: row.comparison || null,++ field: row.field || null,++ remark: row.remark,++ proposed_prompt: row.proposed_prompt || null,++ source_url: row.source_url || null,++ status: row.status,++ subject: row.subject || null,++ reviewer_note: row.reviewer_note || null,++ reviewed_by: row.reviewed_by || null,++ reviewed_at: row.reviewed_at || null,++ created_at: row.created_at,++ };++}++++module.exports = { createCompare, KIND, QUALITY_KINDS, splitVersion, versionDiff, lineagesOf };@@@ -134,6 +134,13 @@ function load(env = process.env) {maxChars: Math.max(200, int(env.EXCHANGE_CHAT_MAX_CHARS, 8000)),},++ // openvibe.compare (plan 297): the comparison site is served by this service behind its own name. The++ // host is what makes a request the compare site's; the API is also reachable on openvibe.exchange.++ compare: {++ host: String(env.COMPARE_HOST || 'openvibe.compare').toLowerCase(),++ baseUrl: trim(env.COMPARE_BASE_URL || 'https://openvibe.compare'),++ },++// IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl a page that appeared or// changed. Unset → off: nothing mounted, nothing sent.indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },nnew file mode 100644@@@ -0,0 +1,205 @@++'use strict';++++/**++ * /api/v1/compare — openvibe.compare's JSON (plan 297/300/301). Mounted before Exchange v1's 410.++ *++ * GET /api/v1/compare the kinds, the method and the links++ * GET /api/v1/compare/ai/models?q=&family=&kind=&sort=&offset=&limit=++ * GET /api/v1/compare/ai/models/:org/:model one model, its price book, quality and versions++ * GET /api/v1/compare/ai?models=a,b two to four models, side by side++ * GET /api/v1/compare/ai/versions every model line with two or more versions++ * GET /api/v1/compare/ai/versions/:key one line, each version against the one before it++ * POST /api/v1/compare/remarks a reader's remark becomes a pending benchmark case++ * GET /api/v1/compare/remarks?status=pending the review queue (owner only)++ * PATCH /api/v1/compare/remarks/:id accept or reject a pending case (owner only)++ *++ * Everything public is a read of OpenVibe.AI and OpenVibe.Bench, joined in server/compare.js. The remark is the++ * only write, and it is stored in Exchange's own database until a reviewer turns it into a Bench task.++ */++const express = require('express');++const rateLimit = require('express-rate-limit');++const { http } = require('openvibe-contracts');++const { asyncRouter } = require('./router');++const { createCompare, KIND } = require('../compare');++++const FIELDS = new Set(['price', 'speed', 'quality', 'privacy', 'context', 'other']);++++function createCompareApi(ctx) {++ const { config, ai, bench, s, log } = ctx;++ const engine = ctx.compare || createCompare({ s, ai, bench, config, now: s.now, log });++ const r = asyncRouter();++++ r.use(express.json({ limit: '32kb' }));++++ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });++ const failed = (res, err) => {++ log.warn('[OpenVibe.Exchange] Compare failed:', String((err && err.message) || err).slice(0, 200));++ return problem(res, 500, 'compare.failed', 'Compare could not answer. Try again in a moment.');++ };++ const owner = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.role === 'owner';++ const requireOwner = (req, res) => {++ if (!req.viewer || req.viewer.kind !== 'user') {++ problem(res, 401, 'compare.sign_in_required', 'Sign in as the owner to review benchmark cases.');++ return false;++ }++ if (!owner(req)) {++ problem(res, 403, 'compare.owner_required', 'Only the owner can review benchmark cases.');++ return false;++ }++ return true;++ };++ const clean = (v, max) => String(v == null ? '' : v).replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, max);++ const cache = (res, seconds = 60) => res.set('Cache-Control', `public, max-age=${seconds}, stale-while-revalidate=300`);++++ r.get('/', async (_req, res) => {++ cache(res, 300);++ return res.json({++ kind: 'compare',++ title: 'Compare anything, starting with AI models',++ generated_at: new Date(s.now()).toISOString(),++ kinds: [{++ id: KIND,++ title: 'AI models',++ what: 'Models with the price of every provider that serves them, measured speed and latency, Bench quality, privacy and tier.',++ endpoints: {++ models: '/api/v1/compare/ai/models',++ model: '/api/v1/compare/ai/models/:org/:model',++ side_by_side: '/api/v1/compare/ai?models=a,b',++ versions: '/api/v1/compare/ai/versions',++ remarks: '/api/v1/compare/remarks',++ },++ }],++ method: engine.method,++ });++ });++++ r.get('/ai/models', async (req, res) => {++ try {++ const out = await engine.methods.models({++ q: req.query.q, family: req.query.family, kind: req.query.kind, sort: req.query.sort,++ limit: req.query.limit, offset: req.query.offset,++ });++ cache(res, 60);++ return res.json(out);++ } catch (err) { return failed(res, err); }++ });++++ r.get('/ai/models/:org/:model', async (req, res) => {++ try {++ const slug = `${req.params.org}/${req.params.model}`;++ const out = await engine.methods.model(slug);++ if (!out) return problem(res, 404, 'compare.model_not_found', `No such model: ${slug}`);++ cache(res, 60);++ return res.json(out);++ } catch (err) { return failed(res, err); }++ });++++ r.get('/ai', async (req, res) => {++ try {++ const refs = String(req.query.models || '').split(',').map((x) => x.trim()).filter(Boolean);++ const out = await engine.methods.compare(refs);++ if (out.error) return problem(res, out.status || 400, out.error, out.message);++ cache(res, 30);++ return res.json(out);++ } catch (err) { return failed(res, err); }++ });++++ r.get('/ai/versions', async (req, res) => {++ try {++ const out = await engine.methods.versions();++ cache(res, 120);++ return res.json(out);++ } catch (err) { return failed(res, err); }++ });++++ r.get('/ai/versions/:key', async (req, res) => {++ try {++ const out = await engine.methods.versions(req.params.key);++ if (!out.lineages.length) return problem(res, 404, 'compare.lineage_not_found', `No model line with versions: ${req.params.key}`);++ out.count = out.lineages.length;++ cache(res, 120);++ return res.json(out);++ } catch (err) { return failed(res, err); }++ });++++ // ── The reader's remark (plan 300) ──────────────────────++ // Public intake: one plain-text remark, no account needed, rate-limited. It becomes a pending case for review;++ // it is never run, published or treated as a benchmark result before a person accepts it.++ r.post('/remarks', rateLimit({ windowMs: 60_000, limit: 20, standardHeaders: true, legacyHeaders: false }), async (req, res) => {++ try {++ const b = req.body && typeof req.body === 'object' ? req.body : {};++ const catalog = await ai.catalog();++ if (catalog.unavailable) return problem(res, 503, 'compare.catalog_unavailable', 'The model catalog is not reachable right now, so a remark cannot name a real model. Try again shortly.');++ const slugs = (Array.isArray(b.models) ? b.models : String(b.models || '').split(','))++ .map((x) => clean(x, 160)).filter(Boolean);++ const wanted = [...new Set(slugs)].slice(0, 4);++ if (!wanted.length) return problem(res, 400, 'compare.models_required', 'Name at least one model the remark is about.');++ const bySlug = new Map((catalog.models || []).map((m) => [String(m.slug).toLowerCase(), m]));++ const unknown = wanted.filter((x) => !bySlug.has(x.toLowerCase()));++ if (unknown.length) return problem(res, 400, 'compare.model_not_found', `No such model: ${unknown.join(', ')}`);++ const remark = clean(b.remark, 2000);++ if (remark.length < 10) return problem(res, 400, 'compare.remark_too_short', 'A remark needs at least ten characters.');++ const field = b.field == null || b.field === '' ? null : String(b.field).toLowerCase();++ if (field && !FIELDS.has(field)) return problem(res, 400, 'compare.field_invalid', `field must be one of: ${[...FIELDS].join(', ')}`);++ const comparison = b.comparison == null ? null : clean(b.comparison, 200) || null;++ let sourceUrl = null;++ if (b.source_url) {++ try {++ const u = new URL(String(b.source_url));++ if (!['http:', 'https:'].includes(u.protocol)) throw new Error('scheme');++ sourceUrl = u.toString().slice(0, 500);++ } catch (e) { return problem(res, 400, 'compare.source_url_invalid', 'source_url must be an http or https address.'); }++ }++ const names = wanted.map((x) => bySlug.get(x.toLowerCase()).name).join(', ');++ const proposed = clean(b.proposed_prompt, 4000) || clean(`Test ${names}: ${remark}`, 4000);++ const remarkRow = await engine.methods.remarks.create({++ models: wanted.map((x) => bySlug.get(x.toLowerCase()).slug),++ comparison, field, remark, proposedPrompt: proposed, sourceUrl,++ subject: req.viewer && req.viewer.kind === 'user' ? req.viewer.subject : null,++ });++ res.set('Location', `/api/v1/compare/remarks/${remarkRow.id}`);++ res.set('Cache-Control', 'no-store');++ return res.status(201).json({ remark: remarkRow, next: 'A reviewer reads it and turns it into a Bench task; nothing runs or publishes before that.' });++ } catch (err) { return failed(res, err); }++ });++++ r.get('/remarks', async (req, res) => {++ if (!requireOwner(req, res)) return;++ try {++ const status = ['pending', 'accepted', 'rejected'].includes(String(req.query.status)) ? String(req.query.status) : 'pending';++ const items = await engine.methods.remarks.list({ status, limit: req.query.limit });++ res.set('Cache-Control', 'private, no-store');++ return res.json({ status, count: items.length, remarks: items });++ } catch (err) { return failed(res, err); }++ });++++ r.get('/remarks/:id', async (req, res) => {++ if (!requireOwner(req, res)) return;++ try {++ const row = await engine.methods.remarks.get(req.params.id);++ if (!row) return problem(res, 404, 'compare.remark_not_found', 'No such remark.');++ res.set('Cache-Control', 'private, no-store');++ return res.json({ remark: row });++ } catch (err) { return failed(res, err); }++ });++++ r.patch('/remarks/:id', async (req, res) => {++ if (!requireOwner(req, res)) return;++ try {++ const b = req.body && typeof req.body === 'object' ? req.body : {};++ const decision = b.decision === 'accepted' || b.decision === 'rejected' ? b.decision : null;++ if (!decision) return problem(res, 400, 'compare.decision_invalid', 'decision must be accepted or rejected.');++ const row = await engine.methods.remarks.review({++ id: req.params.id, decision, note: clean(b.note, 1000) || null,++ reviewer: req.viewer.subject || req.viewer.username || 'owner',++ });++ if (!row) return problem(res, 404, 'compare.remark_not_found', 'No pending remark with that id.');++ res.set('Cache-Control', 'private, no-store');++ return res.json({ remark: row });++ } catch (err) { return failed(res, err); }++ });++++ return r;++}++++module.exports = { createCompareApi };@@@ -29,6 +29,9 @@ const TABLES = [// goes first, then its round (the vote also cascades from the round).{ table: 'exchange_arena_votes', subject: 'owner', value: (usr) => usr, file: 'arena-votes.json', erase: 'delete' },{ table: 'exchange_arena_rounds', subject: 'owner', value: (usr) => usr, file: 'arena-rounds.json', erase: 'delete' },++ // A reader's remark on a comparison (openvibe.compare, plan 300): a signed-in person's pending benchmark++ // case is theirs, exported and erased with the rest. An anonymous remark has no subject and is not named.++ { table: 'exchange_compare_remarks', subject: 'subject', value: (usr) => usr, file: 'compare-remarks.json', erase: 'delete' },// The companions (2026-11-06): a person's characters, their chats and the messages in them are theirs,// exactly like the chats above. The messages go first, then the chats, then the characters: a character// may be deleted while somebody else's chat still points at it, and that chat keeps its messages.nnew file mode 100644@@@ -0,0 +1,228 @@++'use strict';++/**++ * openvibe.compare's API (plan 296/297/300/301).++ *++ * The comparison is a join of two reads: AI's catalogue (price, speed, privacy, tier) and Bench's boards++ * (quality). These tests hold both stand-ins fixed and check the shape, the dates, the version deltas, the++ * two-to-four side-by-side, and the remark that becomes a pending case and nothing more until the owner reviews it.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { createFakeAi } = require('./helpers/ai');++const { createFakeBench } = require('./helpers/bench');++++(async () => {++ const ai = createFakeAi();++ // A fourth and a fifth model on one line, so versions have something to compare.++ ai.catalogData.models.push(++ {++ slug: 'openai/gpt-5-mini', canonical: 'openai/gpt-5-mini', name: 'gpt-5-mini', family: 'GPT', variant: null,++ uncensored: false, kind: 'chat', context: 400000, max_output: 128000, open_weights: false,++ capabilities: { tools: true, json: true, vision: true, stream: true }, modalities: ['text', 'image'], description: null,++ requests_7d: 50, requests_today: 4, community: false,++ offers: [{++ provider: 'openai', provider_name: 'OpenAI', model: 'gpt-5-mini', id: 'openai/gpt-5-mini@v1', trust: 'first-party',++ access: 'OWN_DEPLOYMENT', free: false, free_tier: false, status: 'up',++ price: { in: 0.25, out: 2, cached: 0.025, per: '1M tokens', currency: 'USD' },++ speed: { p50_ms: 900, p95_ms: 3000 }, uptime: 0.99, requests_7d: 50, requests_today: 4,++ privacy: { label: 'anonymized', retention: '0', training: false, region: 'us', known: true },++ updated_at: '2026-11-06T00:00:00.000Z',++ }],++ },++ {++ slug: 'openai/gpt-5.1-mini', canonical: 'openai/gpt-5.1-mini', name: 'gpt-5.1-mini', family: 'GPT', variant: null,++ uncensored: false, kind: 'chat', context: 400000, max_output: 128000, open_weights: false,++ capabilities: { tools: true, json: true, vision: true, stream: true }, modalities: ['text', 'image'], description: null,++ requests_7d: 80, requests_today: 6, community: false,++ offers: [{++ provider: 'openai', provider_name: 'OpenAI', model: 'gpt-5.1-mini', id: 'openai/gpt-5.1-mini@v1', trust: 'first-party',++ access: 'OWN_DEPLOYMENT', free: false, free_tier: false, status: 'up',++ price: { in: 0.2, out: 1.5, cached: 0.02, per: '1M tokens', currency: 'USD' },++ speed: { p50_ms: 700, p95_ms: 2400 }, uptime: 0.995, requests_7d: 80, requests_today: 6,++ privacy: { label: 'anonymized', retention: '0', training: false, region: 'us', known: true },++ updated_at: '2026-11-06T00:00:00.000Z',++ }],++ },++ );++ ai.catalogData.count = ai.catalogData.models.length;++ // AI's offer carries its own privacy label and the tier fields; the page must not invent one.++ ai.catalogData.models[0].offers[0].privacy = { label: 'anonymized', retention: '0', training: false, region: 'cn', known: true };++ ai.catalogData.models[0].offers[0].on_our_gpus = false;++ ai.catalogData.models[0].offers[1].on_our_gpus = false;++++ const bench = createFakeBench({++ boards: {++ 'chat.arena': [++ { subject: 'deepseek/deepseek-chat', elo: 1040, games: 20, wins: 12, losses: 6, ties: 2, win_rate: 0.65, ci_lower: 1020, ci_upper: 1060, provisional: false },++ { subject: 'qwen/qwen3-32b', elo: 990, games: 12, wins: 5, losses: 6, ties: 1, win_rate: 0.46, ci_lower: 970, ci_upper: 1010, provisional: false },++ { subject: 'openai/gpt-5-mini', elo: 1100, games: 30, wins: 20, losses: 8, ties: 2, win_rate: 0.7, ci_lower: 1080, ci_upper: 1120, provisional: false },++ { subject: 'openai/gpt-5.1-mini', elo: 1120, games: 30, wins: 22, losses: 6, ties: 2, win_rate: 0.77, ci_lower: 1125, ci_upper: 1145, provisional: false },++ ],++ },++ });++++ const t = await boot({ ai, bench });++ const owner = t.network.addUser('owner', { role: 'owner' });++ const reader = t.network.addUser('reader');++++ try {++ await check('the index names the kind, the sources and the method', async () => {++ const r = await t.get('/api/v1/compare');++ assert.strictEqual(r.status, 200, r.text);++ const body = r.json();++ assert.strictEqual(body.kind, 'compare');++ assert.strictEqual(body.kinds[0].id, 'ai.models');++ assert.ok(body.method.price && body.method.quality && body.method.versions);++ });++++ await check('the models list is priced per provider, dated, and honest about missing quality', async () => {++ const r = await t.get('/api/v1/compare/ai/models');++ assert.strictEqual(r.status, 200, r.text);++ const body = r.json();++ assert.strictEqual(body.kind, 'ai.models');++ assert.strictEqual(body.total, 5);++ assert.deepStrictEqual(body.sources.map((s) => s.service), ['OpenVibe.AI', 'OpenVibe.Bench']);++ const deep = body.models.find((m) => m.slug === 'deepseek/deepseek-chat');++ assert.ok(deep, 'DeepSeek is listed');++ assert.strictEqual(deep.price.in, 0.2, 'the cheapest offer wins the headline');++ assert.strictEqual(deep.price.out, 0.3);++ assert.strictEqual(deep.price.provider, 'DeepInfra');++ assert.strictEqual(deep.speed.p50_ms, 400);++ assert.strictEqual(deep.quality.available, true);++ assert.strictEqual(deep.quality.score, 1040);++ assert.strictEqual(deep.quality.games, 20);++ assert.strictEqual(deep.measured_at, '2026-11-06T09:00:00.000Z', 'the AI read carries its date');++ assert.ok(deep.privacy.label, 'a privacy label is read, not invented');++ assert.strictEqual(deep.tiers['first-party'], 2);++ const llama = body.models.find((m) => m.slug === 'meta-llama/llama-3-8b-abliterated');++ assert.strictEqual(llama.quality.available, false);++ assert.match(llama.quality.reason, /No public Bench rounds/);++ assert.strictEqual(llama.quality.score, null, 'no invented score');++ });++++ await check('one model carries its whole price book, quality by board, and its line', async () => {++ const r = await t.get('/api/v1/compare/ai/models/deepseek/deepseek-chat');++ assert.strictEqual(r.status, 200, r.text);++ const body = r.json();++ assert.strictEqual(body.model.slug, 'deepseek/deepseek-chat');++ assert.strictEqual(body.offers.length, 2);++ assert.deepStrictEqual(body.offers.map((o) => o.price.in).sort((a, b) => a - b), [0.2, 0.28]);++ assert.strictEqual(body.offers.find((o) => o.provider === 'deepseek').tier, 'first-party');++ assert.strictEqual(body.quality.by_benchmark.length, 1);++ assert.strictEqual(body.versions, null, 'a model with one version has no version table');++ assert.ok(body.alternatives.length >= 1);++ const missing = await t.get('/api/v1/compare/ai/models/nope/nothing');++ assert.strictEqual(missing.status, 404);++ assert.strictEqual(missing.json().code, 'compare.model_not_found');++ });++++ await check('two to four models are compared on every criterion, with a per-criterion winner', async () => {++ const r = await t.get('/api/v1/compare/ai?models=openai/gpt-5.1-mini,deepseek/deepseek-chat,qwen/qwen3-32b');++ assert.strictEqual(r.status, 200, r.text);++ const body = r.json();++ assert.strictEqual(body.models.length, 3);++ assert.deepStrictEqual(body.models.map((m) => m.slug), ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat', 'qwen/qwen3-32b']);++ const price = body.criteria.find((c) => c.key === 'price_out');++ assert.strictEqual(price.winner, 'deepseek/deepseek-chat', '0.3 output is the cheapest');++ const speed = body.criteria.find((c) => c.key === 'p50_ms');++ assert.strictEqual(speed.winner, 'deepseek/deepseek-chat');++ const quality = body.criteria.find((c) => c.key === 'elo');++ assert.strictEqual(quality.winner, 'openai/gpt-5.1-mini');++ assert.ok(body.price_spread, 'the price spread is said');++ assert.ok(body.verdict.headline.includes('cheapest'));++ assert.ok(body.verdict.caveat.includes('never one invented overall score'));++++ const one = await t.get('/api/v1/compare/ai?models=deepseek/deepseek-chat');++ assert.strictEqual(one.status, 400);++ assert.strictEqual(one.json().code, 'compare.selection');++ const five = await t.get('/api/v1/compare/ai?models=a,b,c,d,e');++ assert.strictEqual(five.status, 400);++ const none = await t.get('/api/v1/compare/ai?models=deepseek/deepseek-chat,nope/nothing');++ assert.strictEqual(none.status, 404);++ });++++ await check('versions of one line are ordered and each says what changed, with the interval caveat', async () => {++ const r = await t.get('/api/v1/compare/ai/versions');++ assert.strictEqual(r.status, 200, r.text);++ const body = r.json();++ const line = body.lineages.find((l) => l.key === 'gpt-mini');++ assert.ok(line, `gpt-mini line in ${body.lineages.map((l) => l.key).join(',')}`);++ assert.deepStrictEqual(line.models.map((m) => m.name), ['gpt-5-mini', 'gpt-5.1-mini']);++ const newer = line.models[1];++ const priceIn = newer.changes.changes.find((c) => c.key === 'price_in');++ assert.strictEqual(priceIn.from, 0.25);++ assert.strictEqual(priceIn.to, 0.2);++ assert.strictEqual(priceIn.direction, 'better');++ const speed = newer.changes.changes.find((c) => c.key === 'p50_ms');++ assert.strictEqual(speed.direction, 'better');++ const elo = newer.changes.changes.find((c) => c.key === 'elo');++ assert.strictEqual(elo.direction, 'better');++ assert.strictEqual(newer.changes.quality.meaningful, true);++ assert.match(newer.changes.summary, /better/);++++ const one = await t.get('/api/v1/compare/ai/versions/gpt-mini');++ assert.strictEqual(one.status, 200);++ assert.strictEqual(one.json().lineages.length, 1);++ const missing = await t.get('/api/v1/compare/ai/versions/nope');++ assert.strictEqual(missing.status, 404);++ });++++ let remarkId;++ await check('a reader’s remark becomes a pending benchmark case, not a result', async () => {++ const r = await t.get('/api/v1/compare/remarks', {++ json: {++ models: ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat'],++ comparison: 'openai/gpt-5.1-mini-vs-deepseek/deepseek-chat',++ field: 'quality',++ remark: 'On a long refactor the newer mini kept the whole file in view where the other dropped the last function.',++ },++ });++ assert.strictEqual(r.status, 201, r.text);++ const body = r.json();++ remarkId = body.remark.id;++ assert.match(remarkId, /^excr_/);++ assert.strictEqual(body.remark.status, 'pending');++ assert.deepStrictEqual(body.remark.models, ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat']);++ assert.strictEqual(body.remark.subject, null, 'an anonymous remark stores no person');++ assert.match(body.remark.proposed_prompt, /^Test gpt-5\.1-mini, DeepSeek Chat:/);++ assert.match(body.next, /nothing runs or publishes/);++++ const short = await t.get('/api/v1/compare/remarks', { json: { models: ['deepseek/deepseek-chat'], remark: 'too short' } });++ assert.strictEqual(short.status, 400);++ const unknown = await t.get('/api/v1/compare/remarks', { json: { models: ['nope/nothing'], remark: 'A remark of enough length.' } });++ assert.strictEqual(unknown.status, 400);++ const badField = await t.get('/api/v1/compare/remarks', { json: { models: ['deepseek/deepseek-chat'], field: 'vibes', remark: 'A remark of enough length.' } });++ assert.strictEqual(badField.status, 400);++ });++++ await check('the pending queue is the owner’s, and reviewing it is the only thing that moves a case', async () => {++ const anon = await t.get('/api/v1/compare/remarks');++ assert.strictEqual(anon.status, 401);++ const normal = await t.get('/api/v1/compare/remarks', { as: reader });++ assert.strictEqual(normal.status, 403);++ const pending = await t.get('/api/v1/compare/remarks', { as: owner });++ assert.strictEqual(pending.status, 200, pending.text);++ assert.strictEqual(pending.json().count, 1);++ assert.strictEqual(pending.json().remarks[0].id, remarkId);++++ const accepted = await t.get(`/api/v1/compare/remarks/${remarkId}`, { as: owner, method: 'PATCH', json: { decision: 'accepted', note: 'A real case: hold the file at 8k tokens.' } });++ assert.strictEqual(accepted.status, 200, accepted.text);++ assert.strictEqual(accepted.json().remark.status, 'accepted');++ assert.strictEqual(accepted.json().remark.reviewer_note, 'A real case: hold the file at 8k tokens.');++ const again = await t.get(`/api/v1/compare/remarks/${remarkId}`, { as: owner, method: 'PATCH', json: { decision: 'rejected' } });++ assert.strictEqual(again.status, 404, 'only a pending case can be reviewed');++ const noneLeft = await t.get('/api/v1/compare/remarks', { as: owner });++ assert.strictEqual(noneLeft.json().count, 0);++ });++++ await check('Exchange v1 still answers its 410 elsewhere, so the compare API is not a hole', async () => {++ const r = await t.get('/api/v1/listings');++ assert.strictEqual(r.status, 410);++ const posted = await t.get('/api/v1/compare/remarks', { json: { models: [], remark: 'A remark long enough to pass.' } });++ assert.strictEqual(posted.status, 400);++ });++ } finally {++ await t.close();++ }++ done();++})();@@@ -3,7 +3,7 @@* A stand-in OpenVibe.Bench client for the Arena tests: it records every round, vote and board read, and answers* in the shape of server/bench.js. A test may pass its own `createRound`, `vote` or `board` to make one fail.*/--function createFakeBench({ createRound = null, vote = null, board = null } = {}) {++function createFakeBench({ createRound = null, vote = null, board = null, boards = null } = {}) {const calls = { rounds: [], votes: [], boards: [] };let n = 0;return {@@@ -30,7 +30,8 @@ function createFakeBench({ createRound = null, vote = null, board = null } = {})async board(args) {calls.boards.push(args);if (board) return board(args);-- return { unavailable: false, kind_id: 'chat.arena', category: args.category, rows: [] };++ const rows = boards && Object.prototype.hasOwnProperty.call(boards, args.kind) ? boards[args.kind] : null;++ return { unavailable: false, kind_id: args.kind || 'chat.arena', category: args.category, rows: rows || [] };},};}
@@ -71,6 +71,11 @@ EXCHANGE_EVENTS_URL=
@@ -71,6 +71,11 @@ EXCHANGE_EVENTS_URL=
# optional: defaults to true in production
# optional: defaults to true in production
# COOKIE_SECURE=true
# COOKIE_SECURE=true
+# openvibe.compare: the comparison site is served by this same service behind its own name (its vhost
+# deploy/nginx/openvibe.compare.conf proxies to this process). The host makes a request the compare site's.
+#COMPARE_HOST=openvibe.compare
+#COMPARE_BASE_URL=https://openvibe.compare
+
# IndexNow (openvibe-shared/indexnow): a key lets a new page or a changed sitemap be submitted for recrawl.
# IndexNow (openvibe-shared/indexnow): a key lets a new page or a changed sitemap be submitted for recrawl.
# Unset → off; nothing is mounted and nothing is sent. The key file is served at /<key>.txt.
# Unset → off; nothing is mounted and nothing is sent. The key file is served at /<key>.txt.
#INDEXNOW_KEY=
#INDEXNOW_KEY=
new file mode 100644
new file mode 100644
@@ -0,0 +1,27 @@
@@ -0,0 +1,27 @@
+-- phase: expand
+-- openvibe.compare (plan 297/300/301): a reader's remark on a comparison becomes a pending benchmark case for
+-- review. The remark is plain text and public metadata only: the person is stored as their Network subject when
+-- they are signed in, and never as an address or a user agent. A case is reviewed by the owner; nothing here is
+-- a credential and nothing is ever written to OpenVibe.Bench by this migration.
+
+CREATE TABLE IF NOT EXISTS exchange_compare_remarks (
+ id text COLLATE "C" PRIMARY KEY, -- excr_ + 26 Crockford base32 characters (ULID)
+ kind_id text COLLATE "C" NOT NULL DEFAULT 'ai.models',
+ models jsonb NOT NULL DEFAULT '[]'::jsonb, -- the model slugs the remark is about
+ comparison text, -- the pair/comparison key, when the remark is on one
+ field text, -- price | speed | quality | privacy | other
+ remark text NOT NULL,
+ proposed_prompt text, -- the benchmark prompt the reviewer would run
+ source_url text,
+ status text COLLATE "C" NOT NULL DEFAULT 'pending',
+ subject text, -- the signed-in person's usr_…, or null
+ reviewer_note text,
+ reviewed_by text,
+ reviewed_at timestamptz,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ CONSTRAINT exchange_compare_remarks_status CHECK (status IN ('pending', 'accepted', 'rejected')),
+ CONSTRAINT exchange_compare_remarks_field CHECK (field IS NULL OR field IN ('price', 'speed', 'quality', 'privacy', 'context', 'other')),
+ CONSTRAINT exchange_compare_remarks_remark CHECK (length(remark) BETWEEN 10 AND 2000)
+);
+CREATE INDEX IF NOT EXISTS exchange_compare_remarks_pending ON exchange_compare_remarks (status, created_at);
+CREATE INDEX IF NOT EXISTS exchange_compare_remarks_models ON exchange_compare_remarks USING gin (models);
@@ -30,6 +30,7 @@ const { createPrincipal } = require('./http/principal');
@@ -30,6 +30,7 @@ const { createPrincipal } = require('./http/principal');
const { createAi } = require('./ai');
const { createAi } = require('./ai');
const { createNetwork } = require('./network');
const { createNetwork } = require('./network');
const { createBench } = require('./bench');
const { createBench } = require('./bench');
+const { createCompare } = require('./compare');
const { createOpenApi } = require('./openapi');
const { createOpenApi } = require('./openapi');
const { createArenaStore } = require('./arena/store');
const { createArenaStore } = require('./arena/store');
const { createCatalogHistory } = require('./catalog-history');
const { createCatalogHistory } = require('./catalog-history');
@@ -46,6 +47,7 @@ const { createNoteTaker } = require('./companions/taking-notes');
@@ -46,6 +47,7 @@ const { createNoteTaker } = require('./companions/taking-notes');
const { createPageRoutes } = require('./http/pages');
const { createPageRoutes } = require('./http/pages');
const { createConsoleRoutes } = require('./http/console');
const { createConsoleRoutes } = require('./http/console');
const { createArenaRoutes } = require('./http/arena');
const { createArenaRoutes } = require('./http/arena');
+const { createCompareApi } = require('./http/compare');
const { createCompanionRoutes } = require('./http/companions');
const { createCompanionRoutes } = require('./http/companions');
const { createServiceReadiness } = require('./observability');
const { createServiceReadiness } = require('./observability');
const { assetVersion, send } = require('./render/layout');
const { assetVersion, send } = require('./render/layout');
@@ -84,6 +86,8 @@ async function createApp(opts = {}) {
@@ -84,6 +86,8 @@ async function createApp(opts = {}) {
// Network, read and changed with the person's own token: their machines and their machine pools.
// Network, read and changed with the person's own token: their machines and their machine pools.
const network = opts.network || createNetwork({ config, fetchImpl, log });
const network = opts.network || createNetwork({ config, fetchImpl, log });
const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });
const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });
+ // openvibe.compare's engine (server/compare.js): the join of AI's price/speed/privacy/tier with Bench's quality.
+ const compare = createCompare({ s, ai, bench, config, now: s.now, log });
const analysis = opts.analysis || require('./analysis').createAnalysis({ config, ai, bench, fetchImpl: opts.aiFetchImpl || globalThis.fetch, now: s.now });
const analysis = opts.analysis || require('./analysis').createAnalysis({ config, ai, bench, fetchImpl: opts.aiFetchImpl || globalThis.fetch, now: s.now });
const arenaStore = createArenaStore(s);
const arenaStore = createArenaStore(s);
const history = createCatalogHistory({ s, log });
const history = createCatalogHistory({ s, log });
@@ -116,7 +120,7 @@ async function createApp(opts = {}) {
@@ -116,7 +120,7 @@ async function createApp(opts = {}) {
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });
const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });
const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });
- const ctx = { config, s, keys, sso, principal, ai, network, bench, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };
+ const ctx = { config, s, keys, sso, principal, ai, network, bench, compare, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
@@ -255,6 +259,10 @@ async function createApp(opts = {}) {
@@ -255,6 +259,10 @@ async function createApp(opts = {}) {
app.use('/api/v1/analysis', rateLimit({ windowMs: 60000, limit: Number(process.env.EXCHANGE_API_RATE_LIMIT_PER_MIN) || 120, standardHeaders: true, legacyHeaders: false }), require('./http/analysis').createAnalysisRoutes(ctx));
app.use('/api/v1/analysis', rateLimit({ windowMs: 60000, limit: Number(process.env.EXCHANGE_API_RATE_LIMIT_PER_MIN) || 120, standardHeaders: true, legacyHeaders: false }), require('./http/analysis').createAnalysisRoutes(ctx));
+ // ── Compare's JSON API (openvibe.compare) ───────────────
+ // Mounted before Exchange v1's catch-all, which answers 410 to everything else under /api/v1.
+ app.use('/api/v1/compare', createCompareApi(ctx));
+
// ── Exchange v1's JSON API: gone ────────────────────────
// ── Exchange v1's JSON API: gone ────────────────────────
app.use('/api/v1', (req, res) => {
app.use('/api/v1', (req, res) => {
res.set('Cache-Control', 'public, max-age=86400');
res.set('Cache-Control', 'public, max-age=86400');
new file mode 100644
new file mode 100644
@@ -0,0 +1,682 @@
@@ -0,0 +1,682 @@
+'use strict';
+
+/**
+ * Compare — the comparison engine behind openvibe.compare (plan 296, 297, 300, 301).
+ *
+ * Why Exchange: Exchange already reads OpenVibe.AI's public catalogue (price, speed, privacy, tier) and
+ * OpenVibe.Bench's boards (quality), and it is the service that serves the public site. Bench owns the runs but
+ * has no prices; a separate service would only re-fetch the same two reads and add a deployment. So the join is
+ * made here, and every field says which service it came from and when it was read.
+ *
+ * A comparison is data (plan 297): things (models), criteria (price, speed, quality, privacy, tier), sources, a
+ * method, and results with dates. Nothing is estimated: a model with no public Bench rounds has no quality score,
+ * and the row says why instead of showing a made-up number.
+ */
+
+const QUALITY_KINDS = Object.freeze([
+ { id: 'chat.arena', name: 'Exchange arena' },
+ { id: 'chat.reply', name: 'Chat replies' },
+ { id: 'chat.willing', name: 'Willingness to answer (18+)' },
+ { id: 'vision.answer', name: 'Vision answers' },
+ { id: 'codes.patch', name: 'Code patches' },
+ { id: 'codes.review', name: 'Code diff review' },
+ { id: 'translate.ja', name: 'Japanese translation' },
+ { id: 'search.answer', name: 'Cited answers' },
+]);
+
+const NOT_CHAT = new Set(['embedding', 'embeddings', 'stt', 'asr', 'tts', 'speech', 'image', 'images', 'rerank', 'moderation']);
+const KIND = 'ai.models';
+
+const finite = (v) => (Number.isFinite(Number(v)) ? Number(v) : null);
+const iso = (v, fallback = null) => {
+ if (!v) return fallback;
+ const d = new Date(v);
+ return Number.isNaN(d.getTime()) ? fallback : d.toISOString();
+};
+const round = (v, places = 4) => (v == null ? null : Math.round(Number(v) * 10 ** places) / 10 ** places);
+const pct = (a, b) => (a == null || b == null || Number(b) === 0 ? null : round(((Number(a) - Number(b)) / Math.abs(Number(b))) * 100, 2));
+const norm = (s) => String(s || '').toLowerCase().replace(/^[^/]+\//, '').replace(/[^a-z0-9]+/g, '');
+
+/** A trailing date is a release stamp, not a version of the line: gpt-5-nano-2025-08-07 is gpt-5-nano. */
+const DATE_SUFFIX = /[-_ ]20\d{2}[-_]\d{1,2}[-_]\d{1,2}$/;
+
+function baseNameOf(model) {
+ return String(model.name || model.slug || '').replace(/@.*$/, '').replace(DATE_SUFFIX, '').trim();
+}
+
+/**
+ * Split a model name into the line it belongs to and its version numbers.
+ * gpt-5-nano → line "gpt-nano", version [5]
+ * gpt-5.1-nano → line "gpt-nano", version [5.1]
+ * claude-opus-4-5 → line "claude-opus", version [4,5]
+ * qwen3-32b → line "qwen-32b", version [3] (32b is a size, not a version)
+ * A token that is only a number, or letters followed by a number (qwen3, o3), carries a version; anything else
+ * (32b, mini, chat) stays in the line's name.
+ */
+function splitVersion(name) {
+ const tokens = String(name || '').toLowerCase().split(/[^a-z0-9.]+/).filter(Boolean);
+ const version = [];
+ const line = [];
+ for (const token of tokens) {
+ if (/^\d+(?:\.\d+)*$/.test(token)) {
+ version.push(...token.split('.').map(Number));
+ continue;
+ }
+ const m = /^([a-z]+)(\d+(?:\.\d+)*)$/.exec(token);
+ if (m) {
+ line.push(m[1]);
+ version.push(...m[2].split('.').map(Number));
+ continue;
+ }
+ line.push(token);
+ }
+ return { line, version, key: line.join('-') || norm(name) };
+}
+
+function compareVersions(a, b) {
+ const x = a.version || [];
+ const y = b.version || [];
+ const len = Math.max(x.length, y.length);
+ for (let i = 0; i < len; i++) {
+ const d = (x[i] || 0) - (y[i] || 0);
+ if (d) return d;
+ }
+ return String(a.name).localeCompare(String(b.name));
+}
+
+function isModel(m) {
+ return Boolean(m && m.slug && !NOT_CHAT.has(String(m.kind || m.type || 'chat').toLowerCase()));
+}
+
+/** An offer's price, speed, privacy and tier, exactly as AI's catalogue says them; nothing is filled in. */
+function offerOf(o, catalog) {
+ const price = o && o.price && Number.isFinite(Number(o.price.in)) && Number.isFinite(Number(o.price.out))
+ ? {
+ in: Number(o.price.in), out: Number(o.price.out),
+ cached: finite(o.price.cached), per: o.price.per || '1M tokens', currency: o.price.currency || 'USD',
+ }
+ : null;
+ const blended = price ? round((price.in + price.out) / 2, 6) : null;
+ const privacy = o && o.privacy && typeof o.privacy === 'object' ? {
+ label: o.privacy.label || null, retention: o.privacy.retention == null ? null : o.privacy.retention,
+ training: o.privacy.training == null ? null : o.privacy.training, region: o.privacy.region || null,
+ known: Boolean(o.privacy.known),
+ } : null;
+ return {
+ provider: o.provider || null,
+ provider_name: o.provider_name || o.provider || null,
+ model: o.model || null,
+ id: o.id || null,
+ trust: o.trust || null,
+ access: o.access || null,
+ // The tier is whose capacity answers (the offer's trust/access), not whether it is free: a free tier on a
+ // first-party model is still first-party capacity. The free flags stay beside it, where they belong.
+ tier: (() => {
+ if (o.trust === 'community') return 'community';
+ if (o.on_our_gpus || o.trust === 'first-party') return 'first-party';
+ if (o.trust === 'authorized' || o.access === 'AUTHORIZED_WHOLESALE') return 'authorized';
+ return 'third-party';
+ })(),
+ free: Boolean(o.free), free_tier: Boolean(o.free_tier), free_now: Boolean(o.free_now),
+ on_our_gpus: Boolean(o.on_our_gpus), fixed_price: Boolean(o.fixed_price),
+ status: o.status || null,
+ price,
+ blended,
+ speed: {
+ p50_ms: finite(o.speed && o.speed.p50_ms),
+ p95_ms: finite(o.speed && o.speed.p95_ms),
+ },
+ uptime: finite(o.uptime),
+ error_rate: finite(o.error_rate),
+ requests_7d: finite(o.requests_7d),
+ requests_today: finite(o.requests_today),
+ cache_hit_share_7d: finite(o.cache_hit_share_7d),
+ privacy,
+ data_policy: o.data_policy || null,
+ refusal_rate: finite(o.refusal_rate),
+ refusal_measured_on: o.refusal_measured_on || null,
+ fidelity: o.fidelity == null ? null : o.fidelity,
+ measured_at: iso(o.price_checked_at) || iso(o.updated_at) || iso(catalog && catalog.generated_at),
+ };
+}
+
+function cheapestOffer(offers) {
+ return offers.filter((o) => o.price).sort((a, b) => a.blended - b.blended)[0] || null;
+}
+
+function fastestOffer(offers) {
+ return offers.filter((o) => o.speed.p50_ms != null).sort((a, b) => a.speed.p50_ms - b.speed.p50_ms)[0] || null;
+}
+
+/** The Bench board rows that name this model: an exact match, or a release date appended to the same name. */
+function subjectsFor(model, rows) {
+ const want = norm(model.slug || model.name);
+ const tail = norm(String(model.slug || '').split('/').pop());
+ return rows.filter((r) => {
+ const got = norm(r.subject);
+ if (!got || got.length < 5) return false;
+ if (got === want || got === tail) return true;
+ if (want.length >= 5 && got.startsWith(want)) return true;
+ if (tail.length >= 5 && got.startsWith(tail)) return true;
+ return false;
+ });
+}
+
+/**
+ * Quality is OpenVibe.Bench's Elo, never a guess. A line with no games has no score: the row carries `available:
+ * false` and the reason, and the page prints that instead of a number.
+ */
+function qualityOf(model, quality) {
+ const rows = subjectsFor(model, quality.rows || []);
+ if (!rows.length) {
+ return {
+ available: false, score: null, games: 0, provisional: true, ci: null,
+ by_benchmark: [], read_at: quality.at || null,
+ reason: quality.unavailable ? 'Bench is not reachable right now.' : 'No public Bench rounds for this model yet.',
+ };
+ }
+ const byKind = new Map();
+ for (const r of rows) {
+ const prev = byKind.get(r.kind_id);
+ if (!prev || (Number(r.games) || 0) > (Number(prev.games) || 0)) byKind.set(r.kind_id, r);
+ }
+ const byBenchmark = [...byKind.values()].map((r) => ({
+ kind_id: r.kind_id,
+ name: (QUALITY_KINDS.find((k) => k.id === r.kind_id) || {}).name || r.kind_id,
+ elo: finite(r.elo), games: finite(r.games) || 0, wins: finite(r.wins), losses: finite(r.losses), ties: finite(r.ties),
+ win_rate: finite(r.win_rate),
+ ci_lower: finite(r.ci_lower), ci_upper: finite(r.ci_upper),
+ provisional: r.provisional !== false,
+ read_at: quality.at || null,
+ })).sort((a, b) => b.games - a.games || a.name.localeCompare(b.name));
+ const games = byBenchmark.reduce((n, r) => n + (r.games || 0), 0);
+ if (!games) {
+ return {
+ available: false, score: null, games: 0, provisional: true, ci: null,
+ by_benchmark: byBenchmark, read_at: quality.at || null,
+ reason: 'Bench has rounds listed for this model, but no judged games yet.',
+ };
+ }
+ const score = round(byBenchmark.reduce((n, r) => n + (r.elo || 0) * (r.games || 0), 0) / games, 1);
+ return {
+ available: true, score, games, provisional: byBenchmark.some((r) => r.provisional || r.games < 10),
+ // A pooled interval across different boards would be invented; each board keeps its own interval below.
+ ci: null,
+ by_benchmark: byBenchmark, read_at: quality.at || null,
+ reason: null,
+ };
+}
+
+function privacyOf(offers) {
+ const labels = new Map();
+ for (const o of offers) {
+ const label = o.privacy && o.privacy.label;
+ if (label) labels.set(label, (labels.get(label) || 0) + 1);
+ }
+ if (!labels.size) return { label: null, labels: [], known: false, reason: 'No offer states a privacy label.' };
+ const sorted = [...labels.entries()].sort((a, b) => b[1] - a[1]);
+ return {
+ label: sorted[0][0], labels: sorted.map(([label, count]) => ({ label, offers: count })),
+ known: true, reason: null,
+ };
+}
+
+function summaryOf(model, catalog, quality) {
+ const offers = (model.offers || []).map((o) => offerOf(o, catalog));
+ const best = cheapestOffer(offers);
+ const fastest = fastestOffer(offers);
+ const priced = offers.filter((o) => o.price);
+ const line = splitVersion(baseNameOf(model));
+ return {
+ slug: model.slug,
+ name: model.name || model.slug,
+ family: model.family || null,
+ kind: String(model.kind || model.type || 'chat').toLowerCase(),
+ variant: model.variant || null,
+ open_weights: Boolean(model.open_weights),
+ uncensored: Boolean(model.uncensored),
+ context: finite(model.context),
+ max_output: finite(model.max_output),
+ capabilities: model.capabilities || null,
+ modalities: Array.isArray(model.modalities) ? model.modalities : [],
+ line: { key: line.key, version: line.version, version_label: line.version.join('.') || null },
+ offers_count: offers.length,
+ priced_offers: priced.length,
+ providers: [...new Set(offers.map((o) => o.provider_name).filter(Boolean))],
+ requests_7d: finite(model.requests_7d),
+ requests_today: finite(model.requests_today),
+ price: best ? {
+ in: best.price.in, out: best.price.out, cached: best.price.cached, blended: best.blended,
+ per: best.price.per, currency: best.price.currency, provider: best.provider_name,
+ } : { available: false, reason: offers.length ? 'No offer has a per-token price.' : 'No live offer.' },
+ price_range: priced.length ? {
+ min_in: Math.min(...priced.map((o) => o.price.in)), max_in: Math.max(...priced.map((o) => o.price.in)),
+ min_out: Math.min(...priced.map((o) => o.price.out)), max_out: Math.max(...priced.map((o) => o.price.out)),
+ } : null,
+ speed: fastest ? {
+ p50_ms: fastest.speed.p50_ms, p95_ms: fastest.speed.p95_ms, provider: fastest.provider_name,
+ measured_at: fastest.measured_at,
+ } : { available: false, reason: 'No measured speed yet.' },
+ uptime: offers.length ? Math.max(...offers.map((o) => (o.uptime == null ? 0 : o.uptime))) : null,
+ free: offers.some((o) => o.free || o.free_now || (o.price && o.price.in === 0 && o.price.out === 0)),
+ privacy: privacyOf(offers),
+ tiers: {
+ 'first-party': offers.filter((o) => o.tier === 'first-party').length,
+ community: offers.filter((o) => o.tier === 'community').length,
+ authorized: offers.filter((o) => o.tier === 'authorized').length,
+ other: offers.filter((o) => o.tier === 'third-party').length,
+ free_offers: offers.filter((o) => o.free || o.free_tier || o.free_now).length,
+ },
+ quality: qualityOf(model, quality),
+ measured_at: iso(catalog && catalog.generated_at),
+ };
+}
+
+/**
+ * Read the Bench boards once: one read per board, cached by the Bench client. Quality is only ever taken from
+ * these rows, and a board that cannot be read is said to be unavailable, never replaced by an estimate.
+ */
+async function readQuality(bench, now) {
+ const reads = await Promise.all(QUALITY_KINDS.map(async (kind) => {
+ try {
+ const board = await bench.board({ kind: kind.id, category: 'all' });
+ return { kind, board: board || { unavailable: true, rows: [] } };
+ } catch (err) {
+ return { kind, board: { unavailable: true, rows: [], reason: String((err && err.message) || err).slice(0, 200) } };
+ }
+ }));
+ const rows = [];
+ let unavailable = 0;
+ for (const { kind, board } of reads) {
+ if (board.unavailable) unavailable++;
+ for (const row of board.rows || []) rows.push({ ...row, kind_id: kind.id, board_name: kind.name });
+ }
+ return { rows, at: new Date(now()).toISOString(), unavailable: unavailable === reads.length, unread_boards: unavailable };
+}
+
+function qualityDelta(previous, current) {
+ const oldRows = new Map((previous.quality.by_benchmark || []).map((r) => [r.kind_id, r]));
+ const rows = [];
+ for (const now of current.quality.by_benchmark || []) {
+ const before = oldRows.get(now.kind_id);
+ if (!before || before.elo == null || now.elo == null) continue;
+ const overlap = before.ci_lower != null && before.ci_upper != null && now.ci_lower != null && now.ci_upper != null
+ && before.ci_lower <= now.ci_upper && now.ci_lower <= before.ci_upper;
+ rows.push({
+ kind_id: now.kind_id, name: now.name, from: before.elo, to: now.elo,
+ delta: round(now.elo - before.elo, 1), interval_overlaps: overlap,
+ });
+ }
+ if (!rows.length) return null;
+ const clear = rows.filter((r) => !r.interval_overlaps);
+ return {
+ from: previous.quality.score, to: current.quality.score,
+ delta: previous.quality.score == null || current.quality.score == null ? null : round(current.quality.score - previous.quality.score, 1),
+ meaningful: clear.length > 0,
+ by_benchmark: rows,
+ reason: clear.length ? null : 'Every changed score is inside its own interval, so this is not a clear quality change.',
+ };
+}
+
+function change(key, label, from, to, { better = 'lower', unit = '', note = null } = {}) {
+ if (from == null || to == null) {
+ return { key, label, available: false, from, to, delta: null, pct: null, direction: 'unknown', unit, note: note || 'Not measured on one of the versions.' };
+ }
+ const delta = round(Number(to) - Number(from), 4);
+ const direction = delta === 0 ? 'same' : (better === 'lower' ? (delta < 0 ? 'better' : 'worse') : (delta > 0 ? 'better' : 'worse'));
+ return {
+ key, label, available: true, from, to, delta, pct: pct(to, from), direction, unit,
+ note: note || null,
+ };
+}
+
+function versionDiff(previous, current) {
+ const changes = [
+ change('price_in', 'Price in, per 1M', previous.price && previous.price.in, current.price && current.price.in, { unit: 'USD' }),
+ change('price_out', 'Price out, per 1M', previous.price && previous.price.out, current.price && current.price.out, { unit: 'USD' }),
+ change('p50_ms', 'Typical response, p50', previous.speed && previous.speed.p50_ms, current.speed && current.speed.p50_ms, { unit: 'ms' }),
+ change('context', 'Context', previous.context, current.context, { better: 'higher', unit: 'tokens' }),
+ change('elo', 'Bench Elo', previous.quality && previous.quality.score, current.quality && current.quality.score, { better: 'higher', unit: 'Elo' }),
+ ];
+ const quality = qualityDelta(previous, current);
+ if (quality && !quality.meaningful) {
+ const elo = changes.find((c) => c.key === 'elo');
+ if (elo && elo.available) elo.note = quality.reason;
+ }
+ const known = changes.filter((c) => c.available);
+ return {
+ from: previous.slug, to: current.slug,
+ from_name: previous.name, to_name: current.name,
+ changes,
+ quality,
+ summary: known.length
+ ? known.filter((c) => c.direction !== 'same').map((c) => `${c.label}: ${c.direction}`).join('; ') || 'No measured change.'
+ : 'No criterion is measured on both versions.',
+ };
+}
+
+function lineagesOf(summaries) {
+ const groups = new Map();
+ for (const s of summaries) {
+ const key = s.line.key || norm(s.name);
+ if (!groups.has(key)) groups.set(key, []);
+ groups.get(key).push(s);
+ }
+ const out = [];
+ for (const [key, models] of groups) {
+ const sorted = [...models].sort(compareVersions);
+ if (sorted.length < 2) continue;
+ const rows = sorted.map((m, i) => ({
+ ...m,
+ changes: i ? versionDiff(sorted[i - 1], m) : null,
+ }));
+ out.push({
+ key,
+ label: sorted[0].line.key || key,
+ family: sorted[0].family,
+ models: rows,
+ latest: rows[rows.length - 1],
+ measured_at: sorted.map((m) => m.measured_at).filter(Boolean).sort().pop() || null,
+ });
+ }
+ return out.sort((a, b) => a.label.localeCompare(b.label));
+}
+
+const SIDES = Object.freeze([
+ { key: 'price_in', label: 'Price in, per 1M', better: 'lower', unit: 'USD', get: (m) => m.price && m.price.in },
+ { key: 'price_out', label: 'Price out, per 1M', better: 'lower', unit: 'USD', get: (m) => m.price && m.price.out },
+ { key: 'p50_ms', label: 'Typical response, p50', better: 'lower', unit: 'ms', get: (m) => m.speed && m.speed.p50_ms },
+ { key: 'p95_ms', label: 'Slow response, p95', better: 'lower', unit: 'ms', get: (m) => m.speed && m.speed.p95_ms },
+ { key: 'elo', label: 'Bench Elo', better: 'higher', unit: 'Elo', get: (m) => m.quality && m.quality.score },
+ { key: 'games', label: 'Judged games', better: null, unit: 'games', get: (m) => m.quality && m.quality.games },
+ { key: 'context', label: 'Context', better: 'higher', unit: 'tokens', get: (m) => m.context },
+ { key: 'uptime', label: 'Uptime, 7 days', better: 'higher', unit: 'share', get: (m) => m.uptime },
+ { key: 'requests_7d', label: 'Requests, 7 days', better: null, unit: 'requests', get: (m) => m.requests_7d },
+ { key: 'offers', label: 'Live offers', better: null, unit: 'offers', get: (m) => m.offers_count },
+]);
+
+function sideBySide(models) {
+ const criteria = SIDES.map((side) => {
+ const values = models.map((m) => {
+ const value = side.get(m);
+ return {
+ slug: m.slug, name: m.name, value,
+ display: value == null ? null : value,
+ available: value != null,
+ reason: value == null ? (side.key === 'elo' ? (m.quality && m.quality.reason) || 'No Bench evidence.' : 'Not measured.') : null,
+ };
+ });
+ const present = values.filter((v) => v.available);
+ let winner = null;
+ if (side.better && present.length > 1) {
+ const best = side.better === 'lower'
+ ? Math.min(...present.map((v) => Number(v.value)))
+ : Math.max(...present.map((v) => Number(v.value)));
+ const atBest = present.filter((v) => Number(v.value) === best);
+ if (atBest.length === 1) winner = atBest[0].slug;
+ }
+ return { key: side.key, label: side.label, better: side.better, unit: side.unit, values, winner };
+ });
+
+ const byKey = Object.fromEntries(criteria.map((c) => [c.key, c]));
+ const winnerNames = (key) => {
+ const c = byKey[key];
+ if (!c || !c.winner) return null;
+ const m = models.find((x) => x.slug === c.winner);
+ return m ? m.name : null;
+ };
+ const cheapest = winnerNames('price_out');
+ const fastest = winnerNames('p50_ms');
+ const best = winnerNames('elo');
+ const priced = models.filter((m) => m.price && m.price.out != null);
+ const priceSpread = priced.length > 1 ? (() => {
+ const sorted = [...priced].sort((a, b) => a.price.out - b.price.out);
+ const low = sorted[0];
+ const high = sorted[sorted.length - 1];
+ return {
+ cheapest: { slug: low.slug, name: low.name, price: low.price },
+ dearest: { slug: high.slug, name: high.name, price: high.price },
+ difference: round(high.price.out - low.price.out, 4),
+ percent: pct(high.price.out, low.price.out),
+ };
+ })() : null;
+
+ const qualityNote = (() => {
+ const scored = models.filter((m) => m.quality && m.quality.available && m.quality.by_benchmark.length);
+ if (scored.length < 2) return 'Fewer than two models have public Bench rounds, so quality is not compared.';
+ const all = scored.flatMap((m) => m.quality.by_benchmark.map((r) => ({ ...r, slug: m.slug })));
+ const overlap = all.some((a) => all.some((b) => a.slug !== b.slug && a.kind_id === b.kind_id
+ && a.ci_lower != null && a.ci_upper != null && b.ci_lower != null && b.ci_upper != null
+ && a.ci_lower <= b.ci_upper && b.ci_lower <= a.ci_upper));
+ return overlap ? 'At least one Bench interval overlaps between models, so a quality difference is not clear on that board.' : null;
+ })();
+
+ return {
+ kind: KIND,
+ models,
+ criteria,
+ verdict: {
+ headline: [
+ cheapest ? `${cheapest} is cheapest on the output price` : null,
+ fastest ? `${fastest} answers fastest at p50` : null,
+ best ? `${best} has the strongest measured Bench Elo` : null,
+ ].filter(Boolean).join('; ') || 'Not enough measured criteria to call a winner.',
+ cheapest, fastest, best,
+ quality_note: qualityNote,
+ caveat: 'A winner is per criterion, never one invented overall score. Read the price against the quality row for your task.',
+ },
+ price_spread: priceSpread,
+ sources: null,
+ method: null,
+ };
+}
+
+function createCompare({ s, ai, bench, config = {}, now = () => Date.now(), log = console } = {}) {
+ const remarks = {
+ async create({ models, comparison = null, field = null, remark, proposedPrompt = null, sourceUrl = null, subject = null }) {
+ const id = s.newId('excr');
+ const row = await s.db.maybe(
+ `INSERT INTO exchange_compare_remarks
+ (id, kind_id, models, comparison, field, remark, proposed_prompt, source_url, status, subject, created_at)
+ VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7, $8, 'pending', $9, $10::timestamptz)
+ RETURNING *`,
+ [id, KIND, JSON.stringify(models), comparison, field, remark, proposedPrompt, sourceUrl, subject, s.iso()],
+ );
+ return wireRemark(row);
+ },
+ async list({ status = 'pending', limit = 50 } = {}) {
+ const rows = await s.db.many(
+ 'SELECT * FROM exchange_compare_remarks WHERE status = $1 ORDER BY created_at DESC, id DESC LIMIT $2',
+ [status, Math.max(1, Math.min(200, Number(limit) || 50))],
+ );
+ return rows.map(wireRemark);
+ },
+ async get(id) {
+ return wireRemark(await s.db.maybe('SELECT * FROM exchange_compare_remarks WHERE id = $1', [String(id || '')]));
+ },
+ async review({ id, decision, note = null, reviewer }) {
+ const row = await s.db.maybe(
+ `UPDATE exchange_compare_remarks
+ SET status = $1, reviewer_note = $2, reviewed_by = $3, reviewed_at = $4::timestamptz
+ WHERE id = $5 AND status = 'pending'
+ RETURNING *`,
+ [decision, note, reviewer, s.iso(), String(id || '')],
+ );
+ return wireRemark(row);
+ },
+ };
+
+ let qualityCache = null;
+ const QUALITY_MS = 60_000;
+ async function quality() {
+ if (qualityCache && now() - qualityCache.atMs < QUALITY_MS) return qualityCache.value;
+ const value = await readQuality(bench, now);
+ qualityCache = { atMs: now(), value };
+ return value;
+ }
+
+ async function catalogAndQuality() {
+ const [catalog, q] = await Promise.all([ai.catalog(), quality()]);
+ return { catalog, quality: q };
+ }
+
+ function sources(catalog, q) {
+ return [
+ {
+ name: 'OpenVibe.AI catalog', service: 'OpenVibe.AI', url: `${config.ai && config.ai.url ? config.ai.url : 'https://ai.openvibe.services'}/api/v1/exchange/catalog`,
+ read_at: iso(catalog && catalog.generated_at) || new Date(now()).toISOString(),
+ fields: ['price', 'speed', 'uptime', 'requests', 'privacy', 'tier', 'context'],
+ },
+ {
+ name: 'OpenVibe.Bench boards', service: 'OpenVibe.Bench', url: `${config.bench && config.bench.url ? config.bench.url : 'https://bench.openvibe.services'}/api/v1/boards`,
+ read_at: q.at, fields: ['quality (Elo)', 'games', 'intervals'],
+ unavailable: Boolean(q.unavailable),
+ },
+ ];
+ }
+
+ const method = {
+ name: 'Measured, dated and not estimated',
+ price: 'The cheapest live offer’s input and output price per million tokens; every offer’s own price is in the price book.',
+ speed: 'The fastest live offer’s measured p50 and p95 from real requests over the last seven days; not a tokens/s figure.',
+ quality: 'OpenVibe.Bench Elo from public pairwise or graded rounds, weighted by judged games. A model with no judged games has no score, and the row says why. Intervals are per board.',
+ privacy: 'The privacy label and the provider’s data policy exactly as AI’s catalog records them; “not stated” is not a guess.',
+ tier: 'First-party (OpenVibe’s own capacity), community (a person’s machine), authorized wholesale, free, or third-party, from the offer’s own trust and access fields.',
+ versions: 'Versions are the same model line (the name without its version numbers). Each row is compared with the one before it on price, speed, context and Bench Elo. An Elo move whose intervals overlap is not called a change.',
+ unavailable: 'When AI or Bench cannot be read, the field says it is unavailable and shows no figure.',
+ };
+
+ async function models({ q = '', family = '', kind = '', sort = 'requests', limit = 100, offset = 0 } = {}) {
+ const { catalog, quality: qq } = await catalogAndQuality();
+ let list = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));
+ if (family) list = list.filter((m) => String(m.family || '').toLowerCase() === String(family).toLowerCase());
+ if (kind) list = list.filter((m) => m.kind === String(kind).toLowerCase());
+ if (q) {
+ const needle = String(q).toLowerCase();
+ list = list.filter((m) => `${m.name} ${m.slug} ${m.family || ''} ${m.providers.join(' ')}`.toLowerCase().includes(needle));
+ }
+ const order = {
+ requests: (a, b) => (b.requests_7d || 0) - (a.requests_7d || 0) || a.name.localeCompare(b.name),
+ cheap: (a, b) => (a.price.in == null ? Infinity : a.price.in) - (b.price.in == null ? Infinity : b.price.in) || a.name.localeCompare(b.name),
+ price: (a, b) => (a.price.out == null ? Infinity : a.price.out) - (b.price.out == null ? Infinity : b.price.out) || a.name.localeCompare(b.name),
+ fast: (a, b) => (a.speed.p50_ms == null ? Infinity : a.speed.p50_ms) - (b.speed.p50_ms == null ? Infinity : b.speed.p50_ms) || a.name.localeCompare(b.name),
+ quality: (a, b) => (b.quality.score == null ? -Infinity : b.quality.score) - (a.quality.score == null ? -Infinity : a.quality.score) || a.name.localeCompare(b.name),
+ name: (a, b) => a.name.localeCompare(b.name),
+ };
+ list.sort(order[sort] || order.requests);
+ const total = list.length;
+ const from = Math.max(0, Number(offset) || 0);
+ const take = Math.max(1, Math.min(250, Number(limit) || 100));
+ const page = list.slice(from, from + take);
+ return {
+ kind: KIND,
+ generated_at: new Date(now()).toISOString(),
+ catalog_generated_at: iso(catalog.generated_at),
+ unavailable: Boolean(catalog.unavailable),
+ count: page.length, total, offset: from, limit: take,
+ facets: {
+ families: [...new Set(list.map((m) => m.family).filter(Boolean))].sort(),
+ kinds: [...new Set(list.map((m) => m.kind).filter(Boolean))].sort(),
+ providers: [...new Set(list.flatMap((m) => m.providers))].sort(),
+ },
+ method,
+ sources: sources(catalog, qq),
+ models: page,
+ };
+ }
+
+ async function model(slug) {
+ const { catalog, quality: qq } = await catalogAndQuality();
+ const found = (catalog.models || []).find((m) => String(m.slug).toLowerCase() === String(slug || '').toLowerCase());
+ if (!found) return null;
+ const summary = summaryOf(found, catalog, qq);
+ const offers = (found.offers || []).map((o) => offerOf(o, catalog));
+ const all = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));
+ const lineage = lineagesOf(all).find((l) => l.key === summary.line.key) || null;
+ const alternatives = all
+ .filter((m) => m.slug !== summary.slug && m.kind === summary.kind)
+ .sort((a, b) => {
+ const da = a.price && a.price.in != null && summary.price.in != null ? Math.abs(a.price.in - summary.price.in) : Infinity;
+ const db = b.price && b.price.in != null && summary.price.in != null ? Math.abs(b.price.in - summary.price.in) : Infinity;
+ return da - db;
+ })
+ .slice(0, 6)
+ .map((m) => ({ slug: m.slug, name: m.name, price: m.price, speed: m.speed, quality: m.quality }));
+ return {
+ kind: KIND,
+ generated_at: new Date(now()).toISOString(),
+ model: summary,
+ offers,
+ quality: { ...summary.quality, by_benchmark: summary.quality.by_benchmark },
+ versions: lineage,
+ alternatives,
+ method,
+ sources: sources(catalog, qq),
+ };
+ }
+
+ async function compare(refs) {
+ const { catalog, quality: qq } = await catalogAndQuality();
+ const wanted = (Array.isArray(refs) ? refs : []).map((x) => String(x || '').trim()).filter(Boolean);
+ if (wanted.length < 2 || wanted.length > 4) {
+ return { error: 'compare.selection', message: 'Choose two to four models.', status: 400 };
+ }
+ const all = (catalog.models || []).filter(isModel);
+ const found = [];
+ for (const ref of wanted) {
+ const m = all.find((x) => String(x.slug).toLowerCase() === ref.toLowerCase())
+ || all.find((x) => String(x.name).toLowerCase() === ref.toLowerCase())
+ || all.find((x) => String(x.slug).split('/').pop().toLowerCase() === ref.toLowerCase());
+ if (!m) return { error: 'compare.model_not_found', message: `No such model: ${ref}`, status: 404 };
+ if (!found.some((x) => x.slug === m.slug)) found.push(m);
+ }
+ if (found.length < 2) return { error: 'compare.selection', message: 'Choose two different models.', status: 400 };
+ const summaries = found.map((m) => summaryOf(m, catalog, qq));
+ const out = sideBySide(summaries);
+ return {
+ ...out,
+ generated_at: new Date(now()).toISOString(),
+ catalog_generated_at: iso(catalog.generated_at),
+ method,
+ sources: sources(catalog, qq),
+ };
+ }
+
+ async function versions(key = null) {
+ const { catalog, quality: qq } = await catalogAndQuality();
+ const all = (catalog.models || []).filter(isModel).map((m) => summaryOf(m, catalog, qq));
+ const list = lineagesOf(all);
+ return {
+ kind: `${KIND}.versions`,
+ generated_at: new Date(now()).toISOString(),
+ count: list.length,
+ lineages: key ? list.filter((l) => l.key === String(key)) : list,
+ method,
+ sources: sources(catalog, qq),
+ };
+ }
+
+ return { kind: KIND, methods: { models, model, compare, versions, quality, remarks }, method, qualityKinds: QUALITY_KINDS };
+}
+
+function wireRemark(row) {
+ if (!row) return null;
+ return {
+ id: row.id,
+ kind: row.kind_id,
+ models: row.models || [],
+ comparison: row.comparison || null,
+ field: row.field || null,
+ remark: row.remark,
+ proposed_prompt: row.proposed_prompt || null,
+ source_url: row.source_url || null,
+ status: row.status,
+ subject: row.subject || null,
+ reviewer_note: row.reviewer_note || null,
+ reviewed_by: row.reviewed_by || null,
+ reviewed_at: row.reviewed_at || null,
+ created_at: row.created_at,
+ };
+}
+
+module.exports = { createCompare, KIND, QUALITY_KINDS, splitVersion, versionDiff, lineagesOf };
@@ -134,6 +134,13 @@ function load(env = process.env) {
@@ -134,6 +134,13 @@ function load(env = process.env) {
maxChars: Math.max(200, int(env.EXCHANGE_CHAT_MAX_CHARS, 8000)),
maxChars: Math.max(200, int(env.EXCHANGE_CHAT_MAX_CHARS, 8000)),
},
},
+ // openvibe.compare (plan 297): the comparison site is served by this service behind its own name. The
+ // host is what makes a request the compare site's; the API is also reachable on openvibe.exchange.
+ compare: {
+ host: String(env.COMPARE_HOST || 'openvibe.compare').toLowerCase(),
+ baseUrl: trim(env.COMPARE_BASE_URL || 'https://openvibe.compare'),
+ },
+
// IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl a page that appeared or
// IndexNow (openvibe-shared/indexnow): a key makes search engines recrawl a page that appeared or
// changed. Unset → off: nothing mounted, nothing sent.
// changed. Unset → off: nothing mounted, nothing sent.
indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
new file mode 100644
new file mode 100644
@@ -0,0 +1,205 @@
@@ -0,0 +1,205 @@
+'use strict';
+
+/**
+ * /api/v1/compare — openvibe.compare's JSON (plan 297/300/301). Mounted before Exchange v1's 410.
+ *
+ * GET /api/v1/compare the kinds, the method and the links
+ * GET /api/v1/compare/ai/models?q=&family=&kind=&sort=&offset=&limit=
+ * GET /api/v1/compare/ai/models/:org/:model one model, its price book, quality and versions
+ * GET /api/v1/compare/ai?models=a,b two to four models, side by side
+ * GET /api/v1/compare/ai/versions every model line with two or more versions
+ * GET /api/v1/compare/ai/versions/:key one line, each version against the one before it
+ * POST /api/v1/compare/remarks a reader's remark becomes a pending benchmark case
+ * GET /api/v1/compare/remarks?status=pending the review queue (owner only)
+ * PATCH /api/v1/compare/remarks/:id accept or reject a pending case (owner only)
+ *
+ * Everything public is a read of OpenVibe.AI and OpenVibe.Bench, joined in server/compare.js. The remark is the
+ * only write, and it is stored in Exchange's own database until a reviewer turns it into a Bench task.
+ */
+const express = require('express');
+const rateLimit = require('express-rate-limit');
+const { http } = require('openvibe-contracts');
+const { asyncRouter } = require('./router');
+const { createCompare, KIND } = require('../compare');
+
+const FIELDS = new Set(['price', 'speed', 'quality', 'privacy', 'context', 'other']);
+
+function createCompareApi(ctx) {
+ const { config, ai, bench, s, log } = ctx;
+ const engine = ctx.compare || createCompare({ s, ai, bench, config, now: s.now, log });
+ const r = asyncRouter();
+
+ r.use(express.json({ limit: '32kb' }));
+
+ const problem = (res, status, code, detail) => http.sendProblem(res, status, code, { detail, ctx: res.req && res.req.ov });
+ const failed = (res, err) => {
+ log.warn('[OpenVibe.Exchange] Compare failed:', String((err && err.message) || err).slice(0, 200));
+ return problem(res, 500, 'compare.failed', 'Compare could not answer. Try again in a moment.');
+ };
+ const owner = (req) => req.viewer && req.viewer.kind === 'user' && req.viewer.role === 'owner';
+ const requireOwner = (req, res) => {
+ if (!req.viewer || req.viewer.kind !== 'user') {
+ problem(res, 401, 'compare.sign_in_required', 'Sign in as the owner to review benchmark cases.');
+ return false;
+ }
+ if (!owner(req)) {
+ problem(res, 403, 'compare.owner_required', 'Only the owner can review benchmark cases.');
+ return false;
+ }
+ return true;
+ };
+ const clean = (v, max) => String(v == null ? '' : v).replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, max);
+ const cache = (res, seconds = 60) => res.set('Cache-Control', `public, max-age=${seconds}, stale-while-revalidate=300`);
+
+ r.get('/', async (_req, res) => {
+ cache(res, 300);
+ return res.json({
+ kind: 'compare',
+ title: 'Compare anything, starting with AI models',
+ generated_at: new Date(s.now()).toISOString(),
+ kinds: [{
+ id: KIND,
+ title: 'AI models',
+ what: 'Models with the price of every provider that serves them, measured speed and latency, Bench quality, privacy and tier.',
+ endpoints: {
+ models: '/api/v1/compare/ai/models',
+ model: '/api/v1/compare/ai/models/:org/:model',
+ side_by_side: '/api/v1/compare/ai?models=a,b',
+ versions: '/api/v1/compare/ai/versions',
+ remarks: '/api/v1/compare/remarks',
+ },
+ }],
+ method: engine.method,
+ });
+ });
+
+ r.get('/ai/models', async (req, res) => {
+ try {
+ const out = await engine.methods.models({
+ q: req.query.q, family: req.query.family, kind: req.query.kind, sort: req.query.sort,
+ limit: req.query.limit, offset: req.query.offset,
+ });
+ cache(res, 60);
+ return res.json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/ai/models/:org/:model', async (req, res) => {
+ try {
+ const slug = `${req.params.org}/${req.params.model}`;
+ const out = await engine.methods.model(slug);
+ if (!out) return problem(res, 404, 'compare.model_not_found', `No such model: ${slug}`);
+ cache(res, 60);
+ return res.json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/ai', async (req, res) => {
+ try {
+ const refs = String(req.query.models || '').split(',').map((x) => x.trim()).filter(Boolean);
+ const out = await engine.methods.compare(refs);
+ if (out.error) return problem(res, out.status || 400, out.error, out.message);
+ cache(res, 30);
+ return res.json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/ai/versions', async (req, res) => {
+ try {
+ const out = await engine.methods.versions();
+ cache(res, 120);
+ return res.json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/ai/versions/:key', async (req, res) => {
+ try {
+ const out = await engine.methods.versions(req.params.key);
+ if (!out.lineages.length) return problem(res, 404, 'compare.lineage_not_found', `No model line with versions: ${req.params.key}`);
+ out.count = out.lineages.length;
+ cache(res, 120);
+ return res.json(out);
+ } catch (err) { return failed(res, err); }
+ });
+
+ // ── The reader's remark (plan 300) ──────────────────────
+ // Public intake: one plain-text remark, no account needed, rate-limited. It becomes a pending case for review;
+ // it is never run, published or treated as a benchmark result before a person accepts it.
+ r.post('/remarks', rateLimit({ windowMs: 60_000, limit: 20, standardHeaders: true, legacyHeaders: false }), async (req, res) => {
+ try {
+ const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const catalog = await ai.catalog();
+ if (catalog.unavailable) return problem(res, 503, 'compare.catalog_unavailable', 'The model catalog is not reachable right now, so a remark cannot name a real model. Try again shortly.');
+ const slugs = (Array.isArray(b.models) ? b.models : String(b.models || '').split(','))
+ .map((x) => clean(x, 160)).filter(Boolean);
+ const wanted = [...new Set(slugs)].slice(0, 4);
+ if (!wanted.length) return problem(res, 400, 'compare.models_required', 'Name at least one model the remark is about.');
+ const bySlug = new Map((catalog.models || []).map((m) => [String(m.slug).toLowerCase(), m]));
+ const unknown = wanted.filter((x) => !bySlug.has(x.toLowerCase()));
+ if (unknown.length) return problem(res, 400, 'compare.model_not_found', `No such model: ${unknown.join(', ')}`);
+ const remark = clean(b.remark, 2000);
+ if (remark.length < 10) return problem(res, 400, 'compare.remark_too_short', 'A remark needs at least ten characters.');
+ const field = b.field == null || b.field === '' ? null : String(b.field).toLowerCase();
+ if (field && !FIELDS.has(field)) return problem(res, 400, 'compare.field_invalid', `field must be one of: ${[...FIELDS].join(', ')}`);
+ const comparison = b.comparison == null ? null : clean(b.comparison, 200) || null;
+ let sourceUrl = null;
+ if (b.source_url) {
+ try {
+ const u = new URL(String(b.source_url));
+ if (!['http:', 'https:'].includes(u.protocol)) throw new Error('scheme');
+ sourceUrl = u.toString().slice(0, 500);
+ } catch (e) { return problem(res, 400, 'compare.source_url_invalid', 'source_url must be an http or https address.'); }
+ }
+ const names = wanted.map((x) => bySlug.get(x.toLowerCase()).name).join(', ');
+ const proposed = clean(b.proposed_prompt, 4000) || clean(`Test ${names}: ${remark}`, 4000);
+ const remarkRow = await engine.methods.remarks.create({
+ models: wanted.map((x) => bySlug.get(x.toLowerCase()).slug),
+ comparison, field, remark, proposedPrompt: proposed, sourceUrl,
+ subject: req.viewer && req.viewer.kind === 'user' ? req.viewer.subject : null,
+ });
+ res.set('Location', `/api/v1/compare/remarks/${remarkRow.id}`);
+ res.set('Cache-Control', 'no-store');
+ return res.status(201).json({ remark: remarkRow, next: 'A reviewer reads it and turns it into a Bench task; nothing runs or publishes before that.' });
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/remarks', async (req, res) => {
+ if (!requireOwner(req, res)) return;
+ try {
+ const status = ['pending', 'accepted', 'rejected'].includes(String(req.query.status)) ? String(req.query.status) : 'pending';
+ const items = await engine.methods.remarks.list({ status, limit: req.query.limit });
+ res.set('Cache-Control', 'private, no-store');
+ return res.json({ status, count: items.length, remarks: items });
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.get('/remarks/:id', async (req, res) => {
+ if (!requireOwner(req, res)) return;
+ try {
+ const row = await engine.methods.remarks.get(req.params.id);
+ if (!row) return problem(res, 404, 'compare.remark_not_found', 'No such remark.');
+ res.set('Cache-Control', 'private, no-store');
+ return res.json({ remark: row });
+ } catch (err) { return failed(res, err); }
+ });
+
+ r.patch('/remarks/:id', async (req, res) => {
+ if (!requireOwner(req, res)) return;
+ try {
+ const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const decision = b.decision === 'accepted' || b.decision === 'rejected' ? b.decision : null;
+ if (!decision) return problem(res, 400, 'compare.decision_invalid', 'decision must be accepted or rejected.');
+ const row = await engine.methods.remarks.review({
+ id: req.params.id, decision, note: clean(b.note, 1000) || null,
+ reviewer: req.viewer.subject || req.viewer.username || 'owner',
+ });
+ if (!row) return problem(res, 404, 'compare.remark_not_found', 'No pending remark with that id.');
+ res.set('Cache-Control', 'private, no-store');
+ return res.json({ remark: row });
+ } catch (err) { return failed(res, err); }
+ });
+
+ return r;
+}
+
+module.exports = { createCompareApi };
@@ -29,6 +29,9 @@ const TABLES = [
@@ -29,6 +29,9 @@ const TABLES = [
// goes first, then its round (the vote also cascades from the round).
// goes first, then its round (the vote also cascades from the round).
{ table: 'exchange_arena_votes', subject: 'owner', value: (usr) => usr, file: 'arena-votes.json', erase: 'delete' },
{ table: 'exchange_arena_votes', subject: 'owner', value: (usr) => usr, file: 'arena-votes.json', erase: 'delete' },
{ table: 'exchange_arena_rounds', subject: 'owner', value: (usr) => usr, file: 'arena-rounds.json', erase: 'delete' },
{ table: 'exchange_arena_rounds', subject: 'owner', value: (usr) => usr, file: 'arena-rounds.json', erase: 'delete' },
+ // A reader's remark on a comparison (openvibe.compare, plan 300): a signed-in person's pending benchmark
+ // case is theirs, exported and erased with the rest. An anonymous remark has no subject and is not named.
+ { table: 'exchange_compare_remarks', subject: 'subject', value: (usr) => usr, file: 'compare-remarks.json', erase: 'delete' },
// The companions (2026-11-06): a person's characters, their chats and the messages in them are theirs,
// The companions (2026-11-06): a person's characters, their chats and the messages in them are theirs,
// exactly like the chats above. The messages go first, then the chats, then the characters: a character
// exactly like the chats above. The messages go first, then the chats, then the characters: a character
// may be deleted while somebody else's chat still points at it, and that chat keeps its messages.
// may be deleted while somebody else's chat still points at it, and that chat keeps its messages.
new file mode 100644
new file mode 100644
@@ -0,0 +1,228 @@
@@ -0,0 +1,228 @@
+'use strict';
+/**
+ * openvibe.compare's API (plan 296/297/300/301).
+ *
+ * The comparison is a join of two reads: AI's catalogue (price, speed, privacy, tier) and Bench's boards
+ * (quality). These tests hold both stand-ins fixed and check the shape, the dates, the version deltas, the
+ * two-to-four side-by-side, and the remark that becomes a pending case and nothing more until the owner reviews it.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { createFakeAi } = require('./helpers/ai');
+const { createFakeBench } = require('./helpers/bench');
+
+(async () => {
+ const ai = createFakeAi();
+ // A fourth and a fifth model on one line, so versions have something to compare.
+ ai.catalogData.models.push(
+ {
+ slug: 'openai/gpt-5-mini', canonical: 'openai/gpt-5-mini', name: 'gpt-5-mini', family: 'GPT', variant: null,
+ uncensored: false, kind: 'chat', context: 400000, max_output: 128000, open_weights: false,
+ capabilities: { tools: true, json: true, vision: true, stream: true }, modalities: ['text', 'image'], description: null,
+ requests_7d: 50, requests_today: 4, community: false,
+ offers: [{
+ provider: 'openai', provider_name: 'OpenAI', model: 'gpt-5-mini', id: 'openai/gpt-5-mini@v1', trust: 'first-party',
+ access: 'OWN_DEPLOYMENT', free: false, free_tier: false, status: 'up',
+ price: { in: 0.25, out: 2, cached: 0.025, per: '1M tokens', currency: 'USD' },
+ speed: { p50_ms: 900, p95_ms: 3000 }, uptime: 0.99, requests_7d: 50, requests_today: 4,
+ privacy: { label: 'anonymized', retention: '0', training: false, region: 'us', known: true },
+ updated_at: '2026-11-06T00:00:00.000Z',
+ }],
+ },
+ {
+ slug: 'openai/gpt-5.1-mini', canonical: 'openai/gpt-5.1-mini', name: 'gpt-5.1-mini', family: 'GPT', variant: null,
+ uncensored: false, kind: 'chat', context: 400000, max_output: 128000, open_weights: false,
+ capabilities: { tools: true, json: true, vision: true, stream: true }, modalities: ['text', 'image'], description: null,
+ requests_7d: 80, requests_today: 6, community: false,
+ offers: [{
+ provider: 'openai', provider_name: 'OpenAI', model: 'gpt-5.1-mini', id: 'openai/gpt-5.1-mini@v1', trust: 'first-party',
+ access: 'OWN_DEPLOYMENT', free: false, free_tier: false, status: 'up',
+ price: { in: 0.2, out: 1.5, cached: 0.02, per: '1M tokens', currency: 'USD' },
+ speed: { p50_ms: 700, p95_ms: 2400 }, uptime: 0.995, requests_7d: 80, requests_today: 6,
+ privacy: { label: 'anonymized', retention: '0', training: false, region: 'us', known: true },
+ updated_at: '2026-11-06T00:00:00.000Z',
+ }],
+ },
+ );
+ ai.catalogData.count = ai.catalogData.models.length;
+ // AI's offer carries its own privacy label and the tier fields; the page must not invent one.
+ ai.catalogData.models[0].offers[0].privacy = { label: 'anonymized', retention: '0', training: false, region: 'cn', known: true };
+ ai.catalogData.models[0].offers[0].on_our_gpus = false;
+ ai.catalogData.models[0].offers[1].on_our_gpus = false;
+
+ const bench = createFakeBench({
+ boards: {
+ 'chat.arena': [
+ { subject: 'deepseek/deepseek-chat', elo: 1040, games: 20, wins: 12, losses: 6, ties: 2, win_rate: 0.65, ci_lower: 1020, ci_upper: 1060, provisional: false },
+ { subject: 'qwen/qwen3-32b', elo: 990, games: 12, wins: 5, losses: 6, ties: 1, win_rate: 0.46, ci_lower: 970, ci_upper: 1010, provisional: false },
+ { subject: 'openai/gpt-5-mini', elo: 1100, games: 30, wins: 20, losses: 8, ties: 2, win_rate: 0.7, ci_lower: 1080, ci_upper: 1120, provisional: false },
+ { subject: 'openai/gpt-5.1-mini', elo: 1120, games: 30, wins: 22, losses: 6, ties: 2, win_rate: 0.77, ci_lower: 1125, ci_upper: 1145, provisional: false },
+ ],
+ },
+ });
+
+ const t = await boot({ ai, bench });
+ const owner = t.network.addUser('owner', { role: 'owner' });
+ const reader = t.network.addUser('reader');
+
+ try {
+ await check('the index names the kind, the sources and the method', async () => {
+ const r = await t.get('/api/v1/compare');
+ assert.strictEqual(r.status, 200, r.text);
+ const body = r.json();
+ assert.strictEqual(body.kind, 'compare');
+ assert.strictEqual(body.kinds[0].id, 'ai.models');
+ assert.ok(body.method.price && body.method.quality && body.method.versions);
+ });
+
+ await check('the models list is priced per provider, dated, and honest about missing quality', async () => {
+ const r = await t.get('/api/v1/compare/ai/models');
+ assert.strictEqual(r.status, 200, r.text);
+ const body = r.json();
+ assert.strictEqual(body.kind, 'ai.models');
+ assert.strictEqual(body.total, 5);
+ assert.deepStrictEqual(body.sources.map((s) => s.service), ['OpenVibe.AI', 'OpenVibe.Bench']);
+ const deep = body.models.find((m) => m.slug === 'deepseek/deepseek-chat');
+ assert.ok(deep, 'DeepSeek is listed');
+ assert.strictEqual(deep.price.in, 0.2, 'the cheapest offer wins the headline');
+ assert.strictEqual(deep.price.out, 0.3);
+ assert.strictEqual(deep.price.provider, 'DeepInfra');
+ assert.strictEqual(deep.speed.p50_ms, 400);
+ assert.strictEqual(deep.quality.available, true);
+ assert.strictEqual(deep.quality.score, 1040);
+ assert.strictEqual(deep.quality.games, 20);
+ assert.strictEqual(deep.measured_at, '2026-11-06T09:00:00.000Z', 'the AI read carries its date');
+ assert.ok(deep.privacy.label, 'a privacy label is read, not invented');
+ assert.strictEqual(deep.tiers['first-party'], 2);
+ const llama = body.models.find((m) => m.slug === 'meta-llama/llama-3-8b-abliterated');
+ assert.strictEqual(llama.quality.available, false);
+ assert.match(llama.quality.reason, /No public Bench rounds/);
+ assert.strictEqual(llama.quality.score, null, 'no invented score');
+ });
+
+ await check('one model carries its whole price book, quality by board, and its line', async () => {
+ const r = await t.get('/api/v1/compare/ai/models/deepseek/deepseek-chat');
+ assert.strictEqual(r.status, 200, r.text);
+ const body = r.json();
+ assert.strictEqual(body.model.slug, 'deepseek/deepseek-chat');
+ assert.strictEqual(body.offers.length, 2);
+ assert.deepStrictEqual(body.offers.map((o) => o.price.in).sort((a, b) => a - b), [0.2, 0.28]);
+ assert.strictEqual(body.offers.find((o) => o.provider === 'deepseek').tier, 'first-party');
+ assert.strictEqual(body.quality.by_benchmark.length, 1);
+ assert.strictEqual(body.versions, null, 'a model with one version has no version table');
+ assert.ok(body.alternatives.length >= 1);
+ const missing = await t.get('/api/v1/compare/ai/models/nope/nothing');
+ assert.strictEqual(missing.status, 404);
+ assert.strictEqual(missing.json().code, 'compare.model_not_found');
+ });
+
+ await check('two to four models are compared on every criterion, with a per-criterion winner', async () => {
+ const r = await t.get('/api/v1/compare/ai?models=openai/gpt-5.1-mini,deepseek/deepseek-chat,qwen/qwen3-32b');
+ assert.strictEqual(r.status, 200, r.text);
+ const body = r.json();
+ assert.strictEqual(body.models.length, 3);
+ assert.deepStrictEqual(body.models.map((m) => m.slug), ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat', 'qwen/qwen3-32b']);
+ const price = body.criteria.find((c) => c.key === 'price_out');
+ assert.strictEqual(price.winner, 'deepseek/deepseek-chat', '0.3 output is the cheapest');
+ const speed = body.criteria.find((c) => c.key === 'p50_ms');
+ assert.strictEqual(speed.winner, 'deepseek/deepseek-chat');
+ const quality = body.criteria.find((c) => c.key === 'elo');
+ assert.strictEqual(quality.winner, 'openai/gpt-5.1-mini');
+ assert.ok(body.price_spread, 'the price spread is said');
+ assert.ok(body.verdict.headline.includes('cheapest'));
+ assert.ok(body.verdict.caveat.includes('never one invented overall score'));
+
+ const one = await t.get('/api/v1/compare/ai?models=deepseek/deepseek-chat');
+ assert.strictEqual(one.status, 400);
+ assert.strictEqual(one.json().code, 'compare.selection');
+ const five = await t.get('/api/v1/compare/ai?models=a,b,c,d,e');
+ assert.strictEqual(five.status, 400);
+ const none = await t.get('/api/v1/compare/ai?models=deepseek/deepseek-chat,nope/nothing');
+ assert.strictEqual(none.status, 404);
+ });
+
+ await check('versions of one line are ordered and each says what changed, with the interval caveat', async () => {
+ const r = await t.get('/api/v1/compare/ai/versions');
+ assert.strictEqual(r.status, 200, r.text);
+ const body = r.json();
+ const line = body.lineages.find((l) => l.key === 'gpt-mini');
+ assert.ok(line, `gpt-mini line in ${body.lineages.map((l) => l.key).join(',')}`);
+ assert.deepStrictEqual(line.models.map((m) => m.name), ['gpt-5-mini', 'gpt-5.1-mini']);
+ const newer = line.models[1];
+ const priceIn = newer.changes.changes.find((c) => c.key === 'price_in');
+ assert.strictEqual(priceIn.from, 0.25);
+ assert.strictEqual(priceIn.to, 0.2);
+ assert.strictEqual(priceIn.direction, 'better');
+ const speed = newer.changes.changes.find((c) => c.key === 'p50_ms');
+ assert.strictEqual(speed.direction, 'better');
+ const elo = newer.changes.changes.find((c) => c.key === 'elo');
+ assert.strictEqual(elo.direction, 'better');
+ assert.strictEqual(newer.changes.quality.meaningful, true);
+ assert.match(newer.changes.summary, /better/);
+
+ const one = await t.get('/api/v1/compare/ai/versions/gpt-mini');
+ assert.strictEqual(one.status, 200);
+ assert.strictEqual(one.json().lineages.length, 1);
+ const missing = await t.get('/api/v1/compare/ai/versions/nope');
+ assert.strictEqual(missing.status, 404);
+ });
+
+ let remarkId;
+ await check('a reader’s remark becomes a pending benchmark case, not a result', async () => {
+ const r = await t.get('/api/v1/compare/remarks', {
+ json: {
+ models: ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat'],
+ comparison: 'openai/gpt-5.1-mini-vs-deepseek/deepseek-chat',
+ field: 'quality',
+ remark: 'On a long refactor the newer mini kept the whole file in view where the other dropped the last function.',
+ },
+ });
+ assert.strictEqual(r.status, 201, r.text);
+ const body = r.json();
+ remarkId = body.remark.id;
+ assert.match(remarkId, /^excr_/);
+ assert.strictEqual(body.remark.status, 'pending');
+ assert.deepStrictEqual(body.remark.models, ['openai/gpt-5.1-mini', 'deepseek/deepseek-chat']);
+ assert.strictEqual(body.remark.subject, null, 'an anonymous remark stores no person');
+ assert.match(body.remark.proposed_prompt, /^Test gpt-5\.1-mini, DeepSeek Chat:/);
+ assert.match(body.next, /nothing runs or publishes/);
+
+ const short = await t.get('/api/v1/compare/remarks', { json: { models: ['deepseek/deepseek-chat'], remark: 'too short' } });
+ assert.strictEqual(short.status, 400);
+ const unknown = await t.get('/api/v1/compare/remarks', { json: { models: ['nope/nothing'], remark: 'A remark of enough length.' } });
+ assert.strictEqual(unknown.status, 400);
+ const badField = await t.get('/api/v1/compare/remarks', { json: { models: ['deepseek/deepseek-chat'], field: 'vibes', remark: 'A remark of enough length.' } });
+ assert.strictEqual(badField.status, 400);
+ });
+
+ await check('the pending queue is the owner’s, and reviewing it is the only thing that moves a case', async () => {
+ const anon = await t.get('/api/v1/compare/remarks');
+ assert.strictEqual(anon.status, 401);
+ const normal = await t.get('/api/v1/compare/remarks', { as: reader });
+ assert.strictEqual(normal.status, 403);
+ const pending = await t.get('/api/v1/compare/remarks', { as: owner });
+ assert.strictEqual(pending.status, 200, pending.text);
+ assert.strictEqual(pending.json().count, 1);
+ assert.strictEqual(pending.json().remarks[0].id, remarkId);
+
+ const accepted = await t.get(`/api/v1/compare/remarks/${remarkId}`, { as: owner, method: 'PATCH', json: { decision: 'accepted', note: 'A real case: hold the file at 8k tokens.' } });
+ assert.strictEqual(accepted.status, 200, accepted.text);
+ assert.strictEqual(accepted.json().remark.status, 'accepted');
+ assert.strictEqual(accepted.json().remark.reviewer_note, 'A real case: hold the file at 8k tokens.');
+ const again = await t.get(`/api/v1/compare/remarks/${remarkId}`, { as: owner, method: 'PATCH', json: { decision: 'rejected' } });
+ assert.strictEqual(again.status, 404, 'only a pending case can be reviewed');
+ const noneLeft = await t.get('/api/v1/compare/remarks', { as: owner });
+ assert.strictEqual(noneLeft.json().count, 0);
+ });
+
+ await check('Exchange v1 still answers its 410 elsewhere, so the compare API is not a hole', async () => {
+ const r = await t.get('/api/v1/listings');
+ assert.strictEqual(r.status, 410);
+ const posted = await t.get('/api/v1/compare/remarks', { json: { models: [], remark: 'A remark long enough to pass.' } });
+ assert.strictEqual(posted.status, 400);
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})();
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
* A stand-in OpenVibe.Bench client for the Arena tests: it records every round, vote and board read, and answers
* A stand-in OpenVibe.Bench client for the Arena tests: it records every round, vote and board read, and answers
* in the shape of server/bench.js. A test may pass its own `createRound`, `vote` or `board` to make one fail.
* in the shape of server/bench.js. A test may pass its own `createRound`, `vote` or `board` to make one fail.
*/
*/
-function createFakeBench({ createRound = null, vote = null, board = null } = {}) {
+function createFakeBench({ createRound = null, vote = null, board = null, boards = null } = {}) {
const calls = { rounds: [], votes: [], boards: [] };
const calls = { rounds: [], votes: [], boards: [] };
let n = 0;
let n = 0;
return {
return {
@@ -30,7 +30,8 @@ function createFakeBench({ createRound = null, vote = null, board = null } = {})
@@ -30,7 +30,8 @@ function createFakeBench({ createRound = null, vote = null, board = null } = {})
async board(args) {
async board(args) {
calls.boards.push(args);
calls.boards.push(args);
if (board) return board(args);
if (board) return board(args);
- return { unavailable: false, kind_id: 'chat.arena', category: args.category, rows: [] };
+ const rows = boards && Object.prototype.hasOwnProperty.call(boards, args.kind) ? boards[args.kind] : null;
+ return { unavailable: false, kind_id: args.kind || 'chat.arena', category: args.category, rows: rows || [] };
},
},
};
};
}
}