openvibers/openvibe.exchange Commits
Compare measured performance and shared Bench ratings (#177)
Changes
@@@ -9,11 +9,11 @@"port": "5150","domain": "openvibe.exchange","contracts": "openvibe-contracts v0.384.0",-- "sdk": "openvibe-sdk v0.133.0",++ "sdk": "openvibe-sdk v0.145.0","shared": "openvibe-shared v4.197.0","packages": ["openvibe-contracts v0.384.0",-- "openvibe-sdk v0.133.0",++ "openvibe-sdk v0.145.0","openvibe-shared v4.197.0"],"summary": "OpenVibe.Exchange is the network's AI front. It reads OpenVibe.AI's public catalog and aggregate usage and renders: the home (popular now, cheapest per million, uncensored, community machines, live numbers), /models and /m/<org>/<model> (every offer with price in/out, measured p50/p95, uptime and data policy, a Try it box, curl and the SDK), /chat (one to three models side by side, system prompt, temperature, history in the account, on openvibe-shared/thread), /api, /sell (Link's offer to the network; what is not sold, said plainly), /uncensored (an 18+ confirmation on the account and the hard floor) and /ai (which products run which models, and the daily cost). Chat and Try it are ai.complete runs on the person's behalf (on_behalf_of) with the service's ai.run.create token. Exchange v1 (free/swap listings, passports, sandboxes, installs, offers) is retired: every v1 page answers 410 with a pointer, its JSON API answers 410 exchange.retired, and its tables stay read-only; its tests live in test/retired/.",nnew file mode 100644@@@ -0,0 +1,27 @@++# Shared analysis++++- `/api/v1/analysis/compare?models=openai/gpt-5-nano,openai/gpt-5-mini&kind=other&rating_kind=chat.arena`++ reads AI receipt metrics for up to six models and attaches each unchanged Bench rating.++- `/api/v1/analysis/models/openai%2Fgpt-5-nano?days=30&rating_kind=chat.arena`++ returns the daily series and the same rating used by the model page. Days may be 30 or 90.++- `/api/v1/analysis/ratings/:kind` reads the same public Bench board as Report.++++Reads cache for sixty seconds, allow public browser GETs, and use SDK clients.++Metrics remain separate by day/provider/task; medians are never averaged.++Ratings below twenty eligible games and receipt cells below twenty runs are++withheld. Missing evidence is null or empty. Historical TTFT can remain null.++++The model join first uses an exact catalogue/offer subject, then an unambiguous++dated API name. It returns the Bench subject with the score. Several rated++revisions never become an invented family score. Design and Exchange already++send their arena votes to Bench. Exchange's local arena tables still retain++round/result links and vote deduplication, not a second rating computation.++Design's active store already reads fitted Bench boards; its old rating table++is retained only for migration compatibility. Existing provisional Bench/Design++views remain; the new cross-site reads use the twenty-game floor.++++For Fable: the existing model page has plain kit tables for the latest completed++day's runs, p50/p95 latency, TTFT and cost per run, and a plain Bench rating block++with its interval and games. Comparison JSON supplies those fields per model and++provider, plus `rating_kind` and a shared rating. The source-unavailable, cached++rating, sparse-data and ambiguous-model states need their look. No CSS was added.@@@ -18,7 +18,7 @@"multer": "^2.5.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0","pg": "^8.23.0"},@@@ -1221,9 +1221,9 @@}},"node_modules/openvibe-sdk": {-- "version": "0.133.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",-- "integrity": "sha512-qn+y/yQnlYkfm0+tDpKLwUw04L0fZydcyYiEM6OaXzc7iclgp2HAcBB/OkmPXWOCwXGyv2CBQAJeEVesgkWBFA==",++ "version": "0.145.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",++ "integrity": "sha512-3kEEiEQSzgRHoCNdOQ/xmqNRNNf/QmJ269E6rPASDHpuciUTOwZkjIVMMTfGIf6VGBe73TdYdLjqMAMThSn93g==","license": "MIT","bin": {"openvibe-download": "bin/openvibe-download.js"@@@ -2,7 +2,7 @@"name": "openvibe-exchange","version": "0.1.0","private": true,-- "description": "OpenVibe.Exchange — trade what people make.",++ "description": "OpenVibe.Exchange \u2014 trade what people make.","main": "server/index.js","scripts": {"start": "node server/index.js",@@@ -27,7 +27,7 @@"multer": "^2.5.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0","pg": "^8.23.0"},nnew file mode 100644@@@ -0,0 +1,56 @@++'use strict';++const { createClient } = require('openvibe-sdk/core');++const { createAiClient } = require('openvibe-sdk/ai');++++const bare = (v) => String(v || '').toLowerCase().replace(/^[^/]+\//, '').replace(/[-_@:](\d{4}-?\d{2}-?\d{2}|latest|free)$/, '');++const validKind = (kind) => typeof kind === 'string' && /^[a-z0-9][a-z0-9_.-]{0,79}$/.test(kind);++function ratingFor(model, rows = []) {++ const candidates = [model.slug, model.canonical, model.id, ...(model.offers || []).map((offer) => offer.model)].filter(Boolean);++ const eligible = rows.filter((row) => row.games >= 20 && Number.isFinite(row.elo));++ const exact = eligible.filter((row) => candidates.includes(row.subject));++ const matches = exact.length ? exact : eligible.filter((row) => candidates.some((id) => bare(id) === bare(row.subject)));++ // A family with several rated revisions has no single rating: keep each Bench subject distinct.++ return matches.length === 1 ? matches[0] : null;++}++++function createAnalysis({ config, ai, bench, fetchImpl = globalThis.fetch, now = () => Date.now(), metrics = null }) {++ const source = metrics || createAiClient(createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl: config.ai.url }).metrics;++ const cache = new Map();++ async function read(key, loader) {++ const hit = cache.get(key);++ if (hit && hit.until > now()) return hit.value;++ const value = await loader();++ cache.set(key, { until: now() + 60000, value });++ if (cache.size > 128) cache.delete(cache.keys().next().value);++ return value;++ }++ function checkedKind(value = 'chat.arena') {++ if (!validKind(value)) throw new TypeError('Use a Bench task kind.');++ return value;++ }++ const ratingSource = (board, kind) => ({ source: 'OpenVibe.Bench', kind, minimum_samples: 20, unavailable: Boolean(board?.unavailable), stale: Boolean(board?.stale) });++ async function model(model, { rating_kind = 'chat.arena', days = 30 } = {}) {++ const kind = checkedKind(rating_kind);++ return read(`model:${model.slug}:${kind}:${days}`, async () => {++ const [measured, board] = await Promise.all([++ source.model(model.slug, { days }, { signal: AbortSignal.timeout(config.ai.timeoutMs) }), bench.board({ kind }),++ ]);++ return { schema: 'exchange.analysis@1', model: model.slug, measured: { ...measured, series: (measured.series || []).filter((cell) => cell.runs >= 20) }, rating: ratingFor(model, board.rows), ratings: ratingSource(board, kind) };++ });++ }++ async function compare({ models, measures, kind, day, rating_kind = 'chat.arena' }) {++ const ratingKind = checkedKind(rating_kind);++ if (!Array.isArray(models) || !models.length || models.length > 6 || new Set(models).size !== models.length) throw new TypeError('Choose one to six distinct models.');++ return read(`compare:${JSON.stringify([models, measures, kind, day, ratingKind])}`, async () => {++ const [measured, board, catalog] = await Promise.all([++ source.compare({ models, measures, kind, day }, { signal: AbortSignal.timeout(config.ai.timeoutMs) }),++ bench.board({ kind: ratingKind }), ai.catalog(),++ ]);++ return { ...measured, schema: 'exchange.analysis@1', ratings: ratingSource(board, ratingKind), models: (measured.models || []).map((item) => ({++ ...item, cells: (item.cells || []).filter((cell) => cell.runs >= 20), rating: ratingFor((catalog.models || []).find((entry) => entry.slug === item.model) || { slug: item.model }, board.rows),++ })) };++ });++ }++ return { model, compare, metrics: source };++}++module.exports = { createAnalysis, ratingFor };@@@ -84,6 +84,7 @@ async function createApp(opts = {}) {// Network, read and changed with the person's own token: their machines and their machine pools.const network = opts.network || createNetwork({ config, fetchImpl, log });const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });++ const analysis = opts.analysis || require('./analysis').createAnalysis({ config, ai, bench, fetchImpl: opts.aiFetchImpl || globalThis.fetch, now: s.now });const arenaStore = createArenaStore(s);const history = createCatalogHistory({ s, log });const chats = createChatStore(s);@@@ -115,7 +116,7 @@ async function createApp(opts = {}) {: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });-- const ctx = { config, s, keys, sso, principal, ai, network, bench, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };++ const ctx = { config, s, keys, sso, principal, ai, network, bench, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };const app = express();app.disable('x-powered-by');@@@ -252,6 +253,8 @@ async function createApp(opts = {}) {return res.redirect(301, `https://openvibe.chat/vibe${suffix}`);});++ app.use('/api/v1/analysis', rateLimit({ windowMs: 60000, limit: Number(process.env.EXCHANGE_API_RATE_LIMIT_PER_MIN) || 120, standardHeaders: true, legacyHeaders: false }), require('./http/analysis').createAnalysisRoutes(ctx));++// ── Exchange v1's JSON API: gone ────────────────────────app.use('/api/v1', (req, res) => {res.set('Cache-Control', 'public, max-age=86400');@@@ -21,6 +21,7 @@ async function jsonOf(res) {}function createBench({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {++ const shared = require('openvibe-sdk/bench').createBenchClient(require('openvibe-sdk/core').createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl: config.bench.url });const tokens = config.oauth.clientSecret ? createServiceTokenClient({tokenUrl: `${config.networkInternalUrl}/oauth/token`,clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,@@@ -82,8 +83,8 @@ function createBench({ config, fetchImpl = globalThis.fetch, log = console, nowconst hit = boardHit.get(key);if (hit && now() - hit.at < BOARD_MS) return hit.value;try {-- const body = await call('GET', `/api/v1/boards/${encodeURIComponent(kind)}?category=${encodeURIComponent(category)}`);-- const value = body && body.board ? body.board : { unavailable: true, kind_id: kind, category, rows: [] };++ const body = await shared.ratings.get(kind, { category, signal: AbortSignal.timeout(config.bench.timeoutMs) });++ const value = body && body.board ? { ...body.board, rows: (body.board.rows || []).filter((row) => row.games >= 20) } : { unavailable: true, kind_id: kind, category, rows: [] };boardHit.set(key, { at: now(), value });if (boardHit.size > 16) boardHit.delete(boardHit.keys().next().value);return value;nnew file mode 100644@@@ -0,0 +1,33 @@++'use strict';++const { asyncRouter } = require('./router');++function createAnalysisRoutes({ analysis, ai, bench }) {++ const router = asyncRouter();++ router.use((_req, res, next) => {++ res.set('Access-Control-Allow-Origin', '*').set('Access-Control-Allow-Methods', 'GET, OPTIONS').set('Access-Control-Allow-Headers', 'Accept, Content-Type, traceparent, X-OpenVibe-Request-Id');++ res.set('Cache-Control', 'public, max-age=60');++ next();++ });++ router.options('*', (_req, res) => res.status(204).end());++ const fail = (res, err) => res.set('Cache-Control', 'no-store').status(err instanceof TypeError || err.status === 422 ? 422 : 503).json({ error: { code: err instanceof TypeError || err.status === 422 ? 'analysis.invalid' : 'analysis.unavailable', detail: err instanceof TypeError ? err.message : 'Analysis is unavailable.' } });++ router.get('/compare', async (req, res) => {++ try {++ if (req.originalUrl.length > 2400 || Object.values(req.query).some((value) => typeof value !== 'string')) throw new TypeError('Use one value per comparison parameter.');++ return res.json(await analysis.compare({ ...req.query, models: String(req.query.models || '').split(',').filter(Boolean), measures: req.query.measures?.split(',') }));++ } catch (err) { return fail(res, err); }++ });++ router.get('/models/:model', async (req, res) => {++ try {++ const model = (await ai.catalog()).models?.find((entry) => entry.slug === req.params.model);++ if (!model) return res.status(404).json({ error: { code: 'model.not_found' } });++ return res.json(await analysis.model(model, { rating_kind: req.query.rating_kind, days: Number(req.query.days || 30) }));++ } catch (err) { return fail(res, err); }++ });++ router.get('/ratings/:kind', async (req, res) => {++ if (!/^[a-z0-9][a-z0-9_.-]{0,79}$/.test(req.params.kind)) return fail(res, new TypeError('Use a Bench task kind.'));++ const board = await bench.board({ kind: req.params.kind });++ if (board.unavailable) res.set('Cache-Control', 'no-store');++ return res.status(board.unavailable ? 503 : 200).json({ board });++ });++ return router;++}++module.exports = { createAnalysisRoutes };@@@ -140,6 +140,9 @@ function createPageRoutes(ctx) {return page(req, res, { title: 'Not found', index: false, body: raw(`<div class="ex-wrap ex-narrow">${kit.empty({ icon: 'search', title: 'No such model', text: `Nothing here serves ${esc(`${req.params.org}/${req.params.model}`)}. <a href="/models">Browse the models</a>.` })}</div>`) }, 404);}const isAdult = signedIn(req) ? await adult.confirmed(owner(req)) : false;++ let analysis = null;++ try { analysis = await ctx.analysis.model(model, { rating_kind: typeof req.query.rating_kind === 'string' ? req.query.rating_kind : 'chat.arena' }); }++ catch (err) { log.warn('[exchange] model analysis unavailable: ' + String(err.message || err).slice(0, 160)); }const jsonLd = seo.jsonLd.webPage({name: `${model.name} on OpenVibe.Exchange`, url: `${config.baseUrl}${modelPath(model)}`,description: `${model.name}: ${model.offers.length} live offers with price, speed and uptime.`.slice(0, 160), siteUrl: config.baseUrl,@@@ -148,7 +151,7 @@ function createPageRoutes(ctx) {index: true, cache: req.method === 'GET' ? PUBLIC_CACHE : null, agentLayer: true,title: model.name, description: `${model.name}: price, speed, uptime and data policy of every offer.`,jsonLd: [jsonLd], head: '',-- body: raw(views.modelPage({ model, viewer: req.viewer, result, problem, text, adult: isAdult, community })),++ body: raw(views.modelPage({ model, viewer: req.viewer, result, problem, text, adult: isAdult, community, analysis })),}, status);}@@@ -14,9 +14,20 @@ const FALLBACK = (config) => ({description: 'One OpenAI- and Anthropic-compatible endpoint over every model OpenVibe routes.',},servers: [{ url: config.api.publicUrl }],-- paths: {},++ paths: analysisPaths(config),});++function analysisPaths(config) {++ const make = (id, parameters) => ({ get: { operationId: id, security: [], servers: [{ url: config.baseUrl }], parameters, responses: { '200': { description: 'Public measured performance and shared Bench ratings; minimum twenty samples' }, '422': { description: 'Invalid selection' }, '503': { description: 'Source unavailable' } } } });++ const query = (name) => ({ name, in: 'query', schema: { type: 'string' } });++ const param = (name) => ({ name, in: 'path', required: true, schema: { type: 'string' } });++ return {++ '/api/v1/analysis/compare': make('exchange.analysis.compare', ['models', 'measures', 'kind', 'day', 'rating_kind'].map(query)),++ '/api/v1/analysis/models/{model}': make('exchange.analysis.model', [param('model'), query('days'), query('rating_kind')]),++ '/api/v1/analysis/ratings/{kind}': make('exchange.analysis.ratings', [param('kind')]),++ };++}++function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {const ttl = config.api.openapiCacheMs;let at = 0;@@@ -31,6 +42,7 @@ function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, noconst doc = await res.json();if (!doc || typeof doc !== 'object' || typeof doc.openapi !== 'string') throw new Error('openapi document is not OpenAPI');doc.servers = [{ url: config.api.publicUrl }];++ doc.paths = { ...(doc.paths || {}), ...analysisPaths(config) };return doc;}nnew file mode 100644@@@ -0,0 +1,20 @@++'use strict';++const kit = require('openvibe-shared/kit');++const { esc } = require('../render/html');++const number = (value, digits = 2) => value == null ? '—' : Number(value).toLocaleString('en-US', { maximumFractionDigits: digits });++function modelFacts(analysis) {++ if (!analysis) return kit.section({ title: 'Measured performance', body: '<p>Measured performance is unavailable.</p>' });++ const series = (analysis.measured?.series || []).filter((row) => row.runs >= 20);++ const latest = series.reduce((day, row) => row.day > day ? row.day : day, '');++ const rows = series.filter((row) => row.day === latest);++ const measured = rows.length ? kit.table({ caption: `Measured runs on ${latest}`, columns: [++ { key: 'provider', label: 'Provider' }, { key: 'task', label: 'Task kind' }, { key: 'runs', label: 'Runs', num: true },++ { key: 'median', label: 'Median latency (ms)', num: true, fmt: number }, { key: 'p95', label: '95th percentile (ms)', num: true, fmt: number },++ { key: 'first', label: 'First token (ms)', num: true, fmt: number }, { key: 'cost', label: 'Cost per run (USD)', num: true, fmt: (value) => number(value, 8) },++ ], rows: rows.map((row) => ({ provider: row.provider, task: row.task_kind, runs: row.runs, median: row.latency_p50_ms, p95: row.latency_p95_ms, first: row.ttft_p50_ms, cost: row.cost_per_run_usd })) }) : '<p>No completed day has twenty qualifying runs for this model.</p>';++ const rating = analysis.rating;++ const quality = rating ? `<p>${esc(rating.subject)}: ${number(rating.elo)} (${number(rating.ci_lower)}–${number(rating.ci_upper)}), ${number(rating.games, 0)} games.</p>` : `<p>${analysis.ratings?.unavailable ? 'The rating source is unavailable.' : 'No single model rating has twenty qualifying games.'}</p>`;++ return kit.section({ title: 'Measured performance', body: measured + `<p><a href="/api/v1/analysis/models/${encodeURIComponent(analysis.model)}?rating_kind=${encodeURIComponent(analysis.ratings?.kind || 'chat.arena')}">Metrics and rating data</a></p>` })++ + kit.section({ title: `Bench rating: ${analysis.ratings?.kind || 'chat.arena'}`, body: quality + (analysis.ratings?.stale ? '<p>This is the last cached rating; Bench is unavailable.</p>' : '') + '<p>Source: OpenVibe.Bench. Latency and cost come from OpenVibe.AI receipts.</p>' });++}++module.exports = { modelFacts };@@@ -310,7 +310,7 @@ ${communityPools(community)}}// ── One model ──────────────────────────────────────────────--function modelPage({ model, viewer, result = null, problem = null, text = '', adult = false, community = null }) {++function modelPage({ model, viewer, result = null, problem = null, text = '', adult = false, community = null, analysis = null }) {const offers = model.offers || [];const best = cheapest(model);const badges = [model.uncensored ? kit.badge('18+', 'bad') : '', model.community ? kit.badge('Community', 'accent') : '', offers.some(freeTier) ? kit.badge('Free to try') : (offers.some(freePaused) ? kit.badge('Free paused') : '')].join('');@@@ -367,6 +367,7 @@ ${stats.length ? kit.stats(stats) : ''}${model.description ? `<p class="ex-prose">${esc(model.description)}</p>` : ''}${panel({ title: 'Every offer that serves it', note: 'Price in, out and cached per million tokens; speed and uptime from the last seven days of real requests; the privacy label and the provider’s own data policy.', id: 'offers', body: kit.table({ columns, rows, label: `Offers for ${model.name}`, empty: 'No live offer serves this model right now.' }) })}${activityPanel(model.activity)}++${require('./analysis').modelFacts(analysis)}${panel({ title: 'Presets', note: 'A preset fixes the router’s floor, ceiling and policy, and you name it as the model.', id: 'presets', body: presetsBox() })}${panel({ title: 'Try it', id: 'try', body: `<p class="ex-actions">${kit.button({ href: chatModel(model), label: `Chat with ${model.name}` })} ${kit.button({ href: `https://openvibe.report/ai/models/${encodeURIComponent(model.slug || model.canonical)}`, label: 'How it ranks', kind: 'quiet' })}</p>` })}${panel({ title: 'Use it from code', id: 'code', body: `${kit.snippet({ code, id: 'model-curl' })}nnew file mode 100644@@@ -0,0 +1,63 @@++'use strict';++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { createFakeAi } = require('./helpers/ai');++const { createAnalysis, ratingFor } = require('../server/analysis');++const { createBench } = require('../server/bench');++(async () => {++ const ai = createFakeAi();++ const rating = { subject: 'deepseek-chat', kind_id: 'chat.arena', category: 'all', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 };++ const bench = { board: async () => ({ minimum_samples: 20, rows: [rating, { subject: 'qwen3-32b', games: 19, elo: 1500 }] }) };++ const cell = { day: '2026-11-09', model: 'deepseek/deepseek-chat', provider: 'deepseek', task_kind: 'chat', runs: 40, latency_p50_ms: 1234, latency_p95_ms: 2500, ttft_p50_ms: null, cost_per_run_usd: 0.001 };++ let reads = 0;++ const analysis = createAnalysis({ config: { ai: { url: 'https://unused.test', timeoutMs: 2000 } }, ai, bench, metrics: {++ model: async () => ({ series: [cell, { ...cell, provider: 'sparse', runs: 19 }] }),++ compare: async ({ models }) => { reads++; return { day: cell.day, minimum_samples: 20, models: models.map((model) => ({ model, cells: [{ ...cell, model }] })) }; },++ } });++ const t = await boot({ ai, bench, analysis });++ try {++ await check('comparison joins unchanged Bench scores to measured cells and withholds small samples', async () => {++ const response = await t.get('/api/v1/analysis/compare?models=deepseek/deepseek-chat,qwen/qwen3-32b');++ assert.strictEqual(response.status, 200, response.text);++ assert.strictEqual(response.headers.get('access-control-allow-origin'), '*');++ const preflight = await t.get('/api/v1/analysis/compare', { method: 'OPTIONS' });++ assert.strictEqual(preflight.status, 204);++ assert.match(preflight.headers.get('access-control-allow-headers'), /traceparent/);++ const data = response.json();++ assert.deepStrictEqual(data.models[0].rating, rating);++ assert.strictEqual(data.models[1].rating, null);++ assert.strictEqual(data.models[0].cells[0].latency_p50_ms, 1234);++ await t.get('/api/v1/analysis/compare?models=deepseek/deepseek-chat,qwen/qwen3-32b');++ assert.strictEqual(reads, 1, 'public comparison is cached');++ assert.strictEqual((await t.get('/api/v1/analysis/compare?models=a,b,c,d,e,f,g')).status, 422);++ assert.strictEqual((await t.get('/api/v1/analysis/compare?models=a&models=b')).status, 422);++ });++ await check('model page and JSON show the same measured latency and Bench rating', async () => {++ const data = (await t.get('/api/v1/analysis/models/deepseek%2Fdeepseek-chat')).json();++ assert.strictEqual(data.measured.series.length, 1);++ assert.deepStrictEqual(data.rating, rating);++ const page = await t.get('/m/deepseek/deepseek-chat');++ assert.strictEqual(page.status, 200, page.text);++ assert(page.text.includes('1,234') && page.text.includes('1,100') && page.text.includes('chat.arena'));++ assert(!page.text.includes('data-label="Provider">sparse'));++ });++ await check('several dated subjects never become an invented family rating', async () => {++ const model = { slug: 'openai/gpt-5-mini', offers: [{ model: 'gpt-5-mini' }] };++ const one = { ...rating, subject: 'gpt-5-mini-2025-08-07' };++ assert.deepStrictEqual(ratingFor(model, [one]), one);++ assert.strictEqual(ratingFor(model, [one, { ...one, subject: 'gpt-5-mini-2026-08-07' }]), null);++ });++ await check('a public Bench read works without service credentials', async () => {++ let path;++ const publicBench = createBench({ config: { oauth: {}, bench: { url: 'https://bench.test', timeoutMs: 2000 } }, fetchImpl: async (url, init) => {++ path = String(url);++ assert(!new Headers(init.headers).has('authorization'));++ return new Response(JSON.stringify({ board: { rows: [rating, { ...rating, games: 19 }] } }), { headers: { 'Content-Type': 'application/json' } });++ } });++ const data = await publicBench.board();++ assert.strictEqual(path, 'https://bench.test/api/v1/ratings/chat.arena?category=all');++ assert.deepStrictEqual(data.rows, [rating]);++ });++ } finally { await t.close(); }++ done();++})().catch((err) => { console.error(err); process.exitCode = 1; });@@@ -44,7 +44,8 @@ async function boot(opts = {}) {// The M2 memory client goes out with this fetch. A test that does not stand one up gets an empty store, so// no test ever waits on the network (the notes tests inject their own, in test/helpers or the file itself).const emptyMemory = async () => ({ ok: true, status: 200, text: async () => '{"items":[]}', json: async () => ({ items: [] }) });-- const built = await createApp({ config, store, log, telemetrySink: opts.telemetrySink, accountSend: opts.accountSend, ai: opts.ai, aiFetchImpl: opts.aiFetchImpl || emptyMemory, bench: opts.bench, benchFetchImpl: opts.benchFetchImpl, openapiFetch: opts.openapiFetch, indexnow: opts.indexnow, fetchImpl: opts.fetchImpl });++ const emptyRatings = async () => new Response(JSON.stringify({ board: { minimum_samples: 20, rows: [] } }), { headers: { 'content-type': 'application/json' } });++ const built = await createApp({ config, store, log, telemetrySink: opts.telemetrySink, accountSend: opts.accountSend, ai: opts.ai, aiFetchImpl: opts.aiFetchImpl || emptyMemory, bench: opts.bench, benchFetchImpl: opts.benchFetchImpl || emptyRatings, analysis: opts.analysis, openapiFetch: opts.openapiFetch, indexnow: opts.indexnow, fetchImpl: opts.fetchImpl });await built.ctx.keys.ensure();const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });const base = `http://127.0.0.1:${server.address().port}`;
@@ -9,11 +9,11 @@
@@ -9,11 +9,11 @@
"port": "5150",
"port": "5150",
"domain": "openvibe.exchange",
"domain": "openvibe.exchange",
"contracts": "openvibe-contracts v0.384.0",
"contracts": "openvibe-contracts v0.384.0",
- "sdk": "openvibe-sdk v0.133.0",
+ "sdk": "openvibe-sdk v0.145.0",
"shared": "openvibe-shared v4.197.0",
"shared": "openvibe-shared v4.197.0",
"packages": [
"packages": [
"openvibe-contracts v0.384.0",
"openvibe-contracts v0.384.0",
- "openvibe-sdk v0.133.0",
+ "openvibe-sdk v0.145.0",
"openvibe-shared v4.197.0"
"openvibe-shared v4.197.0"
],
],
"summary": "OpenVibe.Exchange is the network's AI front. It reads OpenVibe.AI's public catalog and aggregate usage and renders: the home (popular now, cheapest per million, uncensored, community machines, live numbers), /models and /m/<org>/<model> (every offer with price in/out, measured p50/p95, uptime and data policy, a Try it box, curl and the SDK), /chat (one to three models side by side, system prompt, temperature, history in the account, on openvibe-shared/thread), /api, /sell (Link's offer to the network; what is not sold, said plainly), /uncensored (an 18+ confirmation on the account and the hard floor) and /ai (which products run which models, and the daily cost). Chat and Try it are ai.complete runs on the person's behalf (on_behalf_of) with the service's ai.run.create token. Exchange v1 (free/swap listings, passports, sandboxes, installs, offers) is retired: every v1 page answers 410 with a pointer, its JSON API answers 410 exchange.retired, and its tables stay read-only; its tests live in test/retired/.",
"summary": "OpenVibe.Exchange is the network's AI front. It reads OpenVibe.AI's public catalog and aggregate usage and renders: the home (popular now, cheapest per million, uncensored, community machines, live numbers), /models and /m/<org>/<model> (every offer with price in/out, measured p50/p95, uptime and data policy, a Try it box, curl and the SDK), /chat (one to three models side by side, system prompt, temperature, history in the account, on openvibe-shared/thread), /api, /sell (Link's offer to the network; what is not sold, said plainly), /uncensored (an 18+ confirmation on the account and the hard floor) and /ai (which products run which models, and the daily cost). Chat and Try it are ai.complete runs on the person's behalf (on_behalf_of) with the service's ai.run.create token. Exchange v1 (free/swap listings, passports, sandboxes, installs, offers) is retired: every v1 page answers 410 with a pointer, its JSON API answers 410 exchange.retired, and its tables stay read-only; its tests live in test/retired/.",
new file mode 100644
new file mode 100644
@@ -0,0 +1,27 @@
@@ -0,0 +1,27 @@
+# Shared analysis
+
+- `/api/v1/analysis/compare?models=openai/gpt-5-nano,openai/gpt-5-mini&kind=other&rating_kind=chat.arena`
+ reads AI receipt metrics for up to six models and attaches each unchanged Bench rating.
+- `/api/v1/analysis/models/openai%2Fgpt-5-nano?days=30&rating_kind=chat.arena`
+ returns the daily series and the same rating used by the model page. Days may be 30 or 90.
+- `/api/v1/analysis/ratings/:kind` reads the same public Bench board as Report.
+
+Reads cache for sixty seconds, allow public browser GETs, and use SDK clients.
+Metrics remain separate by day/provider/task; medians are never averaged.
+Ratings below twenty eligible games and receipt cells below twenty runs are
+withheld. Missing evidence is null or empty. Historical TTFT can remain null.
+
+The model join first uses an exact catalogue/offer subject, then an unambiguous
+dated API name. It returns the Bench subject with the score. Several rated
+revisions never become an invented family score. Design and Exchange already
+send their arena votes to Bench. Exchange's local arena tables still retain
+round/result links and vote deduplication, not a second rating computation.
+Design's active store already reads fitted Bench boards; its old rating table
+is retained only for migration compatibility. Existing provisional Bench/Design
+views remain; the new cross-site reads use the twenty-game floor.
+
+For Fable: the existing model page has plain kit tables for the latest completed
+day's runs, p50/p95 latency, TTFT and cost per run, and a plain Bench rating block
+with its interval and games. Comparison JSON supplies those fields per model and
+provider, plus `rating_kind` and a shared rating. The source-unavailable, cached
+rating, sparse-data and ambiguous-model states need their look. No CSS was added.
@@ -18,7 +18,7 @@
@@ -18,7 +18,7 @@
"multer": "^2.5.0",
"multer": "^2.5.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
@@ -1221,9 +1221,9 @@
@@ -1221,9 +1221,9 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.133.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",
- "integrity": "sha512-qn+y/yQnlYkfm0+tDpKLwUw04L0fZydcyYiEM6OaXzc7iclgp2HAcBB/OkmPXWOCwXGyv2CBQAJeEVesgkWBFA==",
+ "version": "0.145.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
+ "integrity": "sha512-3kEEiEQSzgRHoCNdOQ/xmqNRNNf/QmJ269E6rPASDHpuciUTOwZkjIVMMTfGIf6VGBe73TdYdLjqMAMThSn93g==",
"license": "MIT",
"license": "MIT",
"bin": {
"bin": {
"openvibe-download": "bin/openvibe-download.js"
"openvibe-download": "bin/openvibe-download.js"
@@ -2,7 +2,7 @@
@@ -2,7 +2,7 @@
"name": "openvibe-exchange",
"name": "openvibe-exchange",
"version": "0.1.0",
"version": "0.1.0",
"private": true,
"private": true,
- "description": "OpenVibe.Exchange — trade what people make.",
+ "description": "OpenVibe.Exchange \u2014 trade what people make.",
"main": "server/index.js",
"main": "server/index.js",
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
@@ -27,7 +27,7 @@
@@ -27,7 +27,7 @@
"multer": "^2.5.0",
"multer": "^2.5.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.384.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.133.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.145.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.197.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
new file mode 100644
new file mode 100644
@@ -0,0 +1,56 @@
@@ -0,0 +1,56 @@
+'use strict';
+const { createClient } = require('openvibe-sdk/core');
+const { createAiClient } = require('openvibe-sdk/ai');
+
+const bare = (v) => String(v || '').toLowerCase().replace(/^[^/]+\//, '').replace(/[-_@:](\d{4}-?\d{2}-?\d{2}|latest|free)$/, '');
+const validKind = (kind) => typeof kind === 'string' && /^[a-z0-9][a-z0-9_.-]{0,79}$/.test(kind);
+function ratingFor(model, rows = []) {
+ const candidates = [model.slug, model.canonical, model.id, ...(model.offers || []).map((offer) => offer.model)].filter(Boolean);
+ const eligible = rows.filter((row) => row.games >= 20 && Number.isFinite(row.elo));
+ const exact = eligible.filter((row) => candidates.includes(row.subject));
+ const matches = exact.length ? exact : eligible.filter((row) => candidates.some((id) => bare(id) === bare(row.subject)));
+ // A family with several rated revisions has no single rating: keep each Bench subject distinct.
+ return matches.length === 1 ? matches[0] : null;
+}
+
+function createAnalysis({ config, ai, bench, fetchImpl = globalThis.fetch, now = () => Date.now(), metrics = null }) {
+ const source = metrics || createAiClient(createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl: config.ai.url }).metrics;
+ const cache = new Map();
+ async function read(key, loader) {
+ const hit = cache.get(key);
+ if (hit && hit.until > now()) return hit.value;
+ const value = await loader();
+ cache.set(key, { until: now() + 60000, value });
+ if (cache.size > 128) cache.delete(cache.keys().next().value);
+ return value;
+ }
+ function checkedKind(value = 'chat.arena') {
+ if (!validKind(value)) throw new TypeError('Use a Bench task kind.');
+ return value;
+ }
+ const ratingSource = (board, kind) => ({ source: 'OpenVibe.Bench', kind, minimum_samples: 20, unavailable: Boolean(board?.unavailable), stale: Boolean(board?.stale) });
+ async function model(model, { rating_kind = 'chat.arena', days = 30 } = {}) {
+ const kind = checkedKind(rating_kind);
+ return read(`model:${model.slug}:${kind}:${days}`, async () => {
+ const [measured, board] = await Promise.all([
+ source.model(model.slug, { days }, { signal: AbortSignal.timeout(config.ai.timeoutMs) }), bench.board({ kind }),
+ ]);
+ return { schema: 'exchange.analysis@1', model: model.slug, measured: { ...measured, series: (measured.series || []).filter((cell) => cell.runs >= 20) }, rating: ratingFor(model, board.rows), ratings: ratingSource(board, kind) };
+ });
+ }
+ async function compare({ models, measures, kind, day, rating_kind = 'chat.arena' }) {
+ const ratingKind = checkedKind(rating_kind);
+ if (!Array.isArray(models) || !models.length || models.length > 6 || new Set(models).size !== models.length) throw new TypeError('Choose one to six distinct models.');
+ return read(`compare:${JSON.stringify([models, measures, kind, day, ratingKind])}`, async () => {
+ const [measured, board, catalog] = await Promise.all([
+ source.compare({ models, measures, kind, day }, { signal: AbortSignal.timeout(config.ai.timeoutMs) }),
+ bench.board({ kind: ratingKind }), ai.catalog(),
+ ]);
+ return { ...measured, schema: 'exchange.analysis@1', ratings: ratingSource(board, ratingKind), models: (measured.models || []).map((item) => ({
+ ...item, cells: (item.cells || []).filter((cell) => cell.runs >= 20), rating: ratingFor((catalog.models || []).find((entry) => entry.slug === item.model) || { slug: item.model }, board.rows),
+ })) };
+ });
+ }
+ return { model, compare, metrics: source };
+}
+module.exports = { createAnalysis, ratingFor };
@@ -84,6 +84,7 @@ async function createApp(opts = {}) {
@@ -84,6 +84,7 @@ async function createApp(opts = {}) {
// Network, read and changed with the person's own token: their machines and their machine pools.
// Network, read and changed with the person's own token: their machines and their machine pools.
const network = opts.network || createNetwork({ config, fetchImpl, log });
const network = opts.network || createNetwork({ config, fetchImpl, log });
const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });
const bench = opts.bench || createBench({ config, fetchImpl: opts.benchFetchImpl || globalThis.fetch, log, now: s.now });
+ const analysis = opts.analysis || require('./analysis').createAnalysis({ config, ai, bench, fetchImpl: opts.aiFetchImpl || globalThis.fetch, now: s.now });
const arenaStore = createArenaStore(s);
const arenaStore = createArenaStore(s);
const history = createCatalogHistory({ s, log });
const history = createCatalogHistory({ s, log });
const chats = createChatStore(s);
const chats = createChatStore(s);
@@ -115,7 +116,7 @@ async function createApp(opts = {}) {
@@ -115,7 +116,7 @@ async function createApp(opts = {}) {
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });
: async () => { throw new Error('OV_OAUTH_CLIENT_SECRET is not set: Exchange cannot answer account events'); });
const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });
const openapi = createOpenApi({ config, fetchImpl: opts.openapiFetch || globalThis.fetch, log });
- const ctx = { config, s, keys, sso, principal, ai, network, bench, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };
+ const ctx = { config, s, keys, sso, principal, ai, network, bench, analysis, arenaStore, history, openapi, chats, adult, adultAccount, characters, companionChats, groups, voice, sharing, accountData, accountSend, memory, notes, lorebooks, locker, noteTaker, indexnow, log };
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
@@ -252,6 +253,8 @@ async function createApp(opts = {}) {
@@ -252,6 +253,8 @@ async function createApp(opts = {}) {
return res.redirect(301, `https://openvibe.chat/vibe${suffix}`);
return res.redirect(301, `https://openvibe.chat/vibe${suffix}`);
});
});
+ app.use('/api/v1/analysis', rateLimit({ windowMs: 60000, limit: Number(process.env.EXCHANGE_API_RATE_LIMIT_PER_MIN) || 120, standardHeaders: true, legacyHeaders: false }), require('./http/analysis').createAnalysisRoutes(ctx));
+
// ── Exchange v1's JSON API: gone ────────────────────────
// ── Exchange v1's JSON API: gone ────────────────────────
app.use('/api/v1', (req, res) => {
app.use('/api/v1', (req, res) => {
res.set('Cache-Control', 'public, max-age=86400');
res.set('Cache-Control', 'public, max-age=86400');
@@ -21,6 +21,7 @@ async function jsonOf(res) {
@@ -21,6 +21,7 @@ async function jsonOf(res) {
}
}
function createBench({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {
function createBench({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {
+ const shared = require('openvibe-sdk/bench').createBenchClient(require('openvibe-sdk/core').createClient({ fetch: fetchImpl, retries: 0 }), { baseUrl: config.bench.url });
const tokens = config.oauth.clientSecret ? createServiceTokenClient({
const tokens = config.oauth.clientSecret ? createServiceTokenClient({
tokenUrl: `${config.networkInternalUrl}/oauth/token`,
tokenUrl: `${config.networkInternalUrl}/oauth/token`,
clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
@@ -82,8 +83,8 @@ function createBench({ config, fetchImpl = globalThis.fetch, log = console, now
@@ -82,8 +83,8 @@ function createBench({ config, fetchImpl = globalThis.fetch, log = console, now
const hit = boardHit.get(key);
const hit = boardHit.get(key);
if (hit && now() - hit.at < BOARD_MS) return hit.value;
if (hit && now() - hit.at < BOARD_MS) return hit.value;
try {
try {
- const body = await call('GET', `/api/v1/boards/${encodeURIComponent(kind)}?category=${encodeURIComponent(category)}`);
- const value = body && body.board ? body.board : { unavailable: true, kind_id: kind, category, rows: [] };
+ const body = await shared.ratings.get(kind, { category, signal: AbortSignal.timeout(config.bench.timeoutMs) });
+ const value = body && body.board ? { ...body.board, rows: (body.board.rows || []).filter((row) => row.games >= 20) } : { unavailable: true, kind_id: kind, category, rows: [] };
boardHit.set(key, { at: now(), value });
boardHit.set(key, { at: now(), value });
if (boardHit.size > 16) boardHit.delete(boardHit.keys().next().value);
if (boardHit.size > 16) boardHit.delete(boardHit.keys().next().value);
return value;
return value;
new file mode 100644
new file mode 100644
@@ -0,0 +1,33 @@
@@ -0,0 +1,33 @@
+'use strict';
+const { asyncRouter } = require('./router');
+function createAnalysisRoutes({ analysis, ai, bench }) {
+ const router = asyncRouter();
+ router.use((_req, res, next) => {
+ res.set('Access-Control-Allow-Origin', '*').set('Access-Control-Allow-Methods', 'GET, OPTIONS').set('Access-Control-Allow-Headers', 'Accept, Content-Type, traceparent, X-OpenVibe-Request-Id');
+ res.set('Cache-Control', 'public, max-age=60');
+ next();
+ });
+ router.options('*', (_req, res) => res.status(204).end());
+ const fail = (res, err) => res.set('Cache-Control', 'no-store').status(err instanceof TypeError || err.status === 422 ? 422 : 503).json({ error: { code: err instanceof TypeError || err.status === 422 ? 'analysis.invalid' : 'analysis.unavailable', detail: err instanceof TypeError ? err.message : 'Analysis is unavailable.' } });
+ router.get('/compare', async (req, res) => {
+ try {
+ if (req.originalUrl.length > 2400 || Object.values(req.query).some((value) => typeof value !== 'string')) throw new TypeError('Use one value per comparison parameter.');
+ return res.json(await analysis.compare({ ...req.query, models: String(req.query.models || '').split(',').filter(Boolean), measures: req.query.measures?.split(',') }));
+ } catch (err) { return fail(res, err); }
+ });
+ router.get('/models/:model', async (req, res) => {
+ try {
+ const model = (await ai.catalog()).models?.find((entry) => entry.slug === req.params.model);
+ if (!model) return res.status(404).json({ error: { code: 'model.not_found' } });
+ return res.json(await analysis.model(model, { rating_kind: req.query.rating_kind, days: Number(req.query.days || 30) }));
+ } catch (err) { return fail(res, err); }
+ });
+ router.get('/ratings/:kind', async (req, res) => {
+ if (!/^[a-z0-9][a-z0-9_.-]{0,79}$/.test(req.params.kind)) return fail(res, new TypeError('Use a Bench task kind.'));
+ const board = await bench.board({ kind: req.params.kind });
+ if (board.unavailable) res.set('Cache-Control', 'no-store');
+ return res.status(board.unavailable ? 503 : 200).json({ board });
+ });
+ return router;
+}
+module.exports = { createAnalysisRoutes };
@@ -140,6 +140,9 @@ function createPageRoutes(ctx) {
@@ -140,6 +140,9 @@ function createPageRoutes(ctx) {
return page(req, res, { title: 'Not found', index: false, body: raw(`<div class="ex-wrap ex-narrow">${kit.empty({ icon: 'search', title: 'No such model', text: `Nothing here serves ${esc(`${req.params.org}/${req.params.model}`)}. <a href="/models">Browse the models</a>.` })}</div>`) }, 404);
return page(req, res, { title: 'Not found', index: false, body: raw(`<div class="ex-wrap ex-narrow">${kit.empty({ icon: 'search', title: 'No such model', text: `Nothing here serves ${esc(`${req.params.org}/${req.params.model}`)}. <a href="/models">Browse the models</a>.` })}</div>`) }, 404);
}
}
const isAdult = signedIn(req) ? await adult.confirmed(owner(req)) : false;
const isAdult = signedIn(req) ? await adult.confirmed(owner(req)) : false;
+ let analysis = null;
+ try { analysis = await ctx.analysis.model(model, { rating_kind: typeof req.query.rating_kind === 'string' ? req.query.rating_kind : 'chat.arena' }); }
+ catch (err) { log.warn('[exchange] model analysis unavailable: ' + String(err.message || err).slice(0, 160)); }
const jsonLd = seo.jsonLd.webPage({
const jsonLd = seo.jsonLd.webPage({
name: `${model.name} on OpenVibe.Exchange`, url: `${config.baseUrl}${modelPath(model)}`,
name: `${model.name} on OpenVibe.Exchange`, url: `${config.baseUrl}${modelPath(model)}`,
description: `${model.name}: ${model.offers.length} live offers with price, speed and uptime.`.slice(0, 160), siteUrl: config.baseUrl,
description: `${model.name}: ${model.offers.length} live offers with price, speed and uptime.`.slice(0, 160), siteUrl: config.baseUrl,
@@ -148,7 +151,7 @@ function createPageRoutes(ctx) {
@@ -148,7 +151,7 @@ function createPageRoutes(ctx) {
index: true, cache: req.method === 'GET' ? PUBLIC_CACHE : null, agentLayer: true,
index: true, cache: req.method === 'GET' ? PUBLIC_CACHE : null, agentLayer: true,
title: model.name, description: `${model.name}: price, speed, uptime and data policy of every offer.`,
title: model.name, description: `${model.name}: price, speed, uptime and data policy of every offer.`,
jsonLd: [jsonLd], head: '',
jsonLd: [jsonLd], head: '',
- body: raw(views.modelPage({ model, viewer: req.viewer, result, problem, text, adult: isAdult, community })),
+ body: raw(views.modelPage({ model, viewer: req.viewer, result, problem, text, adult: isAdult, community, analysis })),
}, status);
}, status);
}
}
@@ -14,9 +14,20 @@ const FALLBACK = (config) => ({
@@ -14,9 +14,20 @@ const FALLBACK = (config) => ({
description: 'One OpenAI- and Anthropic-compatible endpoint over every model OpenVibe routes.',
description: 'One OpenAI- and Anthropic-compatible endpoint over every model OpenVibe routes.',
},
},
servers: [{ url: config.api.publicUrl }],
servers: [{ url: config.api.publicUrl }],
- paths: {},
+ paths: analysisPaths(config),
});
});
+function analysisPaths(config) {
+ const make = (id, parameters) => ({ get: { operationId: id, security: [], servers: [{ url: config.baseUrl }], parameters, responses: { '200': { description: 'Public measured performance and shared Bench ratings; minimum twenty samples' }, '422': { description: 'Invalid selection' }, '503': { description: 'Source unavailable' } } } });
+ const query = (name) => ({ name, in: 'query', schema: { type: 'string' } });
+ const param = (name) => ({ name, in: 'path', required: true, schema: { type: 'string' } });
+ return {
+ '/api/v1/analysis/compare': make('exchange.analysis.compare', ['models', 'measures', 'kind', 'day', 'rating_kind'].map(query)),
+ '/api/v1/analysis/models/{model}': make('exchange.analysis.model', [param('model'), query('days'), query('rating_kind')]),
+ '/api/v1/analysis/ratings/{kind}': make('exchange.analysis.ratings', [param('kind')]),
+ };
+}
+
function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {
function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() }) {
const ttl = config.api.openapiCacheMs;
const ttl = config.api.openapiCacheMs;
let at = 0;
let at = 0;
@@ -31,6 +42,7 @@ function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, no
@@ -31,6 +42,7 @@ function createOpenApi({ config, fetchImpl = globalThis.fetch, log = console, no
const doc = await res.json();
const doc = await res.json();
if (!doc || typeof doc !== 'object' || typeof doc.openapi !== 'string') throw new Error('openapi document is not OpenAPI');
if (!doc || typeof doc !== 'object' || typeof doc.openapi !== 'string') throw new Error('openapi document is not OpenAPI');
doc.servers = [{ url: config.api.publicUrl }];
doc.servers = [{ url: config.api.publicUrl }];
+ doc.paths = { ...(doc.paths || {}), ...analysisPaths(config) };
return doc;
return doc;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,20 @@
@@ -0,0 +1,20 @@
+'use strict';
+const kit = require('openvibe-shared/kit');
+const { esc } = require('../render/html');
+const number = (value, digits = 2) => value == null ? '—' : Number(value).toLocaleString('en-US', { maximumFractionDigits: digits });
+function modelFacts(analysis) {
+ if (!analysis) return kit.section({ title: 'Measured performance', body: '<p>Measured performance is unavailable.</p>' });
+ const series = (analysis.measured?.series || []).filter((row) => row.runs >= 20);
+ const latest = series.reduce((day, row) => row.day > day ? row.day : day, '');
+ const rows = series.filter((row) => row.day === latest);
+ const measured = rows.length ? kit.table({ caption: `Measured runs on ${latest}`, columns: [
+ { key: 'provider', label: 'Provider' }, { key: 'task', label: 'Task kind' }, { key: 'runs', label: 'Runs', num: true },
+ { key: 'median', label: 'Median latency (ms)', num: true, fmt: number }, { key: 'p95', label: '95th percentile (ms)', num: true, fmt: number },
+ { key: 'first', label: 'First token (ms)', num: true, fmt: number }, { key: 'cost', label: 'Cost per run (USD)', num: true, fmt: (value) => number(value, 8) },
+ ], rows: rows.map((row) => ({ provider: row.provider, task: row.task_kind, runs: row.runs, median: row.latency_p50_ms, p95: row.latency_p95_ms, first: row.ttft_p50_ms, cost: row.cost_per_run_usd })) }) : '<p>No completed day has twenty qualifying runs for this model.</p>';
+ const rating = analysis.rating;
+ const quality = rating ? `<p>${esc(rating.subject)}: ${number(rating.elo)} (${number(rating.ci_lower)}–${number(rating.ci_upper)}), ${number(rating.games, 0)} games.</p>` : `<p>${analysis.ratings?.unavailable ? 'The rating source is unavailable.' : 'No single model rating has twenty qualifying games.'}</p>`;
+ return kit.section({ title: 'Measured performance', body: measured + `<p><a href="/api/v1/analysis/models/${encodeURIComponent(analysis.model)}?rating_kind=${encodeURIComponent(analysis.ratings?.kind || 'chat.arena')}">Metrics and rating data</a></p>` })
+ + kit.section({ title: `Bench rating: ${analysis.ratings?.kind || 'chat.arena'}`, body: quality + (analysis.ratings?.stale ? '<p>This is the last cached rating; Bench is unavailable.</p>' : '') + '<p>Source: OpenVibe.Bench. Latency and cost come from OpenVibe.AI receipts.</p>' });
+}
+module.exports = { modelFacts };
@@ -310,7 +310,7 @@ ${communityPools(community)}
@@ -310,7 +310,7 @@ ${communityPools(community)}
}
}
// ── One model ──────────────────────────────────────────────
// ── One model ──────────────────────────────────────────────
-function modelPage({ model, viewer, result = null, problem = null, text = '', adult = false, community = null }) {
+function modelPage({ model, viewer, result = null, problem = null, text = '', adult = false, community = null, analysis = null }) {
const offers = model.offers || [];
const offers = model.offers || [];
const best = cheapest(model);
const best = cheapest(model);
const badges = [model.uncensored ? kit.badge('18+', 'bad') : '', model.community ? kit.badge('Community', 'accent') : '', offers.some(freeTier) ? kit.badge('Free to try') : (offers.some(freePaused) ? kit.badge('Free paused') : '')].join('');
const badges = [model.uncensored ? kit.badge('18+', 'bad') : '', model.community ? kit.badge('Community', 'accent') : '', offers.some(freeTier) ? kit.badge('Free to try') : (offers.some(freePaused) ? kit.badge('Free paused') : '')].join('');
@@ -367,6 +367,7 @@ ${stats.length ? kit.stats(stats) : ''}
@@ -367,6 +367,7 @@ ${stats.length ? kit.stats(stats) : ''}
${model.description ? `<p class="ex-prose">${esc(model.description)}</p>` : ''}
${model.description ? `<p class="ex-prose">${esc(model.description)}</p>` : ''}
${panel({ title: 'Every offer that serves it', note: 'Price in, out and cached per million tokens; speed and uptime from the last seven days of real requests; the privacy label and the provider’s own data policy.', id: 'offers', body: kit.table({ columns, rows, label: `Offers for ${model.name}`, empty: 'No live offer serves this model right now.' }) })}
${panel({ title: 'Every offer that serves it', note: 'Price in, out and cached per million tokens; speed and uptime from the last seven days of real requests; the privacy label and the provider’s own data policy.', id: 'offers', body: kit.table({ columns, rows, label: `Offers for ${model.name}`, empty: 'No live offer serves this model right now.' }) })}
${activityPanel(model.activity)}
${activityPanel(model.activity)}
+${require('./analysis').modelFacts(analysis)}
${panel({ title: 'Presets', note: 'A preset fixes the router’s floor, ceiling and policy, and you name it as the model.', id: 'presets', body: presetsBox() })}
${panel({ title: 'Presets', note: 'A preset fixes the router’s floor, ceiling and policy, and you name it as the model.', id: 'presets', body: presetsBox() })}
${panel({ title: 'Try it', id: 'try', body: `<p class="ex-actions">${kit.button({ href: chatModel(model), label: `Chat with ${model.name}` })} ${kit.button({ href: `https://openvibe.report/ai/models/${encodeURIComponent(model.slug || model.canonical)}`, label: 'How it ranks', kind: 'quiet' })}</p>` })}
${panel({ title: 'Try it', id: 'try', body: `<p class="ex-actions">${kit.button({ href: chatModel(model), label: `Chat with ${model.name}` })} ${kit.button({ href: `https://openvibe.report/ai/models/${encodeURIComponent(model.slug || model.canonical)}`, label: 'How it ranks', kind: 'quiet' })}</p>` })}
${panel({ title: 'Use it from code', id: 'code', body: `${kit.snippet({ code, id: 'model-curl' })}
${panel({ title: 'Use it from code', id: 'code', body: `${kit.snippet({ code, id: 'model-curl' })}
new file mode 100644
new file mode 100644
@@ -0,0 +1,63 @@
@@ -0,0 +1,63 @@
+'use strict';
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { createFakeAi } = require('./helpers/ai');
+const { createAnalysis, ratingFor } = require('../server/analysis');
+const { createBench } = require('../server/bench');
+(async () => {
+ const ai = createFakeAi();
+ const rating = { subject: 'deepseek-chat', kind_id: 'chat.arena', category: 'all', games: 20, elo: 1100, ci_lower: 1050, ci_upper: 1150 };
+ const bench = { board: async () => ({ minimum_samples: 20, rows: [rating, { subject: 'qwen3-32b', games: 19, elo: 1500 }] }) };
+ const cell = { day: '2026-11-09', model: 'deepseek/deepseek-chat', provider: 'deepseek', task_kind: 'chat', runs: 40, latency_p50_ms: 1234, latency_p95_ms: 2500, ttft_p50_ms: null, cost_per_run_usd: 0.001 };
+ let reads = 0;
+ const analysis = createAnalysis({ config: { ai: { url: 'https://unused.test', timeoutMs: 2000 } }, ai, bench, metrics: {
+ model: async () => ({ series: [cell, { ...cell, provider: 'sparse', runs: 19 }] }),
+ compare: async ({ models }) => { reads++; return { day: cell.day, minimum_samples: 20, models: models.map((model) => ({ model, cells: [{ ...cell, model }] })) }; },
+ } });
+ const t = await boot({ ai, bench, analysis });
+ try {
+ await check('comparison joins unchanged Bench scores to measured cells and withholds small samples', async () => {
+ const response = await t.get('/api/v1/analysis/compare?models=deepseek/deepseek-chat,qwen/qwen3-32b');
+ assert.strictEqual(response.status, 200, response.text);
+ assert.strictEqual(response.headers.get('access-control-allow-origin'), '*');
+ const preflight = await t.get('/api/v1/analysis/compare', { method: 'OPTIONS' });
+ assert.strictEqual(preflight.status, 204);
+ assert.match(preflight.headers.get('access-control-allow-headers'), /traceparent/);
+ const data = response.json();
+ assert.deepStrictEqual(data.models[0].rating, rating);
+ assert.strictEqual(data.models[1].rating, null);
+ assert.strictEqual(data.models[0].cells[0].latency_p50_ms, 1234);
+ await t.get('/api/v1/analysis/compare?models=deepseek/deepseek-chat,qwen/qwen3-32b');
+ assert.strictEqual(reads, 1, 'public comparison is cached');
+ assert.strictEqual((await t.get('/api/v1/analysis/compare?models=a,b,c,d,e,f,g')).status, 422);
+ assert.strictEqual((await t.get('/api/v1/analysis/compare?models=a&models=b')).status, 422);
+ });
+ await check('model page and JSON show the same measured latency and Bench rating', async () => {
+ const data = (await t.get('/api/v1/analysis/models/deepseek%2Fdeepseek-chat')).json();
+ assert.strictEqual(data.measured.series.length, 1);
+ assert.deepStrictEqual(data.rating, rating);
+ const page = await t.get('/m/deepseek/deepseek-chat');
+ assert.strictEqual(page.status, 200, page.text);
+ assert(page.text.includes('1,234') && page.text.includes('1,100') && page.text.includes('chat.arena'));
+ assert(!page.text.includes('data-label="Provider">sparse'));
+ });
+ await check('several dated subjects never become an invented family rating', async () => {
+ const model = { slug: 'openai/gpt-5-mini', offers: [{ model: 'gpt-5-mini' }] };
+ const one = { ...rating, subject: 'gpt-5-mini-2025-08-07' };
+ assert.deepStrictEqual(ratingFor(model, [one]), one);
+ assert.strictEqual(ratingFor(model, [one, { ...one, subject: 'gpt-5-mini-2026-08-07' }]), null);
+ });
+ await check('a public Bench read works without service credentials', async () => {
+ let path;
+ const publicBench = createBench({ config: { oauth: {}, bench: { url: 'https://bench.test', timeoutMs: 2000 } }, fetchImpl: async (url, init) => {
+ path = String(url);
+ assert(!new Headers(init.headers).has('authorization'));
+ return new Response(JSON.stringify({ board: { rows: [rating, { ...rating, games: 19 }] } }), { headers: { 'Content-Type': 'application/json' } });
+ } });
+ const data = await publicBench.board();
+ assert.strictEqual(path, 'https://bench.test/api/v1/ratings/chat.arena?category=all');
+ assert.deepStrictEqual(data.rows, [rating]);
+ });
+ } finally { await t.close(); }
+ done();
+})().catch((err) => { console.error(err); process.exitCode = 1; });
@@ -44,7 +44,8 @@ async function boot(opts = {}) {
@@ -44,7 +44,8 @@ async function boot(opts = {}) {
// The M2 memory client goes out with this fetch. A test that does not stand one up gets an empty store, so
// The M2 memory client goes out with this fetch. A test that does not stand one up gets an empty store, so
// no test ever waits on the network (the notes tests inject their own, in test/helpers or the file itself).
// no test ever waits on the network (the notes tests inject their own, in test/helpers or the file itself).
const emptyMemory = async () => ({ ok: true, status: 200, text: async () => '{"items":[]}', json: async () => ({ items: [] }) });
const emptyMemory = async () => ({ ok: true, status: 200, text: async () => '{"items":[]}', json: async () => ({ items: [] }) });
- const built = await createApp({ config, store, log, telemetrySink: opts.telemetrySink, accountSend: opts.accountSend, ai: opts.ai, aiFetchImpl: opts.aiFetchImpl || emptyMemory, bench: opts.bench, benchFetchImpl: opts.benchFetchImpl, openapiFetch: opts.openapiFetch, indexnow: opts.indexnow, fetchImpl: opts.fetchImpl });
+ const emptyRatings = async () => new Response(JSON.stringify({ board: { minimum_samples: 20, rows: [] } }), { headers: { 'content-type': 'application/json' } });
+ const built = await createApp({ config, store, log, telemetrySink: opts.telemetrySink, accountSend: opts.accountSend, ai: opts.ai, aiFetchImpl: opts.aiFetchImpl || emptyMemory, bench: opts.bench, benchFetchImpl: opts.benchFetchImpl || emptyRatings, analysis: opts.analysis, openapiFetch: opts.openapiFetch, indexnow: opts.indexnow, fetchImpl: opts.fetchImpl });
await built.ctx.keys.ensure();
await built.ctx.keys.ensure();
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const base = `http://127.0.0.1:${server.address().port}`;
const base = `http://127.0.0.1:${server.address().port}`;