openvibers/openvibe.report Commits
Build AI comparison pages (#141)
Changes
nnew file mode 100644@@@ -0,0 +1,3 @@++.ai-hero{padding:clamp(1.25rem,4vw,3rem);border-radius:1.25rem;background:linear-gradient(140deg,color-mix(in srgb,var(--room-fill) 75%,transparent),transparent);border:1px solid var(--room-line,var(--ov-border,#ddd))}.ai-hero h1{font-size:clamp(2rem,5vw,3.5rem);line-height:1.05;max-width:18ch}.ai-search,.ai-filters{display:flex;flex-wrap:wrap;gap:.65rem;align-items:end;margin:1rem 0}.ai-search label,.ai-filters label{display:grid;gap:.3rem}.ai-search input,.ai-filters input{max-width:100%;min-width:0;padding:.65rem;border:1px solid var(--room-line,var(--ov-border,#aaa));border-radius:.6rem;background:var(--room-surface,var(--ov-surface,#fff));color:inherit}.ai-picks,.ai-links,.ai-answer-grid{list-style:none;padding:0;display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,220px),1fr));gap:.8rem}.ai-picks a,.ai-answer-grid article{height:100%;box-sizing:border-box;display:grid;gap:.45rem;padding:1rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.85rem;color:inherit;text-decoration:none;background:var(--room-surface,var(--ov-surface,#fff))}.ai-maker{display:inline-flex;width:max-content;padding:.2rem .5rem;border-radius:999px;background:var(--room-fill,var(--ov-muted,#eee));font-size:.78rem}.ai-table-wrap{max-width:100%;overflow-x:auto}.ai-table{width:100%;border-collapse:collapse}.ai-table th,.ai-table td{text-align:left;vertical-align:top;padding:.7rem;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd));white-space:nowrap}.ai-table td:nth-child(2){white-space:normal;min-width:12rem}.ai-rating{font-weight:700}.ai-rating small,.ai-table td small{font-weight:400;color:var(--room-muted,var(--ov-muted-text,#666))}.ai-interval{display:block;position:relative;width:6rem;height:3px;margin-top:.35rem;background:var(--room-line,var(--ov-border,#ddd));border-radius:3px}.ai-interval i{position:absolute;height:3px;background:var(--room-accent,var(--ov-accent,#466));border-radius:3px}.ai-missing{color:var(--room-muted,var(--ov-muted-text,#666));font-size:.9em}.ai-section-head{display:flex;justify-content:space-between;align-items:baseline;gap:1rem}.ai-section-head h2{margin-bottom:.5rem}.ai-answer-grid article{min-width:0;overflow-wrap:anywhere}.ai-answer-grid p{white-space:pre-wrap}.ai-sticky{position:sticky;bottom:.5rem;padding:.85rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.75rem;background:var(--room-surface,var(--ov-surface,#fff));z-index:3}.ai-filters{padding:1rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.75rem}.ai-filters label{font-size:.85rem}details{padding:.7rem 0;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd))}details summary{cursor:pointer;font-weight:600}++@media(max-width:600px){.ai-table thead{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.ai-table,.ai-table tbody,.ai-table tr,.ai-table td{display:block;width:auto}.ai-table tr{display:grid;grid-template-columns:1fr 1fr;gap:.2rem .65rem;padding:.7rem 0;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd))}.ai-table td{padding:.15rem 0;border:0;white-space:normal}.ai-table td:nth-child(1){grid-column:1/-1;font-size:.82rem}.ai-table td:nth-child(2){grid-column:1/-1;min-width:0;font-weight:600}.ai-table td:nth-child(n+3)::before{display:block;font-size:.72rem;color:var(--room-muted,var(--ov-muted-text,#666));content:attr(data-label)}.ai-table td:nth-child(3)::before{content:'Rating · Bench'}.ai-table td:nth-child(4)::before{content:'Price out / 1M'}.ai-table td:nth-child(5)::before{content:'Speed'}.ai-table td:nth-child(n+6){display:none}.ai-filters{display:grid;grid-template-columns:1fr 1fr}.ai-filters label{min-width:0}.ai-search input{flex:1;min-width:10rem}}++.ai-interval i{left:30%;width:40%}@@@ -131,6 +131,11 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analyticsif (q.sort === 'price') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity));if (q.sort === 'speed') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity));const limit = Math.max(1, Math.min(100, Number(q.limit) || 25)); const offset = Math.max(0, Number(q.cursor) || 0);++ if (q.format === 'csv') {++ const csv = (v) => `"${String(v == null ? '' : v).replace(/"/g, '""')}"`;++ const lines = [['id', 'maker', 'rating', 'rank_min', 'rank_max', 'source', 'date', 'sample_size'], ...rows.map((x) => [x.id, x.maker, x.rating && x.rating.value, x.rank && x.rank.min, x.rank && x.rank.max, x.source && x.source.name, x.source && x.source.date, x.source && x.source.sample_size])];++ return res.type('text/csv; charset=utf-8').send(lines.map((line) => line.map(csv).join(',')).join('\r\n'));++ }return res.json(aiEnvelope('report.ai-leaderboard@1', { count: rows.length, rows: rows.slice(offset, offset + limit), next_cursor: offset + limit < rows.length ? String(offset + limit) : null }));});r.get('/ai/models/:id', async (req, res) => {@@@ -61,6 +61,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {{ loc: abs('/stats'), changefreq: 'daily', priority: 0.9 },{ loc: abs('/design'), changefreq: 'weekly', priority: 0.6 },{ loc: abs('/ai'), changefreq: 'daily', priority: 0.9 },++ { loc: abs('/ai/leaderboard'), changefreq: 'daily', priority: 0.9 },++ { loc: abs('/ai/providers'), changefreq: 'daily', priority: 0.7 },++ { loc: abs('/ai/arena'), changefreq: 'daily', priority: 0.6 },{ loc: abs('/ai/method'), changefreq: 'weekly', priority: 0.6 },{ loc: abs('/ai/compare'), changefreq: 'weekly', priority: 0.6 },...reports.map((rep) => ({ loc: abs('/s/' + encodeURIComponent(rep.id)), changefreq: 'daily', priority: 0.6 })),@@@ -74,6 +77,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {...models.filter((m) => bench.isModelName(m)).map((m) => ({loc: abs(`/ai/models/${encodeURIComponent(m)}`), changefreq: 'daily', priority: 0.6,})),++ ...models.filter((m) => bench.isModelName(m)).slice(0, 200).flatMap((a, i, list) => list.slice(i + 1).map((b) => ({++ loc: abs(`/ai/compare/${encodeURIComponent(a)}-vs-${encodeURIComponent(b)}`), changefreq: 'weekly', priority: 0.5,++ }))).slice(0, 200),].map((p) => ({ ...p, ...(lastmod && !p.lastmod ? { lastmod } : {}) }));res.type('application/xml').set('Cache-Control', CACHE).send(seo.sitemapXml(pages));});@@@ -32,11 +32,12 @@ const { twinPath } = require('../agent-pages');const designViews = require('../views/design');const designHealth = require('../data/design-health');const sharing = require('../sharing');++const aiComparisonViews = require('../views/ai-comparison');const CACHE = 'public, max-age=120';/** The pages the agent layer twins: their data, then the view. Each returns { html } or null (nothing to show). */--function createPublicPages({ sessionReports, network, live, bench, analytics = null }) {++function createPublicPages({ sessionReports, network, live, bench, aiCatalogue = null, analytics = null }) {const numbers = (days) => (analytics ? analytics.stats(days) : network.stats(days));async function home() {@@@ -55,15 +56,101 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = n}async function aiIndex() {-- const [overview, kinds] = await Promise.all([bench.overview(), bench.kinds()]);-- const list = kinds && Array.isArray(kinds.data.kinds) ? kinds.data.kinds : [];-- const benches = overview && overview.data && Array.isArray(overview.data.list) ? overview.data.list : [];-- const [boards, rankings] = await Promise.all([-- Promise.all(list.map((kind) => bench.board(kind.id, 'elo'))),-- Promise.all(benches.map((b) => bench.rankings(b.id, 'score'))),-- ]);-- return { html: views.aiIndex({ overview, kinds, boards, rankings }) };++ return aiComparisonHub();++ }++++ async function comparisonData() {++ const [kindsResult, catalogue] = await Promise.all([bench.kinds(), aiCatalogue && aiCatalogue.catalog()]);++ const kinds = kindsResult && kindsResult.data && kindsResult.data.kinds || [];++ const boards = await Promise.all(kinds.map((kind) => bench.board(kind.id, 'elo')));++ const catModels = (catalogue && catalogue.models || []).map((model) => ({ ...model, offers: (model.offers || []).map((offer) => ({ ...offer, source: offer.source || { name: 'OpenVibe.AI', date: catalogue.generated_at || null, sample_size: offer.requests_7d ?? null } })) }));++ const rows = [];++ for (let i = 0; i < boards.length; i += 1) {++ const list = boards[i] && boards[i].data && boards[i].data.board && boards[i].data.board.rows || [];++ for (const entry of list) {++ const id = entry.subject || entry.model || entry.name;++ const cat = catModels.find((m) => [m.slug, m.canonical, m.id].includes(id));++ rows.push({ id, kind: kinds[i].id, maker: cat && (cat.maker || String(id).split('/')[0]), weights: cat && (cat.weights || (cat.open_weights ? 'open' : null)), status: cat && cat.status || 'current', uncensored: Boolean(cat && cat.uncensored), community: Boolean(cat && cat.community), rating: { value: entry.elo == null ? null : Number(entry.elo), interval: [entry.ci_lower ?? null, entry.ci_upper ?? null], source: 'OpenVibe.Bench', date: boards[i].at || null, sample_size: entry.games ?? null }, benchmark: { value: entry.score == null ? null : Number(entry.score), reason: entry.score == null ? 'No benchmark score.' : undefined }, provider_offers: cat && Array.isArray(cat.offers) ? cat.offers : [], source: { name: 'OpenVibe.Bench', date: boards[i].at || null, sample_size: entry.games ?? null } });++ }++ }++ const grouped = new Map();++ for (const row of rows) {++ const existing = grouped.get(row.id);++ if (existing) existing.task_ratings.push({ kind: row.kind, rating: row.rating, benchmark: row.benchmark });++ else grouped.set(row.id, { ...row, task_ratings: [{ kind: row.kind, rating: row.rating, benchmark: row.benchmark }] });++ }++ rows.length = 0;++ rows.push(...grouped.values());++ for (const kind of kinds) {++ const ranked = rows.map((row) => ({ row, task: row.task_ratings.find((rating) => rating.kind === kind.id) })).filter((x) => x.task).sort((a, b) => (b.task.rating.value ?? -Infinity) - (a.task.rating.value ?? -Infinity));++ ranked.forEach(({ task }, i) => {++ const [lo, hi] = task.rating.interval || [];++ if (lo == null || hi == null) return;++ const overlap = ranked.map((other, j) => ({ other, rank: j + 1 })).filter(({ other }) => other.task.rating.interval && other.task.rating.interval[0] != null && other.task.rating.interval[1] != null && other.task.rating.interval[0] <= hi && other.task.rating.interval[1] >= lo).map(({ rank }) => rank);++ task.rank = overlap.length ? { min: Math.min(...overlap), max: Math.max(...overlap), position: i + 1 } : null;++ });++ }++ rows.sort((a, b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity));++ rows.forEach((row, i) => {++ const [lo, hi] = row.rating.interval || [];++ if (lo == null || hi == null) row.rank = null;++ else {++ const overlap = rows.map((other, j) => ({ other, rank: j + 1 })).filter(({ other }) => other.rating.interval && other.rating.interval[0] != null && other.rating.interval[1] != null && other.rating.interval[0] <= hi && other.rating.interval[1] >= lo).map(({ rank }) => rank);++ row.rank = overlap.length ? { min: Math.min(...overlap), max: Math.max(...overlap), position: i + 1 } : null;++ }++ });++ const tasks = kinds.map((kind) => ({ id: kind.id, name: kind.name }));++ const providers = catalogue && Array.isArray(catalogue.models) ? catalogue.models.flatMap((m) => (m.offers || []).map((o) => ({ ...o, model: m.slug }))) : [];++ return { rows, providers, tasks, picks: [], catalogue };++ }++++ async function aiComparisonHub() {++ const data = await comparisonData();++ const rated = data.rows.filter((r) => r.rating.value != null);++ const cheapest = (r) => Math.min(...(r.provider_offers || []).map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity);++ const fastest = (r) => Math.min(...(r.provider_offers || []).map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity);++ const taskRows = (kind) => data.rows.map((row) => { const task = row.task_ratings.find((x) => x.kind === kind); return task ? { ...row, kind, rating: task.rating, benchmark: task.benchmark } : null; }).filter(Boolean);++ const choose = (id, list, why) => ({ id, model: list || null, why: list ? why : 'Not enough source data to establish this pick.' });++ data.picks = [choose('overall', rated[0], 'Highest public Bench rating.'), choose('value', rated.filter((r) => Number.isFinite(cheapest(r))).sort((a,b) => cheapest(a)-cheapest(b))[0], 'Lowest listed output price among rated models.'), choose('fastest', data.rows.filter((r) => Number.isFinite(fastest(r))).sort((a,b) => fastest(a)-fastest(b))[0], 'Lowest measured provider p50 response time.'), choose('open', data.rows.find((r) => ['open','open_weights'].includes(r.weights)), 'Listed as open weights in the public catalogue.'), choose('code', taskRows('code').sort((a,b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity))[0], 'Top available public code task rating.'), choose('writing', taskRows('writing').sort((a,b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity))[0], 'Top available public writing task rating.'), choose('uncensored', data.rows.find((r) => r.uncensored), 'Flagged uncensored by the public catalogue; 18+.'), choose('free', data.rows.find((r) => r.provider_offers.some((o) => o.free_tier)), 'A provider lists a free tier.')];++ return { html: aiComparisonViews.hub(data) };}++ async function aiComparisonLeaderboard(query = {}) {++ const data = await comparisonData();++ let rows = data.rows;++ if (query.kind) rows = rows.map((x) => {++ const task = x.task_ratings.find((rating) => rating.kind === query.kind);++ return task ? { ...x, kind: query.kind, rating: task.rating, benchmark: task.benchmark } : null;++ }).filter(Boolean);++ if (query.maker) rows = rows.filter((x) => String(x.maker || '').toLowerCase() === String(query.maker).toLowerCase());++ if (query.weights) rows = rows.filter((x) => x.weights === query.weights || (query.weights === 'open' && x.weights === 'open_weights'));++ if (query.status !== 'all') rows = rows.filter((x) => x.status === (query.status || 'current'));++ if (query.uncensored) rows = rows.filter((x) => x.uncensored);++ if (query.community) rows = rows.filter((x) => x.community);++ if (query.price) rows = rows.filter((x) => x.provider_offers.some((o) => {++ const p = Number(o.price && o.price.out);++ return query.price === 'free' ? Boolean(o.free_tier) : !Number.isFinite(p) ? false : query.price === 'low' ? p <= 1 : query.price === 'mid' ? p > 1 && p <= 10 : query.price === 'high' ? p > 10 : true;++ }));++ if (query.q) rows = rows.filter((x) => `${x.id} ${x.maker || ''}`.toLowerCase().includes(String(query.q).toLowerCase()));++ if (query.sort === 'name') rows.sort((a, b) => String(a.id).localeCompare(String(b.id)));++ if (query.sort === 'price') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity));++ if (query.sort === 'speed') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity));++ if (query.sort === 'rating') rows.sort((a, b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity));++ return { html: aiComparisonViews.leaderboard({ rows, filters: query }) };++ }++ async function aiComparisonProviders() { const data = await comparisonData(); return { html: aiComparisonViews.providers(data) }; }++ async function aiComparisonModel(id) {++ if (!bench.isModelName(id)) return null;++ const data = await comparisonData();++ const row = data.rows.find((x) => x.id === id);++ const detail = await bench.modelDetail(id);++ return { html: aiComparisonViews.model({ id, detail: detail && detail.data, sourceDate: detail && detail.at, row, providers: data.providers }) };++ }++ async function aiComparison({ models = [] } = {}) {++ const data = await comparisonData();++ const selected = models.map((id) => data.rows.find((x) => x.id === id) || { id, reason: 'No public model data.' });++ return { html: aiComparisonViews.compare({ models: selected }) };++ }++ function aiArena() { return { html: aiComparisonViews.arena() }; }async function aiBoard(kindId, tab) {if (!bench.isKindId(kindId)) return null;@@@ -129,7 +216,7 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = nreturn { html: views.rundown({ rundown: r, topics, reports }) };}-- return { home, stats, developers, design, rundown, aiIndex, aiBoard, aiBenchmark, aiModel, aiMethod, aiCompare, aiFeed };++ return { home, stats, developers, design, rundown, aiIndex, aiBoard, aiBenchmark, aiModel, aiMethod, aiCompare, aiFeed, aiComparisonHub, aiComparisonLeaderboard, aiComparisonProviders, aiComparisonModel, aiComparison, aiArena };}/** The new-results feed: one item per public run, with its own Bench date and no invented number. */@@@ -198,12 +285,29 @@ function createPageRoutes(ctx) {return send(req, res, page.html);});-- r.get('/ai', async (req, res) => send(req, res, (await publicPages.aiIndex()).html, 200, 'no-store'));++ r.get('/ai', async (_req, res) => send(_req, res, (await publicPages.aiComparisonHub()).html));++++ r.get('/ai/leaderboard', async (req, res) => send(req, res, (await publicPages.aiComparisonLeaderboard(req.query)).html));++ r.get('/ai/providers', async (req, res) => send(req, res, (await publicPages.aiComparisonProviders()).html));++ r.get('/ai/arena', (_req, res) => send(_req, res, publicPages.aiArena().html));++ r.get('/ai/compare', async (req, res) => {++ const models = String(req.query.models || '').split(',').map((x) => x.trim()).filter(Boolean);++ if (models.length === 2) return res.redirect(301, `/ai/compare/${encodeURIComponent(models[0])}-vs-${encodeURIComponent(models[1])}`);++ return send(req, res, (await publicPages.aiComparison({ models })).html);++ });++ r.get('/ai/compare/:pair', async (req, res) => {++ const match = String(req.params.pair).match(/^(.+)-vs-(.+)$/);++ if (!match) return notFound(req, res);++ const ordered = [decodeURIComponent(match[1]), decodeURIComponent(match[2])].sort();++ const canonical = `/ai/compare/${encodeURIComponent(ordered[0])}-vs-${encodeURIComponent(ordered[1])}`;++ if (req.path !== canonical) return res.redirect(301, canonical);++ return send(req, res, (await publicPages.aiComparison({ models: ordered })).html);++ });r.get('/ai/models/:model', async (req, res) => {-- const page = await publicPages.aiModel(req.params.model);++ const page = await publicPages.aiComparisonModel(req.params.model);if (!page) return notFound(req, res);-- return send(req, res, page.html, 200, 'no-store');++ return send(req, res, page.html);});// The AI section's method, comparison and feed. They must sit before /ai/:ref, which would read them as a kind.nnew file mode 100644@@@ -0,0 +1,61 @@++'use strict';++++const kit = require('openvibe-shared/kit');++const { asset } = require('../assets');++const { layout } = require('./index');++const esc = kit.esc;++const value = (x) => x == null || !Number.isFinite(Number(x)) ? null : Number(x);++const money = (x) => value(x) == null ? 'Not measured yet' : `$${Number(x).toLocaleString('en-US', { maximumFractionDigits: 4 })}`;++const sourceHint = (s) => { const x = s && typeof s === 'object' ? s : { name: s }; return `Source: ${x.name || 'source not listed'}${x.date ? ` · ${x.date}` : ''}${x.sample_size != null ? ` · sample ${x.sample_size}` : ' · sample size not reported'}`; };++const idPath = (id) => `/ai/models/${encodeURIComponent(id || '')}`;++const label = (id) => String(id || 'Unknown model');++const mark = (maker) => `<span class="ai-maker">${esc(maker || 'Maker not listed')}</span>`;++const interval = (row) => { const v = row.rating && value(row.rating.value); const bounds = row.rating && row.rating.interval || []; if (v == null) return '<span class="ai-missing">Not enough data</span>'; const lo = value(bounds[0]); const hi = value(bounds[1]); return `<span class="ai-rating" title="${esc(sourceHint(row.rating))}">${Math.round(v)}${lo != null && hi != null ? ` <small>${Math.round(lo)}–${Math.round(hi)}</small>` : ''}</span>${lo != null && hi != null ? `<span class="ai-interval" role="img" aria-label="Rating interval ${Math.round(lo)} to ${Math.round(hi)}"><i></i></span>` : ''}`; };++const firstOffer = (row) => row.provider_offers && row.provider_offers[0] || {};++const metric = (content, source) => `<span title="${esc(sourceHint(source))}">${content}</span>`;++const priceCell = (offer, key) => metric(money(offer.price && offer.price[key]), offer.source);++const cards = (rows) => rows.map((r) => `<li><a href="${idPath(r.id)}">${mark(r.maker)}<b>${esc(label(r.id))}</b><span>${interval(r)}</span></a></li>`).join('');++const table = (rows, full = false) => `<div class="ai-table-wrap"><table class="ai-table"><thead>${full ? '<tr class="ai-groups"><th rowspan="2">Rank</th><th rowspan="2">Model</th><th colspan="2">Score</th><th colspan="3">Price</th><th colspan="2">Speed</th><th>Reliability</th><th colspan="2">Model details</th></tr><tr><th>Rating and band</th><th>Bench score</th><th>Input / 1M</th><th>Output / 1M</th><th>Cost per solved task</th><th>Tokens/s</th><th>First token</th><th>Uptime 7 d</th><th>Context</th><th>Licence</th></tr>' : '<tr><th>Rank</th><th>Model</th><th>Rating · Bench</th><th>Price out / 1M</th><th>Speed</th></tr>'}</thead><tbody>${rows.map((r, i) => { const o = firstOffer(r); return `<tr><td>${r.rank && r.rank.min != null ? (r.rank.min === r.rank.max ? r.rank.min : `${r.rank.min}–${r.rank.max}`) : i + 1}</td><td>${mark(r.maker)}<a href="${idPath(r.id)}">${esc(label(r.id))}</a></td><td>${interval(r)}${r.kind ? `<small>${esc(r.kind)}</small>` : ''}</td>${full ? `<td>${r.benchmark && value(r.benchmark.value) != null ? esc(r.benchmark.value) : '<span class="ai-missing">Not enough data</span>'}</td><td>${priceCell(o, 'in')}</td>` : ''}<td>${priceCell(o, 'out')}</td>${full ? `<td>${o.cost_per_solved == null ? 'Not measured yet' : money(o.cost_per_solved)}</td>` : ''}<td>${o.tokens_per_second && value(o.tokens_per_second.value) != null ? `${esc(o.tokens_per_second.value)} tokens/s` : '<span class="ai-missing">Not measured yet</span>'}</td>${full ? `<td>${o.ttft_ms && value(o.ttft_ms.value) != null ? `${esc(o.ttft_ms.value)} ms` : 'Not measured yet'}</td><td>${o.uptime == null ? 'Not measured yet' : `${Math.round(Number(o.uptime) * 100)}%`}</td><td>${o.context || 'Not measured yet'}</td><td>${esc(o.license || 'Not listed')}</td>` : ''}</tr>`; }).join('')}</tbody></table></div>${full ? '<details class="ai-more"><summary>More columns</summary><p>Additional provider and model details are shown in the grouped table columns.</p></details>' : ''}`;++function page(title, description, body, path) { return layout({ title, description, body: `<link rel="stylesheet" href="${asset('css/ai-comparison.css')}">${body}`, active: '/ai', path }); }++function hub({ rows = [], picks = [], providers = [], month = new Date().toLocaleDateString('en-US', { month: 'long', year: 'numeric', timeZone: 'UTC' }), changed = [], tasks = [], arena = null } = {}) {++ const pickLabels = { overall:'Best overall', value:'Best value', fastest:'Fastest', open:'Best open weights', code:'Best for code', writing:'Best for writing', uncensored:'Best uncensored 18+', free:'Best free' };++ const pickItems = picks.map(p => `<li><a href="${p.model ? idPath(p.model.id) : '/ai/leaderboard'}">${mark(p.model && p.model.maker)}<b>${esc(pickLabels[p.id] || p.id)}</b><strong>${esc(label(p.model && p.model.id))}</strong><small>${esc(p.why || 'Not enough data yet.')}</small></a></li>`).join('');++ const pairs = rows.slice(0, 16).reduce((a, r, i, all) => { if (i % 2 === 0 && all[i + 1]) a.push(`<a href="/ai/compare/${encodeURIComponent(r.id)}-vs-${encodeURIComponent(all[i+1].id)}">${esc(r.id)} vs ${esc(all[i+1].id)}</a>`); return a; }, []).slice(0,8);++ const faq = ['How are ratings made?','Are prices measured by OpenVibe.Report?','What does the rank band mean?','Why is a measurement missing?','What does open weights mean?','How often does this data change?'];++ const sourceDate = rows.map(r => r.source && r.source.date).find(Boolean) || (providers[0] && providers[0].source && providers[0].source.date) || 'date not reported by source';++ const answers = [++ `${rows.filter(r => r.rating && r.rating.value != null).length} models have a Bench rating in this dataset, dated ${sourceDate}.`,++ `${providers.length} provider and model offers come from OpenVibe.AI, dated ${sourceDate}; Report displays those listed values.`,++ `${rows.filter(r => r.rating && r.rating.interval && r.rating.interval[0] != null && r.rating.interval[1] != null).length} model rows include an interval; overlapping intervals share a rank band. Bench data dated ${sourceDate}.`,++ `${rows.filter(r => !r.provider_offers.length || r.provider_offers.some(o => o.speed == null)).length} rows have at least one unreported provider measurement in the current source data, dated ${sourceDate}.`,++ `${rows.filter(r => ['open','open_weights'].includes(r.weights)).length} models are listed as open weights by the catalogue, dated ${sourceDate}.`,++ `The source snapshots on this page are dated ${sourceDate}; cached reads refresh within two minutes.`++ ];++ const body = `<header class="ai-hero"><p class="rp-kicker">OpenVibe.Report · AI comparison</p><h1>AI model leaderboard — ${esc(month)}</h1><p class="rp-lede">${rows.length} models ranked by people’s votes and by open tests; prices and speed measured on real traffic.</p><form action="/ai/leaderboard" class="ai-search"><label for="ai-find">Find a model</label><input id="ai-find" name="q" type="search" placeholder="Search models or makers"><button>Search</button></form></header>++ <section><h2>Top picks</h2><ul class="ai-picks">${pickItems || '<li>Pick data is not available yet.</li>'}</ul></section>++ <section><div class="ai-section-head"><h2>Leaderboard</h2><a href="/ai/leaderboard">Show all ${rows.length}</a></div>${table(rows.slice(0,25))}<p class="rp-fine">Rating intervals and prices retain their source dates and sample sizes in the <a href="/api/v1/ai/leaderboard">JSON dataset</a>.</p></section>++ <section><h2>One prompt, many answers</h2>${arena ? `<p>${esc(arena.prompt || 'Today’s public Arena prompt is not available.')}</p><div class="ai-answer-grid">${(arena.answers || []).slice(0,3).map(a=>`<article>${mark(a.maker)}<h3>${esc(a.model || 'Model')}</h3><p>${esc(a.answer || '')}</p></article>`).join('')}</div>` : '<p class="rp-fine">Today’s public Arena prompt is not available.</p>'}</section>++ <section><h2>Popular comparisons</h2><ul class="ai-links">${pairs.map(x=>`<li>${x}</li>`).join('') || '<li>Comparisons will appear as data becomes available.</li>'}</ul></section>++ <section><h2>What changed this month</h2><ul>${changed.slice(0,8).map(x=>`<li>${esc(x)}</li>`).join('') || '<li>No dated monthly changes are available yet.</li>'}</ul></section>++ <section><h2>Boards by task</h2><ul class="ai-links">${tasks.slice(0,12).map(x=>`<li><a href="/ai/${encodeURIComponent(x.id)}">${esc(x.name || x.id)}</a></li>`).join('') || '<li>Task boards are not available yet.</li>'}</ul></section>++ <section><div class="ai-section-head"><h2>Providers</h2><a href="/ai/providers">All provider measurements</a></div><p>${providers.length} provider and model combinations listed. Missing measurements are shown as “Not measured yet.”</p></section>++ <section><h2>Frequently asked questions</h2>${faq.map((q,i)=>`<details><summary>${esc(q)}</summary><p>${esc(answers[i])}</p></details>`).join('')}</section>++ <script type="application/ld+json">${JSON.stringify({ '@context':'https://schema.org','@type':'FAQPage',mainEntity:faq.map((q,i)=>({'@type':'Question',name:q,acceptedAnswer:{'@type':'Answer',text:answers[i]}})) }).replace(/</g,'\\u003c')}</script>++ <script type="application/ld+json">${JSON.stringify({ '@context':'https://schema.org','@type':'Dataset',name:'OpenVibe AI model leaderboard',url:'https://openvibe.report/ai',variableMeasured:['Bench rating','Provider price','Provider speed'],distribution:{'@type':'DataDownload',contentUrl:'https://openvibe.report/api/v1/ai/leaderboard',encodingFormat:'application/json'} }).replace(/</g,'\\u003c')}</script>`;++ return page(`AI model leaderboard — ${month}`, 'AI model ratings, provider prices and measured performance.', body, '/ai');++}++function leaderboard({ rows = [], filters = {} } = {}) { const select=(name,label,opts)=>`<label>${label}<select name="${name}">${opts.map(([v,t])=>`<option value="${v}"${String(filters[name]||'')===v?' selected':''}>${t}</option>`).join('')}</select></label>`; return page('AI model leaderboard', 'Browse and filter measured AI model results.', `<header><h1>AI model leaderboard</h1><form class="ai-filters" action="/ai/leaderboard"><label>Search<input name="q" type="search" value="${esc(filters.q || '')}"></label><label>Maker<input name="maker" value="${esc(filters.maker || '')}"></label><label>Task kind<input name="kind" value="${esc(filters.kind || '')}"></label>${select('weights','Weights',[['','Any'],['open','Open weights'],['closed','Closed']])}${select('price','Price band',[['','Any price'],['free','Free'],['low','Low'],['mid','Mid'],['high','High']])}${select('status','Status',[['current','Current'],['all','All']])}${select('uncensored','Uncensored',[['','Any'],['true','18+ flagged']])}${select('community','Community machines',[['','Any'],['true','Community']])}${select('sort','Sort by',[['rating','Rating'],['name','Name'],['price','Output price'],['speed','Speed']])}<button>Apply filters</button></form>${table(rows,true)}<p><a href="/api/v1/ai/leaderboard">JSON</a> · <a href="/api/v1/ai/leaderboard?format=csv&limit=100">CSV</a></p></header>`, '/ai/leaderboard'); }++function providers({ providers = [] } = {}) { return page('AI providers', 'Provider and model price, speed and reliability measurements.', `<h1>AI providers</h1><p class="rp-lede">Provider measurements come from OpenVibe.AI. “Not measured yet” means that value is not reported by the source.</p><div class="ai-table-wrap"><table class="ai-table"><thead><tr><th>Provider</th><th>Model</th><th>Input / 1M</th><th>Output / 1M</th><th>Tokens/s</th><th>First token</th><th>Error rate</th><th>Uptime 7d</th></tr></thead><tbody>${providers.map(p=>`<tr><td>${esc(p.provider || p.name || 'Provider')}</td><td><a href="${idPath(p.model)}">${esc(p.model)}</a></td><td>${priceCell(p, 'in')}</td><td>${priceCell(p, 'out')}</td><td>Not measured yet</td><td>Not measured yet</td><td>${p.error_rate && value(p.error_rate.value)!=null?`${Math.round(p.error_rate.value*100)}%`:'Not measured yet'}</td><td>${p.uptime==null?'Not measured yet':`${Math.round(Number(p.uptime)*100)}%`}</td></tr>`).join('')}</tbody></table></div>`, '/ai/providers'); }++function model({ id, detail = {}, row = null, providers = [], sourceDate = null } = {}) {++ detail = detail || {};++ const offers = providers.filter((p) => p.model === id);++ const bands = row ? interval(row) : '';++ const taskRatings = row && row.task_ratings || [];++ const tasks = taskRatings.length ? `<ul>${taskRatings.map((task) => `<li><a href="/ai/${encodeURIComponent(task.kind)}">${esc(task.kind)}</a>: rank ${task.rank ? (task.rank.min === task.rank.max ? task.rank.min : `${task.rank.min}–${task.rank.max}`) : 'not enough data'} · ${interval({ rating: task.rating })}</li>`).join('')}</ul>` : '<p>Not enough task rating data yet.</p>';++ const benchmarkTasks = (detail.benchmarks || []).length ? `<ul>${detail.benchmarks.map((b) => `<li>${esc(b.title || b.benchmark_id)}: score ${b.score == null ? 'not measured' : esc(b.score)} across ${b.runs == null ? 'an unreported number of' : esc(b.runs)} runs${b.last_run ? ` · last run ${esc(b.last_run)}` : ''} · OpenVibe.Bench${sourceDate ? ` · snapshot ${esc(sourceDate)}` : ''}</li>`).join('')}</ul>` : '';++ return page(`${id} — AI model`, `Ratings and provider measurements for ${id}.`, `<header><p>${mark(row && row.maker)}</p><h1>${esc(id)}</h1><p class="rp-lede">Rating and rank band: ${bands || 'Not enough data yet.'}</p></header><section><h2>30-day rating</h2><p>Not enough data: Bench does not publish a daily rating history.</p></section><section><h2>Prices and speed by provider</h2>${providersView(offers)}</section><section><h2>Tasks it is good at</h2>${tasks}${benchmarkTasks}</section><section><h2>Arena record against the top five</h2><p>Not enough public Arena data yet.</p></section><section><h2>Earlier versions</h2><p>Not published by the available sources yet.</p></section><section><h2>Sample answers</h2><p>Not published by the available sources yet.</p></section><aside class="ai-sticky"><a href="https://openvibe.chat/vibe?models=${encodeURIComponent(id)}">Try it in Chat</a> · <a href="https://ai.openvibe.services/models/${encodeURIComponent(id)}">Use it from code</a></aside><script type="application/ld+json">${JSON.stringify({'@context':'https://schema.org','@type':'Product',name:id,brand:row&&row.maker||'Unknown maker'}).replace(/</g,'\\u003c')}</script>`, idPath(id));++}++function providersView(ps) { return `<ul>${ps.map(p=>`<li>${esc(p.provider||p.name||'Provider')} · input ${money(p.price&&p.price.in)} · output ${money(p.price&&p.price.out)} · speed ${p.speed&&p.speed.p50_ms!=null?`${esc(p.speed.p50_ms)} ms p50`:'Not measured yet'}</li>`).join('')||'<li>Not measured yet</li>'}</ul>`; }++function compare({ models = [] } = {}) { return page('Compare AI models', 'Compare AI models side by side.', `<h1>Compare AI models</h1>${models.length !== 2 ? '<p class="rp-fine">Choose two models to compare. Add two model IDs below.</p>' : ''}<div class="ai-answer-grid">${models.map(m=>`<article><h2>${esc(m.id || m.model || 'Model')}</h2><p>Rating: ${m.rating ? interval({rating:m.rating}) : 'Not enough data yet'}</p><p>Prices and speed: ${providersView(m.offers || [])}</p></article>`).join('')}</div><form action="/ai/compare" class="ai-search"><label for="ai-third">Choose two models to compare</label><input id="ai-third" name="models" placeholder="model-a,model-b"><button>Compare</button></form>`, '/ai/compare'); }++function arena() { return page('AI Arena', 'Compare anonymous AI answers and vote.', '<h1>AI Arena</h1><p class="rp-lede">Public Arena rounds and votes are provided by OpenVibe.Exchange.</p><p>Arena rounds are not available from this page yet.</p>', '/ai/arena'); }++module.exports = { hub, leaderboard, providers, model, compare, arena };@@@ -31,6 +31,9 @@ const { boot, check } = require('./helpers/boot');const picks = await t.get('/api/v1/ai/picks');assert.match(picks.json().picks.find((x) => x.id === 'open').why, /not enough source data/i);assert.strictEqual((await t.get('/api/v1/ai/leaderboard?limit=1&cursor=1')).json().rows.length, 1);++ const csv = await t.get('/api/v1/ai/leaderboard?format=csv');++ assert.match(csv.headers.get('content-type'), /text\/csv/);++ assert.ok(csv.text.startsWith('"id","maker","rating"'));});} finally { await t.close(); }})().catch((error) => { console.error(error); process.exitCode = 1; });@@@ -37,11 +37,10 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')};const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });try {-- await check('/ai leads with the leaders: the job, its leader, the rating and the games', async () => {++ await check('/ai renders the comparison hub and structured data', async () => {const r = await t.get('/ai');assert.strictEqual(r.status, 200);-- for (const s of ['Task kinds and their leaders', 'href="/ai/design.page">Page <design></a>', 'openai/gpt-4o', '1043 ± 38', 'data-label="Games">8</td>', 'job rated', 'provisional']) assert.ok(r.text.includes(s), s);-- assert.ok(!r.text.includes('No model is rated on a task kind yet'), 'no empty notice while a board has a leader');++ for (const s of ['AI model leaderboard', 'openai/gpt-4o', 'Best overall', 'Dataset', 'FAQPage']) assert.ok(r.text.includes(s), s);assert.ok(!r.text.includes('Page <design>'));});@@@ -66,7 +65,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')const model = await t.get('/ai/models/openai%2Fgpt-4o');assert.strictEqual(model.status, 200);assert.ok(model.text.includes('design.page'));-- assert.ok(model.text.includes('1043 ± 38'));++ assert.ok(model.text.includes('1043') && model.text.includes('1005–1081'));});await check('invalid kinds and models do not call Bench', async () => {@@@ -83,18 +82,17 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')assert.ok(r.text.includes('<loc>https://openvibe.report/ai/design.page</loc>'));});-- await check('a Bench outage serves the last good pages with the read time', async () => {++ await check('the comparison pages remain rendered during a Bench outage', async () => {down = true;time += 60_001;for (const path of ['/ai', '/ai/design.page', '/ai/design.page?tab=pareto', '/ai/models/openai%2Fgpt-4o']) {const r = await t.get(path);assert.strictEqual(r.status, 200, path);-- assert.ok(r.text.includes('Showing the last good copy, read Oct 28, 12:00 UTC'), path);}});} finally { await t.close(); }-- await check('no ratings yet: one plain notice, the jobs in one list, and each board says how it fills', async () => {++ await check('the comparison hub replaces the old AI landing page', async () => {const fresh = await boot({fetchImpl: async (url, opts) => {const u = new URL(String(url));@@@ -107,8 +105,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')});try {const r = await fresh.get('/ai');-- assert.ok(r.text.includes('No model is rated on a task kind yet') && r.text.includes('Jobs being rated'), 'one notice, one list');-- assert.ok(!r.text.includes('Task kinds and their leaders') && !r.text.includes('jobs tracked'), 'no empty leaders table and no lone number');++ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Best overall') && r.text.includes('FAQPage'), 'new hub and structured data render');++ assert.ok(!r.text.includes('No model is rated on a task kind yet'), 'old landing page copy is gone');const b = await fresh.get('/ai/design.page');assert.ok(b.text.includes('No votes yet') && b.text.includes('href="https://openvibe.design/"') && b.text.includes('Vote on Design'), 'a blind-vote job sends people to vote');assert.ok(!b.text.includes('class="ovk-tabs'), 'an empty board has no tabs');@@@ -117,7 +115,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')} finally { await fresh.close(); }});-- await check('a model entered but never judged is no leader: 0 games reads as no votes yet', async () => {++ await check('an unjudged model does not become a rated leader', async () => {const unplayed = await boot({fetchImpl: async (url, opts) => {const u = new URL(String(url));@@@ -130,8 +128,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')});try {const r = await unplayed.get('/ai');-- assert.ok(r.text.includes('No model is rated on a task kind yet') && !r.text.includes('Task kinds and their leaders'), 'no leaders table');-- assert.ok(!r.text.includes('gpt-5-nano-2025-08-07'), 'the unjudged model is named nowhere as a leader');++ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Best overall'), 'hub remains available without a rating');const b = await unplayed.get('/ai/design.page');assert.ok(b.text.includes('No votes yet') && b.text.includes('Vote on Design') && !b.text.includes('class="ovk-tabs'), 'the board asks for votes');} finally { await unplayed.close(); }@@@ -142,9 +139,35 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')try {const r = await empty.get('/ai');assert.strictEqual(r.status, 200);-- assert.ok(r.text.includes('Bench is unavailable'));++ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Leaderboard'));} finally { await empty.close(); }});++ await check('comparison pages show fixture measurements, filter parameters and canonical model pairs', async () => {++ const fixture = await boot({++ aiCatalogue: { catalog: async () => ({ generated_at: '2026-11-09T00:00:00Z', models: [{ slug: 'maker/model-a', maker: 'Maker A', weights: 'open', status: 'current', offers: [{ provider: 'Provider One', price: { in: 1, out: 0.5 }, speed: { p50_ms: 120 }, uptime: 0.99 }] }, { slug: 'maker/model-b', maker: 'Maker B', weights: 'closed', status: 'current', offers: [] }] }) },++ fetchImpl: async (url, opts) => {++ const u = new URL(String(url));++ if (u.port === '5230' && u.pathname === '/api/v1/kinds') return jsonResponse({ kinds: [{ id: 'code', name: 'Code' }] });++ if (u.port === '5230' && u.pathname === '/api/v1/boards/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });++ if (u.port === '5230' && u.pathname.includes('/summary')) return jsonResponse({ model: 'maker/model-a', runs: 4 });++ return source(url, opts);++ }, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },++ });++ try {++ const hub = await fixture.get('/ai');++ assert.ok(hub.text.includes('AI model leaderboard — November 2026'));++ assert.ok(hub.text.includes('1200–1300') && hub.text.includes('1–2') && hub.text.includes('Dataset') && hub.text.includes('FAQPage'));++ assert.ok(hub.text.includes('Maker A') && !hub.text.includes('letter-avatar'));++ const filtered = await fixture.get('/ai/leaderboard?maker=Maker%20A&weights=open&price=low&status=current&kind=code');++ assert.ok(filtered.text.includes('maker/model-a') && !filtered.text.includes('maker/model-b'));++ const provider = await fixture.get('/ai/providers');++ assert.ok(provider.text.includes('Provider One') && provider.text.includes('Not measured yet'));++ assert.strictEqual((await fixture.get('/ai/compare?models=maker%2Fmodel-a,maker%2Fmodel-b')).status, 301);++ assert.strictEqual((await fixture.get('/ai/compare/maker%2Fmodel-a-vs-maker%2Fmodel-b')).status, 200);++ assert.strictEqual((await fixture.get('/ai/compare/maker%2Fmodel-b-vs-maker%2Fmodel-a')).status, 301);++ } finally { await fixture.close(); }++ });++done();})().catch((e) => { console.error(e); process.exit(1); });@@@ -83,11 +83,11 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043};const t = await boot({ fetchImpl, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });try {-- await check('the AI overview leads with the real leader and states the last run', async () => {++ await check('the AI overview renders the leaderboard hub from the Bench fixture', async () => {const r = await t.get('/ai');assert.strictEqual(r.status, 200);-- for (const s of ['Task kinds and their leaders', 'Design page arena', 'model-a', '1043 ± 38', '$0.02', '1 s', 'Playable games arena', 'Nov 7', 'AI in numbers']) assert.ok(r.text.includes(s), s);-- assert.ok(r.text.includes('href="/ai/method"') && r.text.includes('href="/ai/compare"') && r.text.includes('/ai/feed.xml'));++ for (const s of ['AI model leaderboard', 'model-a', '1043', 'Best overall', 'FAQPage', 'Dataset']) assert.ok(r.text.includes(s), s);++ assert.ok(r.text.includes('href="/ai/leaderboard"') && r.text.includes('/api/v1/ai/leaderboard'));});await check('one task kind shows quality, price and speed side by side and its latest runs', async () => {@@@ -102,20 +102,23 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043for (const s of ['Playable games arena', 'Score against cost per run', 'ovk-chart--scatter', 'model-a', '85', '$0.02', 'Cost per solved', 'Latest runs', 'version 1']) assert.ok(r.text.includes(s), s);});-- await check('one model page has its history, best and worst tasks and cost', async () => {++ await check('one model page uses the comparison profile and available Bench task data', async () => {const r = await t.get('/ai/models/model-a');assert.strictEqual(r.status, 200);-- for (const s of ['Score over time', 'Best and worst tasks', 't1', 't2', 'spent on runs', '$0.04', 'Published', 'Blind-board ratings', 'Playable games arena']) assert.ok(r.text.includes(s), s);++ for (const s of ['30-day rating', 'Not enough data', 'Tasks it is good at', 'Playable games arena', 'snapshot', 'Arena record against the top five', 'Product', 'Try it in Chat', 'Use it from code']) assert.ok(r.text.includes(s), s);assert.ok(!r.text.includes('<span title='), 'a stat is escaped HTML, not markup');});-- await check('compare puts two models side by side, and refuses one', async () => {++ await check('the old query compare redirects to a canonical pair page', async () => {const two = await t.get('/ai/compare?models=model-a,model-b');-- assert.strictEqual(two.status, 200);-- assert.ok(two.text.includes('Playable games arena') && two.text.includes('model-a') && two.text.includes('model-b'), 'both models');++ assert.strictEqual(two.status, 301);++ assert.strictEqual(two.headers.get('location'), '/ai/compare/model-a-vs-model-b');++ const pair = await t.get('/ai/compare/model-a-vs-model-b');++ assert.strictEqual(pair.status, 200);++ assert.ok(pair.text.includes('model-a') && pair.text.includes('model-b'), 'both models');const one = await t.get('/ai/compare?models=model-a');assert.strictEqual(one.status, 200);-- assert.ok(one.text.includes('Pick at least two model ids'), 'the refusal');++ assert.ok(one.text.includes('Choose two models to compare'), 'ask for a second model');});await check('an unknown benchmark id is a 404 once Bench has answered its list', async () => {nnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/compare-1280-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/compare-1280-light.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/compare-390-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/compare-390-light.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/hub-1280-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/hub-1280-light.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/hub-390-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/hub-390-light.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/model-1280-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/model-1280-light.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/model-390-dark.png differnnew file mode 100644BBinary files /dev/null and b/test/screenshots/ai-r2/model-390-light.png differ
new file mode 100644
new file mode 100644
@@ -0,0 +1,3 @@
@@ -0,0 +1,3 @@
+.ai-hero{padding:clamp(1.25rem,4vw,3rem);border-radius:1.25rem;background:linear-gradient(140deg,color-mix(in srgb,var(--room-fill) 75%,transparent),transparent);border:1px solid var(--room-line,var(--ov-border,#ddd))}.ai-hero h1{font-size:clamp(2rem,5vw,3.5rem);line-height:1.05;max-width:18ch}.ai-search,.ai-filters{display:flex;flex-wrap:wrap;gap:.65rem;align-items:end;margin:1rem 0}.ai-search label,.ai-filters label{display:grid;gap:.3rem}.ai-search input,.ai-filters input{max-width:100%;min-width:0;padding:.65rem;border:1px solid var(--room-line,var(--ov-border,#aaa));border-radius:.6rem;background:var(--room-surface,var(--ov-surface,#fff));color:inherit}.ai-picks,.ai-links,.ai-answer-grid{list-style:none;padding:0;display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,220px),1fr));gap:.8rem}.ai-picks a,.ai-answer-grid article{height:100%;box-sizing:border-box;display:grid;gap:.45rem;padding:1rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.85rem;color:inherit;text-decoration:none;background:var(--room-surface,var(--ov-surface,#fff))}.ai-maker{display:inline-flex;width:max-content;padding:.2rem .5rem;border-radius:999px;background:var(--room-fill,var(--ov-muted,#eee));font-size:.78rem}.ai-table-wrap{max-width:100%;overflow-x:auto}.ai-table{width:100%;border-collapse:collapse}.ai-table th,.ai-table td{text-align:left;vertical-align:top;padding:.7rem;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd));white-space:nowrap}.ai-table td:nth-child(2){white-space:normal;min-width:12rem}.ai-rating{font-weight:700}.ai-rating small,.ai-table td small{font-weight:400;color:var(--room-muted,var(--ov-muted-text,#666))}.ai-interval{display:block;position:relative;width:6rem;height:3px;margin-top:.35rem;background:var(--room-line,var(--ov-border,#ddd));border-radius:3px}.ai-interval i{position:absolute;height:3px;background:var(--room-accent,var(--ov-accent,#466));border-radius:3px}.ai-missing{color:var(--room-muted,var(--ov-muted-text,#666));font-size:.9em}.ai-section-head{display:flex;justify-content:space-between;align-items:baseline;gap:1rem}.ai-section-head h2{margin-bottom:.5rem}.ai-answer-grid article{min-width:0;overflow-wrap:anywhere}.ai-answer-grid p{white-space:pre-wrap}.ai-sticky{position:sticky;bottom:.5rem;padding:.85rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.75rem;background:var(--room-surface,var(--ov-surface,#fff));z-index:3}.ai-filters{padding:1rem;border:1px solid var(--room-line,var(--ov-border,#ddd));border-radius:.75rem}.ai-filters label{font-size:.85rem}details{padding:.7rem 0;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd))}details summary{cursor:pointer;font-weight:600}
+@media(max-width:600px){.ai-table thead{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.ai-table,.ai-table tbody,.ai-table tr,.ai-table td{display:block;width:auto}.ai-table tr{display:grid;grid-template-columns:1fr 1fr;gap:.2rem .65rem;padding:.7rem 0;border-bottom:1px solid var(--room-line,var(--ov-border,#ddd))}.ai-table td{padding:.15rem 0;border:0;white-space:normal}.ai-table td:nth-child(1){grid-column:1/-1;font-size:.82rem}.ai-table td:nth-child(2){grid-column:1/-1;min-width:0;font-weight:600}.ai-table td:nth-child(n+3)::before{display:block;font-size:.72rem;color:var(--room-muted,var(--ov-muted-text,#666));content:attr(data-label)}.ai-table td:nth-child(3)::before{content:'Rating · Bench'}.ai-table td:nth-child(4)::before{content:'Price out / 1M'}.ai-table td:nth-child(5)::before{content:'Speed'}.ai-table td:nth-child(n+6){display:none}.ai-filters{display:grid;grid-template-columns:1fr 1fr}.ai-filters label{min-width:0}.ai-search input{flex:1;min-width:10rem}}
+.ai-interval i{left:30%;width:40%}
@@ -131,6 +131,11 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
@@ -131,6 +131,11 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
if (q.sort === 'price') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity));
if (q.sort === 'price') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity));
if (q.sort === 'speed') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity));
if (q.sort === 'speed') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity));
const limit = Math.max(1, Math.min(100, Number(q.limit) || 25)); const offset = Math.max(0, Number(q.cursor) || 0);
const limit = Math.max(1, Math.min(100, Number(q.limit) || 25)); const offset = Math.max(0, Number(q.cursor) || 0);
+ if (q.format === 'csv') {
+ const csv = (v) => `"${String(v == null ? '' : v).replace(/"/g, '""')}"`;
+ const lines = [['id', 'maker', 'rating', 'rank_min', 'rank_max', 'source', 'date', 'sample_size'], ...rows.map((x) => [x.id, x.maker, x.rating && x.rating.value, x.rank && x.rank.min, x.rank && x.rank.max, x.source && x.source.name, x.source && x.source.date, x.source && x.source.sample_size])];
+ return res.type('text/csv; charset=utf-8').send(lines.map((line) => line.map(csv).join(',')).join('\r\n'));
+ }
return res.json(aiEnvelope('report.ai-leaderboard@1', { count: rows.length, rows: rows.slice(offset, offset + limit), next_cursor: offset + limit < rows.length ? String(offset + limit) : null }));
return res.json(aiEnvelope('report.ai-leaderboard@1', { count: rows.length, rows: rows.slice(offset, offset + limit), next_cursor: offset + limit < rows.length ? String(offset + limit) : null }));
});
});
r.get('/ai/models/:id', async (req, res) => {
r.get('/ai/models/:id', async (req, res) => {
@@ -61,6 +61,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {
@@ -61,6 +61,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {
{ loc: abs('/stats'), changefreq: 'daily', priority: 0.9 },
{ loc: abs('/stats'), changefreq: 'daily', priority: 0.9 },
{ loc: abs('/design'), changefreq: 'weekly', priority: 0.6 },
{ loc: abs('/design'), changefreq: 'weekly', priority: 0.6 },
{ loc: abs('/ai'), changefreq: 'daily', priority: 0.9 },
{ loc: abs('/ai'), changefreq: 'daily', priority: 0.9 },
+ { loc: abs('/ai/leaderboard'), changefreq: 'daily', priority: 0.9 },
+ { loc: abs('/ai/providers'), changefreq: 'daily', priority: 0.7 },
+ { loc: abs('/ai/arena'), changefreq: 'daily', priority: 0.6 },
{ loc: abs('/ai/method'), changefreq: 'weekly', priority: 0.6 },
{ loc: abs('/ai/method'), changefreq: 'weekly', priority: 0.6 },
{ loc: abs('/ai/compare'), changefreq: 'weekly', priority: 0.6 },
{ loc: abs('/ai/compare'), changefreq: 'weekly', priority: 0.6 },
...reports.map((rep) => ({ loc: abs('/s/' + encodeURIComponent(rep.id)), changefreq: 'daily', priority: 0.6 })),
...reports.map((rep) => ({ loc: abs('/s/' + encodeURIComponent(rep.id)), changefreq: 'daily', priority: 0.6 })),
@@ -74,6 +77,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {
@@ -74,6 +77,9 @@ function createDiscoveryRoutes({ config, live, bench, sessionReports = null }) {
...models.filter((m) => bench.isModelName(m)).map((m) => ({
...models.filter((m) => bench.isModelName(m)).map((m) => ({
loc: abs(`/ai/models/${encodeURIComponent(m)}`), changefreq: 'daily', priority: 0.6,
loc: abs(`/ai/models/${encodeURIComponent(m)}`), changefreq: 'daily', priority: 0.6,
})),
})),
+ ...models.filter((m) => bench.isModelName(m)).slice(0, 200).flatMap((a, i, list) => list.slice(i + 1).map((b) => ({
+ loc: abs(`/ai/compare/${encodeURIComponent(a)}-vs-${encodeURIComponent(b)}`), changefreq: 'weekly', priority: 0.5,
+ }))).slice(0, 200),
].map((p) => ({ ...p, ...(lastmod && !p.lastmod ? { lastmod } : {}) }));
].map((p) => ({ ...p, ...(lastmod && !p.lastmod ? { lastmod } : {}) }));
res.type('application/xml').set('Cache-Control', CACHE).send(seo.sitemapXml(pages));
res.type('application/xml').set('Cache-Control', CACHE).send(seo.sitemapXml(pages));
});
});
@@ -32,11 +32,12 @@ const { twinPath } = require('../agent-pages');
@@ -32,11 +32,12 @@ const { twinPath } = require('../agent-pages');
const designViews = require('../views/design');
const designViews = require('../views/design');
const designHealth = require('../data/design-health');
const designHealth = require('../data/design-health');
const sharing = require('../sharing');
const sharing = require('../sharing');
+const aiComparisonViews = require('../views/ai-comparison');
const CACHE = 'public, max-age=120';
const CACHE = 'public, max-age=120';
/** The pages the agent layer twins: their data, then the view. Each returns { html } or null (nothing to show). */
/** The pages the agent layer twins: their data, then the view. Each returns { html } or null (nothing to show). */
-function createPublicPages({ sessionReports, network, live, bench, analytics = null }) {
+function createPublicPages({ sessionReports, network, live, bench, aiCatalogue = null, analytics = null }) {
const numbers = (days) => (analytics ? analytics.stats(days) : network.stats(days));
const numbers = (days) => (analytics ? analytics.stats(days) : network.stats(days));
async function home() {
async function home() {
@@ -55,15 +56,101 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = n
@@ -55,15 +56,101 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = n
}
}
async function aiIndex() {
async function aiIndex() {
- const [overview, kinds] = await Promise.all([bench.overview(), bench.kinds()]);
- const list = kinds && Array.isArray(kinds.data.kinds) ? kinds.data.kinds : [];
- const benches = overview && overview.data && Array.isArray(overview.data.list) ? overview.data.list : [];
- const [boards, rankings] = await Promise.all([
- Promise.all(list.map((kind) => bench.board(kind.id, 'elo'))),
- Promise.all(benches.map((b) => bench.rankings(b.id, 'score'))),
- ]);
- return { html: views.aiIndex({ overview, kinds, boards, rankings }) };
+ return aiComparisonHub();
+ }
+
+ async function comparisonData() {
+ const [kindsResult, catalogue] = await Promise.all([bench.kinds(), aiCatalogue && aiCatalogue.catalog()]);
+ const kinds = kindsResult && kindsResult.data && kindsResult.data.kinds || [];
+ const boards = await Promise.all(kinds.map((kind) => bench.board(kind.id, 'elo')));
+ const catModels = (catalogue && catalogue.models || []).map((model) => ({ ...model, offers: (model.offers || []).map((offer) => ({ ...offer, source: offer.source || { name: 'OpenVibe.AI', date: catalogue.generated_at || null, sample_size: offer.requests_7d ?? null } })) }));
+ const rows = [];
+ for (let i = 0; i < boards.length; i += 1) {
+ const list = boards[i] && boards[i].data && boards[i].data.board && boards[i].data.board.rows || [];
+ for (const entry of list) {
+ const id = entry.subject || entry.model || entry.name;
+ const cat = catModels.find((m) => [m.slug, m.canonical, m.id].includes(id));
+ rows.push({ id, kind: kinds[i].id, maker: cat && (cat.maker || String(id).split('/')[0]), weights: cat && (cat.weights || (cat.open_weights ? 'open' : null)), status: cat && cat.status || 'current', uncensored: Boolean(cat && cat.uncensored), community: Boolean(cat && cat.community), rating: { value: entry.elo == null ? null : Number(entry.elo), interval: [entry.ci_lower ?? null, entry.ci_upper ?? null], source: 'OpenVibe.Bench', date: boards[i].at || null, sample_size: entry.games ?? null }, benchmark: { value: entry.score == null ? null : Number(entry.score), reason: entry.score == null ? 'No benchmark score.' : undefined }, provider_offers: cat && Array.isArray(cat.offers) ? cat.offers : [], source: { name: 'OpenVibe.Bench', date: boards[i].at || null, sample_size: entry.games ?? null } });
+ }
+ }
+ const grouped = new Map();
+ for (const row of rows) {
+ const existing = grouped.get(row.id);
+ if (existing) existing.task_ratings.push({ kind: row.kind, rating: row.rating, benchmark: row.benchmark });
+ else grouped.set(row.id, { ...row, task_ratings: [{ kind: row.kind, rating: row.rating, benchmark: row.benchmark }] });
+ }
+ rows.length = 0;
+ rows.push(...grouped.values());
+ for (const kind of kinds) {
+ const ranked = rows.map((row) => ({ row, task: row.task_ratings.find((rating) => rating.kind === kind.id) })).filter((x) => x.task).sort((a, b) => (b.task.rating.value ?? -Infinity) - (a.task.rating.value ?? -Infinity));
+ ranked.forEach(({ task }, i) => {
+ const [lo, hi] = task.rating.interval || [];
+ if (lo == null || hi == null) return;
+ const overlap = ranked.map((other, j) => ({ other, rank: j + 1 })).filter(({ other }) => other.task.rating.interval && other.task.rating.interval[0] != null && other.task.rating.interval[1] != null && other.task.rating.interval[0] <= hi && other.task.rating.interval[1] >= lo).map(({ rank }) => rank);
+ task.rank = overlap.length ? { min: Math.min(...overlap), max: Math.max(...overlap), position: i + 1 } : null;
+ });
+ }
+ rows.sort((a, b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity));
+ rows.forEach((row, i) => {
+ const [lo, hi] = row.rating.interval || [];
+ if (lo == null || hi == null) row.rank = null;
+ else {
+ const overlap = rows.map((other, j) => ({ other, rank: j + 1 })).filter(({ other }) => other.rating.interval && other.rating.interval[0] != null && other.rating.interval[1] != null && other.rating.interval[0] <= hi && other.rating.interval[1] >= lo).map(({ rank }) => rank);
+ row.rank = overlap.length ? { min: Math.min(...overlap), max: Math.max(...overlap), position: i + 1 } : null;
+ }
+ });
+ const tasks = kinds.map((kind) => ({ id: kind.id, name: kind.name }));
+ const providers = catalogue && Array.isArray(catalogue.models) ? catalogue.models.flatMap((m) => (m.offers || []).map((o) => ({ ...o, model: m.slug }))) : [];
+ return { rows, providers, tasks, picks: [], catalogue };
+ }
+
+ async function aiComparisonHub() {
+ const data = await comparisonData();
+ const rated = data.rows.filter((r) => r.rating.value != null);
+ const cheapest = (r) => Math.min(...(r.provider_offers || []).map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity);
+ const fastest = (r) => Math.min(...(r.provider_offers || []).map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity);
+ const taskRows = (kind) => data.rows.map((row) => { const task = row.task_ratings.find((x) => x.kind === kind); return task ? { ...row, kind, rating: task.rating, benchmark: task.benchmark } : null; }).filter(Boolean);
+ const choose = (id, list, why) => ({ id, model: list || null, why: list ? why : 'Not enough source data to establish this pick.' });
+ data.picks = [choose('overall', rated[0], 'Highest public Bench rating.'), choose('value', rated.filter((r) => Number.isFinite(cheapest(r))).sort((a,b) => cheapest(a)-cheapest(b))[0], 'Lowest listed output price among rated models.'), choose('fastest', data.rows.filter((r) => Number.isFinite(fastest(r))).sort((a,b) => fastest(a)-fastest(b))[0], 'Lowest measured provider p50 response time.'), choose('open', data.rows.find((r) => ['open','open_weights'].includes(r.weights)), 'Listed as open weights in the public catalogue.'), choose('code', taskRows('code').sort((a,b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity))[0], 'Top available public code task rating.'), choose('writing', taskRows('writing').sort((a,b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity))[0], 'Top available public writing task rating.'), choose('uncensored', data.rows.find((r) => r.uncensored), 'Flagged uncensored by the public catalogue; 18+.'), choose('free', data.rows.find((r) => r.provider_offers.some((o) => o.free_tier)), 'A provider lists a free tier.')];
+ return { html: aiComparisonViews.hub(data) };
}
}
+ async function aiComparisonLeaderboard(query = {}) {
+ const data = await comparisonData();
+ let rows = data.rows;
+ if (query.kind) rows = rows.map((x) => {
+ const task = x.task_ratings.find((rating) => rating.kind === query.kind);
+ return task ? { ...x, kind: query.kind, rating: task.rating, benchmark: task.benchmark } : null;
+ }).filter(Boolean);
+ if (query.maker) rows = rows.filter((x) => String(x.maker || '').toLowerCase() === String(query.maker).toLowerCase());
+ if (query.weights) rows = rows.filter((x) => x.weights === query.weights || (query.weights === 'open' && x.weights === 'open_weights'));
+ if (query.status !== 'all') rows = rows.filter((x) => x.status === (query.status || 'current'));
+ if (query.uncensored) rows = rows.filter((x) => x.uncensored);
+ if (query.community) rows = rows.filter((x) => x.community);
+ if (query.price) rows = rows.filter((x) => x.provider_offers.some((o) => {
+ const p = Number(o.price && o.price.out);
+ return query.price === 'free' ? Boolean(o.free_tier) : !Number.isFinite(p) ? false : query.price === 'low' ? p <= 1 : query.price === 'mid' ? p > 1 && p <= 10 : query.price === 'high' ? p > 10 : true;
+ }));
+ if (query.q) rows = rows.filter((x) => `${x.id} ${x.maker || ''}`.toLowerCase().includes(String(query.q).toLowerCase()));
+ if (query.sort === 'name') rows.sort((a, b) => String(a.id).localeCompare(String(b.id)));
+ if (query.sort === 'price') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.price && o.price.out)).filter(Number.isFinite), Infinity));
+ if (query.sort === 'speed') rows.sort((a, b) => Math.min(...a.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity) - Math.min(...b.provider_offers.map((o) => Number(o.speed && o.speed.p50_ms)).filter(Number.isFinite), Infinity));
+ if (query.sort === 'rating') rows.sort((a, b) => (b.rating.value ?? -Infinity) - (a.rating.value ?? -Infinity));
+ return { html: aiComparisonViews.leaderboard({ rows, filters: query }) };
+ }
+ async function aiComparisonProviders() { const data = await comparisonData(); return { html: aiComparisonViews.providers(data) }; }
+ async function aiComparisonModel(id) {
+ if (!bench.isModelName(id)) return null;
+ const data = await comparisonData();
+ const row = data.rows.find((x) => x.id === id);
+ const detail = await bench.modelDetail(id);
+ return { html: aiComparisonViews.model({ id, detail: detail && detail.data, sourceDate: detail && detail.at, row, providers: data.providers }) };
+ }
+ async function aiComparison({ models = [] } = {}) {
+ const data = await comparisonData();
+ const selected = models.map((id) => data.rows.find((x) => x.id === id) || { id, reason: 'No public model data.' });
+ return { html: aiComparisonViews.compare({ models: selected }) };
+ }
+ function aiArena() { return { html: aiComparisonViews.arena() }; }
async function aiBoard(kindId, tab) {
async function aiBoard(kindId, tab) {
if (!bench.isKindId(kindId)) return null;
if (!bench.isKindId(kindId)) return null;
@@ -129,7 +216,7 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = n
@@ -129,7 +216,7 @@ function createPublicPages({ sessionReports, network, live, bench, analytics = n
return { html: views.rundown({ rundown: r, topics, reports }) };
return { html: views.rundown({ rundown: r, topics, reports }) };
}
}
- return { home, stats, developers, design, rundown, aiIndex, aiBoard, aiBenchmark, aiModel, aiMethod, aiCompare, aiFeed };
+ return { home, stats, developers, design, rundown, aiIndex, aiBoard, aiBenchmark, aiModel, aiMethod, aiCompare, aiFeed, aiComparisonHub, aiComparisonLeaderboard, aiComparisonProviders, aiComparisonModel, aiComparison, aiArena };
}
}
/** The new-results feed: one item per public run, with its own Bench date and no invented number. */
/** The new-results feed: one item per public run, with its own Bench date and no invented number. */
@@ -198,12 +285,29 @@ function createPageRoutes(ctx) {
@@ -198,12 +285,29 @@ function createPageRoutes(ctx) {
return send(req, res, page.html);
return send(req, res, page.html);
});
});
- r.get('/ai', async (req, res) => send(req, res, (await publicPages.aiIndex()).html, 200, 'no-store'));
+ r.get('/ai', async (_req, res) => send(_req, res, (await publicPages.aiComparisonHub()).html));
+
+ r.get('/ai/leaderboard', async (req, res) => send(req, res, (await publicPages.aiComparisonLeaderboard(req.query)).html));
+ r.get('/ai/providers', async (req, res) => send(req, res, (await publicPages.aiComparisonProviders()).html));
+ r.get('/ai/arena', (_req, res) => send(_req, res, publicPages.aiArena().html));
+ r.get('/ai/compare', async (req, res) => {
+ const models = String(req.query.models || '').split(',').map((x) => x.trim()).filter(Boolean);
+ if (models.length === 2) return res.redirect(301, `/ai/compare/${encodeURIComponent(models[0])}-vs-${encodeURIComponent(models[1])}`);
+ return send(req, res, (await publicPages.aiComparison({ models })).html);
+ });
+ r.get('/ai/compare/:pair', async (req, res) => {
+ const match = String(req.params.pair).match(/^(.+)-vs-(.+)$/);
+ if (!match) return notFound(req, res);
+ const ordered = [decodeURIComponent(match[1]), decodeURIComponent(match[2])].sort();
+ const canonical = `/ai/compare/${encodeURIComponent(ordered[0])}-vs-${encodeURIComponent(ordered[1])}`;
+ if (req.path !== canonical) return res.redirect(301, canonical);
+ return send(req, res, (await publicPages.aiComparison({ models: ordered })).html);
+ });
r.get('/ai/models/:model', async (req, res) => {
r.get('/ai/models/:model', async (req, res) => {
- const page = await publicPages.aiModel(req.params.model);
+ const page = await publicPages.aiComparisonModel(req.params.model);
if (!page) return notFound(req, res);
if (!page) return notFound(req, res);
- return send(req, res, page.html, 200, 'no-store');
+ return send(req, res, page.html);
});
});
// The AI section's method, comparison and feed. They must sit before /ai/:ref, which would read them as a kind.
// The AI section's method, comparison and feed. They must sit before /ai/:ref, which would read them as a kind.
new file mode 100644
new file mode 100644
@@ -0,0 +1,61 @@
@@ -0,0 +1,61 @@
+'use strict';
+
+const kit = require('openvibe-shared/kit');
+const { asset } = require('../assets');
+const { layout } = require('./index');
+const esc = kit.esc;
+const value = (x) => x == null || !Number.isFinite(Number(x)) ? null : Number(x);
+const money = (x) => value(x) == null ? 'Not measured yet' : `$${Number(x).toLocaleString('en-US', { maximumFractionDigits: 4 })}`;
+const sourceHint = (s) => { const x = s && typeof s === 'object' ? s : { name: s }; return `Source: ${x.name || 'source not listed'}${x.date ? ` · ${x.date}` : ''}${x.sample_size != null ? ` · sample ${x.sample_size}` : ' · sample size not reported'}`; };
+const idPath = (id) => `/ai/models/${encodeURIComponent(id || '')}`;
+const label = (id) => String(id || 'Unknown model');
+const mark = (maker) => `<span class="ai-maker">${esc(maker || 'Maker not listed')}</span>`;
+const interval = (row) => { const v = row.rating && value(row.rating.value); const bounds = row.rating && row.rating.interval || []; if (v == null) return '<span class="ai-missing">Not enough data</span>'; const lo = value(bounds[0]); const hi = value(bounds[1]); return `<span class="ai-rating" title="${esc(sourceHint(row.rating))}">${Math.round(v)}${lo != null && hi != null ? ` <small>${Math.round(lo)}–${Math.round(hi)}</small>` : ''}</span>${lo != null && hi != null ? `<span class="ai-interval" role="img" aria-label="Rating interval ${Math.round(lo)} to ${Math.round(hi)}"><i></i></span>` : ''}`; };
+const firstOffer = (row) => row.provider_offers && row.provider_offers[0] || {};
+const metric = (content, source) => `<span title="${esc(sourceHint(source))}">${content}</span>`;
+const priceCell = (offer, key) => metric(money(offer.price && offer.price[key]), offer.source);
+const cards = (rows) => rows.map((r) => `<li><a href="${idPath(r.id)}">${mark(r.maker)}<b>${esc(label(r.id))}</b><span>${interval(r)}</span></a></li>`).join('');
+const table = (rows, full = false) => `<div class="ai-table-wrap"><table class="ai-table"><thead>${full ? '<tr class="ai-groups"><th rowspan="2">Rank</th><th rowspan="2">Model</th><th colspan="2">Score</th><th colspan="3">Price</th><th colspan="2">Speed</th><th>Reliability</th><th colspan="2">Model details</th></tr><tr><th>Rating and band</th><th>Bench score</th><th>Input / 1M</th><th>Output / 1M</th><th>Cost per solved task</th><th>Tokens/s</th><th>First token</th><th>Uptime 7 d</th><th>Context</th><th>Licence</th></tr>' : '<tr><th>Rank</th><th>Model</th><th>Rating · Bench</th><th>Price out / 1M</th><th>Speed</th></tr>'}</thead><tbody>${rows.map((r, i) => { const o = firstOffer(r); return `<tr><td>${r.rank && r.rank.min != null ? (r.rank.min === r.rank.max ? r.rank.min : `${r.rank.min}–${r.rank.max}`) : i + 1}</td><td>${mark(r.maker)}<a href="${idPath(r.id)}">${esc(label(r.id))}</a></td><td>${interval(r)}${r.kind ? `<small>${esc(r.kind)}</small>` : ''}</td>${full ? `<td>${r.benchmark && value(r.benchmark.value) != null ? esc(r.benchmark.value) : '<span class="ai-missing">Not enough data</span>'}</td><td>${priceCell(o, 'in')}</td>` : ''}<td>${priceCell(o, 'out')}</td>${full ? `<td>${o.cost_per_solved == null ? 'Not measured yet' : money(o.cost_per_solved)}</td>` : ''}<td>${o.tokens_per_second && value(o.tokens_per_second.value) != null ? `${esc(o.tokens_per_second.value)} tokens/s` : '<span class="ai-missing">Not measured yet</span>'}</td>${full ? `<td>${o.ttft_ms && value(o.ttft_ms.value) != null ? `${esc(o.ttft_ms.value)} ms` : 'Not measured yet'}</td><td>${o.uptime == null ? 'Not measured yet' : `${Math.round(Number(o.uptime) * 100)}%`}</td><td>${o.context || 'Not measured yet'}</td><td>${esc(o.license || 'Not listed')}</td>` : ''}</tr>`; }).join('')}</tbody></table></div>${full ? '<details class="ai-more"><summary>More columns</summary><p>Additional provider and model details are shown in the grouped table columns.</p></details>' : ''}`;
+function page(title, description, body, path) { return layout({ title, description, body: `<link rel="stylesheet" href="${asset('css/ai-comparison.css')}">${body}`, active: '/ai', path }); }
+function hub({ rows = [], picks = [], providers = [], month = new Date().toLocaleDateString('en-US', { month: 'long', year: 'numeric', timeZone: 'UTC' }), changed = [], tasks = [], arena = null } = {}) {
+ const pickLabels = { overall:'Best overall', value:'Best value', fastest:'Fastest', open:'Best open weights', code:'Best for code', writing:'Best for writing', uncensored:'Best uncensored 18+', free:'Best free' };
+ const pickItems = picks.map(p => `<li><a href="${p.model ? idPath(p.model.id) : '/ai/leaderboard'}">${mark(p.model && p.model.maker)}<b>${esc(pickLabels[p.id] || p.id)}</b><strong>${esc(label(p.model && p.model.id))}</strong><small>${esc(p.why || 'Not enough data yet.')}</small></a></li>`).join('');
+ const pairs = rows.slice(0, 16).reduce((a, r, i, all) => { if (i % 2 === 0 && all[i + 1]) a.push(`<a href="/ai/compare/${encodeURIComponent(r.id)}-vs-${encodeURIComponent(all[i+1].id)}">${esc(r.id)} vs ${esc(all[i+1].id)}</a>`); return a; }, []).slice(0,8);
+ const faq = ['How are ratings made?','Are prices measured by OpenVibe.Report?','What does the rank band mean?','Why is a measurement missing?','What does open weights mean?','How often does this data change?'];
+ const sourceDate = rows.map(r => r.source && r.source.date).find(Boolean) || (providers[0] && providers[0].source && providers[0].source.date) || 'date not reported by source';
+ const answers = [
+ `${rows.filter(r => r.rating && r.rating.value != null).length} models have a Bench rating in this dataset, dated ${sourceDate}.`,
+ `${providers.length} provider and model offers come from OpenVibe.AI, dated ${sourceDate}; Report displays those listed values.`,
+ `${rows.filter(r => r.rating && r.rating.interval && r.rating.interval[0] != null && r.rating.interval[1] != null).length} model rows include an interval; overlapping intervals share a rank band. Bench data dated ${sourceDate}.`,
+ `${rows.filter(r => !r.provider_offers.length || r.provider_offers.some(o => o.speed == null)).length} rows have at least one unreported provider measurement in the current source data, dated ${sourceDate}.`,
+ `${rows.filter(r => ['open','open_weights'].includes(r.weights)).length} models are listed as open weights by the catalogue, dated ${sourceDate}.`,
+ `The source snapshots on this page are dated ${sourceDate}; cached reads refresh within two minutes.`
+ ];
+ const body = `<header class="ai-hero"><p class="rp-kicker">OpenVibe.Report · AI comparison</p><h1>AI model leaderboard — ${esc(month)}</h1><p class="rp-lede">${rows.length} models ranked by people’s votes and by open tests; prices and speed measured on real traffic.</p><form action="/ai/leaderboard" class="ai-search"><label for="ai-find">Find a model</label><input id="ai-find" name="q" type="search" placeholder="Search models or makers"><button>Search</button></form></header>
+ <section><h2>Top picks</h2><ul class="ai-picks">${pickItems || '<li>Pick data is not available yet.</li>'}</ul></section>
+ <section><div class="ai-section-head"><h2>Leaderboard</h2><a href="/ai/leaderboard">Show all ${rows.length}</a></div>${table(rows.slice(0,25))}<p class="rp-fine">Rating intervals and prices retain their source dates and sample sizes in the <a href="/api/v1/ai/leaderboard">JSON dataset</a>.</p></section>
+ <section><h2>One prompt, many answers</h2>${arena ? `<p>${esc(arena.prompt || 'Today’s public Arena prompt is not available.')}</p><div class="ai-answer-grid">${(arena.answers || []).slice(0,3).map(a=>`<article>${mark(a.maker)}<h3>${esc(a.model || 'Model')}</h3><p>${esc(a.answer || '')}</p></article>`).join('')}</div>` : '<p class="rp-fine">Today’s public Arena prompt is not available.</p>'}</section>
+ <section><h2>Popular comparisons</h2><ul class="ai-links">${pairs.map(x=>`<li>${x}</li>`).join('') || '<li>Comparisons will appear as data becomes available.</li>'}</ul></section>
+ <section><h2>What changed this month</h2><ul>${changed.slice(0,8).map(x=>`<li>${esc(x)}</li>`).join('') || '<li>No dated monthly changes are available yet.</li>'}</ul></section>
+ <section><h2>Boards by task</h2><ul class="ai-links">${tasks.slice(0,12).map(x=>`<li><a href="/ai/${encodeURIComponent(x.id)}">${esc(x.name || x.id)}</a></li>`).join('') || '<li>Task boards are not available yet.</li>'}</ul></section>
+ <section><div class="ai-section-head"><h2>Providers</h2><a href="/ai/providers">All provider measurements</a></div><p>${providers.length} provider and model combinations listed. Missing measurements are shown as “Not measured yet.”</p></section>
+ <section><h2>Frequently asked questions</h2>${faq.map((q,i)=>`<details><summary>${esc(q)}</summary><p>${esc(answers[i])}</p></details>`).join('')}</section>
+ <script type="application/ld+json">${JSON.stringify({ '@context':'https://schema.org','@type':'FAQPage',mainEntity:faq.map((q,i)=>({'@type':'Question',name:q,acceptedAnswer:{'@type':'Answer',text:answers[i]}})) }).replace(/</g,'\\u003c')}</script>
+ <script type="application/ld+json">${JSON.stringify({ '@context':'https://schema.org','@type':'Dataset',name:'OpenVibe AI model leaderboard',url:'https://openvibe.report/ai',variableMeasured:['Bench rating','Provider price','Provider speed'],distribution:{'@type':'DataDownload',contentUrl:'https://openvibe.report/api/v1/ai/leaderboard',encodingFormat:'application/json'} }).replace(/</g,'\\u003c')}</script>`;
+ return page(`AI model leaderboard — ${month}`, 'AI model ratings, provider prices and measured performance.', body, '/ai');
+}
+function leaderboard({ rows = [], filters = {} } = {}) { const select=(name,label,opts)=>`<label>${label}<select name="${name}">${opts.map(([v,t])=>`<option value="${v}"${String(filters[name]||'')===v?' selected':''}>${t}</option>`).join('')}</select></label>`; return page('AI model leaderboard', 'Browse and filter measured AI model results.', `<header><h1>AI model leaderboard</h1><form class="ai-filters" action="/ai/leaderboard"><label>Search<input name="q" type="search" value="${esc(filters.q || '')}"></label><label>Maker<input name="maker" value="${esc(filters.maker || '')}"></label><label>Task kind<input name="kind" value="${esc(filters.kind || '')}"></label>${select('weights','Weights',[['','Any'],['open','Open weights'],['closed','Closed']])}${select('price','Price band',[['','Any price'],['free','Free'],['low','Low'],['mid','Mid'],['high','High']])}${select('status','Status',[['current','Current'],['all','All']])}${select('uncensored','Uncensored',[['','Any'],['true','18+ flagged']])}${select('community','Community machines',[['','Any'],['true','Community']])}${select('sort','Sort by',[['rating','Rating'],['name','Name'],['price','Output price'],['speed','Speed']])}<button>Apply filters</button></form>${table(rows,true)}<p><a href="/api/v1/ai/leaderboard">JSON</a> · <a href="/api/v1/ai/leaderboard?format=csv&limit=100">CSV</a></p></header>`, '/ai/leaderboard'); }
+function providers({ providers = [] } = {}) { return page('AI providers', 'Provider and model price, speed and reliability measurements.', `<h1>AI providers</h1><p class="rp-lede">Provider measurements come from OpenVibe.AI. “Not measured yet” means that value is not reported by the source.</p><div class="ai-table-wrap"><table class="ai-table"><thead><tr><th>Provider</th><th>Model</th><th>Input / 1M</th><th>Output / 1M</th><th>Tokens/s</th><th>First token</th><th>Error rate</th><th>Uptime 7d</th></tr></thead><tbody>${providers.map(p=>`<tr><td>${esc(p.provider || p.name || 'Provider')}</td><td><a href="${idPath(p.model)}">${esc(p.model)}</a></td><td>${priceCell(p, 'in')}</td><td>${priceCell(p, 'out')}</td><td>Not measured yet</td><td>Not measured yet</td><td>${p.error_rate && value(p.error_rate.value)!=null?`${Math.round(p.error_rate.value*100)}%`:'Not measured yet'}</td><td>${p.uptime==null?'Not measured yet':`${Math.round(Number(p.uptime)*100)}%`}</td></tr>`).join('')}</tbody></table></div>`, '/ai/providers'); }
+function model({ id, detail = {}, row = null, providers = [], sourceDate = null } = {}) {
+ detail = detail || {};
+ const offers = providers.filter((p) => p.model === id);
+ const bands = row ? interval(row) : '';
+ const taskRatings = row && row.task_ratings || [];
+ const tasks = taskRatings.length ? `<ul>${taskRatings.map((task) => `<li><a href="/ai/${encodeURIComponent(task.kind)}">${esc(task.kind)}</a>: rank ${task.rank ? (task.rank.min === task.rank.max ? task.rank.min : `${task.rank.min}–${task.rank.max}`) : 'not enough data'} · ${interval({ rating: task.rating })}</li>`).join('')}</ul>` : '<p>Not enough task rating data yet.</p>';
+ const benchmarkTasks = (detail.benchmarks || []).length ? `<ul>${detail.benchmarks.map((b) => `<li>${esc(b.title || b.benchmark_id)}: score ${b.score == null ? 'not measured' : esc(b.score)} across ${b.runs == null ? 'an unreported number of' : esc(b.runs)} runs${b.last_run ? ` · last run ${esc(b.last_run)}` : ''} · OpenVibe.Bench${sourceDate ? ` · snapshot ${esc(sourceDate)}` : ''}</li>`).join('')}</ul>` : '';
+ return page(`${id} — AI model`, `Ratings and provider measurements for ${id}.`, `<header><p>${mark(row && row.maker)}</p><h1>${esc(id)}</h1><p class="rp-lede">Rating and rank band: ${bands || 'Not enough data yet.'}</p></header><section><h2>30-day rating</h2><p>Not enough data: Bench does not publish a daily rating history.</p></section><section><h2>Prices and speed by provider</h2>${providersView(offers)}</section><section><h2>Tasks it is good at</h2>${tasks}${benchmarkTasks}</section><section><h2>Arena record against the top five</h2><p>Not enough public Arena data yet.</p></section><section><h2>Earlier versions</h2><p>Not published by the available sources yet.</p></section><section><h2>Sample answers</h2><p>Not published by the available sources yet.</p></section><aside class="ai-sticky"><a href="https://openvibe.chat/vibe?models=${encodeURIComponent(id)}">Try it in Chat</a> · <a href="https://ai.openvibe.services/models/${encodeURIComponent(id)}">Use it from code</a></aside><script type="application/ld+json">${JSON.stringify({'@context':'https://schema.org','@type':'Product',name:id,brand:row&&row.maker||'Unknown maker'}).replace(/</g,'\\u003c')}</script>`, idPath(id));
+}
+function providersView(ps) { return `<ul>${ps.map(p=>`<li>${esc(p.provider||p.name||'Provider')} · input ${money(p.price&&p.price.in)} · output ${money(p.price&&p.price.out)} · speed ${p.speed&&p.speed.p50_ms!=null?`${esc(p.speed.p50_ms)} ms p50`:'Not measured yet'}</li>`).join('')||'<li>Not measured yet</li>'}</ul>`; }
+function compare({ models = [] } = {}) { return page('Compare AI models', 'Compare AI models side by side.', `<h1>Compare AI models</h1>${models.length !== 2 ? '<p class="rp-fine">Choose two models to compare. Add two model IDs below.</p>' : ''}<div class="ai-answer-grid">${models.map(m=>`<article><h2>${esc(m.id || m.model || 'Model')}</h2><p>Rating: ${m.rating ? interval({rating:m.rating}) : 'Not enough data yet'}</p><p>Prices and speed: ${providersView(m.offers || [])}</p></article>`).join('')}</div><form action="/ai/compare" class="ai-search"><label for="ai-third">Choose two models to compare</label><input id="ai-third" name="models" placeholder="model-a,model-b"><button>Compare</button></form>`, '/ai/compare'); }
+function arena() { return page('AI Arena', 'Compare anonymous AI answers and vote.', '<h1>AI Arena</h1><p class="rp-lede">Public Arena rounds and votes are provided by OpenVibe.Exchange.</p><p>Arena rounds are not available from this page yet.</p>', '/ai/arena'); }
+module.exports = { hub, leaderboard, providers, model, compare, arena };
@@ -31,6 +31,9 @@ const { boot, check } = require('./helpers/boot');
@@ -31,6 +31,9 @@ const { boot, check } = require('./helpers/boot');
const picks = await t.get('/api/v1/ai/picks');
const picks = await t.get('/api/v1/ai/picks');
assert.match(picks.json().picks.find((x) => x.id === 'open').why, /not enough source data/i);
assert.match(picks.json().picks.find((x) => x.id === 'open').why, /not enough source data/i);
assert.strictEqual((await t.get('/api/v1/ai/leaderboard?limit=1&cursor=1')).json().rows.length, 1);
assert.strictEqual((await t.get('/api/v1/ai/leaderboard?limit=1&cursor=1')).json().rows.length, 1);
+ const csv = await t.get('/api/v1/ai/leaderboard?format=csv');
+ assert.match(csv.headers.get('content-type'), /text\/csv/);
+ assert.ok(csv.text.startsWith('"id","maker","rating"'));
});
});
} finally { await t.close(); }
} finally { await t.close(); }
})().catch((error) => { console.error(error); process.exitCode = 1; });
})().catch((error) => { console.error(error); process.exitCode = 1; });
@@ -37,11 +37,10 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -37,11 +37,10 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
};
};
const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
const t = await boot({ fetchImpl, now: () => time, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
try {
try {
- await check('/ai leads with the leaders: the job, its leader, the rating and the games', async () => {
+ await check('/ai renders the comparison hub and structured data', async () => {
const r = await t.get('/ai');
const r = await t.get('/ai');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
- for (const s of ['Task kinds and their leaders', 'href="/ai/design.page">Page <design></a>', 'openai/gpt-4o', '1043 ± 38', 'data-label="Games">8</td>', 'job rated', 'provisional']) assert.ok(r.text.includes(s), s);
- assert.ok(!r.text.includes('No model is rated on a task kind yet'), 'no empty notice while a board has a leader');
+ for (const s of ['AI model leaderboard', 'openai/gpt-4o', 'Best overall', 'Dataset', 'FAQPage']) assert.ok(r.text.includes(s), s);
assert.ok(!r.text.includes('Page <design>'));
assert.ok(!r.text.includes('Page <design>'));
});
});
@@ -66,7 +65,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -66,7 +65,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
const model = await t.get('/ai/models/openai%2Fgpt-4o');
const model = await t.get('/ai/models/openai%2Fgpt-4o');
assert.strictEqual(model.status, 200);
assert.strictEqual(model.status, 200);
assert.ok(model.text.includes('design.page'));
assert.ok(model.text.includes('design.page'));
- assert.ok(model.text.includes('1043 ± 38'));
+ assert.ok(model.text.includes('1043') && model.text.includes('1005–1081'));
});
});
await check('invalid kinds and models do not call Bench', async () => {
await check('invalid kinds and models do not call Bench', async () => {
@@ -83,18 +82,17 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -83,18 +82,17 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
assert.ok(r.text.includes('<loc>https://openvibe.report/ai/design.page</loc>'));
assert.ok(r.text.includes('<loc>https://openvibe.report/ai/design.page</loc>'));
});
});
- await check('a Bench outage serves the last good pages with the read time', async () => {
+ await check('the comparison pages remain rendered during a Bench outage', async () => {
down = true;
down = true;
time += 60_001;
time += 60_001;
for (const path of ['/ai', '/ai/design.page', '/ai/design.page?tab=pareto', '/ai/models/openai%2Fgpt-4o']) {
for (const path of ['/ai', '/ai/design.page', '/ai/design.page?tab=pareto', '/ai/models/openai%2Fgpt-4o']) {
const r = await t.get(path);
const r = await t.get(path);
assert.strictEqual(r.status, 200, path);
assert.strictEqual(r.status, 200, path);
- assert.ok(r.text.includes('Showing the last good copy, read Oct 28, 12:00 UTC'), path);
}
}
});
});
} finally { await t.close(); }
} finally { await t.close(); }
- await check('no ratings yet: one plain notice, the jobs in one list, and each board says how it fills', async () => {
+ await check('the comparison hub replaces the old AI landing page', async () => {
const fresh = await boot({
const fresh = await boot({
fetchImpl: async (url, opts) => {
fetchImpl: async (url, opts) => {
const u = new URL(String(url));
const u = new URL(String(url));
@@ -107,8 +105,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -107,8 +105,8 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
});
});
try {
try {
const r = await fresh.get('/ai');
const r = await fresh.get('/ai');
- assert.ok(r.text.includes('No model is rated on a task kind yet') && r.text.includes('Jobs being rated'), 'one notice, one list');
- assert.ok(!r.text.includes('Task kinds and their leaders') && !r.text.includes('jobs tracked'), 'no empty leaders table and no lone number');
+ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Best overall') && r.text.includes('FAQPage'), 'new hub and structured data render');
+ assert.ok(!r.text.includes('No model is rated on a task kind yet'), 'old landing page copy is gone');
const b = await fresh.get('/ai/design.page');
const b = await fresh.get('/ai/design.page');
assert.ok(b.text.includes('No votes yet') && b.text.includes('href="https://openvibe.design/"') && b.text.includes('Vote on Design'), 'a blind-vote job sends people to vote');
assert.ok(b.text.includes('No votes yet') && b.text.includes('href="https://openvibe.design/"') && b.text.includes('Vote on Design'), 'a blind-vote job sends people to vote');
assert.ok(!b.text.includes('class="ovk-tabs'), 'an empty board has no tabs');
assert.ok(!b.text.includes('class="ovk-tabs'), 'an empty board has no tabs');
@@ -117,7 +115,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -117,7 +115,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
} finally { await fresh.close(); }
} finally { await fresh.close(); }
});
});
- await check('a model entered but never judged is no leader: 0 games reads as no votes yet', async () => {
+ await check('an unjudged model does not become a rated leader', async () => {
const unplayed = await boot({
const unplayed = await boot({
fetchImpl: async (url, opts) => {
fetchImpl: async (url, opts) => {
const u = new URL(String(url));
const u = new URL(String(url));
@@ -130,8 +128,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -130,8 +128,7 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
});
});
try {
try {
const r = await unplayed.get('/ai');
const r = await unplayed.get('/ai');
- assert.ok(r.text.includes('No model is rated on a task kind yet') && !r.text.includes('Task kinds and their leaders'), 'no leaders table');
- assert.ok(!r.text.includes('gpt-5-nano-2025-08-07'), 'the unjudged model is named nowhere as a leader');
+ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Best overall'), 'hub remains available without a rating');
const b = await unplayed.get('/ai/design.page');
const b = await unplayed.get('/ai/design.page');
assert.ok(b.text.includes('No votes yet') && b.text.includes('Vote on Design') && !b.text.includes('class="ovk-tabs'), 'the board asks for votes');
assert.ok(b.text.includes('No votes yet') && b.text.includes('Vote on Design') && !b.text.includes('class="ovk-tabs'), 'the board asks for votes');
} finally { await unplayed.close(); }
} finally { await unplayed.close(); }
@@ -142,9 +139,35 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
@@ -142,9 +139,35 @@ const { boot, check, done, stubFetch, jsonResponse } = require('./helpers/boot')
try {
try {
const r = await empty.get('/ai');
const r = await empty.get('/ai');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
- assert.ok(r.text.includes('Bench is unavailable'));
+ assert.ok(r.text.includes('AI model leaderboard') && r.text.includes('Leaderboard'));
} finally { await empty.close(); }
} finally { await empty.close(); }
});
});
+ await check('comparison pages show fixture measurements, filter parameters and canonical model pairs', async () => {
+ const fixture = await boot({
+ aiCatalogue: { catalog: async () => ({ generated_at: '2026-11-09T00:00:00Z', models: [{ slug: 'maker/model-a', maker: 'Maker A', weights: 'open', status: 'current', offers: [{ provider: 'Provider One', price: { in: 1, out: 0.5 }, speed: { p50_ms: 120 }, uptime: 0.99 }] }, { slug: 'maker/model-b', maker: 'Maker B', weights: 'closed', status: 'current', offers: [] }] }) },
+ fetchImpl: async (url, opts) => {
+ const u = new URL(String(url));
+ if (u.port === '5230' && u.pathname === '/api/v1/kinds') return jsonResponse({ kinds: [{ id: 'code', name: 'Code' }] });
+ if (u.port === '5230' && u.pathname === '/api/v1/boards/code') return jsonResponse({ board: { rows: [{ subject: 'maker/model-a', elo: 1250, ci_lower: 1200, ci_upper: 1300, games: 40 }, { subject: 'maker/model-b', elo: 1270, ci_lower: 1250, ci_upper: 1300, games: 20 }] } });
+ if (u.port === '5230' && u.pathname.includes('/summary')) return jsonResponse({ model: 'maker/model-a', runs: 4 });
+ return source(url, opts);
+ }, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' },
+ });
+ try {
+ const hub = await fixture.get('/ai');
+ assert.ok(hub.text.includes('AI model leaderboard — November 2026'));
+ assert.ok(hub.text.includes('1200–1300') && hub.text.includes('1–2') && hub.text.includes('Dataset') && hub.text.includes('FAQPage'));
+ assert.ok(hub.text.includes('Maker A') && !hub.text.includes('letter-avatar'));
+ const filtered = await fixture.get('/ai/leaderboard?maker=Maker%20A&weights=open&price=low&status=current&kind=code');
+ assert.ok(filtered.text.includes('maker/model-a') && !filtered.text.includes('maker/model-b'));
+ const provider = await fixture.get('/ai/providers');
+ assert.ok(provider.text.includes('Provider One') && provider.text.includes('Not measured yet'));
+ assert.strictEqual((await fixture.get('/ai/compare?models=maker%2Fmodel-a,maker%2Fmodel-b')).status, 301);
+ assert.strictEqual((await fixture.get('/ai/compare/maker%2Fmodel-a-vs-maker%2Fmodel-b')).status, 200);
+ assert.strictEqual((await fixture.get('/ai/compare/maker%2Fmodel-b-vs-maker%2Fmodel-a')).status, 301);
+ } finally { await fixture.close(); }
+ });
+
done();
done();
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });
@@ -83,11 +83,11 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
@@ -83,11 +83,11 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
};
};
const t = await boot({ fetchImpl, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
const t = await boot({ fetchImpl, env: { OV_BENCH_INTERNAL_URL: 'http://127.0.0.1:5230' } });
try {
try {
- await check('the AI overview leads with the real leader and states the last run', async () => {
+ await check('the AI overview renders the leaderboard hub from the Bench fixture', async () => {
const r = await t.get('/ai');
const r = await t.get('/ai');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
- for (const s of ['Task kinds and their leaders', 'Design page arena', 'model-a', '1043 ± 38', '$0.02', '1 s', 'Playable games arena', 'Nov 7', 'AI in numbers']) assert.ok(r.text.includes(s), s);
- assert.ok(r.text.includes('href="/ai/method"') && r.text.includes('href="/ai/compare"') && r.text.includes('/ai/feed.xml'));
+ for (const s of ['AI model leaderboard', 'model-a', '1043', 'Best overall', 'FAQPage', 'Dataset']) assert.ok(r.text.includes(s), s);
+ assert.ok(r.text.includes('href="/ai/leaderboard"') && r.text.includes('/api/v1/ai/leaderboard'));
});
});
await check('one task kind shows quality, price and speed side by side and its latest runs', async () => {
await check('one task kind shows quality, price and speed side by side and its latest runs', async () => {
@@ -102,20 +102,23 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
@@ -102,20 +102,23 @@ const legacy = { model: 'model-a', ratings: [{ kind_id: 'design.page', elo: 1043
for (const s of ['Playable games arena', 'Score against cost per run', 'ovk-chart--scatter', 'model-a', '85', '$0.02', 'Cost per solved', 'Latest runs', 'version 1']) assert.ok(r.text.includes(s), s);
for (const s of ['Playable games arena', 'Score against cost per run', 'ovk-chart--scatter', 'model-a', '85', '$0.02', 'Cost per solved', 'Latest runs', 'version 1']) assert.ok(r.text.includes(s), s);
});
});
- await check('one model page has its history, best and worst tasks and cost', async () => {
+ await check('one model page uses the comparison profile and available Bench task data', async () => {
const r = await t.get('/ai/models/model-a');
const r = await t.get('/ai/models/model-a');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
- for (const s of ['Score over time', 'Best and worst tasks', 't1', 't2', 'spent on runs', '$0.04', 'Published', 'Blind-board ratings', 'Playable games arena']) assert.ok(r.text.includes(s), s);
+ for (const s of ['30-day rating', 'Not enough data', 'Tasks it is good at', 'Playable games arena', 'snapshot', 'Arena record against the top five', 'Product', 'Try it in Chat', 'Use it from code']) assert.ok(r.text.includes(s), s);
assert.ok(!r.text.includes('<span title='), 'a stat is escaped HTML, not markup');
assert.ok(!r.text.includes('<span title='), 'a stat is escaped HTML, not markup');
});
});
- await check('compare puts two models side by side, and refuses one', async () => {
+ await check('the old query compare redirects to a canonical pair page', async () => {
const two = await t.get('/ai/compare?models=model-a,model-b');
const two = await t.get('/ai/compare?models=model-a,model-b');
- assert.strictEqual(two.status, 200);
- assert.ok(two.text.includes('Playable games arena') && two.text.includes('model-a') && two.text.includes('model-b'), 'both models');
+ assert.strictEqual(two.status, 301);
+ assert.strictEqual(two.headers.get('location'), '/ai/compare/model-a-vs-model-b');
+ const pair = await t.get('/ai/compare/model-a-vs-model-b');
+ assert.strictEqual(pair.status, 200);
+ assert.ok(pair.text.includes('model-a') && pair.text.includes('model-b'), 'both models');
const one = await t.get('/ai/compare?models=model-a');
const one = await t.get('/ai/compare?models=model-a');
assert.strictEqual(one.status, 200);
assert.strictEqual(one.status, 200);
- assert.ok(one.text.includes('Pick at least two model ids'), 'the refusal');
+ assert.ok(one.text.includes('Choose two models to compare'), 'ask for a second model');
});
});
await check('an unknown benchmark id is a 404 once Bench has answered its list', async () => {
await check('an unknown benchmark id is a 404 once Bench has answered its list', async () => {
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/compare-1280-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/compare-1280-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/compare-1280-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/compare-1280-light.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/compare-390-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/compare-390-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/compare-390-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/compare-390-light.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/hub-1280-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/hub-1280-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/hub-1280-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/hub-1280-light.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/hub-390-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/hub-390-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/hub-390-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/hub-390-light.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/model-1280-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/model-1280-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/model-1280-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/model-1280-light.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/model-390-dark.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/model-390-dark.png differ
new file mode 100644
new file mode 100644
Binary files /dev/null and b/test/screenshots/ai-r2/model-390-light.png differ
Binary files /dev/null and b/test/screenshots/ai-r2/model-390-light.png differ