openvibers/openvibe.ai Commits

Expose the full own-page model market with metadata provenance (#409)

OP OpenVibers committed 4 days ago · 6cab022638 · 12 files, +851 −0

* Expose own-page model market with sourced metadata

* Retain provider preset flags for newly discovered models

* Document the market endpoint in AI OpenAPI

* Limit market provider identities to public pricebook sources

Changes

ddiff --git a/docs/exchange-platform.md b/docs/exchange-platform.md
iindex 6897b78..e9ec27c 100644
---- a/docs/exchange-platform.md
++++ b/docs/exchange-platform.md
@@@ -39,3 +39,27 @@ It requires the same operator capability as provider configuration. The parser r
days) and missing throughput never cause a rent proposal. Recording listings does not rent anything or contact providers.
Migration 0052 is expand-only. These changes need money/security review before deployment; this PR does not deploy.
++
++## Public market observations
++
++`GET https://api.openvibe.exchange/api/v1/exchange/market` returns
++`exchange.market@1` with a 60-second public/process cache. The existing nginx
++`/api/v1/exchange/` prefix serves this read. `limit` is 1–500 (default 100);
++`next_cursor` continues a page with the same filters (`maker`, `open_weights`,
++`kind`). Counts describe the filtered offers; model counts use Report's normalized
++model identity (maker prefix and date/channel suffix removed).
++
++The feed combines provider presets, pricebook-owned registry rows and every
++own-page observation in `exchange_market_prices`, including disabled and
++unregistered models. The weekly pricebook writes observations without enabling
++models or approving routing prices. Unknown prices stay null; a pending routing
++price change can already be the provider's current advertised price. OpenRouter
++cross-reference prices do not become another provider's own-page prices.
++
++Model metadata comes from https://models.dev/api.json (MIT), refreshed on the
++first read each day. `exchange_model_metadata` persists the last successful copy;
++failed refreshes retain it with `metadata.stale: true` and retry no more than once
++per minute. Every known metadata field carries its own source URL. Provider-page
++facts take precedence over models.dev. Tests disable live refresh and inject
++recorded responses. The market projects public fields explicitly; credentials,
++account ids, internal provider metadata and price-change actors are never returned.
ddiff --git a/migrations/0057_exchange_model_metadata.sql b/migrations/0057_exchange_model_metadata.sql
nnew file mode 100644
iindex 0000000..b4acb95
---- /dev/null
++++ b/migrations/0057_exchange_model_metadata.sql
@@@ -0,0 +1,17 @@
++-- phase: expand
++-- Last successful public models.dev read survives restarts and failed refreshes.
++CREATE TABLE IF NOT EXISTS exchange_model_metadata (
++ id TEXT PRIMARY KEY,
++ payload JSONB NOT NULL,
++ checked_at TIMESTAMPTZ NOT NULL
++);
++
++-- Own-page observations include models that have never been enabled in the router.
++CREATE TABLE IF NOT EXISTS exchange_market_prices (
++ provider_key TEXT NOT NULL,
++ model_key TEXT NOT NULL,
++ price JSONB NOT NULL,
++ source TEXT NOT NULL,
++ checked_at TEXT NOT NULL,
++ PRIMARY KEY (provider_key, model_key)
++);
ddiff --git a/server/api/exchange.js b/server/api/exchange.js
iindex 6c0b696..9ac21ec 100644
---- a/server/api/exchange.js
++++ b/server/api/exchange.js
@@@ -48,6 +48,7 @@ function exchangeRouter({ db, registry, pool, quotas = null, env = process.env,
return hits.get(key)();
};
const all = () => allFor(null);
++ const market = require('../exchange/market').createMarket({ db, registry, catalog: all, config, clock });
const usageByWindow = new Map();
const r = express.Router();
@@@ -148,6 +149,16 @@ function exchangeRouter({ db, registry, pool, quotas = null, env = process.env,
r.get('/api/v1/receipts/:id', read, readReceipt);
}
++ r.get('/api/v1/exchange/market', publicCache(60), async (req, res, next) => {
++ try {
++ // Validate before fetching or caching upstream data.
++ require('../exchange/market').pageMarket({ offers: [] }, req.query);
++ } catch (error) {
++ return http.sendProblem(res, 422, 'exchange.market_filter_invalid', { detail: error.message, ctx: req.ov });
++ }
++ try { res.json(await market.read(req.query)); } catch (error) { next(error); }
++ });
++
r.get('/api/v1/exchange/catalog', publicCache(30), async (req, res, next) => {
try {
const kind = req.query.kind == null || req.query.kind === '' ? null : String(req.query.kind).toLowerCase();
ddiff --git a/server/exchange/market.js b/server/exchange/market.js
nnew file mode 100644
iindex 0000000..0f4344b
---- /dev/null
++++ b/server/exchange/market.js
@@@ -0,0 +1,129 @@
++'use strict';
++const { PROVIDERS } = require('../providers/pool-providers');
++const { SOURCES } = require('../pricebook/sources');
++const { canonicalOf, slugOf, kindOf } = require('./catalog');
++const { createModelMetadata, lookupMetadata, bare, URL } = require('./model-metadata');
++const TTL_MS = 60_000;
++const recordKey = (provider, model) => `${provider}\0${String(model).toLowerCase()}`;
++const num = (v) => v == null || v === '' || !Number.isFinite(Number(v)) || Number(v) < 0 ? null : Number(v);
++const webUrl = (v) => typeof v === 'string' && /^https:\/\//.test(v) ? v : null;
++
++function buildOffers({ providers = [], rows = [], changes = [], catalogue = {}, metadata = {}, observations = [], presets = PROVIDERS }) {
++ const publicKeys = new Set([...presets.map(p => p.key), ...SOURCES.filter(s => s.role === 'own').map(s => s.provider)]);
++ const byProvider = new Map(providers.filter(p => publicKeys.has(p.key)).map((p) => [p.key, p]));
++ const seeds = new Map();
++ const ownUrls = new Set(SOURCES.filter(s => s.role === 'own').map(s => s.url));
++ for (const p of presets) for (const m of p.models || []) {
++ // OpenRouter is a cross-reference in this pricebook, never an own-page price.
++ if (p.role === 'cross-reference' || p.key === 'openrouter') continue;
++ if (m.source) ownUrls.add(m.source);
++ seeds.set(recordKey(p.key, m.model_key), { p, m });
++ }
++ const records = new Map([...seeds].map(([key, { p, m }]) => [key, {
++ provider_key: p.key, model_key: m.model_key, display_name: m.display_name,
++ type: m.type || 'chat', context_window: m.context_window, max_output: m.max_output,
++ cost: { ...m.cost, source: m.source, verified_at: m.checked }, metadata: m,
++ }]));
++ for (const row of rows) {
++ if (publicKeys.has(row.provider_key) && (ownUrls.has(row.cost?.source) || ownUrls.has(row.metadata?.price_book?.source))) records.set(recordKey(row.provider_key, row.model_key), row);
++ }
++ for (const observed of observations) {
++ const key = [...records.keys()].find(k => {
++ const r = records.get(k);
++ return r.provider_key === observed.provider_key && bare(r.model_key) === bare(observed.model_key);
++ }) || recordKey(observed.provider_key, observed.model_key);
++ const price = typeof observed.price === 'string' ? JSON.parse(observed.price) : observed.price;
++ const old = records.get(key);
++ records.set(key, { ...old, provider_key: observed.provider_key, model_key: old?.model_key || observed.model_key,
++ cost: { in_per_mtok: price.in, out_per_mtok: price.out, cached_per_mtok: price.cached,
++ source: observed.source, verified_at: observed.checked_at } });
++ }
++ const routed = (catalogue.models || []).flatMap(m => (m.offers || []).map(o => ({ ...o, canonical: m.canonical || m.slug })));
++ const offers = [];
++ for (const [key, row] of records) {
++ const seed = seeds.get(key);
++ const p = byProvider.get(row.provider_key) || presets.find(p => p.key === row.provider_key)
++ || (SOURCES.some(s => s.role === 'own' && s.provider === row.provider_key) ? { key: row.provider_key } : null);
++ if (!p || p.metadata?.hidden || p.metadata?.synthetic || p.provider_access === 'CUSTOMER_BYOK') continue;
++ const facts = { ...seed?.m, ...row.metadata };
++ const canonical = canonicalOf(row, p); const model = slugOf(canonical);
++ const meta = lookupMetadata(metadata.entries || [], row.model_key, row.provider_key);
++ // Explicit projection only: no provider credentials, account ids or arbitrary metadata.
++ const offer = {
++ model, provider: p.key, provider_name: p.display_name || p.key,
++ provider_model: row.model_key, name: row.display_name || row.model_key, kind: kindOf(row),
++ price: { in: num(row.cost?.in_per_mtok), out: num(row.cost?.out_per_mtok), cached: num(row.cost?.cached_per_mtok), currency: 'USD', per: '1M tokens' },
++ price_source: webUrl(row.cost?.source), price_checked_at: row.cost?.verified_at || facts.checked || null,
++ routed: routed.some(o => o.provider === p.key && bare(o.model) === bare(row.model_key)),
++ uncensored: seed?.m.uncensored === true || presets.find(preset => preset.key === p.key)?.uncensored === true,
++ provider_access: p.provider_access || 'UNVERIFIED', field_sources: {},
++ };
++ // Pending price changes are the latest observed own-page price, even before the router
++ // approves charging it. Never publish the change log's actor, note or account data.
++ const observed = changes.filter(c => c.provider_key === p.key && c.model_key === row.model_key && ownUrls.has(c.source))
++ .sort((a,b) => String(b.checked_at).localeCompare(String(a.checked_at)));
++ for (const [field, short] of [['in_per_mtok','in'], ['out_per_mtok','out'], ['cached_per_mtok','cached']]) {
++ const change = observed.find(c => c.field === field);
++ if (change && (!offer.price_checked_at || String(change.checked_at) >= String(offer.price_checked_at))) offer.price[short] = num(change.new_price);
++ }
++ if (observed[0] && (!offer.price_checked_at || String(observed[0].checked_at) >= String(offer.price_checked_at))) {
++ offer.price_source = webUrl(observed[0].source); offer.price_checked_at = observed[0].checked_at;
++ }
++ if (!offer.price_source) offer.price = { in: null, out: null, cached: null, currency: 'USD', per: '1M tokens' };
++ const own = { maker: facts.maker, context: row.context_window, max_output: row.max_output,
++ open_weights: facts.open_weights, licence: facts.licence || facts.license,
++ release_date: facts.release_date, knowledge_cutoff: facts.knowledge_cutoff, modalities: facts.modalities };
++ for (const field of Object.keys(own)) {
++ const fromProvider = own[field] != null && webUrl(seed?.m.source || row.cost?.source);
++ const value = (fromProvider ? own[field] : meta[field]) ?? null;
++ offer[field] = ['context', 'max_output'].includes(field) ? num(value)
++ : field === 'open_weights' ? (typeof value === 'boolean' ? value : null)
++ : field === 'modalities' ? (value && typeof value === 'object' ? {
++ input: Array.isArray(value.input) ? value.input.filter(v => typeof v === 'string') : [],
++ output: Array.isArray(value.output) ? value.output.filter(v => typeof v === 'string') : [],
++ } : null) : (typeof value === 'string' ? value : null);
++ if (offer[field] != null) offer.field_sources[field] = fromProvider ? (seed?.m.source || row.cost.source) : URL;
++ }
++ offers.push(offer);
++ }
++ return offers.sort((a,b) => a.model.localeCompare(b.model) || a.provider.localeCompare(b.provider) || a.provider_model.localeCompare(b.provider_model));
++}
++
++function pageMarket(snapshot, query = {}) {
++ const limit = query.limit == null ? 100 : Number(query.limit);
++ if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error('limit must be an integer from 1 to 500');
++ if (query.open_weights != null && !['true','false'].includes(String(query.open_weights))) throw new Error('open_weights must be true or false');
++ if (query.kind != null && !['chat','embedding','stt','image'].includes(query.kind)) throw new Error('unknown kind');
++ let offset = 0;
++ if (query.cursor) {
++ if (!/^\d+$/.test(query.cursor) || !Number.isSafeInteger(Number(query.cursor))) throw new Error('invalid cursor');
++ offset = Number(query.cursor);
++ }
++ const offers = snapshot.offers.filter(o => (!query.maker || o.maker?.toLowerCase() === String(query.maker).toLowerCase()) &&
++ (query.open_weights == null || o.open_weights === (String(query.open_weights) === 'true')) && (!query.kind || o.kind === query.kind));
++ return { $schema: 'exchange.market@1', generated_at: snapshot.generated_at, metadata: snapshot.metadata,
++ total_models: new Set(offers.map(o => bare(o.model))).size, total_offers: offers.length,
++ offers: offers.slice(offset, offset + limit), next_cursor: offset + limit < offers.length ? String(offset + limit) : null };
++}
++
++function createMarket({ db, registry, catalog, config, clock = { now: Date.now }, metadata = null }) {
++ const source = metadata || createModelMetadata({ db, now: () => clock.now(), enabled: config?.nodeEnv !== 'test' && process.env.NODE_ENV !== 'test' });
++ let hit = null; let pending = null;
++ async function snapshot() {
++ if (hit && clock.now() < hit.until) return hit.value;
++ if (!pending) pending = (async () => {
++ const [providers, rows, changes, catalogue, meta, observations] = await Promise.all([
++ registry.listProviders(), registry.listModels(),
++ db.prepare('SELECT provider_key, model_key, field, new_price, source, checked_at FROM price_changes ORDER BY checked_at DESC').all(),
++ catalog(), source.read(),
++ db.prepare('SELECT provider_key, model_key, price, source, checked_at FROM exchange_market_prices').all(),
++ ]);
++ const value = { generated_at: new Date(clock.now()).toISOString(), metadata: { source: URL, checked_at: meta.checked_at, stale: meta.stale },
++ offers: buildOffers({ providers, rows, changes, catalogue, metadata: meta, observations }) };
++ hit = { value, until: clock.now() + TTL_MS }; return value;
++ })().finally(() => { pending = null; });
++ return pending;
++ }
++ return { async read(query) { return pageMarket(await snapshot(), query); } };
++}
++module.exports = { buildOffers, pageMarket, createMarket, TTL_MS };
ddiff --git a/server/exchange/model-metadata.js b/server/exchange/model-metadata.js
nnew file mode 100644
iindex 0000000..c38f240
---- /dev/null
++++ b/server/exchange/model-metadata.js
@@@ -0,0 +1,73 @@
++'use strict';
++
++const URL = 'https://models.dev/api.json';
++const DAY_MS = 86_400_000;
++// Keep identical to Report server/data/ai-join.js.
++const bare = (v) => String(v || '').toLowerCase()
++ .replace(/^[^/]+\//, '')
++ .replace(/[-_@:](\d{4}-?\d{2}-?\d{2}|latest|free)$/, '');
++
++function indexMetadata(data) {
++ const entries = [];
++ for (const [provider, p] of Object.entries(data || {})) {
++ for (const [id, m] of Object.entries(p.models || {})) {
++ if (!m || typeof m !== 'object') continue;
++ const canonical = m.canonical_model_id || m.id || id;
++ // A reseller's name is not the maker. Prefer explicit canonical identity, then a
++ // first-party record; unprefixed reseller-only records leave maker unknown.
++ const maker = m.maker || (canonical.includes('/') ? canonical.split('/')[0] : null);
++ entries.push({ id: m.id || id, provider, maker, model: m });
++ }
++ }
++ return entries;
++}
++
++function lookupMetadata(entries, id, provider) {
++ const matches = entries.filter((e) => bare(e.id) === bare(id));
++ const found = matches.find((e) => e.provider === provider && e.id === id)
++ || matches.find((e) => e.provider === provider)
++ || matches.find((e) => e.id === id) || matches[0];
++ if (!found) return {};
++ const m = found.model;
++ const firstParty = matches.find((e) => !e.id.includes('/') && e.provider !== 'openrouter' &&
++ matches.some((other) => other.maker === e.provider));
++ return {
++ maker: found.maker || (firstParty && firstParty.provider) || null,
++ release_date: m.release_date || null, knowledge_cutoff: m.knowledge || null,
++ licence: typeof (m.license || m.licence) === 'string' ? (m.license || m.licence) : null,
++ context: m.limit?.context ?? null, max_output: m.limit?.output ?? null,
++ modalities: m.modalities ? { input: m.modalities.input || [], output: m.modalities.output || [] } : null,
++ open_weights: typeof m.open_weights === 'boolean' ? m.open_weights : null,
++ };
++}
++
++function createModelMetadata({ db, fetchImpl = globalThis.fetch, now = Date.now, enabled = true, log = console }) {
++ let snapshot = null; let loaded = false; let retryAt = 0; let pending = null;
++ async function refresh() {
++ if (!loaded) {
++ const row = await db.prepare('SELECT payload, checked_at FROM exchange_model_metadata WHERE id = ?').get('models.dev');
++ if (row) snapshot = { entries: indexMetadata(typeof row.payload === 'string' ? JSON.parse(row.payload) : row.payload), checked_at: new Date(row.checked_at).toISOString() };
++ loaded = true;
++ }
++ const expired = !snapshot || now() - Date.parse(snapshot.checked_at) >= DAY_MS;
++ if (expired && enabled && now() >= retryAt) {
++ retryAt = now() + 60_000;
++ try {
++ const response = await fetchImpl(URL, { signal: AbortSignal.timeout(15_000) });
++ if (!response.ok) throw new Error('metadata HTTP failure');
++ const raw = await response.text();
++ if (raw.length > 32 * 1024 * 1024) throw new Error('metadata exceeds size limit');
++ const data = JSON.parse(raw); const entries = indexMetadata(data);
++ if (!entries.length) throw new Error('metadata has no models');
++ const checked_at = new Date(now()).toISOString();
++ await db.prepare(`INSERT INTO exchange_model_metadata (id, payload, checked_at) VALUES (?, ?::jsonb, ?)
++ ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload, checked_at = EXCLUDED.checked_at`).run('models.dev', raw, checked_at);
++ snapshot = { entries, checked_at };
++ } catch (_) { log.warn?.('models.dev metadata refresh failed; keeping last copy'); }
++ }
++ return { entries: snapshot?.entries || [], source: URL, checked_at: snapshot?.checked_at || null,
++ stale: !snapshot || now() - Date.parse(snapshot.checked_at) >= DAY_MS };
++ }
++ return { read() { if (!pending) pending = refresh().finally(() => { pending = null; }); return pending; } };
++}
++module.exports = { createModelMetadata, indexMetadata, lookupMetadata, bare, URL, DAY_MS };
ddiff --git a/server/openapi.js b/server/openapi.js
iindex 70927d6..65098d6 100644
---- a/server/openapi.js
++++ b/server/openapi.js
@@@ -468,6 +468,32 @@ const exchangeModel = {
offers: arrayOf(exchangeOffer),
},
};
++route('get', '/exchange/market', {
++ operationId: 'exchangeMarket', summary: 'Every own-page model and provider offer, enabled or not',
++ tags: ['exchange'], security: [], servers: [{ url: 'https://api.openvibe.exchange' }],
++ description: 'Public, cached 60 seconds. exchange.market@1 lists provider-page prices per million tokens, routing and preset flags, and model metadata with per-field sources. Metadata refreshes daily; a failed read retains the last copy marked stale. Observed prices may differ from approved routing prices.',
++ parameters: [query('limit', integer({ minimum: 1, maximum: 500, default: 100 })),
++ query('cursor', str(), 'Continue with next_cursor and the same filters'), query('maker', str()),
++ query('open_weights', { type: 'boolean' }), query('kind', str({ enum: ['chat', 'embedding', 'stt', 'image'] }))],
++ responses: { 200: json('Market offers with price and metadata provenance', {
++ type: 'object', required: ['$schema', 'generated_at', 'metadata', 'total_models', 'total_offers', 'offers', 'next_cursor'],
++ properties: {
++ $schema: { const: 'exchange.market@1' }, generated_at: str({ format: 'date-time' }),
++ total_models: integer({ minimum: 0 }), total_offers: integer({ minimum: 0 }), next_cursor: nullable('string'),
++ metadata: { type: 'object', properties: { source: str({ format: 'uri' }), checked_at: nullable('string', { format: 'date-time' }), stale: { type: 'boolean' } } },
++ offers: arrayOf({ type: 'object', properties: {
++ model: str(), provider: str(), provider_name: str(), provider_model: str(), name: str(), kind: str(),
++ price: { type: 'object', properties: { in: nullable('number'), out: nullable('number'), cached: nullable('number'), currency: { const: 'USD' }, per: { const: '1M tokens' } } },
++ price_source: nullable('string', { format: 'uri' }), price_checked_at: nullable('string'),
++ routed: { type: 'boolean' }, uncensored: { type: 'boolean' }, provider_access: str(),
++ maker: nullable('string'), context: nullable('number'), max_output: nullable('number'),
++ open_weights: nullable('boolean'), licence: nullable('string'), release_date: nullable('string'), knowledge_cutoff: nullable('string'),
++ modalities: { type: ['object', 'null'], properties: { input: arrayOf(str()), output: arrayOf(str()) } },
++ field_sources: { type: 'object', additionalProperties: str({ format: 'uri' }) },
++ } }),
++ },
++ }), ...errors([422]) },
++});
route('get', '/exchange/catalog', {
operationId: 'exchangeCatalog', summary: 'Every model and every offer that serves it', tags: ['exchange'], security: [],
description: 'Public. The live offers: price in, out and cached per million tokens, measured p50/p95, uptime, requests, and the provider data policy. Speed and uptime are read from the last seven days of real runs (time to the first token where a receipt recorded one, else the answering attempt\'s total); cached answers are counted apart and excluded from speed, uptime and requests. Uptime is succeeded / (succeeded + failed); null when no data, never 0. A prepaid offer has no price (free: true); free_tier says whether the free allowance and the anonymous Try it may use an offer at all (a named AI_FREE_PROVIDERS source under the output price ceiling). Models carry a kind (chat, embedding, stt, image); chat lists must use kind "chat" and never mix embeddings or speech-to-text.',
ddiff --git a/server/pricebook/job.js b/server/pricebook/job.js
iindex abf71fa..c7e127e 100644
---- a/server/pricebook/job.js
++++ b/server/pricebook/job.js
@@@ -371,6 +371,7 @@ function createPriceBook({ db, registry, config = {}, clock = { now: () => Date.
}
report.sources.push({ source: source.key, url: source.url, role: source.role, listed: Object.keys(prices).length });
if (source.role === 'cross-reference') { await crossCheck(prices, readings, report); continue; }
++ await require('./market-store').recordMarketPrices(db, source, prices, checkedAt);
readings[source.provider] = prices;
await priceProvider(source, prices, report, checkedAt);
}
ddiff --git a/server/pricebook/market-store.js b/server/pricebook/market-store.js
nnew file mode 100644
iindex 0000000..b1231cc
---- /dev/null
++++ b/server/pricebook/market-store.js
@@@ -0,0 +1,15 @@
++'use strict';
++// Persist every parsed own-page offer, including ids that are not in the routing registry.
++// This does not approve a routing price or enable a provider.
++async function recordMarketPrices(db, source, prices, checkedAt) {
++ if (source.role !== 'own') return;
++ for (const [model, price] of Object.entries(prices)) {
++ const number = v => typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : null;
++ const publicPrice = { in: number(price.in), out: number(price.out), cached: number(price.cached) };
++ await db.prepare(`INSERT INTO exchange_market_prices (provider_key, model_key, price, source, checked_at)
++ VALUES (?, ?, ?::jsonb, ?, ?) ON CONFLICT (provider_key, model_key)
++ DO UPDATE SET price = EXCLUDED.price, source = EXCLUDED.source, checked_at = EXCLUDED.checked_at`)
++ .run(source.provider, model, JSON.stringify(publicPrice), source.url, checkedAt);
++ }
++}
++module.exports = { recordMarketPrices };
ddiff --git a/test/exchange-market.test.js b/test/exchange-market.test.js
nnew file mode 100644
iindex 0000000..040bd89
---- /dev/null
++++ b/test/exchange-market.test.js
@@@ -0,0 +1,74 @@
++'use strict';
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { buildOffers, pageMarket, createMarket } = require('../server/exchange/market');
++const { indexMetadata, createModelMetadata, lookupMetadata, DAY_MS, bare } = require('../server/exchange/model-metadata');
++const { recordMarketPrices } = require('../server/pricebook/market-store');
++const { SOURCES } = require('../server/pricebook/sources');
++const { PROVIDERS } = require('../server/providers/pool-providers');
++const { boot, request, suite } = require('./helpers');
++const dev = require('./fixtures/market/models-dev.json');
++const files = { deepseek:'deepseek.md', qwen:'qwen.html', kimi:'kimi.html', glm:'glm.md', minimax:'minimax.md', deepinfra:'deepinfra.json', novita:'novita.json', featherless:'featherless.json', refuseless:'refuseless.md', abliteration:'abliteration.md' };
++const observations = [];
++for (const s of SOURCES.filter(s => s.role === 'own')) {
++ const raw = fs.readFileSync(path.join(__dirname, 'fixtures/pricebook', files[s.key]), 'utf8');
++ for (const [model_key, price] of Object.entries(s.parse(s.kind === 'json' ? JSON.parse(raw) : raw))) observations.push({ provider_key: s.provider, model_key, price, source: s.url, checked_at: '2026-11-09' });
++}
++const t = suite('exchange-market'); let h;
++t.test('production presets and recorded pages include disabled and unregistered offers', () => {
++ const offers = buildOffers({ observations, metadata: { entries: indexMetadata(dev) } });
++ const snapshot = { generated_at: '2026-11-09T00:00:00Z', metadata: { source: 'https://models.dev/api.json', checked_at: null, stale: true }, offers };
++ const page = pageMarket(snapshot, { limit: 500 });
++ assert.strictEqual(page.total_models, 52); assert.strictEqual(page.total_offers, 61); assert(offers.every(o => !o.routed));
++ assert(offers.some(o => o.price.in === null)); assert(offers.some(o => o.uncensored));
++ const providerFlag = buildOffers({ observations: [{ provider_key: 'refuseless', model_key: 'new-fixture-model', price: { in: null, out: null, cached: null }, source: 'https://refuseless.com', checked_at: '2026-11-09' }] });
++ assert.strictEqual(providerFlag.find(o => o.provider_model === 'new-fixture-model').uncensored, true, 'new models retain their provider preset flag');
++ const first = pageMarket(snapshot, { limit: 3 }); const second = pageMarket(snapshot, { limit: 3, cursor: first.next_cursor });
++ assert.deepStrictEqual([...first.offers,...second.offers], offers.slice(0,6));
++ assert.throws(() => pageMarket(snapshot,{limit:'no'})); assert.throws(() => pageMarket(snapshot,{cursor:'bad'}));
++ assert.throws(() => pageMarket(snapshot,{open_weights:'unknown'}));
++ console.log(`Production-like market fixture: ${page.total_models} models, ${page.total_offers} offers`);
++});
++t.test('routing belongs to a provider offer; metadata joins dated names, prices stay own-page', () => {
++ const p = PROVIDERS.find(p => p.key === 'qwen'); const m = p.models[0];
++ const entries = indexMetadata({ qwen: { models: { [m.model_key]: { id: 'qwen/'+m.model_key, open_weights: true, release_date:'2026-11-01', knowledge:'2026-01', license:'Apache-2.0', limit:{context:1,output:42}, modalities:{input:['text'],output:['text']}, cost:{input:999} } } } });
++ const offers = buildOffers({ metadata: { entries }, catalogue: { models: [{ offers:[{provider:p.key, model:m.model_key}] }] },
++ providers: [{ ...p, status:'disabled', secret_ref:'must-not-be-exported', metadata:{account_id:'private-account'} }] });
++ const offer = offers.find(o => o.provider === p.key && o.provider_model === m.model_key);
++ assert(offer.routed); assert.strictEqual(offer.context,m.context_window); assert.strictEqual(offer.max_output,42);
++ assert.strictEqual(offer.licence,'Apache-2.0'); assert.strictEqual(offer.field_sources.licence,'https://models.dev/api.json');
++ assert.strictEqual(offer.field_sources.context,m.source); assert.strictEqual(offer.price.in,m.cost.in_per_mtok);
++ assert.strictEqual(bare('qwen/'+m.model_key+'-2026-11-09'),bare(m.model_key));
++ const page = pageMarket({offers},{maker:'qwen',open_weights:'true',kind:'chat'}); assert(page.offers.includes(offer));
++ const latest = buildOffers({ changes: [{ provider_key:p.key, model_key:m.model_key, field:'in_per_mtok', new_price:9, source:m.source, checked_at:'2026-11-09', status:'pending', actor:'private-actor' }] }).find(o => o.provider === p.key && o.provider_model === m.model_key);
++ assert.strictEqual(latest.price.in,9); assert.strictEqual(latest.price_checked_at,'2026-11-09');
++ const privateRows = buildOffers({ providers: [{ key:'private-provider', provider_access:'UNVERIFIED' }], rows: [{ provider_key:'private-provider', model_key:'private-model', cost:{source:m.source} }] });
++ assert(!privateRows.some(o => o.provider === 'private-provider'), 'private endpoints cannot impersonate a pricebook provider by reusing its URL');
++ assert(!JSON.stringify(offers).includes('private-account')); assert(!JSON.stringify(offers).includes('secret_ref'));
++ assert.strictEqual(lookupMetadata(entries,m.model_key+'-latest','qwen').licence,'Apache-2.0');
++});
++t.test('metadata persists, refreshes daily, coalesces and survives failure/restart as stale', async () => {
++ let time=Date.parse('2026-11-09'); let calls=0; let fail=false;
++ const options={db:h.db,now:()=>time,log:{warn(){}},fetchImpl:async()=>{calls++; if(fail)throw Error('offline'); return new Response(JSON.stringify(dev));}};
++ const meta=createModelMetadata(options);
++ const [one,two]=await Promise.all([meta.read(),meta.read()]); assert.strictEqual(calls,1); assert.deepStrictEqual(one,two); assert(!one.stale);
++ await createModelMetadata(options).read(); assert.strictEqual(calls,1);
++ time+=DAY_MS+1; fail=true; const stale=await createModelMetadata(options).read(); assert(stale.stale); assert(stale.entries.length); assert.strictEqual(stale.checked_at,one.checked_at);
++ fail=false; time+=60_001; assert(!(await meta.read()).stale); assert.strictEqual(calls,3);
++});
++t.test('unregistered pricebook rows survive and the public route is cached and paginated', async () => {
++ const s=SOURCES.find(s=>s.key==='qwen');
++ await recordMarketPrices(h.db,s,{'unregistered-fixture':{in:0,out:1,cached:null}},'2026-11-09');
++ const r=await request(h.base,'GET','/api/v1/exchange/market?limit=500'); assert.strictEqual(r.status,200);
++ assert.strictEqual(r.headers.get('cache-control'),'public, max-age=60');
++ assert(r.body.offers.some(o=>o.provider_model==='unregistered-fixture' && o.price.in===0 && !o.routed));
++ assert.strictEqual((await request(h.base,'GET','/api/v1/exchange/market?open_weights=bad')).status,422);
++ const nginx=fs.readFileSync(path.join(__dirname,'../deploy/nginx/api.openvibe.exchange.conf'),'utf8');
++ assert(nginx.includes('location ^~ /api/v1/exchange/'));
++ let time=100; let reads=0;
++ const market=createMarket({db:h.db,registry:{listProviders:async()=>{reads++;return[];},listModels:async()=>[]},catalog:async()=>({models:[]}),clock:{now:()=>time},metadata:{read:async()=>({entries:[],checked_at:null,stale:true})}});
++ await Promise.all([market.read({limit:1}),market.read({limit:2})]); assert.strictEqual(reads,1);
++ time+=60_001; await market.read({}); assert.strictEqual(reads,2);
++});
++(async()=>{h=await boot(); try{await t.run();}finally{await h.close();}})().catch(e=>{console.error(e);process.exitCode=1;});
ddiff --git a/test/fixtures/market/README.md b/test/fixtures/market/README.md
nnew file mode 100644
iindex 0000000..5486567
---- /dev/null
++++ b/test/fixtures/market/README.md
@@@ -0,0 +1,4 @@
++models-dev.json is a small unmodified selection from https://models.dev/api.json,
++read 2026-11-09. Upstream: https://github.com/sst/models.dev (MIT).
++Tests combine it with the existing recorded provider pages in ../pricebook/ and
++production provider presets. No test requests a live listing.
ddiff --git a/test/fixtures/market/models-dev.LICENSE b/test/fixtures/market/models-dev.LICENSE
nnew file mode 100644
iindex 0000000..9ef0008
---- /dev/null
++++ b/test/fixtures/market/models-dev.LICENSE
@@@ -0,0 +1,21 @@
++MIT License
++
++Copyright (c) 2025 models.dev
++
++Permission is hereby granted, free of charge, to any person obtaining a copy
++of this software and associated documentation files (the "Software"), to deal
++in the Software without restriction, including without limitation the rights
++to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
++copies of the Software, and to permit persons to whom the Software is
++furnished to do so, subject to the following conditions:
++
++The above copyright notice and this permission notice shall be included in all
++copies or substantial portions of the Software.
++
++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
++AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++SOFTWARE.
ddiff --git a/test/fixtures/market/models-dev.json b/test/fixtures/market/models-dev.json
nnew file mode 100644
iindex 0000000..a2102d8
---- /dev/null
++++ b/test/fixtures/market/models-dev.json
@@@ -0,0 +1,456 @@
++{
++ "openai": {
++ "id": "openai",
++ "models": {
++ "gpt-5.1": {
++ "id": "gpt-5.1",
++ "name": "GPT-5.1",
++ "description": "Sharper GPT-5 generation for coding, product work, and tool-assisted tasks",
++ "family": "gpt",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "none",
++ "low",
++ "medium",
++ "high"
++ ]
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2024-09-30",
++ "release_date": "2025-11-13",
++ "last_updated": "2025-11-13",
++ "modalities": {
++ "input": [
++ "text",
++ "image"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 400000,
++ "input": 272000,
++ "output": 128000
++ },
++ "experimental": {
++ "modes": {
++ "fast": {
++ "cost": {
++ "input": 2.5,
++ "output": 20,
++ "cache_read": 0.25
++ },
++ "provider": {
++ "body": {
++ "service_tier": "priority"
++ }
++ }
++ },
++ "flex": {
++ "cost": {
++ "input": 0.625,
++ "output": 5,
++ "cache_read": 0.0625
++ },
++ "provider": {
++ "body": {
++ "service_tier": "flex"
++ }
++ }
++ }
++ }
++ },
++ "cost": {
++ "input": 1.25,
++ "output": 10,
++ "cache_read": 0.125
++ }
++ },
++ "gpt-5-pro": {
++ "id": "gpt-5-pro",
++ "name": "GPT-5 Pro",
++ "description": "Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning",
++ "family": "gpt-pro",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "high"
++ ]
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": false,
++ "knowledge": "2024-09-30",
++ "release_date": "2025-10-06",
++ "last_updated": "2025-10-06",
++ "modalities": {
++ "input": [
++ "text",
++ "image"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 400000,
++ "input": 272000,
++ "output": 272000
++ },
++ "cost": {
++ "input": 15,
++ "output": 120
++ }
++ },
++ "gpt-4o": {
++ "id": "gpt-4o",
++ "name": "GPT-4o",
++ "description": "Omni-era GPT for multimodal chat, practical coding, and general assistants",
++ "family": "gpt",
++ "attachment": true,
++ "reasoning": false,
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2023-09",
++ "release_date": "2024-05-13",
++ "last_updated": "2024-08-06",
++ "modalities": {
++ "input": [
++ "text",
++ "image",
++ "pdf"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 128000,
++ "output": 16384
++ },
++ "experimental": {
++ "modes": {
++ "fast": {
++ "cost": {
++ "input": 4.25,
++ "output": 17,
++ "cache_read": 2.125
++ },
++ "provider": {
++ "body": {
++ "service_tier": "priority"
++ }
++ }
++ }
++ }
++ },
++ "cost": {
++ "input": 2.5,
++ "output": 10,
++ "cache_read": 1.25
++ }
++ }
++ }
++ },
++ "deepinfra": {
++ "id": "deepinfra",
++ "models": {
++ "openai/gpt-oss-20b": {
++ "id": "openai/gpt-oss-20b",
++ "name": "GPT OSS 20B",
++ "description": "Open-weight GPT model for self-hosted reasoning and instruction-following workloads",
++ "family": "gpt-oss",
++ "attachment": false,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "low",
++ "medium",
++ "high"
++ ]
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2024-06",
++ "release_date": "2025-08-05",
++ "last_updated": "2025-08-05",
++ "modalities": {
++ "input": [
++ "text"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": true,
++ "limit": {
++ "context": 131072,
++ "output": 16384
++ },
++ "cost": {
++ "input": 0.03,
++ "output": 0.14
++ },
++ "canonical_model_id": "openai/gpt-oss-20b"
++ },
++ "openai/gpt-oss-120b": {
++ "id": "openai/gpt-oss-120b",
++ "name": "GPT OSS 120B",
++ "description": "Open-weight GPT model for self-hosted reasoning and instruction-following workloads",
++ "family": "gpt-oss",
++ "attachment": false,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "low",
++ "medium",
++ "high"
++ ]
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2024-06",
++ "release_date": "2025-08-05",
++ "last_updated": "2025-08-05",
++ "modalities": {
++ "input": [
++ "text"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": true,
++ "limit": {
++ "context": 131072,
++ "output": 16384
++ },
++ "cost": {
++ "input": 0.037,
++ "output": 0.17
++ },
++ "canonical_model_id": "openai/gpt-oss-120b"
++ },
++ "moonshotai/Kimi-K2.6": {
++ "id": "moonshotai/Kimi-K2.6",
++ "name": "Kimi K2.6",
++ "description": "Kimi multimodal agent model for visual understanding, coding, and planning",
++ "family": "kimi-k2",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "toggle"
++ }
++ ],
++ "tool_call": true,
++ "interleaved": {
++ "field": "reasoning_content"
++ },
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2025-01",
++ "release_date": "2026-04-20",
++ "last_updated": "2026-04-20",
++ "modalities": {
++ "input": [
++ "text",
++ "image",
++ "video"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": true,
++ "limit": {
++ "context": 262144,
++ "output": 16384
++ },
++ "cost": {
++ "input": 0.75,
++ "output": 3.5,
++ "cache_read": 0.15
++ },
++ "canonical_model_id": "moonshotai/kimi-k2.6"
++ }
++ }
++ },
++ "anthropic": {
++ "id": "anthropic",
++ "models": {
++ "claude-opus-4-5": {
++ "id": "claude-opus-4-5",
++ "name": "Claude Opus 4.5 (latest)",
++ "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents",
++ "family": "claude-opus",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "low",
++ "medium",
++ "high"
++ ]
++ },
++ {
++ "type": "budget_tokens",
++ "min": 1024
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2025-05",
++ "release_date": "2025-11-24",
++ "last_updated": "2025-11-24",
++ "modalities": {
++ "input": [
++ "text",
++ "image",
++ "pdf"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 200000,
++ "output": 64000
++ },
++ "cost": {
++ "input": 5,
++ "output": 25,
++ "cache_read": 0.5,
++ "cache_write": 6.25
++ },
++ "canonical_model_id": "anthropic/claude-opus-4-5"
++ },
++ "claude-sonnet-4-6": {
++ "id": "claude-sonnet-4-6",
++ "name": "Claude Sonnet 4.6",
++ "description": "Claude workhorse for coding agents, careful analysis, and production cost control",
++ "family": "claude-sonnet",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "low",
++ "medium",
++ "high",
++ "max"
++ ]
++ },
++ {
++ "type": "budget_tokens",
++ "min": 1024
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": true,
++ "knowledge": "2025-08-31",
++ "release_date": "2026-02-17",
++ "last_updated": "2026-03-13",
++ "modalities": {
++ "input": [
++ "text",
++ "image",
++ "pdf"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 1000000,
++ "output": 128000
++ },
++ "cost": {
++ "input": 3,
++ "output": 15,
++ "cache_read": 0.3,
++ "cache_write": 3.75
++ }
++ },
++ "claude-fable-5-1": {
++ "id": "claude-fable-5-1",
++ "name": "Claude Fable 5.1",
++ "description": "Claude model for demanding reasoning and long-horizon agentic work",
++ "family": "claude-fable",
++ "attachment": true,
++ "reasoning": true,
++ "reasoning_options": [
++ {
++ "type": "effort",
++ "values": [
++ "low",
++ "medium",
++ "high",
++ "xhigh",
++ "max"
++ ]
++ }
++ ],
++ "tool_call": true,
++ "structured_output": true,
++ "temperature": false,
++ "knowledge": "2026-06",
++ "release_date": "2026-09-01",
++ "last_updated": "2026-09-01",
++ "modalities": {
++ "input": [
++ "text",
++ "image",
++ "pdf"
++ ],
++ "output": [
++ "text"
++ ]
++ },
++ "open_weights": false,
++ "limit": {
++ "context": 1000000,
++ "output": 128000
++ },
++ "cost": {
++ "input": 10,
++ "output": 50,
++ "cache_read": 0.25,
++ "cache_write": 12.5
++ },
++ "canonical_model_id": "anthropic/claude-fable-5-1"
++ }
++ }
++ }
++}