openvibers/openvibe.exchange Commits

Publish every registry model with status and crawlable pages (#204)

OP OpenVibers committed 2 days ago · 309263e03c · 8 files, +138 −23

* List every registry model with status and crawlable pages

* Merge main model API and preserve live offers

* Include sourced sightings in Exchange model pages

Changes

ddiff --git a/server/agent-pages.js b/server/agent-pages.js
iindex 0765382..423ad30 100644
---- a/server/agent-pages.js
++++ b/server/agent-pages.js
@@@ -30,8 +30,16 @@ function createAgentPages(ctx) {
          const q = query instanceof URLSearchParams ? query : new URLSearchParams();
          if (p === '/') return home('/');
          if (p === '/models') {
--            const catalog = await ai.catalog();
--            return page(p, { index: true, title: 'Models', description: 'Every model with its live offers: price, speed, uptime, context and data policy.', body: raw(views.modelsPage({ catalog, q: q.get('q') || '', family: q.get('family') || '', sort: q.get('sort') || '' })) });
++            const [catalog, directory] = await Promise.all([ai.catalog(), ai.directory?.() || { models: [] }]);
++            return page(p, { index: true, title: 'Models', description: 'Every model with its live offers: price, speed, uptime, context and data policy.', body: raw(views.modelsPage({ catalog, q: q.get('q') || '', family: q.get('family') || '', sort: q.get('sort') || '' })
++                + require('./model-directory').directoryList(directory.models || [], Object.fromEntries(q), (catalog.models || []).map(m => m.slug))) });
++        }
++        if (p.startsWith('/models/')) {
++            const id = decodeURIComponent(p.slice('/models/'.length)).toLowerCase();
++            const directory = await ai.directory?.();
++            const model = (directory?.models || []).find(m => m.id.toLowerCase() === id);
++            if (model) return page(p, { index: true, title: model.name, description: `${model.name}: status, limits and dated prices.`,
++                body: raw(require('./model-directory').directoryDetail(model)) });
          }
          const m = p.match(/^\/m\/([A-Za-z0-9._~-]{1,80})\/([A-Za-z0-9._~:-]{1,120})$/);
          if (m) {
ddiff --git a/server/ai.js b/server/ai.js
iindex 9695e94..4f5358e 100644
---- a/server/ai.js
++++ b/server/ai.js
@@@ -39,6 +39,7 @@ function createAi({ config, fetchImpl = globalThis.fetch, log = console, now = (
  
      const empty = (what) => ({ generated_at: new Date(now()).toISOString(), unavailable: true, count: 0, models: [], families: [], by_day: [], by_model: [], by_product: [], today: null, totals: null, what });
      let catalogHit = { at: 0, value: null };
++    let directoryHit = { at: 0, value: null };
      let usageHit = new Map();
      let communityHit = { at: 0, value: null };
      let warnedAt = 0;
@@@ -87,6 +88,32 @@ function createAi({ config, fetchImpl = globalThis.fetch, log = console, now = (
          }
      }
  
++    async function directory() {
++        if (directoryHit.value && now() - directoryHit.at < CATALOG_MS) return directoryHit.value;
++        try {
++            const body = await get('/api/v1/exchange/directory', 'model directory');
++            if (!Array.isArray(body.models)) throw new Error('invalid model directory');
++            let spots = [];
++            try {
++                const spotted = await get('/api/v1/exchange/spotted', 'spotted models');
++                spots = Array.isArray(spotted.models) ? spotted.models : [];
++            } catch { /* The listed directory works while the spotting service is unavailable. */ }
++            const listed = new Set(body.models.map(m => m.id));
++            const models = [...body.models, ...spots.filter(m => !listed.has(m.id)).map(m => ({
++                id: m.id, provider: m.provider, model: m.model, name: m.name, status: 'spotted',
++                routing_enabled: false, first_seen_at: m.first_seen_at, updated_at: m.last_seen_at,
++                source: m.sources?.[0]?.url || null, sources: m.sources || [], price: {},
++            }))];
++            const value = { ...body, models, count: models.length };
++            directoryHit = { at: now(), value };
++            return value;
++        } catch (err) {
++            warned('model directory', err);
++            if (directoryHit.value && now() - directoryHit.at < 5 * 60_000) return { ...directoryHit.value, stale: true };
++            return { unavailable: true, models: [] };
++        }
++    }
++
      /** AI's search engines (GET /api/v1/exchange/search/engines, public): ids, kinds, prices, availability. */
      let enginesHit = { at: 0, value: null };
      async function searchEngines() {
@@@ -328,7 +355,7 @@ function createAi({ config, fetchImpl = globalThis.fetch, log = console, now = (
          return createCredentialsClient(client, { baseUrl: config.ai.url });
      }
  
--    return { catalog, searchEngines, usage, communityOffers, chat, presetsFor, setKeyPrivacy, ownedProjects, gpuQuote, ownStatus, ownList, ownCreate, ownStop, adminRequest, meRequest, credentialsFor, configured: Boolean(tokens) };
++    return { catalog, directory, searchEngines, usage, communityOffers, chat, presetsFor, setKeyPrivacy, ownedProjects, gpuQuote, ownStatus, ownList, ownCreate, ownStop, adminRequest, meRequest, credentialsFor, configured: Boolean(tokens) };
  }
  
  /**
ddiff --git a/server/app.js b/server/app.js
iindex 235f851..eea7eea 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -277,21 +277,24 @@ async function createApp(opts = {}) {
          }
      });
  
--    // Public model directory reader. The site and SDK clients share the same live catalog.
++    // Registry rows stay discoverable when an offer is off; live offers enrich the same public row.
++    const directoryRows = async () => {
++        const [directory, catalog] = await Promise.all([ai.directory(), ai.catalog()]);
++        if (directory.unavailable) return null;
++        const live = new Map((catalog.models || []).map(m => [m.slug, m]));
++        return directory.models.map(m => ({ ...m, slug: m.id, offers: live.get(m.id)?.offers || [] }));
++    };
      app.get('/api/v1/models', async (req, res) => {
--        const catalog = await ai.catalog();
--        if (catalog.unavailable) return res.status(503).json({ error: 'catalog_unavailable' });
--        const q = String(req.query.q || '').toLowerCase().slice(0, 120);
--        const provider = String(req.query.provider || '').toLowerCase().slice(0, 80);
--        const models = (catalog.models || []).filter(m => (!q || `${m.name} ${m.slug} ${m.family}`.toLowerCase().includes(q))
--            && (!provider || (m.offers || []).some(o => String(o.provider || o.provider_key || '').toLowerCase() === provider)));
++        const rows = await directoryRows();
++        if (!rows) return res.status(503).json({ error: 'directory_unavailable' });
++        const models = require('./model-directory').select(rows, req.query);
          return res.set('Cache-Control', 'public, max-age=30').json({ count: models.length, models });
      });
      const modelDirectoryDetail = async (req, res) => {
--        const catalog = await ai.catalog();
--        if (catalog.unavailable) return res.status(503).json({ error: 'catalog_unavailable' });
++        const rows = await directoryRows();
++        if (!rows) return res.status(503).json({ error: 'directory_unavailable' });
          const id = (req.params.id || `${req.params.org}/${req.params.model}`).toLowerCase();
--        const model = (catalog.models || []).find(m => m.slug === id);
++        const model = rows.find(m => m.id.toLowerCase() === id);
          return model ? res.set('Cache-Control', 'public, max-age=30').json({ model }) : res.status(404).json({ error: 'model_not_found' });
      };
      app.get('/api/v1/models/:id', modelDirectoryDetail);
ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex b3171bb..e57f1d5 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -76,13 +76,18 @@ function createDiscoveryRoutes(ctx) {
      }
  
      async function modelPages() {
--        const catalog = await ai.catalog();
--        return (catalog.models || []).slice(0, 5000).map((m) => ({
++        const [catalog, directory] = await Promise.all([ai.catalog(), ai.directory?.() || { models: [] }]);
++        const active = new Set((catalog.models || []).map(m => m.slug));
++        return [...(catalog.models || []).slice(0, 5000).map((m) => ({
              path: `/m/${m.slug.split('/').map(encodeURIComponent).join('/')}`,
              title: m.name,
              text: `${m.family} · ${m.offers.length} offers${m.uncensored ? ' · 18+' : ''}`,
              changefreq: 'daily', priority: 0.7,
--        }));
++        })), ...(directory.models || []).filter(m => !active.has(m.id)).slice(0, 5000).map(m => ({
++            path: require('../model-directory').href(m.id), title: m.name,
++            text: `${m.provider_name || m.provider} · ${m.status} · ${m.context_window || 'unknown'} context`,
++            changefreq: m.status === 'retired' ? 'monthly' : 'daily', priority: 0.5,
++        }))];
      }
  
      r.get('/robots.txt', (_req, res) => {
@@@ -95,7 +100,7 @@ function createDiscoveryRoutes(ctx) {
      r.get('/llms.txt', async (_req, res) => {
          let models = [];
          let characters = [];
--        try { models = (await ai.catalog()).models || []; } catch (e) { models = []; }
++        try { models = await modelPages(); } catch (e) { models = []; }
          try { characters = await characterPages(); } catch (e) { characters = []; }
          res.type('text/plain').set('Cache-Control', TEXT).send(seo.llmsTxt({
              name: SITE_NAME,
@@@ -117,10 +122,7 @@ function createDiscoveryRoutes(ctx) {
                      { title: 'AI in OpenVibe', url: abs('/ai') },
                  ] },
                  { title: 'Characters', links: [{ title: 'The gallery', url: abs('/companions'), note: 'character cards' }, ...characters.slice(0, 40).map((c) => ({ title: c.title, url: abs(c.path), note: c.text }))] },
--                { title: 'Models', links: models.slice(0, 60).map((m) => ({
--                    title: m.name, url: abs(`/m/${m.slug.split('/').map(encodeURIComponent).join('/')}`),
--                    note: [m.family, m.uncensored ? '18+' : null, m.offers.length ? `${m.offers.length} offers` : null].filter(Boolean).join(' · '),
--                })) },
++                { title: 'Models', links: models.slice(0, 60).map((m) => ({ title: m.title, url: abs(m.path), note: m.text })) },
              ],
              optional: [
                  { title: 'Sitemap', url: abs('/sitemap.xml') },
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex 5bc626d..1cd4486 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -71,6 +71,7 @@ function createPageRoutes(ctx) {
          await history.capture(catalog).catch((err) => log.warn('[OpenVibe.Exchange] catalogue snapshot failed:', err.message));
          return catalog;
      };
++    const directoryOf = async () => typeof ai.directory === 'function' ? ai.directory() : { models: [] };
  
      r.get('/compute', (req, res) => {
          const { prices, checked_at } = require('../compute-prices');
@@@ -118,7 +119,7 @@ function createPageRoutes(ctx) {
  
      // ── Models ───────────────────────────────────────────────
      r.get('/models', async (req, res) => {
--        const [catalog, community] = await Promise.all([catalogOf(), communityOffers()]);
++        const [catalog, community, directory] = await Promise.all([catalogOf(), communityOffers(), directoryOf()]);
          const q = typeof req.query.q === 'string' ? req.query.q.slice(0, 120) : '';
          const family = typeof req.query.family === 'string' ? req.query.family.slice(0, 60) : '';
          const sort = ['cheap', 'fast'].includes(req.query.sort) ? req.query.sort : '';
@@@ -138,7 +139,8 @@ function createPageRoutes(ctx) {
              index: true, cache: PUBLIC_CACHE, agentLayer: true,
              title: q ? `Models: ${q}` : 'Models',
              description: 'Every model on OpenVibe.Exchange with its live offers: price in and out per million tokens, measured speed and uptime, context length and data policy.',
--            body: raw(platform.modelsPage({ catalog, query: req.query, searchResults })),
++            body: raw(platform.modelsPage({ catalog: req.query.status && req.query.status !== 'available' ? { ...catalog, models: [] } : catalog, query: req.query, searchResults })
++                + require('../model-directory').directoryList(directory.models || [], req.query, (catalog.models || []).map(m => m.slug))),
          });
      });
  
@@@ -152,6 +154,14 @@ function createPageRoutes(ctx) {
          const [catalog, community] = await Promise.all([catalogOf(), communityOffers()]);
          const model = await modelFrom(catalog, req.params.org, req.params.model);
          if (!model) {
++            const directory = await directoryOf();
++            const id = `${req.params.org}/${req.params.model}`.toLowerCase();
++            const listed = (directory.models || []).find(m => m.id.toLowerCase() === id);
++            if (listed) return page(req, res, { index: true, cache: PUBLIC_CACHE, title: listed.name,
++                description: `${listed.name}: status, limits, dated prices and sources.`,
++                jsonLd: [{ '@context': 'https://schema.org', '@type': 'SoftwareApplication', name: listed.name,
++                    url: `${config.baseUrl}${require('../model-directory').href(listed.id)}` }],
++                body: raw(require('../model-directory').directoryDetail(listed)) });
              if (catalog.unavailable) {
                  return page(req, res, { title: 'Models are not available', index: false, body: raw(`<div class="ex-wrap ex-narrow">${kit.notice({ kind: 'warn', text: 'The live catalog is not reachable right now. Reload in a moment.' })}<p><a href="/models">All models</a></p></div>`) }, 503);
              }
ddiff --git a/server/model-directory.js b/server/model-directory.js
nnew file mode 100644
iindex 0000000..18efeb6
---- /dev/null
++++ b/server/model-directory.js
@@@ -0,0 +1,37 @@
++'use strict';
++const { esc } = require('./render/html');
++
++const href = id => `/models/${encodeURIComponent(id)}`;
++const allowedStatus = new Set(['available', 'preview', 'spotted', 'retired']);
++function select(models, query = {}) {
++    const q = String(query.q || '').toLowerCase().slice(0, 120);
++    const status = String(query.status || '').toLowerCase();
++    const provider = String(query.provider || '').toLowerCase().slice(0, 80);
++    return models.filter(m => (!q || `${m.name} ${m.id} ${m.family || ''}`.toLowerCase().includes(q))
++        && (!allowedStatus.has(status) || m.status === status) && (!provider || m.provider.toLowerCase() === provider));
++}
++const usd = value => value == null || !Number.isFinite(Number(value)) ? 'Price not listed' : `$${Number(value)} per million tokens`;
++function directoryList(models, query, activeSlugs = []) {
++    const active = new Set(activeSlugs);
++    const rows = select(models, query).filter(m => !active.has(m.id));
++    return `<section class="ex-wrap" aria-labelledby="directory-more"><h2 id="directory-more">More models in the registry</h2>
++<p>Provider-listed models stay here when routing is off or a provider retires them. Prices and availability are dated at their sources.</p>
++<nav aria-label="Model status">${['all', 'available', 'preview', 'spotted', 'retired'].map(s => `<a class="ovk-btn ovk-btn--quiet" href="/models${s === 'all' ? '' : `?status=${s}`}">${esc(s)}</a>`).join(' ')}</nav>
++${rows.length ? `<ul>${rows.map(m => `<li><a href="${href(m.id)}">${esc(m.name)}</a> · ${esc(m.provider_name || m.provider)} · ${esc(m.status)} · ${esc(m.context_window == null ? 'Context not listed' : `${m.context_window} tokens`)} · ${esc(usd(m.price?.output_per_million))}</li>`).join('')}</ul>` : '<p>No additional models match this filter.</p>'}</section>`;
++}
++function directoryDetail(m) {
++    const price = m.price || {};
++    const link = value => { try { const u = new URL(value); return u.protocol === 'https:' ? `<a href="${esc(u.href)}" rel="noopener noreferrer">${esc(u.host)}</a>` : '—'; } catch { return '—'; } };
++    return `<div class="ex-wrap ex-narrow"><p><a href="/models">All models</a></p><h1>${esc(m.name)}</h1>
++<p>${esc(m.provider_name || m.provider)} · ${esc(m.status)}${m.routing_enabled ? ' · Routing on' : ' · Routing off'}</p>
++<dl><dt>Model ID</dt><dd><code>${esc(m.id)}</code></dd><dt>Version</dt><dd>${esc(m.version || 'Not listed')}</dd>
++<dt>Context</dt><dd>${esc(m.context_window == null ? 'Not listed' : `${m.context_window} tokens`)}</dd>
++<dt>Maximum output</dt><dd>${esc(m.max_output == null ? 'Not listed' : `${m.max_output} tokens`)}</dd>
++<dt>Modalities</dt><dd>${esc((m.modalities || []).join(', ') || 'Not listed')}</dd>
++<dt>Input price</dt><dd>${esc(usd(price.input_per_million))}</dd><dt>Output price</dt><dd>${esc(usd(price.output_per_million))}</dd>
++<dt>Price checked</dt><dd>${esc(price.checked_at || 'Not listed')}</dd><dt>Price source</dt><dd>${link(price.source)}</dd>
++<dt>Model source</dt><dd>${link(m.source)}</dd><dt>First seen</dt><dd>${esc(m.first_seen_at || 'Not listed')}</dd>
++${m.retired_at ? `<dt>Retired</dt><dd>${esc(m.retired_at)}</dd>` : ''}</dl>
++${m.sources?.length ? `<h2>Sources</h2><ul>${m.sources.map(s => `<li>${link(s.url)} · seen ${esc(s.observed_at || 'date not listed')}</li>`).join('')}</ul>` : ''}</div>`;
++}
++module.exports = { href, select, directoryList, directoryDetail };
ddiff --git a/test/helpers/ai.js b/test/helpers/ai.js
iindex e3dbd30..45d12c8 100644
---- a/test/helpers/ai.js
++++ b/test/helpers/ai.js
@@@ -72,6 +72,13 @@ function createFakeAi({ chat = null, community = [] } = {}) {
      return {
          calls,
          configured: true,
++        async directory() { return { models: [
++            { id: 'deepseek/deepseek-chat', provider: 'deepseek', model: 'deepseek-chat', name: 'DeepSeek Chat', status: 'available', routing_enabled: true, context_window: 131072, price: { input_per_million: 0.2, output_per_million: 0.3 } },
++            { id: 'google/gemini-preview', provider: 'google', model: 'gemini-preview', name: 'Gemini Preview', status: 'preview', routing_enabled: false, context_window: 1000000, price: { input_per_million: null, output_per_million: null, source: null } },
++            { id: 'google/gemini-old', provider: 'google', model: 'gemini-old', name: 'Gemini Old', status: 'retired', routing_enabled: false, retired_at: '2026-11-10T00:00:00Z', price: {} },
++            { id: 'arena/anonymous-silver', provider: 'arena', model: 'anonymous-silver', name: 'anonymous-silver', status: 'spotted', routing_enabled: false,
++                first_seen_at: '2026-11-11T00:00:00Z', sources: [{ url: 'https://lmarena.ai/leaderboard/text', observed_at: '2026-11-11T00:00:00Z' }], price: {} },
++        ] }; },
          // AI's search engines as GET /api/v1/exchange/search/engines answers them.
          async searchEngines() {
              return [
ddiff --git a/test/pages.test.js b/test/pages.test.js
iindex 45b073f..b20e1a9 100644
---- a/test/pages.test.js
++++ b/test/pages.test.js
@@@ -34,6 +34,27 @@ const { createFakeAi, downAi } = require('./helpers/ai');
              assert.ok(!family.text.includes('Llama 3 8B'));
          });
  
++        await check('disabled and retired registry models have public pages, API rows and crawl links', async () => {
++            const list = await t.get('/api/v1/models?status=retired');
++            assert.strictEqual(list.status, 200, list.text);
++            assert.deepStrictEqual(list.json().models.map(m => m.id), ['google/gemini-old']);
++            const one = await t.get('/api/v1/models/google%2Fgemini-old');
++            assert.strictEqual(one.json().model.status, 'retired');
++            const page = await t.get('/models/google%2Fgemini-old');
++            assert.strictEqual(page.status, 200, page.text);
++            assert.ok(page.text.includes('Gemini Old') && page.text.includes('Retired') && page.text.includes('SoftwareApplication'));
++            const index = await t.get('/models?status=preview');
++            assert.ok(index.text.includes('Gemini Preview') && !index.text.includes('Gemini Old'));
++            const spotted = await t.get('/models?status=spotted');
++            assert.ok(spotted.text.includes('anonymous-silver') && !spotted.text.includes('Gemini Old'));
++            const spotPage = await t.get('/models/arena%2Fanonymous-silver');
++            assert.ok(spotPage.text.includes('lmarena.ai') && spotPage.text.includes('2026-11-11'));
++            const map = await t.get('/sitemap.xml');
++            assert.ok(map.text.includes('google%2Fgemini-old'));
++            const twin = await t.get('/models/google%2Fgemini-old.md');
++            assert.strictEqual(twin.status, 200, twin.text);
++            assert.ok(twin.text.includes('Gemini Old'));
++        });
          await check('public directory JSON and model structured data match the catalog', async () => {
              const list = await t.get('/api/v1/models?q=deepseek');
              assert.strictEqual(list.status, 200, list.text);