openvibers/openvibe.exchange Commits
Show qualified chat candidates for coding task search (#200)
Changes
@@@ -39,7 +39,8 @@ function searchCatalog(catalog, rows, params) {const results = [];for (const model of catalog.models || []) {const caps = model.capabilities || {};-- const hasCap = (key) => key === 'coder' ? Boolean(caps.code || caps.coding || /cod(e|er|ing)/i.test(`${model.name} ${model.slug}`)) : Boolean(caps[key]);++ const explicitCoder = Boolean(caps.code || caps.coding || /cod(e|er|ing)/i.test(`${model.name} ${model.slug}`));++ const hasCap = (key) => key === 'coder' ? (model.type === 'chat' && (explicitCoder || caps.tools || caps.json)) : Boolean(caps[key]);if (capability && !hasCap(capability)) continue;if (intent.capabilities.some((key) => !hasCap(key))) continue;const hay = clean([model.name, model.slug, model.family, model.description, ...(model.offers || []).flatMap((o) => [o.provider, o.provider_name])].join(' '));@@@ -61,7 +62,8 @@ function searchCatalog(catalog, rows, params) {results.push({ model: model.slug, name: model.name, family: model.family, capabilities: caps,provider: best.provider_name || best.provider, price: best.price || null, speed: best.speed || null,quality: rating ? { elo: rating.elo, games: rating.games, kind: params.ratingKind } : null,-- offers: offers.length, score: nameScore + (model.requests_7d || 0) / 10000 });++ offers: offers.length, task_match: intent.capabilities.includes('coder') && !explicitCoder ? 'General chat model; coding quality is not measured here' : null,++ score: nameScore + (model.requests_7d || 0) / 10000 });}const sort = params.sort !== 'relevance' ? params.sort : intent.fast ? 'speed' : intent.cheap ? 'price' : 'relevance';const metric = (r) => sort === 'speed' ? number(r.speed?.p50_ms) ?? Infinity : sort === 'quality' ? -(number(r.quality?.elo) ?? -Infinity) : number(r.price?.out) ?? Infinity;@@@ -56,7 +56,7 @@ function table(models, searchMeta = null) {].filter(Boolean).join(' ');const fact = (label, value) => `<div><dt>${esc(label)}</dt><dd>${esc(value)}</dd></div>`;return `<li class="xm-row"><div class="xm-name"><a href="/models/${encodeURIComponent(id(m))}">${esc(m.name)}</a><span class="xm-maker">${esc(legacy.familyLabel(m))}</span>${badges ? `<span class="xm-badges">${badges}</span>` : ''}</div>`-- + `<dl class="xm-facts">${fact('In / out per M', `${perM(o.price?.in)} / ${perM(o.price?.out)}`)}${fact('Uptime', o.uptime == null ? '–' : `${dash(o.uptime * 100)}%`)}${fact('Speed', o.speed?.tokens_per_second != null ? `${dash(o.speed.tokens_per_second)} tok/s` : o.speed?.p50_ms == null ? '–' : `${dash(o.speed.p50_ms)} ms`)}${found ? fact('Quality', found.quality ? `${dash(found.quality.elo)} Elo · ${found.quality.games} games` : 'Not measured') : ''}${found ? fact('Provider', found.provider) : ''}</dl>`++ + `<dl class="xm-facts">${fact('In / out per M', `${perM(o.price?.in)} / ${perM(o.price?.out)}`)}${fact('Uptime', o.uptime == null ? '–' : `${dash(o.uptime * 100)}%`)}${fact('Speed', o.speed?.tokens_per_second != null ? `${dash(o.speed.tokens_per_second)} tok/s` : o.speed?.p50_ms == null ? '–' : `${dash(o.speed.p50_ms)} ms`)}${found ? fact('Quality', found.quality ? `${dash(found.quality.elo)} Elo · ${found.quality.games} games` : 'Not measured') : ''}${found ? fact('Provider', found.provider) : ''}${found?.task_match ? fact('Task match', found.task_match) : ''}</dl>`+ `<p class="xm-acts">${kit.button({ href: chat(m), label: 'Try in Chat', kind: 'quiet', small: true })} ${kit.button({ href: rank(m), label: 'How it ranks', kind: 'quiet', small: true })}</p></li>`;}).join('')}</ul>`;}@@@ -7,7 +7,9 @@ const { parseSearch, searchCatalog } = require('../server/model-search');const catalog = await createFakeAi().catalog();const fast = searchCatalog(catalog, [], parseSearch({ q: 'fast coder under $1' }));assert.ok(Array.isArray(fast));++ assert.ok(fast.length > 0, 'general chat offers can answer a coding task query');assert.ok(fast.every((r) => r.price.out <= 1));++ assert.ok(fast.every((r) => r.task_match && r.task_match.includes('not measured')));const provider = searchCatalog(catalog, [], parseSearch({ provider: 'DeepInfra', max_price: '0.5' }));assert.equal(provider.length, 1);assert.equal(provider[0].provider, 'DeepInfra');
@@ -39,7 +39,8 @@ function searchCatalog(catalog, rows, params) {
@@ -39,7 +39,8 @@ function searchCatalog(catalog, rows, params) {
const results = [];
const results = [];
for (const model of catalog.models || []) {
for (const model of catalog.models || []) {
const caps = model.capabilities || {};
const caps = model.capabilities || {};
- const hasCap = (key) => key === 'coder' ? Boolean(caps.code || caps.coding || /cod(e|er|ing)/i.test(`${model.name} ${model.slug}`)) : Boolean(caps[key]);
+ const explicitCoder = Boolean(caps.code || caps.coding || /cod(e|er|ing)/i.test(`${model.name} ${model.slug}`));
+ const hasCap = (key) => key === 'coder' ? (model.type === 'chat' && (explicitCoder || caps.tools || caps.json)) : Boolean(caps[key]);
if (capability && !hasCap(capability)) continue;
if (capability && !hasCap(capability)) continue;
if (intent.capabilities.some((key) => !hasCap(key))) continue;
if (intent.capabilities.some((key) => !hasCap(key))) continue;
const hay = clean([model.name, model.slug, model.family, model.description, ...(model.offers || []).flatMap((o) => [o.provider, o.provider_name])].join(' '));
const hay = clean([model.name, model.slug, model.family, model.description, ...(model.offers || []).flatMap((o) => [o.provider, o.provider_name])].join(' '));
@@ -61,7 +62,8 @@ function searchCatalog(catalog, rows, params) {
@@ -61,7 +62,8 @@ function searchCatalog(catalog, rows, params) {
results.push({ model: model.slug, name: model.name, family: model.family, capabilities: caps,
results.push({ model: model.slug, name: model.name, family: model.family, capabilities: caps,
provider: best.provider_name || best.provider, price: best.price || null, speed: best.speed || null,
provider: best.provider_name || best.provider, price: best.price || null, speed: best.speed || null,
quality: rating ? { elo: rating.elo, games: rating.games, kind: params.ratingKind } : null,
quality: rating ? { elo: rating.elo, games: rating.games, kind: params.ratingKind } : null,
- offers: offers.length, score: nameScore + (model.requests_7d || 0) / 10000 });
+ offers: offers.length, task_match: intent.capabilities.includes('coder') && !explicitCoder ? 'General chat model; coding quality is not measured here' : null,
+ score: nameScore + (model.requests_7d || 0) / 10000 });
}
}
const sort = params.sort !== 'relevance' ? params.sort : intent.fast ? 'speed' : intent.cheap ? 'price' : 'relevance';
const sort = params.sort !== 'relevance' ? params.sort : intent.fast ? 'speed' : intent.cheap ? 'price' : 'relevance';
const metric = (r) => sort === 'speed' ? number(r.speed?.p50_ms) ?? Infinity : sort === 'quality' ? -(number(r.quality?.elo) ?? -Infinity) : number(r.price?.out) ?? Infinity;
const metric = (r) => sort === 'speed' ? number(r.speed?.p50_ms) ?? Infinity : sort === 'quality' ? -(number(r.quality?.elo) ?? -Infinity) : number(r.price?.out) ?? Infinity;
@@ -56,7 +56,7 @@ function table(models, searchMeta = null) {
@@ -56,7 +56,7 @@ function table(models, searchMeta = null) {
].filter(Boolean).join(' ');
].filter(Boolean).join(' ');
const fact = (label, value) => `<div><dt>${esc(label)}</dt><dd>${esc(value)}</dd></div>`;
const fact = (label, value) => `<div><dt>${esc(label)}</dt><dd>${esc(value)}</dd></div>`;
return `<li class="xm-row"><div class="xm-name"><a href="/models/${encodeURIComponent(id(m))}">${esc(m.name)}</a><span class="xm-maker">${esc(legacy.familyLabel(m))}</span>${badges ? `<span class="xm-badges">${badges}</span>` : ''}</div>`
return `<li class="xm-row"><div class="xm-name"><a href="/models/${encodeURIComponent(id(m))}">${esc(m.name)}</a><span class="xm-maker">${esc(legacy.familyLabel(m))}</span>${badges ? `<span class="xm-badges">${badges}</span>` : ''}</div>`
- + `<dl class="xm-facts">${fact('In / out per M', `${perM(o.price?.in)} / ${perM(o.price?.out)}`)}${fact('Uptime', o.uptime == null ? '–' : `${dash(o.uptime * 100)}%`)}${fact('Speed', o.speed?.tokens_per_second != null ? `${dash(o.speed.tokens_per_second)} tok/s` : o.speed?.p50_ms == null ? '–' : `${dash(o.speed.p50_ms)} ms`)}${found ? fact('Quality', found.quality ? `${dash(found.quality.elo)} Elo · ${found.quality.games} games` : 'Not measured') : ''}${found ? fact('Provider', found.provider) : ''}</dl>`
+ + `<dl class="xm-facts">${fact('In / out per M', `${perM(o.price?.in)} / ${perM(o.price?.out)}`)}${fact('Uptime', o.uptime == null ? '–' : `${dash(o.uptime * 100)}%`)}${fact('Speed', o.speed?.tokens_per_second != null ? `${dash(o.speed.tokens_per_second)} tok/s` : o.speed?.p50_ms == null ? '–' : `${dash(o.speed.p50_ms)} ms`)}${found ? fact('Quality', found.quality ? `${dash(found.quality.elo)} Elo · ${found.quality.games} games` : 'Not measured') : ''}${found ? fact('Provider', found.provider) : ''}${found?.task_match ? fact('Task match', found.task_match) : ''}</dl>`
+ `<p class="xm-acts">${kit.button({ href: chat(m), label: 'Try in Chat', kind: 'quiet', small: true })} ${kit.button({ href: rank(m), label: 'How it ranks', kind: 'quiet', small: true })}</p></li>`;
+ `<p class="xm-acts">${kit.button({ href: chat(m), label: 'Try in Chat', kind: 'quiet', small: true })} ${kit.button({ href: rank(m), label: 'How it ranks', kind: 'quiet', small: true })}</p></li>`;
}).join('')}</ul>`;
}).join('')}</ul>`;
}
}
@@ -7,7 +7,9 @@ const { parseSearch, searchCatalog } = require('../server/model-search');
@@ -7,7 +7,9 @@ const { parseSearch, searchCatalog } = require('../server/model-search');
const catalog = await createFakeAi().catalog();
const catalog = await createFakeAi().catalog();
const fast = searchCatalog(catalog, [], parseSearch({ q: 'fast coder under $1' }));
const fast = searchCatalog(catalog, [], parseSearch({ q: 'fast coder under $1' }));
assert.ok(Array.isArray(fast));
assert.ok(Array.isArray(fast));
+ assert.ok(fast.length > 0, 'general chat offers can answer a coding task query');
assert.ok(fast.every((r) => r.price.out <= 1));
assert.ok(fast.every((r) => r.price.out <= 1));
+ assert.ok(fast.every((r) => r.task_match && r.task_match.includes('not measured')));
const provider = searchCatalog(catalog, [], parseSearch({ provider: 'DeepInfra', max_price: '0.5' }));
const provider = searchCatalog(catalog, [], parseSearch({ provider: 'DeepInfra', max_price: '0.5' }));
assert.equal(provider.length, 1);
assert.equal(provider.length, 1);
assert.equal(provider[0].provider, 'DeepInfra');
assert.equal(provider[0].provider, 'DeepInfra');