openvibers/openvibe.ai Commits

AI: three suites that never ran now do, and a guard keeps it that way (#328)

OP OpenVibers committed 7 days ago · 1a6677d6ac · 6 files, +53 −8

* AI: three suites that never ran now do, and a guard keeps it that way

exchange-catalog-kinds, free-ceiling and free-providers built a suite() and never
called t.run(), so the runner counted each as a pass while checking nothing.

Running them showed the two free-tier files were written against the http seam
provider, which has never offered `complete` (server/providers/http.js OPS); the
runs they exercise go through the ai.complete workflow, which only an OpenAI-format
provider serves (server/providers/openai.js: supports complete when it declares
chat, since #72). They now register kind openai against an OpenAI-shaped stub, as
every other ai.complete test does; the free-tier assertions are unchanged.

test/suites-run.test.js fails when any test file builds a suite and never runs it.

* exchange-read runs too, and the guard looks for the suite's own run call

The guard accepted any ".run(", so exchange-read (whose database statements call .run) stayed vacuous. With t.run()
its tests found: catalog prices printed as 0.020000000000000004 (now rounded to the micro-dollar), the DeepSeek
offer needing the pool key the catalog reads, DeepInfra's seeded policy, and usage counting the earlier test's runs.

Changes

ddiff --git a/server/exchange/catalog.js b/server/exchange/catalog.js
iindex b45c776..a36b633 100644
---- a/server/exchange/catalog.js
++++ b/server/exchange/catalog.js
@@@ -105,6 +105,8 @@ function modalitiesOf(row) {
return [...out];
}
++const usd6 = (n) => (n == null || !Number.isFinite(Number(n)) ? (n ?? null) : Math.round(Number(n) * 1e6) / 1e6);
++
function createCatalog({ db, registry, pool, env = process.env, clock = { now: () => Date.now() }, config = null }) {
// The free-tier admission rules are the router's (server/free-tier.js): the catalog marks an offer with what
// the free allowance and the anonymous Try it may actually use. config.freeUse is fixed at boot, like the
@@@ -161,7 +163,8 @@ function createCatalog({ db, registry, pool, env = process.env, clock = { now: (
// capacity with no per-token card): both its provider is a named AI_FREE_PROVIDERS source and its
// output price is at or below AI_FREE_MAX_OUTPUT_PRICE_USD_PER_MTOK.
free_tier: allowsOffer({ providerKey: provider.key, providerRecord: provider, outputPrice: price ? price.out : null }, freeUse),
-- price: price ? { in: price.in, out: price.out, cached: price.cached, per: '1M tokens', currency: 'USD' } : null,
++ // Rounded to the micro-dollar: a derived cached price was printed as 0.020000000000000004.
++ price: price ? { in: usd6(price.in), out: usd6(price.out), cached: usd6(price.cached), per: '1M tokens', currency: 'USD' } : null,
// Where the number came from and when someone checked it; null when nobody recorded it.
price_source: cards && cards.out.source && !String(cards.out.source).startsWith('urn:openvibe:ai:unverified:') ? cards.out.source : null,
price_checked_at: cards && cards.out.verified_at && cards.out.verified_at !== rates.UNKNOWN_DATE ? cards.out.verified_at : null,
ddiff --git a/test/exchange-catalog-kinds.test.js b/test/exchange-catalog-kinds.test.js
iindex 5770943..143f342 100644
---- a/test/exchange-catalog-kinds.test.js
++++ b/test/exchange-catalog-kinds.test.js
@@@ -101,3 +101,5 @@ t.test('an unknown shared upstream is no offer at all', async () => {
assert.strictEqual(upstream.serves({ families: ['openai'] }, { key: 'shared' }, { model_key: 'claude-haiku' }), false);
assert.strictEqual(upstream.configured({ status: 'active', auth_mode: 'bearer', secret_ref: 'env:NOT_SET_ANYWHERE' }, {}), false);
});
++
++t.run();
ddiff --git a/test/exchange-read.test.js b/test/exchange-read.test.js
iindex 2ac6821..37dff31 100644
---- a/test/exchange-read.test.js
++++ b/test/exchange-read.test.js
@@@ -33,7 +33,7 @@ async function insertReceipt({ runId, body }) {
t.test('boot', async () => {
// The catalog lists only offers whose key is configured: give the test's providers keys and the shared
// provider a real OpenAI upstream, as production has.
-- h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER: 'openai', AI_API_KEY: 'test-openai-key', DEEPSEEK_KEY: 'test-deepseek-key', DEEPINFRA_KEY: 'test-deepinfra-key' } });
++ h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER: 'openai', AI_API_KEY: 'test-openai-key', DEEPSEEK_KEY: 'test-deepseek-key', DEEPSEEK_API_KEY: 'test-deepseek-key', DEEPINFRA_KEY: 'test-deepinfra-key' } });
});
t.test('a run id is its registry name plus a revision, never another model', async () => {
@@@ -98,8 +98,8 @@ t.test('catalog groups the same model across providers, with prices, speed, upti
assert.strictEqual(cheap.uptime, null, 'no data is unknown, not 0');
assert.strictEqual(cheap.requests_7d, 0);
assert.deepStrictEqual(cheap.speed, { p50_ms: null, p95_ms: null });
-- assert.deepStrictEqual(dear.data_policy, { trains: false, retention_days: 0, region: 'cn' });
-- assert.strictEqual(cheap.data_policy, null, 'unknown stays unknown');
++ assert.deepStrictEqual(dear.data_policy, { trains: false, retention_days: 0, region: 'cn', anonymized: null });
++ assert.strictEqual(cheap.data_policy && cheap.data_policy.trains, false, 'DeepInfra carries its published policy (seeded by AI #307): no training');
assert.strictEqual(m.requests_7d, 10);
assert.strictEqual(m.cached_7d, 3);
});
@@@ -123,6 +123,8 @@ t.test('one model by slug, and a miss is a problem document', async () => {
});
t.test('usage: day, model and product aggregates from runs, and never an attribution', async () => {
++ // The catalog test above recorded runs of its own in this database; usage is counted from a clean table.
++ await h.db.prepare('DELETE FROM runs').run();
// Live's real traffic is gpt-5-nano even though its route names a DeepSeek model: the run keeps the
// model that answered, and the product table must show that one.
for (let i = 0; i < 5; i++) await insertRun({ requester: 'live', provider: 'shared', model: 'gpt-5-nano-2025-08-07', tokensIn: 100, tokensOut: 20, cost: 0.0002 });
@@@ -161,3 +163,5 @@ t.test('usage: day, model and product aggregates from runs, and never an attribu
assert.ok(r.body.by_model.some((m) => m.model === 'deepseek-chat' && m.requests === 6));
assert.doesNotMatch(r.text, /9999|9\.000000/, 'cached and failed runs are not counted');
});
++
++t.run();
ddiff --git a/test/free-ceiling.test.js b/test/free-ceiling.test.js
iindex c3855bb..295a6f1 100644
---- a/test/free-ceiling.test.js
++++ b/test/free-ceiling.test.js
@@@ -20,10 +20,12 @@ t.test('the ceiling is $1.5 by default and configurable', () => {
});
t.test('boot two offers: cheap, and a frontier model over the ceiling', async () => {
-- srv = await seamServer(() => ({ text: 'an answer', model: 'm', usage: { input: 5, output: 2 } }));
++ // OpenAI format: the anonymous Try it and a person's own token reach the service through ai.complete
++ // (a Chat Completions passthrough), which only an OpenAI-format provider offers.
++ srv = await seamServer(() => ({ model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_FREE_PROVIDERS: 'cheap,dear' } });
for (const [key, label] of [['cheap', 'Cheap'], ['dear', 'Dear']]) {
-- await h.registry.upsertProvider({ key, display_name: label, kind: 'http', status: 'active', base_url: srv.url, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
++ await h.registry.upsertProvider({ key, display_name: label, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
await h.registry.upsertModel({ provider_key: key, ...models[key] }, { actor: 'test' });
}
route = await h.registry.createRouteVersion('test.free-ceiling', { capability: 'chat', constraints: { objective: 'correctness' }, timeout_ms: 3000 }, { actor: 'test' });
@@@ -60,3 +62,5 @@ t.test('the anonymous ai.complete run is capped; the same request on a person\'s
});
t.test('stop', async () => { await h.stop(); await srv.close(); });
++
++t.run();
ddiff --git a/test/free-providers.test.js b/test/free-providers.test.js
iindex 4799aae..dd29d24 100644
---- a/test/free-providers.test.js
++++ b/test/free-providers.test.js
@@@ -37,10 +37,12 @@ t.test('the default sources, the local family and the ceiling rule', () => {
});
t.test('boot a named source and a cheaper provider outside the list', async () => {
-- srv = await seamServer(() => ({ text: 'an answer', model: 'm', usage: { input: 5, output: 2 } }));
++ // OpenAI format: the free allowance and a person's own token reach the service through ai.complete
++ // (a Chat Completions passthrough), which only an OpenAI-format provider offers.
++ srv = await seamServer(() => ({ model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0' } });
for (const key of ['deepseek', 'openai']) {
-- await h.registry.upsertProvider({ key, display_name: key, kind: 'http', status: 'active', base_url: srv.url, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
++ await h.registry.upsertProvider({ key, display_name: key, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
await h.registry.upsertModel({ provider_key: key, ...models[key] }, { actor: 'test' });
}
route = await h.registry.createRouteVersion('test.free-providers', { capability: 'chat', constraints: { objective: 'correctness' }, timeout_ms: 3000 }, { actor: 'test' });
@@@ -88,3 +90,5 @@ t.test('`local` covers a second local server (local2) for a free run', async ()
});
t.test('stop', async () => { await h.stop(); await srv.close(); });
++
++t.run();
ddiff --git a/test/suites-run.test.js b/test/suites-run.test.js
nnew file mode 100644
iindex 0000000..fef4818
---- /dev/null
++++ b/test/suites-run.test.js
@@@ -0,0 +1,28 @@
++'use strict';
++// A suite() in this repo only collects tests: nothing runs until `t.run()`. A file that builds one and
++// forgets the call exits 0 having checked nothing, and the runner counts it as a pass. Four did
++// (exchange-catalog-kinds, free-ceiling, free-providers, exchange-read), so a new one fails here instead of passing quietly.
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { suite } = require('./helpers');
++
++const t = suite('suites-run');
++
++t.test('every test file that builds a suite runs it', () => {
++ const suites = fs.readdirSync(__dirname)
++ .filter((name) => name.endsWith('.test.js'))
++ .map((name) => ({ name, source: fs.readFileSync(path.join(__dirname, name), 'utf8') }))
++ .filter((f) => /\bsuite\s*\(/.test(f.source) && !/function suite/.test(f.source));
++ assert.ok(suites.length > 0, 'no suite() file was found: the scan is broken');
++ // The suite's own variable must be run: `const t = suite(...)` needs `t.run(`. A bare `.run(` is not enough,
++ // because a database statement's `.run(...)` would hide a vacuous file (exchange-read did).
++ const neverRun = suites.filter((f) => {
++ const names = [...f.source.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*suite\s*\(/g)].map((m) => m[1]);
++ if (!names.length) return !/\bsuite\s*\([^)]*\)\s*\.run\s*\(/.test(f.source);
++ return names.some((n) => !new RegExp('\\b' + n.replace(/\$/g, '\\$') + '\\.run\\s*\\(').test(f.source));
++ }).map((f) => f.name);
++ assert.deepStrictEqual(neverRun, [], `these files build a suite and never run it: ${neverRun.join(', ')}`);
++});
++
++t.run();