openvibers/openvibe.ai Commits
Routing: an empty answer is a failed attempt, and the next offer answers (#527)
Changes
@@@ -1503,6 +1503,17 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.noreturn [...out].sort();}++ // The operations whose answer is text, JSON or tool calls: one with none of them is empty.++ const EMPTY_CHECKED_OPS = new Set(['chat', 'generate', 'summarize', 'classify', 'extract', 'enrich']);++ function isEmptyAnswer(result) {++ if (!result) return true;++ if (result.json != null) return false;++ if (Array.isArray(result.toolCalls) && result.toolCalls.length) return false;++ if (Array.isArray(result.tool_calls) && result.tool_calls.length) return false;++ if (result.message || result.completion) return false;++ return !String(result.text == null ? '' : result.text).trim();++ }++async function execute(route, operation, req, ctx = {}) {if (ctx.compatRouting && (ctx.credential || ctx.byok || ctx.selfPreferences?.only_my_keys))throw new AiError(400, 'input.invalid', 'Provider routing cannot be combined with an explicit credential or only_my_keys');@@@ -2130,6 +2141,12 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.noif (floorGuard) floorGuard.finish();const floorOut = floor.checkResult(result);if (!floorOut.ok) throw new AiError(403, 'policy.floor_refused', floorOut.detail, { rule: floorOut.rule, policy: contentPolicy });++ // An answer with nothing in it is a failed attempt, not a success: the next offer answers instead, and++ // the provider's record counts it (2026-11-10 to 11-13 a model answered "ok" with no text on most++ // background calls, and every one of them was counted a success).++ if (EMPTY_CHECKED_OPS.has(operation) && isEmptyAnswer(result)) {++ throw new ProviderError('the provider answered with nothing', { code: 'provider.empty' });++ }if (uncensored) { result.policy = 'uncensored'; result.label = tier.LABEL; }const latency = Date.now() - t0;const usage = result.usage || { input: 0, output: 0, cached: 0 };@@@ -66,17 +66,20 @@ t.test('the Exchange chat path (complete) gets the same reasoning room', async (assert.strictEqual(out.result.text, answer);});--t.test('without the reserve the same 48 tokens are all reasoning and the answer is empty', async () => {++t.test('without the reserve the same 48 tokens are all reasoning: the empty answer is a failed attempt, not a success', async () => {await h.registry.upsertModel({provider_key: 'deepseek', model_key: 'deepseek-flash', display_name: 'DeepSeek V4.1 Flash', type: 'chat',supports: { json: true, tools: true, streaming: true }, cost: { in_per_mtok: 0.3, out_per_mtok: 1.2 },metadata: { reasoning: true },}, { actor: 'test' });-- const out = await h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 },-- { routeKey: route.key, routeVersion: route.version });++ // The reproduction of the production run (11-10 to 11-13): the model spends the room on reasoning and answers++ // nothing. The router counts that as a failed attempt (provider.empty), so with no other offer the call fails++ // instead of handing back an empty "success".++ await assert.rejects(++ () => h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 }, { routeKey: route.key, routeVersion: route.version }),++ (err) => /could answer/.test(err.message),++ );assert.strictEqual(srv.last.max_tokens, 48, 'no reserve means no extra room');-- assert.strictEqual(out.result.text, '', 'the reproduction of the production run');-- assert.strictEqual(out.usage.output, 0);});t.test('the seed puts the reasoning room on the model, not on every provider', async () => {@@@ -39,7 +39,7 @@ t.test('the default sources, the local family and the ceiling rule', () => {t.test('boot a named source and a cheaper provider outside the list', async () => {// OpenAI format: the free allowance and a person's own token reach the service through ai.complete// (a Chat Completions passthrough), which only an OpenAI-format provider offers.-- srv = await seamServer(() => ({ model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));++ srv = await seamServer(() => ({ model: 'm', text: 'an answer', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0' } });for (const key of ['deepseek', 'openai']) {await h.registry.upsertProvider({ key, display_name: key, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });@@@ -121,7 +121,9 @@ t.test('a local embedding model makes default.embedding a pool: the failing paidt.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {// llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.-- const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));++ // An OpenAI-shaped answer: the local server speaks Chat Completions (a { text } body read as empty, which was counted++ // a success until empty answers became failures).++ const chat = await seamServer(() => ({ choices: [{ message: { content: 'local chat answer' } }], model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));const embedSrv = await seamServer((body, req) => {assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };@@@ -258,6 +260,23 @@ t.test('an upstream 429 shifts to another provider without opening the circuit',await ok.close(); await busy.close();});++t.test('an answer with nothing in it is a failed attempt: the next provider answers', async () => {++ // 2026-11-10 to 11-13: a model answered "ok" with no text on most background calls and each was counted a success.++ const ok = await seamServer(() => ({ text: 'a real answer', model: 'full-m', usage: { input: 3, output: 4 } }));++ const blank = await seamServer(() => ({ text: '', model: 'blank-m', usage: { input: 3, output: 0 } }));++ await provider({ key: 'blank', base_url: blank.url, capabilities: ['enrich'], default_model: 'blank-m' });++ await model('blank', 'blank-m', 0.4);++ await provider({ key: 'full', base_url: ok.url, capabilities: ['enrich'], default_model: 'full-m' });++ await model('full', 'full-m', 0.45); // next after the empty one, before the other tests' closed servers++ const r = await route('test.pool-empty', { capability: 'enrich', constraints: { objective: 'cheapest' }, timeout_ms: 3000 });++ const exec = await h.pool.execute(r, 'enrich', { messages: [{ role: 'user', content: 'x' }], timeoutMs: 3000 }, { routeKey: r.key, routeVersion: r.version });++ assert.strictEqual(exec.provider, 'full', 'the empty answer gives way to the next provider');++ assert.strictEqual(exec.result.text, 'a real answer');++ const blankTry = (exec.explain.attempts || []).find((a) => a.provider === 'blank');++ assert.ok(blankTry && blankTry.status !== 'ok', 'the empty attempt is recorded as failed, not ok');++ await ok.close(); await blank.close();++});++t.test('explain rides the run response and GET /runs/:id; the request log stays', async () => {const fails = await seamServer(() => ({ status: 500, body: { error: { message: 'down' } } }));const ok = await seamServer(() => ({ text: 'the fallback answer', model: 'p2-m', usage: { input: 4, output: 2 } }));
@@ -1503,6 +1503,17 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
@@ -1503,6 +1503,17 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
return [...out].sort();
return [...out].sort();
}
}
+ // The operations whose answer is text, JSON or tool calls: one with none of them is empty.
+ const EMPTY_CHECKED_OPS = new Set(['chat', 'generate', 'summarize', 'classify', 'extract', 'enrich']);
+ function isEmptyAnswer(result) {
+ if (!result) return true;
+ if (result.json != null) return false;
+ if (Array.isArray(result.toolCalls) && result.toolCalls.length) return false;
+ if (Array.isArray(result.tool_calls) && result.tool_calls.length) return false;
+ if (result.message || result.completion) return false;
+ return !String(result.text == null ? '' : result.text).trim();
+ }
+
async function execute(route, operation, req, ctx = {}) {
async function execute(route, operation, req, ctx = {}) {
if (ctx.compatRouting && (ctx.credential || ctx.byok || ctx.selfPreferences?.only_my_keys))
if (ctx.compatRouting && (ctx.credential || ctx.byok || ctx.selfPreferences?.only_my_keys))
throw new AiError(400, 'input.invalid', 'Provider routing cannot be combined with an explicit credential or only_my_keys');
throw new AiError(400, 'input.invalid', 'Provider routing cannot be combined with an explicit credential or only_my_keys');
@@ -2130,6 +2141,12 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
@@ -2130,6 +2141,12 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
if (floorGuard) floorGuard.finish();
if (floorGuard) floorGuard.finish();
const floorOut = floor.checkResult(result);
const floorOut = floor.checkResult(result);
if (!floorOut.ok) throw new AiError(403, 'policy.floor_refused', floorOut.detail, { rule: floorOut.rule, policy: contentPolicy });
if (!floorOut.ok) throw new AiError(403, 'policy.floor_refused', floorOut.detail, { rule: floorOut.rule, policy: contentPolicy });
+ // An answer with nothing in it is a failed attempt, not a success: the next offer answers instead, and
+ // the provider's record counts it (2026-11-10 to 11-13 a model answered "ok" with no text on most
+ // background calls, and every one of them was counted a success).
+ if (EMPTY_CHECKED_OPS.has(operation) && isEmptyAnswer(result)) {
+ throw new ProviderError('the provider answered with nothing', { code: 'provider.empty' });
+ }
if (uncensored) { result.policy = 'uncensored'; result.label = tier.LABEL; }
if (uncensored) { result.policy = 'uncensored'; result.label = tier.LABEL; }
const latency = Date.now() - t0;
const latency = Date.now() - t0;
const usage = result.usage || { input: 0, output: 0, cached: 0 };
const usage = result.usage || { input: 0, output: 0, cached: 0 };
@@ -66,17 +66,20 @@ t.test('the Exchange chat path (complete) gets the same reasoning room', async (
@@ -66,17 +66,20 @@ t.test('the Exchange chat path (complete) gets the same reasoning room', async (
assert.strictEqual(out.result.text, answer);
assert.strictEqual(out.result.text, answer);
});
});
-t.test('without the reserve the same 48 tokens are all reasoning and the answer is empty', async () => {
+t.test('without the reserve the same 48 tokens are all reasoning: the empty answer is a failed attempt, not a success', async () => {
await h.registry.upsertModel({
await h.registry.upsertModel({
provider_key: 'deepseek', model_key: 'deepseek-flash', display_name: 'DeepSeek V4.1 Flash', type: 'chat',
provider_key: 'deepseek', model_key: 'deepseek-flash', display_name: 'DeepSeek V4.1 Flash', type: 'chat',
supports: { json: true, tools: true, streaming: true }, cost: { in_per_mtok: 0.3, out_per_mtok: 1.2 },
supports: { json: true, tools: true, streaming: true }, cost: { in_per_mtok: 0.3, out_per_mtok: 1.2 },
metadata: { reasoning: true },
metadata: { reasoning: true },
}, { actor: 'test' });
}, { actor: 'test' });
- const out = await h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 },
- { routeKey: route.key, routeVersion: route.version });
+ // The reproduction of the production run (11-10 to 11-13): the model spends the room on reasoning and answers
+ // nothing. The router counts that as a failed attempt (provider.empty), so with no other offer the call fails
+ // instead of handing back an empty "success".
+ await assert.rejects(
+ () => h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 }, { routeKey: route.key, routeVersion: route.version }),
+ (err) => /could answer/.test(err.message),
+ );
assert.strictEqual(srv.last.max_tokens, 48, 'no reserve means no extra room');
assert.strictEqual(srv.last.max_tokens, 48, 'no reserve means no extra room');
- assert.strictEqual(out.result.text, '', 'the reproduction of the production run');
- assert.strictEqual(out.usage.output, 0);
});
});
t.test('the seed puts the reasoning room on the model, not on every provider', async () => {
t.test('the seed puts the reasoning room on the model, not on every provider', async () => {
@@ -39,7 +39,7 @@ t.test('the default sources, the local family and the ceiling rule', () => {
@@ -39,7 +39,7 @@ t.test('the default sources, the local family and the ceiling rule', () => {
t.test('boot a named source and a cheaper provider outside the list', async () => {
t.test('boot a named source and a cheaper provider outside the list', async () => {
// OpenAI format: the free allowance and a person's own token reach the service through ai.complete
// OpenAI format: the free allowance and a person's own token reach the service through ai.complete
// (a Chat Completions passthrough), which only an OpenAI-format provider offers.
// (a Chat Completions passthrough), which only an OpenAI-format provider offers.
- srv = await seamServer(() => ({ model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
+ srv = await seamServer(() => ({ model: 'm', text: 'an answer', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0' } });
h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0' } });
for (const key of ['deepseek', 'openai']) {
for (const key of ['deepseek', 'openai']) {
await h.registry.upsertProvider({ key, display_name: key, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
await h.registry.upsertProvider({ key, display_name: key, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
@@ -121,7 +121,9 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
@@ -121,7 +121,9 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {
t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {
// llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.
// llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.
- const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
+ // An OpenAI-shaped answer: the local server speaks Chat Completions (a { text } body read as empty, which was counted
+ // a success until empty answers became failures).
+ const chat = await seamServer(() => ({ choices: [{ message: { content: 'local chat answer' } }], model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
const embedSrv = await seamServer((body, req) => {
const embedSrv = await seamServer((body, req) => {
assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);
assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);
return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };
return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };
@@ -258,6 +260,23 @@ t.test('an upstream 429 shifts to another provider without opening the circuit',
@@ -258,6 +260,23 @@ t.test('an upstream 429 shifts to another provider without opening the circuit',
await ok.close(); await busy.close();
await ok.close(); await busy.close();
});
});
+t.test('an answer with nothing in it is a failed attempt: the next provider answers', async () => {
+ // 2026-11-10 to 11-13: a model answered "ok" with no text on most background calls and each was counted a success.
+ const ok = await seamServer(() => ({ text: 'a real answer', model: 'full-m', usage: { input: 3, output: 4 } }));
+ const blank = await seamServer(() => ({ text: '', model: 'blank-m', usage: { input: 3, output: 0 } }));
+ await provider({ key: 'blank', base_url: blank.url, capabilities: ['enrich'], default_model: 'blank-m' });
+ await model('blank', 'blank-m', 0.4);
+ await provider({ key: 'full', base_url: ok.url, capabilities: ['enrich'], default_model: 'full-m' });
+ await model('full', 'full-m', 0.45); // next after the empty one, before the other tests' closed servers
+ const r = await route('test.pool-empty', { capability: 'enrich', constraints: { objective: 'cheapest' }, timeout_ms: 3000 });
+ const exec = await h.pool.execute(r, 'enrich', { messages: [{ role: 'user', content: 'x' }], timeoutMs: 3000 }, { routeKey: r.key, routeVersion: r.version });
+ assert.strictEqual(exec.provider, 'full', 'the empty answer gives way to the next provider');
+ assert.strictEqual(exec.result.text, 'a real answer');
+ const blankTry = (exec.explain.attempts || []).find((a) => a.provider === 'blank');
+ assert.ok(blankTry && blankTry.status !== 'ok', 'the empty attempt is recorded as failed, not ok');
+ await ok.close(); await blank.close();
+});
+
t.test('explain rides the run response and GET /runs/:id; the request log stays', async () => {
t.test('explain rides the run response and GET /runs/:id; the request log stays', async () => {
const fails = await seamServer(() => ({ status: 500, body: { error: { message: 'down' } } }));
const fails = await seamServer(() => ({ status: 500, body: { error: { message: 'down' } } }));
const ok = await seamServer(() => ({ text: 'the fallback answer', model: 'p2-m', usage: { input: 4, output: 2 } }));
const ok = await seamServer(() => ({ text: 'the fallback answer', model: 'p2-m', usage: { input: 4, output: 2 } }));