openvibers/openvibe.ai Commits

Routing: an empty answer is a failed attempt, and the next offer answers (#527)

OP OpenVibers committed 52 min ago · a475ecdcbc · 4 files, +46 −7

* Routing: an empty answer is a failed attempt, and the next offer answers

From 11-10 to 11-13 a model answered "ok" with no text on most background calls, and every one was counted a success:
the router never moved on, so summaries, slogans and the Rundown's write-ups came back empty. A chat, generate,
summarize, classify, extract or enrich answer with no text, no JSON and no tool calls now throws provider.empty after
the floor check, so the attempt loop tries the next offer and the provider's record counts the failure.
Test provider-router: the first offer answers nothing, the second answers.

* Tests: an empty answer is a failed attempt (DeepSeek's reproduction fails; the free-run stub answers the http seam too)

Changes

ddiff --git a/server/providers/index.js b/server/providers/index.js
iindex 9aac6fa..ca61b31 100644
---- a/server/providers/index.js
++++ b/server/providers/index.js
@@@ -1503,6 +1503,17 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
return [...out].sort();
}
++ // The operations whose answer is text, JSON or tool calls: one with none of them is empty.
++ const EMPTY_CHECKED_OPS = new Set(['chat', 'generate', 'summarize', 'classify', 'extract', 'enrich']);
++ function isEmptyAnswer(result) {
++ if (!result) return true;
++ if (result.json != null) return false;
++ if (Array.isArray(result.toolCalls) && result.toolCalls.length) return false;
++ if (Array.isArray(result.tool_calls) && result.tool_calls.length) return false;
++ if (result.message || result.completion) return false;
++ return !String(result.text == null ? '' : result.text).trim();
++ }
++
async function execute(route, operation, req, ctx = {}) {
if (ctx.compatRouting && (ctx.credential || ctx.byok || ctx.selfPreferences?.only_my_keys))
throw new AiError(400, 'input.invalid', 'Provider routing cannot be combined with an explicit credential or only_my_keys');
@@@ -2130,6 +2141,12 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
if (floorGuard) floorGuard.finish();
const floorOut = floor.checkResult(result);
if (!floorOut.ok) throw new AiError(403, 'policy.floor_refused', floorOut.detail, { rule: floorOut.rule, policy: contentPolicy });
++ // An answer with nothing in it is a failed attempt, not a success: the next offer answers instead, and
++ // the provider's record counts it (2026-11-10 to 11-13 a model answered "ok" with no text on most
++ // background calls, and every one of them was counted a success).
++ if (EMPTY_CHECKED_OPS.has(operation) && isEmptyAnswer(result)) {
++ throw new ProviderError('the provider answered with nothing', { code: 'provider.empty' });
++ }
if (uncensored) { result.policy = 'uncensored'; result.label = tier.LABEL; }
const latency = Date.now() - t0;
const usage = result.usage || { input: 0, output: 0, cached: 0 };
ddiff --git a/test/deepseek-flash.test.js b/test/deepseek-flash.test.js
iindex 5660f75..a1a3810 100644
---- a/test/deepseek-flash.test.js
++++ b/test/deepseek-flash.test.js
@@@ -66,17 +66,20 @@ t.test('the Exchange chat path (complete) gets the same reasoning room', async (
assert.strictEqual(out.result.text, answer);
});
--t.test('without the reserve the same 48 tokens are all reasoning and the answer is empty', async () => {
++t.test('without the reserve the same 48 tokens are all reasoning: the empty answer is a failed attempt, not a success', async () => {
await h.registry.upsertModel({
provider_key: 'deepseek', model_key: 'deepseek-flash', display_name: 'DeepSeek V4.1 Flash', type: 'chat',
supports: { json: true, tools: true, streaming: true }, cost: { in_per_mtok: 0.3, out_per_mtok: 1.2 },
metadata: { reasoning: true },
}, { actor: 'test' });
-- const out = await h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 },
-- { routeKey: route.key, routeVersion: route.version });
++ // The reproduction of the production run (11-10 to 11-13): the model spends the room on reasoning and answers
++ // nothing. The router counts that as a failed attempt (provider.empty), so with no other offer the call fails
++ // instead of handing back an empty "success".
++ await assert.rejects(
++ () => h.pool.execute(route, 'chat', { messages: MESSAGES, maxTokens: 48, timeoutMs: 3000 }, { routeKey: route.key, routeVersion: route.version }),
++ (err) => /could answer/.test(err.message),
++ );
assert.strictEqual(srv.last.max_tokens, 48, 'no reserve means no extra room');
-- assert.strictEqual(out.result.text, '', 'the reproduction of the production run');
-- assert.strictEqual(out.usage.output, 0);
});
t.test('the seed puts the reasoning room on the model, not on every provider', async () => {
ddiff --git a/test/free-providers.test.js b/test/free-providers.test.js
iindex dd29d24..11eeab1 100644
---- a/test/free-providers.test.js
++++ b/test/free-providers.test.js
@@@ -39,7 +39,7 @@ t.test('the default sources, the local family and the ceiling rule', () => {
t.test('boot a named source and a cheaper provider outside the list', async () => {
// OpenAI format: the free allowance and a person's own token reach the service through ai.complete
// (a Chat Completions passthrough), which only an OpenAI-format provider offers.
-- srv = await seamServer(() => ({ model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
++ srv = await seamServer(() => ({ model: 'm', text: 'an answer', choices: [{ index: 0, message: { role: 'assistant', content: 'an answer' }, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }));
h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0' } });
for (const key of ['deepseek', 'openai']) {
await h.registry.upsertProvider({ key, display_name: key, kind: 'openai', status: 'active', base_url: `${srv.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'generate', 'json'], default_model: models[key].model_key, timeout_ms: 3000 }, { actor: 'test', origin: 'admin' });
ddiff --git a/test/provider-router.test.js b/test/provider-router.test.js
iindex 7224e44..63a39b8 100644
---- a/test/provider-router.test.js
++++ b/test/provider-router.test.js
@@@ -121,7 +121,9 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {
// llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.
-- const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
++ // An OpenAI-shaped answer: the local server speaks Chat Completions (a { text } body read as empty, which was counted
++ // a success until empty answers became failures).
++ const chat = await seamServer(() => ({ choices: [{ message: { content: 'local chat answer' } }], model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
const embedSrv = await seamServer((body, req) => {
assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);
return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };
@@@ -258,6 +260,23 @@ t.test('an upstream 429 shifts to another provider without opening the circuit',
await ok.close(); await busy.close();
});
++t.test('an answer with nothing in it is a failed attempt: the next provider answers', async () => {
++ // 2026-11-10 to 11-13: a model answered "ok" with no text on most background calls and each was counted a success.
++ const ok = await seamServer(() => ({ text: 'a real answer', model: 'full-m', usage: { input: 3, output: 4 } }));
++ const blank = await seamServer(() => ({ text: '', model: 'blank-m', usage: { input: 3, output: 0 } }));
++ await provider({ key: 'blank', base_url: blank.url, capabilities: ['enrich'], default_model: 'blank-m' });
++ await model('blank', 'blank-m', 0.4);
++ await provider({ key: 'full', base_url: ok.url, capabilities: ['enrich'], default_model: 'full-m' });
++ await model('full', 'full-m', 0.45); // next after the empty one, before the other tests' closed servers
++ const r = await route('test.pool-empty', { capability: 'enrich', constraints: { objective: 'cheapest' }, timeout_ms: 3000 });
++ const exec = await h.pool.execute(r, 'enrich', { messages: [{ role: 'user', content: 'x' }], timeoutMs: 3000 }, { routeKey: r.key, routeVersion: r.version });
++ assert.strictEqual(exec.provider, 'full', 'the empty answer gives way to the next provider');
++ assert.strictEqual(exec.result.text, 'a real answer');
++ const blankTry = (exec.explain.attempts || []).find((a) => a.provider === 'blank');
++ assert.ok(blankTry && blankTry.status !== 'ok', 'the empty attempt is recorded as failed, not ok');
++ await ok.close(); await blank.close();
++});
++
t.test('explain rides the run response and GET /runs/:id; the request log stays', async () => {
const fails = await seamServer(() => ({ status: 500, body: { error: { message: 'down' } } }));
const ok = await seamServer(() => ({ text: 'the fallback answer', model: 'p2-m', usage: { input: 4, output: 2 } }));