openvibers/openvibe.actor Commits

Actor: the check works with DeepSeek's reasoning model, sees the cited sources, and falls back to another checker. (#1)

OP OpenVibers committed 8 Oct 2026 · 9d4b5a8eaa · 8 files, +70 −35

Production found it: deepseek-flash reasons by default and spent the checker's 120 tokens on reasoning, so an
OpenAI answer was refused as unchecked. Classifying and checking now run with DeepSeek's thinking disabled (the
runtime keeps reasoning, with a 4000-token budget); the checker sees the sources the agent cites and is told not to
fail recent facts it cannot verify; a checker that gives no verdict passes the check to the next one (same family,
and the task says so) before the task fails unchecked.


Claude-Session: https://claude.ai/code/session_01K8AYn3FF3HE7CPfM262qdh

Co-authored-by: Claude Opus 5.5 <[email protected]>

Changes

ddiff --git a/server/agents/adapters.js b/server/agents/adapters.js
iindex 92d6e46..3af819c 100644
---- a/server/agents/adapters.js
++++ b/server/agents/adapters.js
@@@ -37,6 +37,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
const ovTools = toolsOverride || createOpenVibeTools({ config, fetchImpl, log });
const deepseek = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl }) : null;
++ // Classifying and checking are short, structured answers: DeepSeek's reasoning is off for them.
++ const deepseekQuick = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl, extraBody: { thinking: { type: 'disabled' } } }) : null;
const openaiAgent = p.openai.apiKey ? responsesAgent({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.model, rates: rates.openai, fetchImpl }) : null;
const local = p.local.url && p.local.model ? chatClient({ baseUrl: p.local.url, apiKey: '', model: p.local.model, rates: rates.local, fetchImpl, timeoutMs: 120_000 }) : null;
@@@ -48,7 +50,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
for (let turn = 1; turn <= MAX_TURNS; turn++) {
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
const finalTurn = turn === MAX_TURNS;
-- const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 1800, signal });
++ // The model reasons before it answers, inside the same token budget: room for both.
++ const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 4000, signal });
await meter(r.cost_usd);
messages.push(r.message);
if (r.text && r.text.trim()) { last = r.text.trim(); }
@@@ -92,7 +95,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
/** A cheap model of another family to classify and to check results (null when none is configured). */
const checkers = {
-- deepseek: deepseek,
++ deepseek: deepseekQuick,
openai: p.openai.apiKey ? chatClient({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.checkModel || 'gpt-5-nano', rates: rates.openaiCheck, fetchImpl }) : null,
local,
};
ddiff --git a/server/agents/check.js b/server/agents/check.js
iindex 00782fa..13846da 100644
---- a/server/agents/check.js
++++ b/server/agents/check.js
@@@ -16,32 +16,45 @@ const CHECKER_FOR = { deepseek: ['openai', 'deepseek'], openai: ['deepseek', 'op
const SYSTEM = `You check an AI agent's answer before a person sees it. Reply with JSON only: {"ok": true|false, "reason": "<one short sentence>"}.
ok is true when the answer addresses the task and nothing in it is clearly invented, contradictory or unsafe. It does not have to be perfect.
--ok is false when the answer is empty, refuses without a good reason, answers a different question, or states something clearly false.`;
++ok is false when the answer is empty, refuses without a good reason, answers a different question, contradicts itself, or states something that is plainly impossible.
++The agent may have searched the web or run tools: do not fail an answer only because it states recent facts, versions, prices or dates you cannot verify yourself, especially when it cites a source for them.`;
--/** Which checker reads this agent's answer, and whether it is of another family. */
--function checkerFor(agentId, checkers, { mode }) {
++/** Which checkers may read this agent's answer, best first: another family, then the same family. */
++function checkersFor(agentId, checkers, { mode }) {
const family = FAMILY[agentId];
const order = mode === 'private' ? ['local'] : (CHECKER_FOR[family] || []);
-- for (const f of order) if (checkers[f]) return { family: f, chat: checkers[f], cross: f !== family };
-- return null;
++ return order.filter((f) => checkers[f]).map((f) => ({ family: f, chat: checkers[f], cross: f !== family }));
}
++const checkerFor = (agentId, checkers, opts) => checkersFor(agentId, checkers, opts)[0] || null;
--async function check({ task, answer, agentId, mode, checkers, signal }) {
-- const c = checkerFor(agentId, checkers, { mode });
-- if (!c) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };
-- try {
-- const r = await c.chat({
-- messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}` }],
-- json: true, maxTokens: 120, signal,
-- });
-- let parsed = null;
-- try { parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, '')); } catch { parsed = null; }
-- if (!parsed || typeof parsed.ok !== 'boolean') return { ok: false, unavailable: true, reason: 'the checker gave no verdict', cost_usd: r.cost_usd, by: c.family, cross: c.cross };
-- return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: r.cost_usd, by: c.family, model: r.model, cross: c.cross };
-- } catch (err) {
-- if (signal && signal.aborted) throw err;
-- return { ok: false, unavailable: true, reason: `the check could not run (${(err && err.message) || err})`.slice(0, 300), cost_usd: 0, by: c.family, cross: c.cross };
++/**
++ * Ask each checker in turn until one gives a verdict: a checker that answers nothing usable (a provider error, no JSON)
++ * passes the turn to the next; only when none gives a verdict is the result unavailable (and never delivered).
++ */
++async function check({ task, answer, sources = [], agentId, mode, checkers, signal }) {
++ const list = checkersFor(agentId, checkers, { mode });
++ if (!list.length) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };
++ let cost = 0;
++ let last = 'no checker gave a verdict';
++ const cited = sources.length ? `\n\nSources the agent cites:\n${sources.slice(0, 10).join('\n')}` : '';
++ for (const c of list) {
++ try {
++ const r = await c.chat({
++ messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}${cited}` }],
++ json: true, maxTokens: 160, signal,
++ });
++ cost += r.cost_usd || 0;
++ let parsed = null;
++ const m = String(r.text || '').match(/\{[\s\S]*\}/);
++ try { parsed = m ? JSON.parse(m[0]) : null; } catch { parsed = null; }
++ if (!parsed || typeof parsed.ok !== 'boolean') { last = `${c.family} gave no verdict`; continue; }
++ return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: cost, by: c.family, model: r.model, cross: c.cross };
++ } catch (err) {
++ if (signal && signal.aborted) throw err;
++ last = `${c.family} could not check (${(err && err.message) || err})`.slice(0, 300);
++ }
}
++ return { ok: false, unavailable: true, reason: last, cost_usd: cost };
}
--module.exports = { check, checkerFor, FAMILY };
++module.exports = { check, checkerFor, checkersFor, FAMILY };
ddiff --git a/server/agents/classify.js b/server/agents/classify.js
iindex 23acebe..c436256 100644
---- a/server/agents/classify.js
++++ b/server/agents/classify.js
@@@ -49,9 +49,10 @@ async function classify(task, { chat = null, signal } = {}) {
try {
const r = await chat({
messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: String(task).slice(0, 4000) }],
-- json: true, maxTokens: 30, signal,
++ json: true, maxTokens: 60, signal,
});
-- const parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, ''));
++ const m = String(r.text || '').match(/\{[\s\S]*\}/);
++ const parsed = JSON.parse(m ? m[0] : '');
const cls = CLASSES.includes(parsed.class) ? parsed.class : guess;
return { class: cls, cost_usd: r.cost_usd || 0, by: 'model' };
} catch {
ddiff --git a/server/agents/providers.js b/server/agents/providers.js
iindex e843cde..fb83062 100644
---- a/server/agents/providers.js
++++ b/server/agents/providers.js
@@@ -48,10 +48,14 @@ function priceUsage(rates, { input = 0, cached = 0, output = 0, calls = 0 }) {
return (Math.max(0, input - cached) * (r.input_usd_per_mtok || 0) + cached * cachedRate + output * (r.output_usd_per_mtok || 0)) / 1e6 + calls * (r.per_call_usd || 0);
}
--function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000 }) {
++/**
++ * extraBody: provider-specific fields sent with every call (DeepSeek's `thinking: { type: 'disabled' }` for short
++ * structured calls: its Flash model reasons by default, and a small max_tokens is spent on reasoning with no answer).
++ */
++function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000, extraBody = null }) {
const reasoning = /^(gpt-5|o\d)/i.test(model || '');
return async function chat({ messages, tools = null, json = false, maxTokens = 1500, signal } = {}) {
-- const body = { model, messages };
++ const body = { model, messages, ...(extraBody || {}) };
if (reasoning) { body.max_completion_tokens = maxTokens + 1024; body.reasoning_effort = 'minimal'; } else { body.max_tokens = maxTokens; body.temperature = 0.2; }
if (tools && tools.length) { body.tools = tools; body.tool_choice = 'auto'; }
if (json) body.response_format = { type: 'json_object' };
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex 06f1d36..5acf6a0 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -235,7 +235,7 @@ ${t.explanation ? explanationBlock(t.explanation) : ''}
return html`<section class="answer" aria-labelledby="h-answer"><h2 id="h-answer">Answer</h2>
<div class="prose">${raw(markdown(result.answer, { headingOffset: 2 }))}</div>
${result.sources && result.sources.length ? html`<h3>Sources</h3><ul class="sources">${result.sources.map((u) => html`<li><a href="${u}" rel="nofollow ugc noopener" target="_blank">${u}</a></li>`)}</ul>` : ''}
--<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family: private mode)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p>
++<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p>
</section>`;
}
ddiff --git a/server/tasks/engine.js b/server/tasks/engine.js
iindex 0fecc9f..03264bb 100644
---- a/server/tasks/engine.js
++++ b/server/tasks/engine.js
@@@ -115,9 +115,9 @@ function createEngine({ config, s, adapters, stream, now = () => Date.now(), log
continue;
}
await setState(id, 'verifying', { agentForEvent: agentId });
-- const verdict = await check({ task: row.task, answer: out.text, agentId, mode: row.mode, checkers: adapters.checkers, signal });
++ const verdict = await check({ task: row.task, answer: out.text, sources: out.sources || [], agentId, mode: row.mode, checkers: adapters.checkers, signal });
await meter(verdict.cost_usd);
-- await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family: private mode)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });
++ await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });
if (verdict.ok) {
const result = { answer: out.text, sources: out.sources || [], agent: agentId, model: out.model || catalog.modelOf(agent, config) || null, class: cls, checked: { by: verdict.by, cross_family: !!verdict.cross, reason: verdict.reason } };
return await setState(id, 'succeeded', { result, finished_at: iso() });
ddiff --git a/test/helpers/providers.js b/test/helpers/providers.js
iindex 8bd251d..f99ef54 100644
---- a/test/helpers/providers.js
++++ b/test/helpers/providers.js
@@@ -29,10 +29,11 @@ async function startProviders() {
const chatReply = (res, message, u = usage(1000, 100)) => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ model: 'stand-in', choices: [{ message: { role: 'assistant', ...message }, finish_reason: message.tool_calls ? 'tool_calls' : 'stop' }], usage: u })); };
const userText = (body) => { const m = [...(body.messages || [])].reverse().find((x) => x.role === 'user'); return m ? String(m.content) : ''; };
-- function judge(body, res) {
++ function judge(body, res, family) {
const sys = String(((body.messages || [])[0] || {}).content || '');
if (/sort one task/.test(sys)) return chatReply(res, { content: JSON.stringify({ class: state.cls }) }, usage(200, 10));
if (/check an AI agent's answer/.test(sys)) {
++ if (state.checkerDown === family) return chatReply(res, { content: '' }, usage(300, 160));
const ok = typeof state.checkOk === 'function' ? state.checkOk(body) : state.checkOk;
return chatReply(res, { content: JSON.stringify({ ok, reason: ok ? 'answers the task' : 'it answers a different question' }) }, usage(300, 20));
}
@@@ -49,7 +50,7 @@ async function startProviders() {
requests.push({ path: url.pathname, query: url.search, body, auth: req.headers.authorization || null });
if (url.pathname === '/deepseek/chat/completions') {
-- if (body.response_format) return judge(body, res);
++ if (body.response_format) return judge(body, res, 'deepseek');
if (state.runtime === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ error: { message: 'stand-in model is down' } })); }
if (state.runtime === 'hang') { hanging.add(res); return undefined; }
const tools = (body.messages || []).filter((m) => m.role === 'tool');
@@@ -59,7 +60,7 @@ async function startProviders() {
return chatReply(res, { content: `Runtime answer to: ${userText(body)}${tools.length ? `\n\nTool said: ${tools[0].content.slice(0, 300)}` : ''}\n\nSources\n- OpenVibe.Tools mx` });
}
if (url.pathname === '/openai/chat/completions') {
-- const j = judge(body, res);
++ const j = judge(body, res, 'openai');
if (j !== null) return j;
return chatReply(res, { content: 'nano answer' });
}
@@@ -76,7 +77,7 @@ async function startProviders() {
}));
}
if (url.pathname === '/local/chat/completions') {
-- const j = judge(body, res);
++ const j = judge(body, res, 'local');
if (j !== null) return j;
if (state.local === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end('{}'); }
return chatReply(res, { content: `Local answer to: ${userText(body)}` }, usage(100, 50));
@@@ -114,7 +115,7 @@ async function startProviders() {
};
return {
url, env, state, requests,
-- reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok' }); requests.length = 0; },
++ reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok', checkerDown: null }); requests.length = 0; },
close: () => new Promise((r) => { for (const res of hanging) { try { res.destroy(); } catch { /* gone */ } } server.close(r); }),
};
}
ddiff --git a/test/tasks.test.js b/test/tasks.test.js
iindex 974e438..a327794 100644
---- a/test/tasks.test.js
++++ b/test/tasks.test.js
@@@ -126,6 +126,19 @@ const SAME = { 'sec-fetch-site': 'same-origin' };
assert.strictEqual(task.error.code, 'actor.agents.exhausted');
});
++ await check('a checker that gives no verdict passes the check to the next one (same family, and the task says so)', async () => {
++ t.providers.reset();
++ t.providers.state.checkerDown = 'openai';
++ const id = (await create({ task: 'Say hello' })).json().id;
++ await t.waitFor(id);
++ const task = (await t.get(`/api/v1/tasks/${id}`, { as: kim })).json();
++ assert.strictEqual(task.state, 'succeeded', JSON.stringify(task.error));
++ assert.strictEqual(task.result.agent, 'openvibe-runtime');
++ assert.deepStrictEqual([task.result.checked.by, task.result.checked.cross_family], ['deepseek', false]);
++ const deepseekChecks = t.providers.requests.filter((q) => q.path === '/deepseek/chat/completions' && /check an AI agent/.test(JSON.stringify(q.body)));
++ assert.ok(deepseekChecks.length === 1 && deepseekChecks[0].body.thinking && deepseekChecks[0].body.thinking.type === 'disabled', 'short calls run with DeepSeek reasoning off');
++ });
++
await check('a budget below the cheapest capable agent fails before anything is paid', async () => {
t.providers.reset();
t.providers.state.cls = 'web';