openvibers/openvibe.actor Commits
Actor: the check works with DeepSeek's reasoning model, sees the cited sources, and falls back to another checker. (#1)
Changes
@@@ -37,6 +37,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = conconst ovTools = toolsOverride || createOpenVibeTools({ config, fetchImpl, log });const deepseek = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl }) : null;++ // Classifying and checking are short, structured answers: DeepSeek's reasoning is off for them.++ const deepseekQuick = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl, extraBody: { thinking: { type: 'disabled' } } }) : null;const openaiAgent = p.openai.apiKey ? responsesAgent({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.model, rates: rates.openai, fetchImpl }) : null;const local = p.local.url && p.local.model ? chatClient({ baseUrl: p.local.url, apiKey: '', model: p.local.model, rates: rates.local, fetchImpl, timeoutMs: 120_000 }) : null;@@@ -48,7 +50,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = confor (let turn = 1; turn <= MAX_TURNS; turn++) {if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });const finalTurn = turn === MAX_TURNS;-- const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 1800, signal });++ // The model reasons before it answers, inside the same token budget: room for both.++ const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 4000, signal });await meter(r.cost_usd);messages.push(r.message);if (r.text && r.text.trim()) { last = r.text.trim(); }@@@ -92,7 +95,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con/** A cheap model of another family to classify and to check results (null when none is configured). */const checkers = {-- deepseek: deepseek,++ deepseek: deepseekQuick,openai: p.openai.apiKey ? chatClient({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.checkModel || 'gpt-5-nano', rates: rates.openaiCheck, fetchImpl }) : null,local,};@@@ -16,32 +16,45 @@ const CHECKER_FOR = { deepseek: ['openai', 'deepseek'], openai: ['deepseek', 'opconst SYSTEM = `You check an AI agent's answer before a person sees it. Reply with JSON only: {"ok": true|false, "reason": "<one short sentence>"}.ok is true when the answer addresses the task and nothing in it is clearly invented, contradictory or unsafe. It does not have to be perfect.--ok is false when the answer is empty, refuses without a good reason, answers a different question, or states something clearly false.`;++ok is false when the answer is empty, refuses without a good reason, answers a different question, contradicts itself, or states something that is plainly impossible.++The agent may have searched the web or run tools: do not fail an answer only because it states recent facts, versions, prices or dates you cannot verify yourself, especially when it cites a source for them.`;--/** Which checker reads this agent's answer, and whether it is of another family. */--function checkerFor(agentId, checkers, { mode }) {++/** Which checkers may read this agent's answer, best first: another family, then the same family. */++function checkersFor(agentId, checkers, { mode }) {const family = FAMILY[agentId];const order = mode === 'private' ? ['local'] : (CHECKER_FOR[family] || []);-- for (const f of order) if (checkers[f]) return { family: f, chat: checkers[f], cross: f !== family };-- return null;++ return order.filter((f) => checkers[f]).map((f) => ({ family: f, chat: checkers[f], cross: f !== family }));}++const checkerFor = (agentId, checkers, opts) => checkersFor(agentId, checkers, opts)[0] || null;--async function check({ task, answer, agentId, mode, checkers, signal }) {-- const c = checkerFor(agentId, checkers, { mode });-- if (!c) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };-- try {-- const r = await c.chat({-- messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}` }],-- json: true, maxTokens: 120, signal,-- });-- let parsed = null;-- try { parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, '')); } catch { parsed = null; }-- if (!parsed || typeof parsed.ok !== 'boolean') return { ok: false, unavailable: true, reason: 'the checker gave no verdict', cost_usd: r.cost_usd, by: c.family, cross: c.cross };-- return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: r.cost_usd, by: c.family, model: r.model, cross: c.cross };-- } catch (err) {-- if (signal && signal.aborted) throw err;-- return { ok: false, unavailable: true, reason: `the check could not run (${(err && err.message) || err})`.slice(0, 300), cost_usd: 0, by: c.family, cross: c.cross };++/**++ * Ask each checker in turn until one gives a verdict: a checker that answers nothing usable (a provider error, no JSON)++ * passes the turn to the next; only when none gives a verdict is the result unavailable (and never delivered).++ */++async function check({ task, answer, sources = [], agentId, mode, checkers, signal }) {++ const list = checkersFor(agentId, checkers, { mode });++ if (!list.length) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };++ let cost = 0;++ let last = 'no checker gave a verdict';++ const cited = sources.length ? `\n\nSources the agent cites:\n${sources.slice(0, 10).join('\n')}` : '';++ for (const c of list) {++ try {++ const r = await c.chat({++ messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}${cited}` }],++ json: true, maxTokens: 160, signal,++ });++ cost += r.cost_usd || 0;++ let parsed = null;++ const m = String(r.text || '').match(/\{[\s\S]*\}/);++ try { parsed = m ? JSON.parse(m[0]) : null; } catch { parsed = null; }++ if (!parsed || typeof parsed.ok !== 'boolean') { last = `${c.family} gave no verdict`; continue; }++ return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: cost, by: c.family, model: r.model, cross: c.cross };++ } catch (err) {++ if (signal && signal.aborted) throw err;++ last = `${c.family} could not check (${(err && err.message) || err})`.slice(0, 300);++ }}++ return { ok: false, unavailable: true, reason: last, cost_usd: cost };}--module.exports = { check, checkerFor, FAMILY };++module.exports = { check, checkerFor, checkersFor, FAMILY };@@@ -49,9 +49,10 @@ async function classify(task, { chat = null, signal } = {}) {try {const r = await chat({messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: String(task).slice(0, 4000) }],-- json: true, maxTokens: 30, signal,++ json: true, maxTokens: 60, signal,});-- const parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, ''));++ const m = String(r.text || '').match(/\{[\s\S]*\}/);++ const parsed = JSON.parse(m ? m[0] : '');const cls = CLASSES.includes(parsed.class) ? parsed.class : guess;return { class: cls, cost_usd: r.cost_usd || 0, by: 'model' };} catch {@@@ -48,10 +48,14 @@ function priceUsage(rates, { input = 0, cached = 0, output = 0, calls = 0 }) {return (Math.max(0, input - cached) * (r.input_usd_per_mtok || 0) + cached * cachedRate + output * (r.output_usd_per_mtok || 0)) / 1e6 + calls * (r.per_call_usd || 0);}--function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000 }) {++/**++ * extraBody: provider-specific fields sent with every call (DeepSeek's `thinking: { type: 'disabled' }` for short++ * structured calls: its Flash model reasons by default, and a small max_tokens is spent on reasoning with no answer).++ */++function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000, extraBody = null }) {const reasoning = /^(gpt-5|o\d)/i.test(model || '');return async function chat({ messages, tools = null, json = false, maxTokens = 1500, signal } = {}) {-- const body = { model, messages };++ const body = { model, messages, ...(extraBody || {}) };if (reasoning) { body.max_completion_tokens = maxTokens + 1024; body.reasoning_effort = 'minimal'; } else { body.max_tokens = maxTokens; body.temperature = 0.2; }if (tools && tools.length) { body.tools = tools; body.tool_choice = 'auto'; }if (json) body.response_format = { type: 'json_object' };@@@ -235,7 +235,7 @@ ${t.explanation ? explanationBlock(t.explanation) : ''}return html`<section class="answer" aria-labelledby="h-answer"><h2 id="h-answer">Answer</h2><div class="prose">${raw(markdown(result.answer, { headingOffset: 2 }))}</div>${result.sources && result.sources.length ? html`<h3>Sources</h3><ul class="sources">${result.sources.map((u) => html`<li><a href="${u}" rel="nofollow ugc noopener" target="_blank">${u}</a></li>`)}</ul>` : ''}--<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family: private mode)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p>++<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p></section>`;}@@@ -115,9 +115,9 @@ function createEngine({ config, s, adapters, stream, now = () => Date.now(), logcontinue;}await setState(id, 'verifying', { agentForEvent: agentId });-- const verdict = await check({ task: row.task, answer: out.text, agentId, mode: row.mode, checkers: adapters.checkers, signal });++ const verdict = await check({ task: row.task, answer: out.text, sources: out.sources || [], agentId, mode: row.mode, checkers: adapters.checkers, signal });await meter(verdict.cost_usd);-- await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family: private mode)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });++ await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });if (verdict.ok) {const result = { answer: out.text, sources: out.sources || [], agent: agentId, model: out.model || catalog.modelOf(agent, config) || null, class: cls, checked: { by: verdict.by, cross_family: !!verdict.cross, reason: verdict.reason } };return await setState(id, 'succeeded', { result, finished_at: iso() });@@@ -29,10 +29,11 @@ async function startProviders() {const chatReply = (res, message, u = usage(1000, 100)) => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ model: 'stand-in', choices: [{ message: { role: 'assistant', ...message }, finish_reason: message.tool_calls ? 'tool_calls' : 'stop' }], usage: u })); };const userText = (body) => { const m = [...(body.messages || [])].reverse().find((x) => x.role === 'user'); return m ? String(m.content) : ''; };-- function judge(body, res) {++ function judge(body, res, family) {const sys = String(((body.messages || [])[0] || {}).content || '');if (/sort one task/.test(sys)) return chatReply(res, { content: JSON.stringify({ class: state.cls }) }, usage(200, 10));if (/check an AI agent's answer/.test(sys)) {++ if (state.checkerDown === family) return chatReply(res, { content: '' }, usage(300, 160));const ok = typeof state.checkOk === 'function' ? state.checkOk(body) : state.checkOk;return chatReply(res, { content: JSON.stringify({ ok, reason: ok ? 'answers the task' : 'it answers a different question' }) }, usage(300, 20));}@@@ -49,7 +50,7 @@ async function startProviders() {requests.push({ path: url.pathname, query: url.search, body, auth: req.headers.authorization || null });if (url.pathname === '/deepseek/chat/completions') {-- if (body.response_format) return judge(body, res);++ if (body.response_format) return judge(body, res, 'deepseek');if (state.runtime === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ error: { message: 'stand-in model is down' } })); }if (state.runtime === 'hang') { hanging.add(res); return undefined; }const tools = (body.messages || []).filter((m) => m.role === 'tool');@@@ -59,7 +60,7 @@ async function startProviders() {return chatReply(res, { content: `Runtime answer to: ${userText(body)}${tools.length ? `\n\nTool said: ${tools[0].content.slice(0, 300)}` : ''}\n\nSources\n- OpenVibe.Tools mx` });}if (url.pathname === '/openai/chat/completions') {-- const j = judge(body, res);++ const j = judge(body, res, 'openai');if (j !== null) return j;return chatReply(res, { content: 'nano answer' });}@@@ -76,7 +77,7 @@ async function startProviders() {}));}if (url.pathname === '/local/chat/completions') {-- const j = judge(body, res);++ const j = judge(body, res, 'local');if (j !== null) return j;if (state.local === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end('{}'); }return chatReply(res, { content: `Local answer to: ${userText(body)}` }, usage(100, 50));@@@ -114,7 +115,7 @@ async function startProviders() {};return {url, env, state, requests,-- reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok' }); requests.length = 0; },++ reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok', checkerDown: null }); requests.length = 0; },close: () => new Promise((r) => { for (const res of hanging) { try { res.destroy(); } catch { /* gone */ } } server.close(r); }),};}@@@ -126,6 +126,19 @@ const SAME = { 'sec-fetch-site': 'same-origin' };assert.strictEqual(task.error.code, 'actor.agents.exhausted');});++ await check('a checker that gives no verdict passes the check to the next one (same family, and the task says so)', async () => {++ t.providers.reset();++ t.providers.state.checkerDown = 'openai';++ const id = (await create({ task: 'Say hello' })).json().id;++ await t.waitFor(id);++ const task = (await t.get(`/api/v1/tasks/${id}`, { as: kim })).json();++ assert.strictEqual(task.state, 'succeeded', JSON.stringify(task.error));++ assert.strictEqual(task.result.agent, 'openvibe-runtime');++ assert.deepStrictEqual([task.result.checked.by, task.result.checked.cross_family], ['deepseek', false]);++ const deepseekChecks = t.providers.requests.filter((q) => q.path === '/deepseek/chat/completions' && /check an AI agent/.test(JSON.stringify(q.body)));++ assert.ok(deepseekChecks.length === 1 && deepseekChecks[0].body.thinking && deepseekChecks[0].body.thinking.type === 'disabled', 'short calls run with DeepSeek reasoning off');++ });++await check('a budget below the cheapest capable agent fails before anything is paid', async () => {t.providers.reset();t.providers.state.cls = 'web';
@@ -37,6 +37,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
@@ -37,6 +37,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
const ovTools = toolsOverride || createOpenVibeTools({ config, fetchImpl, log });
const ovTools = toolsOverride || createOpenVibeTools({ config, fetchImpl, log });
const deepseek = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl }) : null;
const deepseek = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl }) : null;
+ // Classifying and checking are short, structured answers: DeepSeek's reasoning is off for them.
+ const deepseekQuick = p.deepseek.apiKey ? chatClient({ baseUrl: p.deepseek.baseUrl, apiKey: p.deepseek.apiKey, model: p.deepseek.model, rates: rates.deepseek, fetchImpl, extraBody: { thinking: { type: 'disabled' } } }) : null;
const openaiAgent = p.openai.apiKey ? responsesAgent({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.model, rates: rates.openai, fetchImpl }) : null;
const openaiAgent = p.openai.apiKey ? responsesAgent({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.model, rates: rates.openai, fetchImpl }) : null;
const local = p.local.url && p.local.model ? chatClient({ baseUrl: p.local.url, apiKey: '', model: p.local.model, rates: rates.local, fetchImpl, timeoutMs: 120_000 }) : null;
const local = p.local.url && p.local.model ? chatClient({ baseUrl: p.local.url, apiKey: '', model: p.local.model, rates: rates.local, fetchImpl, timeoutMs: 120_000 }) : null;
@@ -48,7 +50,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
@@ -48,7 +50,8 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
for (let turn = 1; turn <= MAX_TURNS; turn++) {
for (let turn = 1; turn <= MAX_TURNS; turn++) {
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
if (budgetLeft() <= 0) throw new ProviderError('the task reached its budget before the runtime finished', { code: 'actor.budget.exceeded' });
const finalTurn = turn === MAX_TURNS;
const finalTurn = turn === MAX_TURNS;
- const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 1800, signal });
+ // The model reasons before it answers, inside the same token budget: room for both.
+ const r = await deepseek({ messages, tools: finalTurn ? null : tools, maxTokens: 4000, signal });
await meter(r.cost_usd);
await meter(r.cost_usd);
messages.push(r.message);
messages.push(r.message);
if (r.text && r.text.trim()) { last = r.text.trim(); }
if (r.text && r.text.trim()) { last = r.text.trim(); }
@@ -92,7 +95,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
@@ -92,7 +95,7 @@ function createAdapters({ config, rates, fetchImpl = globalThis.fetch, log = con
/** A cheap model of another family to classify and to check results (null when none is configured). */
/** A cheap model of another family to classify and to check results (null when none is configured). */
const checkers = {
const checkers = {
- deepseek: deepseek,
+ deepseek: deepseekQuick,
openai: p.openai.apiKey ? chatClient({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.checkModel || 'gpt-5-nano', rates: rates.openaiCheck, fetchImpl }) : null,
openai: p.openai.apiKey ? chatClient({ baseUrl: p.openai.baseUrl, apiKey: p.openai.apiKey, model: p.openai.checkModel || 'gpt-5-nano', rates: rates.openaiCheck, fetchImpl }) : null,
local,
local,
};
};
@@ -16,32 +16,45 @@ const CHECKER_FOR = { deepseek: ['openai', 'deepseek'], openai: ['deepseek', 'op
@@ -16,32 +16,45 @@ const CHECKER_FOR = { deepseek: ['openai', 'deepseek'], openai: ['deepseek', 'op
const SYSTEM = `You check an AI agent's answer before a person sees it. Reply with JSON only: {"ok": true|false, "reason": "<one short sentence>"}.
const SYSTEM = `You check an AI agent's answer before a person sees it. Reply with JSON only: {"ok": true|false, "reason": "<one short sentence>"}.
ok is true when the answer addresses the task and nothing in it is clearly invented, contradictory or unsafe. It does not have to be perfect.
ok is true when the answer addresses the task and nothing in it is clearly invented, contradictory or unsafe. It does not have to be perfect.
-ok is false when the answer is empty, refuses without a good reason, answers a different question, or states something clearly false.`;
+ok is false when the answer is empty, refuses without a good reason, answers a different question, contradicts itself, or states something that is plainly impossible.
+The agent may have searched the web or run tools: do not fail an answer only because it states recent facts, versions, prices or dates you cannot verify yourself, especially when it cites a source for them.`;
-/** Which checker reads this agent's answer, and whether it is of another family. */
-function checkerFor(agentId, checkers, { mode }) {
+/** Which checkers may read this agent's answer, best first: another family, then the same family. */
+function checkersFor(agentId, checkers, { mode }) {
const family = FAMILY[agentId];
const family = FAMILY[agentId];
const order = mode === 'private' ? ['local'] : (CHECKER_FOR[family] || []);
const order = mode === 'private' ? ['local'] : (CHECKER_FOR[family] || []);
- for (const f of order) if (checkers[f]) return { family: f, chat: checkers[f], cross: f !== family };
- return null;
+ return order.filter((f) => checkers[f]).map((f) => ({ family: f, chat: checkers[f], cross: f !== family }));
}
}
+const checkerFor = (agentId, checkers, opts) => checkersFor(agentId, checkers, opts)[0] || null;
-async function check({ task, answer, agentId, mode, checkers, signal }) {
- const c = checkerFor(agentId, checkers, { mode });
- if (!c) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };
- try {
- const r = await c.chat({
- messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}` }],
- json: true, maxTokens: 120, signal,
- });
- let parsed = null;
- try { parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, '')); } catch { parsed = null; }
- if (!parsed || typeof parsed.ok !== 'boolean') return { ok: false, unavailable: true, reason: 'the checker gave no verdict', cost_usd: r.cost_usd, by: c.family, cross: c.cross };
- return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: r.cost_usd, by: c.family, model: r.model, cross: c.cross };
- } catch (err) {
- if (signal && signal.aborted) throw err;
- return { ok: false, unavailable: true, reason: `the check could not run (${(err && err.message) || err})`.slice(0, 300), cost_usd: 0, by: c.family, cross: c.cross };
+/**
+ * Ask each checker in turn until one gives a verdict: a checker that answers nothing usable (a provider error, no JSON)
+ * passes the turn to the next; only when none gives a verdict is the result unavailable (and never delivered).
+ */
+async function check({ task, answer, sources = [], agentId, mode, checkers, signal }) {
+ const list = checkersFor(agentId, checkers, { mode });
+ if (!list.length) return { ok: false, unavailable: true, reason: 'no model is available to check this answer', cost_usd: 0 };
+ let cost = 0;
+ let last = 'no checker gave a verdict';
+ const cited = sources.length ? `\n\nSources the agent cites:\n${sources.slice(0, 10).join('\n')}` : '';
+ for (const c of list) {
+ try {
+ const r = await c.chat({
+ messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: `Task:\n${String(task).slice(0, 6000)}\n\nAnswer:\n${String(answer).slice(0, 12000)}${cited}` }],
+ json: true, maxTokens: 160, signal,
+ });
+ cost += r.cost_usd || 0;
+ let parsed = null;
+ const m = String(r.text || '').match(/\{[\s\S]*\}/);
+ try { parsed = m ? JSON.parse(m[0]) : null; } catch { parsed = null; }
+ if (!parsed || typeof parsed.ok !== 'boolean') { last = `${c.family} gave no verdict`; continue; }
+ return { ok: parsed.ok, reason: String(parsed.reason || '').slice(0, 300), cost_usd: cost, by: c.family, model: r.model, cross: c.cross };
+ } catch (err) {
+ if (signal && signal.aborted) throw err;
+ last = `${c.family} could not check (${(err && err.message) || err})`.slice(0, 300);
+ }
}
}
+ return { ok: false, unavailable: true, reason: last, cost_usd: cost };
}
}
-module.exports = { check, checkerFor, FAMILY };
+module.exports = { check, checkerFor, checkersFor, FAMILY };
@@ -49,9 +49,10 @@ async function classify(task, { chat = null, signal } = {}) {
@@ -49,9 +49,10 @@ async function classify(task, { chat = null, signal } = {}) {
try {
try {
const r = await chat({
const r = await chat({
messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: String(task).slice(0, 4000) }],
messages: [{ role: 'system', content: SYSTEM }, { role: 'user', content: String(task).slice(0, 4000) }],
- json: true, maxTokens: 30, signal,
+ json: true, maxTokens: 60, signal,
});
});
- const parsed = JSON.parse(String(r.text || '').trim().replace(/^```(?:json)?|```$/g, ''));
+ const m = String(r.text || '').match(/\{[\s\S]*\}/);
+ const parsed = JSON.parse(m ? m[0] : '');
const cls = CLASSES.includes(parsed.class) ? parsed.class : guess;
const cls = CLASSES.includes(parsed.class) ? parsed.class : guess;
return { class: cls, cost_usd: r.cost_usd || 0, by: 'model' };
return { class: cls, cost_usd: r.cost_usd || 0, by: 'model' };
} catch {
} catch {
@@ -48,10 +48,14 @@ function priceUsage(rates, { input = 0, cached = 0, output = 0, calls = 0 }) {
@@ -48,10 +48,14 @@ function priceUsage(rates, { input = 0, cached = 0, output = 0, calls = 0 }) {
return (Math.max(0, input - cached) * (r.input_usd_per_mtok || 0) + cached * cachedRate + output * (r.output_usd_per_mtok || 0)) / 1e6 + calls * (r.per_call_usd || 0);
return (Math.max(0, input - cached) * (r.input_usd_per_mtok || 0) + cached * cachedRate + output * (r.output_usd_per_mtok || 0)) / 1e6 + calls * (r.per_call_usd || 0);
}
}
-function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000 }) {
+/**
+ * extraBody: provider-specific fields sent with every call (DeepSeek's `thinking: { type: 'disabled' }` for short
+ * structured calls: its Flash model reasons by default, and a small max_tokens is spent on reasoning with no answer).
+ */
+function chatClient({ baseUrl, apiKey, model, rates, fetchImpl = globalThis.fetch, timeoutMs = 90_000, extraBody = null }) {
const reasoning = /^(gpt-5|o\d)/i.test(model || '');
const reasoning = /^(gpt-5|o\d)/i.test(model || '');
return async function chat({ messages, tools = null, json = false, maxTokens = 1500, signal } = {}) {
return async function chat({ messages, tools = null, json = false, maxTokens = 1500, signal } = {}) {
- const body = { model, messages };
+ const body = { model, messages, ...(extraBody || {}) };
if (reasoning) { body.max_completion_tokens = maxTokens + 1024; body.reasoning_effort = 'minimal'; } else { body.max_tokens = maxTokens; body.temperature = 0.2; }
if (reasoning) { body.max_completion_tokens = maxTokens + 1024; body.reasoning_effort = 'minimal'; } else { body.max_tokens = maxTokens; body.temperature = 0.2; }
if (tools && tools.length) { body.tools = tools; body.tool_choice = 'auto'; }
if (tools && tools.length) { body.tools = tools; body.tool_choice = 'auto'; }
if (json) body.response_format = { type: 'json_object' };
if (json) body.response_format = { type: 'json_object' };
@@ -235,7 +235,7 @@ ${t.explanation ? explanationBlock(t.explanation) : ''}
@@ -235,7 +235,7 @@ ${t.explanation ? explanationBlock(t.explanation) : ''}
return html`<section class="answer" aria-labelledby="h-answer"><h2 id="h-answer">Answer</h2>
return html`<section class="answer" aria-labelledby="h-answer"><h2 id="h-answer">Answer</h2>
<div class="prose">${raw(markdown(result.answer, { headingOffset: 2 }))}</div>
<div class="prose">${raw(markdown(result.answer, { headingOffset: 2 }))}</div>
${result.sources && result.sources.length ? html`<h3>Sources</h3><ul class="sources">${result.sources.map((u) => html`<li><a href="${u}" rel="nofollow ugc noopener" target="_blank">${u}</a></li>`)}</ul>` : ''}
${result.sources && result.sources.length ? html`<h3>Sources</h3><ul class="sources">${result.sources.map((u) => html`<li><a href="${u}" rel="nofollow ugc noopener" target="_blank">${u}</a></li>`)}</ul>` : ''}
-<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family: private mode)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p>
+<p class="muted small">By ${agent ? agent.name : result.agent}${result.model ? html` (<code>${result.model}</code>)` : ''} · checked by ${result.checked && result.checked.by}${result.checked && !result.checked.cross_family ? ' (same family)' : ''}${result.checked && result.checked.reason ? `: ${result.checked.reason}` : ''}</p>
</section>`;
</section>`;
}
}
@@ -115,9 +115,9 @@ function createEngine({ config, s, adapters, stream, now = () => Date.now(), log
@@ -115,9 +115,9 @@ function createEngine({ config, s, adapters, stream, now = () => Date.now(), log
continue;
continue;
}
}
await setState(id, 'verifying', { agentForEvent: agentId });
await setState(id, 'verifying', { agentForEvent: agentId });
- const verdict = await check({ task: row.task, answer: out.text, agentId, mode: row.mode, checkers: adapters.checkers, signal });
+ const verdict = await check({ task: row.task, answer: out.text, sources: out.sources || [], agentId, mode: row.mode, checkers: adapters.checkers, signal });
await meter(verdict.cost_usd);
await meter(verdict.cost_usd);
- await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family: private mode)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });
+ await emit(id, { kind: 'output', step: 'check', agent: agentId, text: `${verdict.ok ? 'Checked' : 'Check failed'}${verdict.by ? ` by ${verdict.by}${verdict.cross ? '' : ' (same family)'}` : ''}: ${verdict.reason || ''}`.slice(0, 2000), is_error: !verdict.ok });
if (verdict.ok) {
if (verdict.ok) {
const result = { answer: out.text, sources: out.sources || [], agent: agentId, model: out.model || catalog.modelOf(agent, config) || null, class: cls, checked: { by: verdict.by, cross_family: !!verdict.cross, reason: verdict.reason } };
const result = { answer: out.text, sources: out.sources || [], agent: agentId, model: out.model || catalog.modelOf(agent, config) || null, class: cls, checked: { by: verdict.by, cross_family: !!verdict.cross, reason: verdict.reason } };
return await setState(id, 'succeeded', { result, finished_at: iso() });
return await setState(id, 'succeeded', { result, finished_at: iso() });
@@ -29,10 +29,11 @@ async function startProviders() {
@@ -29,10 +29,11 @@ async function startProviders() {
const chatReply = (res, message, u = usage(1000, 100)) => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ model: 'stand-in', choices: [{ message: { role: 'assistant', ...message }, finish_reason: message.tool_calls ? 'tool_calls' : 'stop' }], usage: u })); };
const chatReply = (res, message, u = usage(1000, 100)) => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ model: 'stand-in', choices: [{ message: { role: 'assistant', ...message }, finish_reason: message.tool_calls ? 'tool_calls' : 'stop' }], usage: u })); };
const userText = (body) => { const m = [...(body.messages || [])].reverse().find((x) => x.role === 'user'); return m ? String(m.content) : ''; };
const userText = (body) => { const m = [...(body.messages || [])].reverse().find((x) => x.role === 'user'); return m ? String(m.content) : ''; };
- function judge(body, res) {
+ function judge(body, res, family) {
const sys = String(((body.messages || [])[0] || {}).content || '');
const sys = String(((body.messages || [])[0] || {}).content || '');
if (/sort one task/.test(sys)) return chatReply(res, { content: JSON.stringify({ class: state.cls }) }, usage(200, 10));
if (/sort one task/.test(sys)) return chatReply(res, { content: JSON.stringify({ class: state.cls }) }, usage(200, 10));
if (/check an AI agent's answer/.test(sys)) {
if (/check an AI agent's answer/.test(sys)) {
+ if (state.checkerDown === family) return chatReply(res, { content: '' }, usage(300, 160));
const ok = typeof state.checkOk === 'function' ? state.checkOk(body) : state.checkOk;
const ok = typeof state.checkOk === 'function' ? state.checkOk(body) : state.checkOk;
return chatReply(res, { content: JSON.stringify({ ok, reason: ok ? 'answers the task' : 'it answers a different question' }) }, usage(300, 20));
return chatReply(res, { content: JSON.stringify({ ok, reason: ok ? 'answers the task' : 'it answers a different question' }) }, usage(300, 20));
}
}
@@ -49,7 +50,7 @@ async function startProviders() {
@@ -49,7 +50,7 @@ async function startProviders() {
requests.push({ path: url.pathname, query: url.search, body, auth: req.headers.authorization || null });
requests.push({ path: url.pathname, query: url.search, body, auth: req.headers.authorization || null });
if (url.pathname === '/deepseek/chat/completions') {
if (url.pathname === '/deepseek/chat/completions') {
- if (body.response_format) return judge(body, res);
+ if (body.response_format) return judge(body, res, 'deepseek');
if (state.runtime === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ error: { message: 'stand-in model is down' } })); }
if (state.runtime === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ error: { message: 'stand-in model is down' } })); }
if (state.runtime === 'hang') { hanging.add(res); return undefined; }
if (state.runtime === 'hang') { hanging.add(res); return undefined; }
const tools = (body.messages || []).filter((m) => m.role === 'tool');
const tools = (body.messages || []).filter((m) => m.role === 'tool');
@@ -59,7 +60,7 @@ async function startProviders() {
@@ -59,7 +60,7 @@ async function startProviders() {
return chatReply(res, { content: `Runtime answer to: ${userText(body)}${tools.length ? `\n\nTool said: ${tools[0].content.slice(0, 300)}` : ''}\n\nSources\n- OpenVibe.Tools mx` });
return chatReply(res, { content: `Runtime answer to: ${userText(body)}${tools.length ? `\n\nTool said: ${tools[0].content.slice(0, 300)}` : ''}\n\nSources\n- OpenVibe.Tools mx` });
}
}
if (url.pathname === '/openai/chat/completions') {
if (url.pathname === '/openai/chat/completions') {
- const j = judge(body, res);
+ const j = judge(body, res, 'openai');
if (j !== null) return j;
if (j !== null) return j;
return chatReply(res, { content: 'nano answer' });
return chatReply(res, { content: 'nano answer' });
}
}
@@ -76,7 +77,7 @@ async function startProviders() {
@@ -76,7 +77,7 @@ async function startProviders() {
}));
}));
}
}
if (url.pathname === '/local/chat/completions') {
if (url.pathname === '/local/chat/completions') {
- const j = judge(body, res);
+ const j = judge(body, res, 'local');
if (j !== null) return j;
if (j !== null) return j;
if (state.local === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end('{}'); }
if (state.local === 'fail') { res.writeHead(500, { 'content-type': 'application/json' }); return res.end('{}'); }
return chatReply(res, { content: `Local answer to: ${userText(body)}` }, usage(100, 50));
return chatReply(res, { content: `Local answer to: ${userText(body)}` }, usage(100, 50));
@@ -114,7 +115,7 @@ async function startProviders() {
@@ -114,7 +115,7 @@ async function startProviders() {
};
};
return {
return {
url, env, state, requests,
url, env, state, requests,
- reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok' }); requests.length = 0; },
+ reset() { Object.assign(state, { cls: 'answer', checkOk: true, runtime: 'tool', openai: 'ok', local: 'ok', checkerDown: null }); requests.length = 0; },
close: () => new Promise((r) => { for (const res of hanging) { try { res.destroy(); } catch { /* gone */ } } server.close(r); }),
close: () => new Promise((r) => { for (const res of hanging) { try { res.destroy(); } catch { /* gone */ } } server.close(r); }),
};
};
}
}
@@ -126,6 +126,19 @@ const SAME = { 'sec-fetch-site': 'same-origin' };
@@ -126,6 +126,19 @@ const SAME = { 'sec-fetch-site': 'same-origin' };
assert.strictEqual(task.error.code, 'actor.agents.exhausted');
assert.strictEqual(task.error.code, 'actor.agents.exhausted');
});
});
+ await check('a checker that gives no verdict passes the check to the next one (same family, and the task says so)', async () => {
+ t.providers.reset();
+ t.providers.state.checkerDown = 'openai';
+ const id = (await create({ task: 'Say hello' })).json().id;
+ await t.waitFor(id);
+ const task = (await t.get(`/api/v1/tasks/${id}`, { as: kim })).json();
+ assert.strictEqual(task.state, 'succeeded', JSON.stringify(task.error));
+ assert.strictEqual(task.result.agent, 'openvibe-runtime');
+ assert.deepStrictEqual([task.result.checked.by, task.result.checked.cross_family], ['deepseek', false]);
+ const deepseekChecks = t.providers.requests.filter((q) => q.path === '/deepseek/chat/completions' && /check an AI agent/.test(JSON.stringify(q.body)));
+ assert.ok(deepseekChecks.length === 1 && deepseekChecks[0].body.thinking && deepseekChecks[0].body.thinking.type === 'disabled', 'short calls run with DeepSeek reasoning off');
+ });
+
await check('a budget below the cheapest capable agent fails before anything is paid', async () => {
await check('a budget below the cheapest capable agent fails before anything is paid', async () => {
t.providers.reset();
t.providers.reset();
t.providers.state.cls = 'web';
t.providers.state.cls = 'web';