openvibers/openvibe.codes Commits

Goals: count the harness's own tokens against the goal budget (#377)

OP OpenVibers committed 3 days ago · 6558c77140 · 2 files, +12 −1

Changes

ddiff --git a/server/domain/run-tasks.js b/server/domain/run-tasks.js
iindex 4aed88d..533d23b 100644
---- a/server/domain/run-tasks.js
++++ b/server/domain/run-tasks.js
@@@ -144,7 +144,8 @@ function usageOf(result) {
})();
return {
cost_usd: typeof d.cost_usd === 'number' && Number.isFinite(d.cost_usd) ? d.cost_usd : null,
-- tokens: tokens || null,
++ // The harness's own token count when its log has no usage lines (the CLI agents report it on the result).
++ tokens: tokens || (Number.isFinite(Number(d.tokens)) ? Number(d.tokens) : null),
steps: steps || null,
turns: turns || null,
session: session || null,
ddiff --git a/test/task-goals.test.js b/test/task-goals.test.js
iindex 69f56a2..df740bf 100644
---- a/test/task-goals.test.js
++++ b/test/task-goals.test.js
@@@ -71,6 +71,7 @@ const { boot, check, done } = require('./helpers/boot');
const row = await rowOf(id);
const result = {
ok: state === 'succeeded', summary: 'I wrote notes.txt.', files: [{ path: 'notes.txt', added: 1, removed: 0 }],
++ tokens: 2000,
turns, log: [{ t: 'agent', agent: row.agent, session }, { t: 'done', ok: state === 'succeeded', text: 'I wrote notes.txt.' }],
...(go ? { goal: go } : {}),
};
@@@ -145,6 +146,15 @@ const { boot, check, done } = require('./helpers/boot');
assert.strictEqual(shown.json.outcome.state, 'stopped_by_budget');
});
++ await check('a spent token budget also ends the task stopped by budget', async () => {
++ next = [{ status: 'rejected', reason: 'still empty', check_output: rejectedCheck.check_output }];
++ const id = await startTask({ ...GOAL, budget: { tokens: 100 } });
++ const { row, attempts } = await endAttempt(id, { go: rejectedCheck });
++ assert.strictEqual(attempts.length, 1);
++ assert.strictEqual(row.end_state, 'stopped_by_budget');
++ assert.match(row.outcome_note, /stopped by budget: the task spent 2000 of its 100 tokens/);
++ });
++
await check('a verifier that could not run leaves the task waiting, never ready', async () => {
next = [{ status: 'failed', reason: 'the model was unavailable', check_output: acceptedCheck.check_output }];
const id = await startTask();