openvibers/openvibe.codes Commits
Goals: count the harness's own tokens against the goal budget (#377)
Changes
@@@ -144,7 +144,8 @@ function usageOf(result) {})();return {cost_usd: typeof d.cost_usd === 'number' && Number.isFinite(d.cost_usd) ? d.cost_usd : null,-- tokens: tokens || null,++ // The harness's own token count when its log has no usage lines (the CLI agents report it on the result).++ tokens: tokens || (Number.isFinite(Number(d.tokens)) ? Number(d.tokens) : null),steps: steps || null,turns: turns || null,session: session || null,@@@ -71,6 +71,7 @@ const { boot, check, done } = require('./helpers/boot');const row = await rowOf(id);const result = {ok: state === 'succeeded', summary: 'I wrote notes.txt.', files: [{ path: 'notes.txt', added: 1, removed: 0 }],++ tokens: 2000,turns, log: [{ t: 'agent', agent: row.agent, session }, { t: 'done', ok: state === 'succeeded', text: 'I wrote notes.txt.' }],...(go ? { goal: go } : {}),};@@@ -145,6 +146,15 @@ const { boot, check, done } = require('./helpers/boot');assert.strictEqual(shown.json.outcome.state, 'stopped_by_budget');});++ await check('a spent token budget also ends the task stopped by budget', async () => {++ next = [{ status: 'rejected', reason: 'still empty', check_output: rejectedCheck.check_output }];++ const id = await startTask({ ...GOAL, budget: { tokens: 100 } });++ const { row, attempts } = await endAttempt(id, { go: rejectedCheck });++ assert.strictEqual(attempts.length, 1);++ assert.strictEqual(row.end_state, 'stopped_by_budget');++ assert.match(row.outcome_note, /stopped by budget: the task spent 2000 of its 100 tokens/);++ });++await check('a verifier that could not run leaves the task waiting, never ready', async () => {next = [{ status: 'failed', reason: 'the model was unavailable', check_output: acceptedCheck.check_output }];const id = await startTask();
@@ -144,7 +144,8 @@ function usageOf(result) {
@@ -144,7 +144,8 @@ function usageOf(result) {
})();
})();
return {
return {
cost_usd: typeof d.cost_usd === 'number' && Number.isFinite(d.cost_usd) ? d.cost_usd : null,
cost_usd: typeof d.cost_usd === 'number' && Number.isFinite(d.cost_usd) ? d.cost_usd : null,
- tokens: tokens || null,
+ // The harness's own token count when its log has no usage lines (the CLI agents report it on the result).
+ tokens: tokens || (Number.isFinite(Number(d.tokens)) ? Number(d.tokens) : null),
steps: steps || null,
steps: steps || null,
turns: turns || null,
turns: turns || null,
session: session || null,
session: session || null,
@@ -71,6 +71,7 @@ const { boot, check, done } = require('./helpers/boot');
@@ -71,6 +71,7 @@ const { boot, check, done } = require('./helpers/boot');
const row = await rowOf(id);
const row = await rowOf(id);
const result = {
const result = {
ok: state === 'succeeded', summary: 'I wrote notes.txt.', files: [{ path: 'notes.txt', added: 1, removed: 0 }],
ok: state === 'succeeded', summary: 'I wrote notes.txt.', files: [{ path: 'notes.txt', added: 1, removed: 0 }],
+ tokens: 2000,
turns, log: [{ t: 'agent', agent: row.agent, session }, { t: 'done', ok: state === 'succeeded', text: 'I wrote notes.txt.' }],
turns, log: [{ t: 'agent', agent: row.agent, session }, { t: 'done', ok: state === 'succeeded', text: 'I wrote notes.txt.' }],
...(go ? { goal: go } : {}),
...(go ? { goal: go } : {}),
};
};
@@ -145,6 +146,15 @@ const { boot, check, done } = require('./helpers/boot');
@@ -145,6 +146,15 @@ const { boot, check, done } = require('./helpers/boot');
assert.strictEqual(shown.json.outcome.state, 'stopped_by_budget');
assert.strictEqual(shown.json.outcome.state, 'stopped_by_budget');
});
});
+ await check('a spent token budget also ends the task stopped by budget', async () => {
+ next = [{ status: 'rejected', reason: 'still empty', check_output: rejectedCheck.check_output }];
+ const id = await startTask({ ...GOAL, budget: { tokens: 100 } });
+ const { row, attempts } = await endAttempt(id, { go: rejectedCheck });
+ assert.strictEqual(attempts.length, 1);
+ assert.strictEqual(row.end_state, 'stopped_by_budget');
+ assert.match(row.outcome_note, /stopped by budget: the task spent 2000 of its 100 tokens/);
+ });
+
await check('a verifier that could not run leaves the task waiting, never ready', async () => {
await check('a verifier that could not run leaves the task waiting, never ready', async () => {
next = [{ status: 'failed', reason: 'the model was unavailable', check_output: acceptedCheck.check_output }];
next = [{ status: 'failed', reason: 'the model was unavailable', check_output: acceptedCheck.check_output }];
const id = await startTask();
const id = await startTask();