openvibers/openvibe.codes Commits

Record every turn in a usage ledger; refuse at the cap (#409)

OP OpenVibers committed 2 days ago · 433eed3b2d · 4 files, +277 −1

Changes

ddiff --git a/engine/index.js b/engine/index.js
iindex 996438e..2feef40 100644
---- a/engine/index.js
++++ b/engine/index.js
@@@ -14,6 +14,7 @@
* engine.jobrunner one attempt of one harness, the way the portal runs one
* engine.prompt the prompt every job gets (the rules, the lessons, the memory, the digest, the brief)
* engine.memory the small memory index: MEMORY.md always, topics by search
++ * engine.ledger the per-turn usage ledger and the pre-call cap gate
* engine.repomap the repo digest a job gets instead of a directory tree
* engine.joblines the lines a task's page shows while it runs
* engine.logins what this machine has: each CLI, signed in or not, and its limit windows
@@@ -39,6 +40,7 @@ const adapters = require('./adapters');
const jobrunner = require('./jobrunner');
const prompt = require('./prompt');
const memory = require('./memory');
++const ledger = require('./ledger');
const repomap = require('./repomap');
const joblines = require('./joblines');
const logins = require('./logins');
@@@ -63,6 +65,7 @@ module.exports = {
jobrunner,
prompt,
memory,
++ ledger,
repomap,
joblines,
logins,
ddiff --git a/engine/jobrunner.js b/engine/jobrunner.js
iindex 082758d..2f437ea 100644
---- a/engine/jobrunner.js
++++ b/engine/jobrunner.js
@@@ -25,6 +25,7 @@ const { assemble } = require('./prompt');
const transcript = require('./transcript');
const joblines = require('./joblines');
const budget = require('./budget');
++const ledger = require('./ledger');
/** The wall clock when the task sets none: no limit, like every other unset limit. */
const DEFAULT_TIMEOUT_MS = 0;
@@@ -41,6 +42,10 @@ const DEFAULT_TIMEOUT_MS = 0;
* @param {string} [opts.permission] 'read' | 'edit' | 'full'
* @param {string} [opts.resume] the session to continue
* @param {object} [opts.budget] the limits ({ steps, minutes, usd }); null means no limit
++ * @param {object[]} [opts.ledger] the caller's per-turn ledger (engine/ledger.js); this attempt appends to it
++ * @param {string} [opts.trigger] what triggered this attempt (ledger entry)
++ * @param {string} [opts.parent] the parent turn when delegated (ledger entry)
++ * @param {boolean} [opts.ledgerWarned] the cap warning already went out (threaded back as result.ledgerWarned)
* @param {boolean} [opts.know] this machine has the shared Know MCP server (Claude Code gets mcp__ov_know)
* @param {string} [opts.claudeConfig] the Claude Code config to read for that check
* @param {string} [opts.knowLessons] the Know store's module (know/lessons.js) for the prompt's lessons
@@@ -75,6 +80,20 @@ async function runAttempt(opts = {}) {
// the builder and the process keeps Command Code, Claude Code and MiniMax Code from their old hard-coded 150/200.
const limits = budget.limitsFor({ budget: opts.budget || {} }, opts.agent, opts.budgetOverride);
const caps = budget.caps(limits);
++ // The per-turn ledger and the pre-call gate (engine/ledger.js): the caller's turns ride in on opts.ledger, the
++ // gate refuses at the cap with a 409 instead of running, and this attempt's spending is appended after the run.
++ const turns = Array.isArray(opts.ledger) ? opts.ledger : [];
++ const gate = ledger.preCall(turns, { turns: limits.steps, usd: limits.usd }, { warned: !!opts.ledgerWarned });
++ if (!gate.allowed) {
++ return {
++ agent: adapter.id, ok: false, sessionId: opts.resume || null, model: opts.model || null,
++ summary: gate.message, promptLayers: prompt.parts,
++ refused: { status: gate.status, code: 'budget.capped', hit: gate.hit, message: gate.message },
++ usage: { input: 0, output: 0, cached: 0 }, cost: null, steps: 0,
++ budgetHit: gate.hit, wrapUp: null, log,
++ ledger: turns, ledgerWarned: gate.warned, ledgerWarning: gate.warning || null,
++ };
++ }
const built = adapter.build({
prompt: prompt.text,
model: opts.model,
@@@ -140,9 +159,22 @@ async function runAttempt(opts = {}) {
const hit = budget.evaluate(used, limits).hit;
const wrap = budget.wrapUp(hit, { done: false });
++ const ok = !!(st.events || []).some((e) => e.t === 'final' && !e.isError);
++ // This attempt's spending joins the caller's ledger: engine and model, effort, tokens with cached called out,
++ // cost where the transcript priced it, what triggered it, the parent when delegated, how it ended.
++ const u = state.usage || {};
++ ledger.add(turns, {
++ engine: adapter.id, model: st.model || opts.model || null, effort: opts.effort || null,
++ inputTokens: u.input != null ? Number(u.input) : null,
++ outputTokens: u.output != null ? Number(u.output) : null,
++ cachedTokens: u.cached != null ? Number(u.cached) : null,
++ costUsd: state.cost != null ? Number(state.cost) : null,
++ costSource: state.cost != null ? 'transcript' : null,
++ trigger: opts.trigger || 'attempt', parent: opts.parent || null, outcome: ok ? 'ok' : 'error',
++ });
const result = {
agent: adapter.id,
-- ok: !!(st.events || []).some((e) => e.t === 'final' && !e.isError),
++ ok,
sessionId: state.sessionId,
model: st.model || opts.model || null,
summary: ((st.events || []).find((e) => e.t === 'final') || {}).text || '',
@@@ -154,6 +186,10 @@ async function runAttempt(opts = {}) {
budgetHit: hit,
wrapUp: wrap.action === 'wrap-up' ? wrap.prompt : null,
log,
++ ledger: turns,
++ ledgerSummary: ledger.summarize(turns),
++ ledgerWarned: gate.warned,
++ ledgerWarning: gate.warning || null,
};
// The wrap-up turn: the same session, one last turn that lands the work.
ddiff --git a/engine/ledger.js b/engine/ledger.js
nnew file mode 100644
iindex 0000000..3b3b443
---- /dev/null
++++ b/engine/ledger.js
@@@ -0,0 +1,105 @@
++'use strict';
++
++/**
++ * The per-turn usage ledger: what each turn spent, and the gate before the next call.
++ *
++ * One entry per turn: the engine and model that ran it, the effort, the tokens (cached called out), the cost and
++ * where the price came from, what triggered the turn, the parent turn when it was delegated, and how it ended. The
++ * root's totals sum every entry including the delegated ones, and the delegated subtotal is reported beside them —
++ * a turn farmed out still counts against the task that ordered it.
++ *
++ * The pre-call gate keeps the money honest: at the cap the call is refused (status 409, never run), and one warning
++ * goes out when spending crosses 80% of a cap, once. An unknown price is no USD limit, never a zero budget.
++ *
++ * add(ledger, turn) the validated entry, appended
++ * summarize(ledger) totals: turns, tokens, cost, delegated share, spend by engine
++ * preCall(ledger, limits, {warned}) { allowed, warned, warning?, hit?, status?, message? }
++ * WARN_AT, CAPPED_STATUS
++ */
++
++const WARN_AT = 0.8;
++const CAPPED_STATUS = 409;
++
++const isObject = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
++const num = (v) => (v == null ? null : (Number.isFinite(Number(v)) && Number(v) >= 0 ? Number(v) : null));
++const str = (v, max) => (v == null ? null : String(v).slice(0, max));
++
++/** One turn's spending, validated. Unknowns stay null: a missing price is no limit, never zero. */
++function add(ledger, turn = {}) {
++ if (!Array.isArray(ledger)) throw new Error('a ledger is a list');
++ if (!isObject(turn)) throw new Error('a ledger turn is an object');
++ const entry = {
++ n: ledger.length + 1,
++ engine: str(turn.engine, 64),
++ model: str(turn.model, 128),
++ effort: str(turn.effort, 32),
++ inputTokens: num(turn.inputTokens),
++ outputTokens: num(turn.outputTokens),
++ cachedTokens: num(turn.cachedTokens),
++ costUsd: num(turn.costUsd),
++ costSource: str(turn.costSource, 64),
++ trigger: str(turn.trigger, 128),
++ parent: turn.parent == null ? null : str(turn.parent, 64),
++ outcome: ['ok', 'error', 'refused'].includes(turn.outcome) ? turn.outcome : null,
++ at: turn.at === undefined ? new Date().toISOString() : String(turn.at),
++ };
++ if (entry.costUsd != null && !entry.costSource) throw new Error('a ledger cost names its source');
++ ledger.push(entry);
++ return entry;
++}
++
++/** The totals: every turn counts, delegated ones twice-reported, never twice-summed. */
++function summarize(ledger = []) {
++ const list = Array.isArray(ledger) ? ledger : [];
++ const sum = { turns: list.length, inputTokens: 0, outputTokens: 0, cachedTokens: 0, costUsd: null, delegatedTurns: 0, delegatedCostUsd: null, byEngine: {} };
++ let cost = 0, hasCost = false, delegatedCost = 0, hasDelegatedCost = false;
++ for (const t of list) {
++ if (!isObject(t)) continue;
++ if (t.inputTokens != null) sum.inputTokens += t.inputTokens;
++ if (t.outputTokens != null) sum.outputTokens += t.outputTokens;
++ if (t.cachedTokens != null) sum.cachedTokens += t.cachedTokens;
++ if (t.costUsd != null) { cost += t.costUsd; hasCost = true; }
++ if (t.parent != null) {
++ sum.delegatedTurns++;
++ if (t.costUsd != null) { delegatedCost += t.costUsd; hasDelegatedCost = true; }
++ }
++ const e = t.engine || 'unknown';
++ sum.byEngine[e] = sum.byEngine[e] || { turns: 0, costUsd: null };
++ sum.byEngine[e].turns++;
++ if (t.costUsd != null) sum.byEngine[e].costUsd = (sum.byEngine[e].costUsd || 0) + t.costUsd;
++ }
++ sum.costUsd = hasCost ? Math.round(cost * 1e6) / 1e6 : null;
++ sum.delegatedCostUsd = hasDelegatedCost ? Math.round(delegatedCost * 1e6) / 1e6 : null;
++ return sum;
++}
++
++/** The gate before the next call: refuse at the cap, warn once at 80% of it. */
++function preCall(ledger = [], limits = {}, { warned = false } = {}) {
++ const s = summarize(ledger);
++ const turnsCap = limits.turns == null ? null : num(limits.turns);
++ const usdCap = limits.usd == null ? null : num(limits.usd);
++ const refused = (hit, message) => ({
++ allowed: false, warned, hit, status: CAPPED_STATUS, message,
++ turns: s.turns, costUsd: s.costUsd,
++ });
++ if (turnsCap != null && turnsCap > 0 && s.turns >= turnsCap) {
++ return refused('turns', `the task already ran ${s.turns} of ${turnsCap} turns: no further call`);
++ }
++ if (usdCap != null && usdCap > 0 && s.costUsd != null && s.costUsd >= usdCap) {
++ return refused('usd', `the task already spent $${s.costUsd} of $${usdCap}: no further call`);
++ }
++ if (!warned) {
++ if (turnsCap != null && turnsCap > 0 && s.turns >= Math.ceil(turnsCap * WARN_AT)) {
++ return { allowed: true, warned: true, hit: null, warning: `the task ran ${s.turns} of ${turnsCap} turns`, turns: s.turns, costUsd: s.costUsd };
++ }
++ if (usdCap != null && usdCap > 0 && s.costUsd != null && s.costUsd >= usdCap * WARN_AT) {
++ return { allowed: true, warned: true, hit: null, warning: `the task spent $${s.costUsd} of $${usdCap}`, turns: s.turns, costUsd: s.costUsd };
++ }
++ }
++ return { allowed: true, warned, hit: null, turns: s.turns, costUsd: s.costUsd };
++}
++
++module.exports = {
++ WARN_AT, CAPPED_STATUS,
++ add, summarize, preCall,
++};
ddiff --git a/test/engine-ledger.test.js b/test/engine-ledger.test.js
nnew file mode 100644
iindex 0000000..db9a7e2
---- /dev/null
++++ b/test/engine-ledger.test.js
@@@ -0,0 +1,132 @@
++'use strict';
++
++/**
++ * Release 2 item 9 (Codes slice): the per-turn usage ledger and the pre-call cap gate.
++ *
++ * Every turn records engine, model, effort, tokens with cached called out, cost and its source, trigger, parent and
++ * outcome. The root's totals sum delegated turns too. Before the next call the gate refuses at the cap (409, never
++ * run) and warns once at 80% of it. The runner appends each attempt and enforces the gate.
++ */
++const assert = require('assert');
++const fs = require('node:fs');
++const os = require('node:os');
++const path = require('node:path');
++const ledger = require('../engine/ledger');
++const { runAttempt } = require('../engine/jobrunner');
++const { check, done } = require('./helpers/boot');
++
++const turn = (over = {}) => ({
++ engine: 'command-code', model: 'm-1', effort: 'max',
++ inputTokens: 100, outputTokens: 50, cachedTokens: 30,
++ costUsd: 0.01, costSource: 'receipt', trigger: 'attempt', parent: null, outcome: 'ok', ...over,
++});
++
++/** A stand-in for a CLI: it prints what the real one prints. */
++function fake(dir, lines) {
++ const bin = path.join(dir, 'bin');
++ fs.mkdirSync(bin, { recursive: true });
++ const script = ['#!/usr/bin/env node', ...lines.map((l) => `console.log(${JSON.stringify(JSON.stringify(l))});`)].join('\n');
++ fs.writeFileSync(path.join(bin, 'cmd'), script);
++ fs.chmodSync(path.join(bin, 'cmd'), 0o755);
++ return bin;
++}
++
++(async () => {
++ try {
++ await check('a turn is recorded with its tokens, cost and lineage', () => {
++ const list = [];
++ const e = ledger.add(list, turn({ parent: 't-0' }));
++ assert.strictEqual(e.n, 1);
++ assert.strictEqual(list.length, 1);
++ assert.strictEqual(e.parent, 't-0');
++ assert.throws(() => ledger.add(list, { ...turn(), costSource: null }), /source/, 'a cost names its source');
++ assert.throws(() => ledger.add(null, turn()), /list/);
++ });
++
++ await check('the root sums delegated turns without double counting', () => {
++ const list = [];
++ ledger.add(list, turn({ costUsd: 0.02 }));
++ ledger.add(list, turn({ costUsd: 0.03, parent: 't-1', engine: 'codex' }));
++ ledger.add(list, { engine: 'codex', outcome: 'error' });
++ const s = ledger.summarize(list);
++ assert.strictEqual(s.turns, 3);
++ assert.strictEqual(s.inputTokens, 200);
++ assert.strictEqual(s.cachedTokens, 60);
++ assert.strictEqual(s.costUsd, 0.05);
++ assert.strictEqual(s.delegatedTurns, 1);
++ assert.strictEqual(s.delegatedCostUsd, 0.03);
++ assert.strictEqual(s.byEngine['command-code'].turns, 1);
++ assert.strictEqual(s.byEngine['codex'].turns, 2);
++ assert.strictEqual(ledger.summarize([]).costUsd, null, 'no priced turn is no USD total, never zero');
++ });
++
++ await check('the gate refuses at the cap and warns once at 80% of it', () => {
++ const turns = [];
++ assert.strictEqual(ledger.preCall(turns, {}).allowed, true, 'no limits, no gate');
++ assert.strictEqual(ledger.preCall(turns, { turns: null, usd: null }).allowed, true);
++ const list = [];
++ for (let i = 0; i < 8; i++) ledger.add(list, turn());
++ const warn = ledger.preCall(list, { turns: 10 });
++ assert.strictEqual(warn.allowed, true);
++ assert.strictEqual(warn.warned, true, '8 of 10 trips the one warning');
++ assert.ok(warn.warning.includes('8 of 10'));
++ assert.strictEqual(ledger.preCall(list, { turns: 10 }, { warned: true }).warned, true, 'already warned stays warned, no second warning');
++ assert.strictEqual(ledger.preCall(list, { turns: 10 }, { warned: true }).warning, undefined);
++ for (let i = 0; i < 2; i++) ledger.add(list, turn());
++ const stop = ledger.preCall(list, { turns: 10 });
++ assert.strictEqual(stop.allowed, false);
++ assert.strictEqual(stop.status, 409);
++ assert.strictEqual(stop.hit, 'turns');
++ const rich = [];
++ ledger.add(rich, turn({ costUsd: 8.5 }));
++ assert.strictEqual(ledger.preCall(rich, { usd: 10 }).warned, true, '$8.50 of $10 warns');
++ ledger.add(rich, turn({ costUsd: 2 }));
++ const broke = ledger.preCall(rich, { usd: 10 });
++ assert.strictEqual(broke.allowed, false);
++ assert.strictEqual(broke.status, 409);
++ assert.strictEqual(ledger.preCall([{ ...turn(), costUsd: null }], { usd: 10 }).allowed, true, 'an unpriced turn never trips the dollar cap');
++ });
++
++ await check('an attempt appends its spending to the caller ledger', async () => {
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lg-'));
++ const bin = fake(dir, [
++ { type: 'event', event: { type: 'run_start', sessionId: 's-9' } },
++ { type: 'result', subtype: 'success', finalText: 'done', sessionId: 's-9' },
++ ]);
++ const turns = [];
++ const res = await runAttempt({
++ agent: 'command-code', prompt: 'do it', cwd: dir, log: path.join(dir, 'f.jsonl'),
++ env: { PATH: `${bin}:${process.env.PATH}` }, ledger: turns, trigger: 'retry', effort: 'max',
++ });
++ assert.strictEqual(res.ok, true);
++ assert.strictEqual(turns.length, 1);
++ assert.strictEqual(turns[0].engine, 'command-code');
++ assert.strictEqual(turns[0].trigger, 'retry');
++ assert.strictEqual(turns[0].outcome, 'ok');
++ assert.strictEqual(res.ledgerSummary.turns, 1);
++ assert.strictEqual(res.ledgerWarned, false);
++ fs.rmSync(dir, { recursive: true, force: true });
++ });
++
++ await check('a capped ledger refuses the call with a 409 and runs nothing', async () => {
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lg-cap-'));
++ const turns = [];
++ for (let i = 0; i < 3; i++) ledger.add(turns, turn());
++ const res = await runAttempt({
++ agent: 'command-code', prompt: 'do it', cwd: dir, log: path.join(dir, 'f.jsonl'),
++ env: { PATH: dir }, ledger: turns, budget: { steps: 3 },
++ });
++ assert.strictEqual(res.ok, false);
++ assert.strictEqual(res.refused.status, 409);
++ assert.strictEqual(res.refused.code, 'budget.capped');
++ assert.strictEqual(turns.length, 3, 'the refused call appends nothing');
++ fs.rmSync(dir, { recursive: true, force: true });
++ });
++
++ console.log('ledger: every turn recorded, the root sums delegated work, the cap refuses before the call');
++ } catch (e) {
++ console.error(e);
++ process.exit(1);
++ }
++ done();
++})();