openvibers/openvibe.report Commits

AI performance: histograms, task cost and time, /api/v1/ai/performance (#212)

OP OpenVibers committed 10 h ago · 77e41500f5 · 9 files, +862 −17

Changes

ddiff --git a/README.md b/README.md
iindex 8452056..c342b0a 100644
---- a/README.md
++++ b/README.md
@@@ -143,6 +143,17 @@ HTML page is gated by the signed-in person's staff role (`staff.site.view`) inst
GET /api/v1/ai/exchange?from=YYYY-MM-DD&to=YYYY-MM-DD the Exchange aggregates, staff only
```
++The AI performance rows (`report.ai-performance@1`, plan L40/L41) answer the speed, reliability, tokens and money of
++the calls Report meters. Model, provider and operation are public and show only a group with at least 20 calls; system,
++task, place and harness need a signed-in person with `staff.site.view` or a service token with
++`report.ai_usage.read`, and are never cached. The percentiles come from log-scale histograms and are the upper bound
++of the bucket a call lands in; a task's cost and time are the sums of the calls that share its `task_id`.
++
++```
++GET /api/v1/ai/performance?group=model|provider|operation&days=7|30|90 public, 20-call floor
++GET /api/v1/ai/performance?group=system|task|place|harness&days=7|30|90 staff, filters allowed
++```
++
### Internal (loopback only)
```
@@@ -260,7 +271,7 @@ project, money) is never stored. `/internal/` is loopback-only, and `POST /inter
HMAC. See [SECURITY.md](SECURITY.md).
<!-- versions:start -->
--- openvibe-contracts: v0.419.0
++- openvibe-contracts: v0.421.0
- openvibe-sdk: v0.161.0
- openvibe-shared: v4.231.0
- openvibe-publishing: v1.7.0
ddiff --git a/STATUS.json b/STATUS.json
iindex a2a3359..6ba6acc 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,11 +8,11 @@
"runtime": "node 22, express 4, PostgreSQL (PGlite in development), no sessions",
"port": "5080",
"domain": "openvibe.report",
-- "contracts": "openvibe-contracts v0.419.0",
++ "contracts": "openvibe-contracts v0.421.0",
"sdk": "openvibe-sdk v0.161.0",
"shared": "openvibe-shared v4.231.0",
"packages": [
-- "openvibe-contracts v0.419.0",
++ "openvibe-contracts v0.421.0",
"openvibe-publishing v1.7.0",
"openvibe-sdk v0.161.0",
"openvibe-shared v4.231.0",
ddiff --git a/migrations/0011_ai_performance.sql b/migrations/0011_ai_performance.sql
nnew file mode 100644
iindex 0000000..b2935da
---- /dev/null
++++ b/migrations/0011_ai_performance.sql
@@@ -0,0 +1,75 @@
++-- phase: expand
++-- AI performance (plan L40/L41, ai-metrics.md): every ai.call.recorded@1 call is also counted by day, system,
++-- provider, model, operation, task, place and harness. The sums answer cost and tokens; the three log-scale
++-- histograms answer p50/p90/p95 for time to first token, latency and tokens per second; errors_by_class answers the
++-- top errors. The task roll-up keeps one row per day, task kind and system with the number of tasks and the cost and
++-- time histograms per task. Raw records stay 90 days; these stay.
++
++-- The 1.1.0 fields (ai.call.recorded@1, Contracts 0.421.0). All optional: rows written before this migration keep
++-- the empty string or NULL and are still counted in the older totals.
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS operation text COLLATE "C" NOT NULL DEFAULT '';
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS task text COLLATE "C" NOT NULL DEFAULT '';
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS task_id text COLLATE "C";
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS task_ms bigint;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS harness text COLLATE "C" NOT NULL DEFAULT '';
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS place text COLLATE "C" NOT NULL DEFAULT '';
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS streamed boolean;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS tps double precision;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS queue_ms bigint;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS error_class text COLLATE "C";
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS fallbacks integer;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS cache_hit boolean;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS tool_calls integer;
++ALTER TABLE report_ai_usage ADD COLUMN IF NOT EXISTS region text COLLATE "C";
++CREATE INDEX IF NOT EXISTS report_ai_usage_task ON report_ai_usage (day, task, system, task_id);
++
++-- One row per day and group: the counters, the token and cost sums, and the log-scale histograms (40 counts each).
++-- The three histograms hold counts over HISTOGRAM_BOUNDS in server/analytics/ai-performance.js; a percentile answers
++-- the upper bound of the bucket it lands in, never a made-up value inside it.
++CREATE TABLE report_ai_perf_days (
++ day date NOT NULL,
++ system text COLLATE "C" NOT NULL DEFAULT '',
++ provider text COLLATE "C" NOT NULL DEFAULT '',
++ model text COLLATE "C" NOT NULL DEFAULT '',
++ operation text COLLATE "C" NOT NULL DEFAULT '',
++ task text COLLATE "C" NOT NULL DEFAULT '',
++ place text COLLATE "C" NOT NULL DEFAULT '',
++ harness text COLLATE "C" NOT NULL DEFAULT '',
++ calls bigint NOT NULL DEFAULT 0,
++ succeeded bigint NOT NULL DEFAULT 0,
++ errors bigint NOT NULL DEFAULT 0,
++ fallbacks bigint NOT NULL DEFAULT 0, -- calls that had at least one fallback (a rate, never a sum > calls)
++ streamed bigint NOT NULL DEFAULT 0,
++ cache_hits bigint NOT NULL DEFAULT 0,
++ tool_calls bigint NOT NULL DEFAULT 0,
++ tokens_input bigint NOT NULL DEFAULT 0,
++ tokens_output bigint NOT NULL DEFAULT 0,
++ tokens_reasoning bigint NOT NULL DEFAULT 0,
++ tokens_cache_read bigint NOT NULL DEFAULT 0,
++ tokens_cache_write bigint NOT NULL DEFAULT 0,
++ charged_usd double precision NOT NULL DEFAULT 0,
++ list_cost_usd double precision NOT NULL DEFAULT 0,
++ cache_saved_usd double precision NOT NULL DEFAULT 0,
++ ttft_hist jsonb NOT NULL DEFAULT '[]'::jsonb, -- 40 counts, TTFT ms
++ latency_hist jsonb NOT NULL DEFAULT '[]'::jsonb, -- 40 counts, latency ms
++ tps_hist jsonb NOT NULL DEFAULT '[]'::jsonb, -- 40 counts, output tokens per second
++ errors_by_class jsonb NOT NULL DEFAULT '{}'::jsonb, -- error_class -> count
++ updated_at timestamptz NOT NULL DEFAULT now(),
++ PRIMARY KEY (day, system, provider, model, operation, task, place, harness)
++);
++CREATE INDEX report_ai_perf_days_day ON report_ai_perf_days (day);
++
++-- One row per day, task kind and system: a task is the calls that share task_id. Its cost is the sum of the calls'
++-- charged_usd; its time is task_ms when any call gives it, else the sum of the calls' latency_ms. The histograms hold
++-- the cost and time of each task, so a p50 is a task's middle cost or time, never a call's.
++CREATE TABLE report_ai_task_days (
++ day date NOT NULL,
++ task text COLLATE "C" NOT NULL DEFAULT '',
++ system text COLLATE "C" NOT NULL DEFAULT '',
++ tasks bigint NOT NULL DEFAULT 0,
++ cost_hist jsonb NOT NULL DEFAULT '[]'::jsonb, -- 40 counts, USD per task
++ time_hist jsonb NOT NULL DEFAULT '[]'::jsonb, -- 40 counts, ms per task
++ updated_at timestamptz NOT NULL DEFAULT now(),
++ PRIMARY KEY (day, task, system)
++);
++CREATE INDEX report_ai_task_days_day ON report_ai_task_days (day);
ddiff --git a/package-lock.json b/package-lock.json
iindex ec893e3..96df4e2 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -13,7 +13,7 @@
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.421.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.231.0",
@@@ -1138,9 +1138,9 @@
}
},
"node_modules/openvibe-contracts": {
-- "version": "0.419.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
-- "integrity": "sha512-VxrTdeHchT8gvl2wWC9ASaId92en7ZKfJ8F/0hKzCAD0f2Y00rPrqgBWF9BNPoLGARoyQvsMbhBEhSkhxGNnjA==",
++ "version": "0.421.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.421.0",
++ "integrity": "sha512-iRSbGGqTS6Lr4XYrAvN5yhoCTkPkxrU/l3dufJCsc5BhbGU9F4ZSNt0FrgpqDseNxRM/8zx8Zu3t/w7vryAfBw==",
"license": "AGPL-3.0-only",
"dependencies": {
"ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 03ebc13..182a1cb 100644
---- a/package.json
++++ b/package.json
@@@ -19,7 +19,7 @@
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.421.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.231.0",
ddiff --git a/server/analytics/ai-performance.js b/server/analytics/ai-performance.js
nnew file mode 100644
iindex 0000000..8db9f6d
---- /dev/null
++++ b/server/analytics/ai-performance.js
@@@ -0,0 +1,441 @@
++'use strict';
++/**
++ * AI performance (plan L40/L41, ai-metrics.md): the per-call roll-ups behind report.ai-performance@1.
++ *
++ * ai.call.recorded@1 carries the 1.1.0 fields: operation, task and task_id, task_ms, harness, place, streamed, tps,
++ * error_class, fallbacks, cache_hit and tool_calls. Report keeps the raw record for 90 days and, inside the same
++ * ingest transaction, adds it to:
++ *
++ * report_ai_perf_days one row per day and system/provider/model/operation/task/place/harness: counts, token and
++ * cost sums, and three histograms. The histograms are 40 counts over the log-scale bounds in
++ * HISTOGRAM_BOUNDS below: time to first token (ms), latency (ms) and output tokens per second.
++ * report_ai_task_days one row per day, task kind and system: how many tasks, and the cost and time of each task
++ * as histograms. A task is the calls that share task_id; its time is task_ms when any call
++ * gives it, else the sum of its calls' latency_ms. Its cost is the sum of the calls'
++ * charged_usd.
++ *
++ * performance() answers the rows of one group. A percentile answers the UPPER BOUND of the bucket the p-th call
++ * falls in, never an invented value inside the bucket: with 40 buckets a p50 is at most one bucket's width off, and
++ * the answer says so by being a bound. The public groups are model, provider and operation, with a floor of 20 calls;
++ * system, task, place and harness are for staff.
++ */
++
++const DAY_MS = 86400000;
++const BUCKET_COUNT = 40;
++const DEFAULT_RANGE_DAYS = 30;
++const MAX_RANGE_DAYS = 400;
++const PUBLIC_CALL_FLOOR = 20;
++
++/** The groups report.ai-performance@1 has. */
++const GROUPS = Object.freeze({
++ model: 'model', provider: 'provider', operation: 'operation', system: 'system', task: 'task', place: 'place', harness: 'harness',
++});
++const PERFORMANCE_GROUPS = Object.freeze(Object.keys(GROUPS));
++/** The groups anyone may ask for, and the groups a staff caller may ask for. */
++const PUBLIC_GROUPS = Object.freeze(['model', 'provider', 'operation']);
++const STAFF_GROUPS = Object.freeze(['system', 'task', 'place', 'harness']);
++/** A filter may name any group's column; the public route only accepts the public three. */
++const FILTERS = Object.freeze(['provider', 'model', 'operation', 'system', 'task', 'place', 'harness']);
++const PUBLIC_FILTERS = Object.freeze(['provider', 'model', 'operation']);
++
++/**
++ * The upper bound of each log-scale bucket, written once. The bounds are per measure because the ranges differ by
++ * orders of magnitude; a value at or below a bound falls in that bucket, above the last bound in the last bucket, and
++ * a percentile answers the bucket's upper bound.
++ */
++function logBounds(min, max, count = BUCKET_COUNT) {
++ const lo = Math.log(min);
++ const hi = Math.log(max);
++ const out = [];
++ for (let i = 1; i <= count; i += 1) out.push(Math.round(Math.exp(lo + ((hi - lo) * i) / count) * 1e6) / 1e6);
++ return Object.freeze(out);
++}
++const HISTOGRAM_BOUNDS = Object.freeze({
++ ttft: logBounds(1, 120000),
++ latency: logBounds(1, 3600000),
++ tps: logBounds(0.1, 10000),
++ task_cost_usd: logBounds(0.000001, 100),
++ task_time_ms: logBounds(1, 3600000),
++});
++
++const num = (v) => (v === null || v === undefined || v === '' ? 0 : (Number.isFinite(Number(v)) ? Number(v) : 0));
++const int = (v) => Math.trunc(num(v));
++/** Six decimals, the way money is written here. */
++const money = (v) => Math.round(num(v) * 1e6) / 1e6;
++const round = (v, places = 6) => {
++ const f = 10 ** places;
++ return Math.round(num(v) * f) / f;
++};
++const dayOf = (v) => (v instanceof Date ? v : new Date(v)).toISOString().slice(0, 10);
++const addDays = (day, n) => new Date(Date.parse(`${day}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);
++const dateRegex = /^\d{4}-\d{2}-\d{2}$/;
++
++/** A text field, cut to its contract's maximum and never null (the empty string means none). */
++function text(v, max) {
++ if (v == null) return '';
++ return String(v).slice(0, max);
++}
++
++/** The bucket a value falls in: 0 for a value at or below the first bound, the last bucket above the last bound. */
++function bucketOf(value, bounds) {
++ const v = Number(value);
++ if (!Number.isFinite(v) || v <= 0) return 0;
++ let lo = 0;
++ let hi = bounds.length - 1;
++ while (lo < hi) {
++ const mid = (lo + hi) >> 1;
++ if (v <= bounds[mid]) hi = mid;
++ else lo = mid + 1;
++ }
++ return lo;
++}
++
++/** A fresh histogram: 40 zero counts. */
++const zeroHist = () => new Array(BUCKET_COUNT).fill(0);
++
++/** Add one value to a histogram; a null (not measured) adds nothing. */
++function bump(hist, value, bounds) {
++ if (value === null || value === undefined || value === '') return false;
++ const v = Number(value);
++ if (!Number.isFinite(v) || v < 0) return false;
++ hist[bucketOf(v, bounds)] += 1;
++ return true;
++}
++
++/** A jsonb histogram from the database as 40 numbers (a missing or short one is padded with zeros). */
++function asHist(value) {
++ const out = zeroHist();
++ let arr = value;
++ if (typeof arr === 'string') {
++ try { arr = JSON.parse(arr); } catch { arr = []; }
++ }
++ if (!Array.isArray(arr)) return out;
++ for (let i = 0; i < Math.min(arr.length, out.length); i += 1) out[i] = int(arr[i]);
++ return out;
++}
++
++/** Merge one histogram into another (the first is changed). */
++function mergeHist(into, from) {
++ for (let i = 0; i < BUCKET_COUNT; i += 1) into[i] += int(from[i]);
++ return into;
++}
++
++/** A jsonb error-class map from the database. */
++function asMap(value) {
++ let obj = value;
++ if (typeof obj === 'string') {
++ try { obj = JSON.parse(obj); } catch { obj = {}; }
++ }
++ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return {};
++ const out = {};
++ for (const [k, v] of Object.entries(obj)) {
++ const key = String(k).slice(0, 60);
++ if (key) out[key] = int(v);
++ }
++ return out;
++}
++
++/**
++ * The p-th percentile of a histogram: the upper bound of the bucket the nearest-rank call lands in. n = 0 answers
++ * null; a p below 1 counts as 1 and above 100 as 100. Never an interpolation, never a value below the data.
++ */
++function percentile(hist, bounds, p) {
++ const counts = Array.isArray(hist) ? hist : [];
++ const n = counts.reduce((s, c) => s + int(c), 0);
++ if (!n) return null;
++ const want = Math.min(n, Math.max(1, Math.ceil((Math.max(1, Math.min(100, Number(p) || 50)) / 100) * n)));
++ let seen = 0;
++ for (let i = 0; i < bounds.length; i += 1) {
++ seen += int(counts[i]);
++ if (seen >= want) return round(bounds[i]);
++ }
++ return round(bounds[bounds.length - 1]);
++}
++
++/** p50, p90, p95 and the number of samples of a histogram. */
++function summary(hist, bounds) {
++ return {
++ p50: percentile(hist, bounds, 50),
++ p90: percentile(hist, bounds, 90),
++ p95: percentile(hist, bounds, 95),
++ n: (Array.isArray(hist) ? hist : []).reduce((s, c) => s + int(c), 0),
++ };
++}
++
++/** The key of a performance row, as SQL compares it. */
++const KEY_WHERE = 'day = ?::date AND system = ? AND provider = ? AND model = ? AND operation = ? AND task = ? AND place = ? AND harness = ?';
++
++const succeededOutcome = (outcome) => outcome === 'succeeded' || outcome === 'cached';
++const failedOutcome = (outcome) => outcome === 'failed' || outcome === 'cancelled';
++
++/**
++ * Add one ai.call.recorded@1 payload to report_ai_perf_days, inside the delivery's own transaction. The caller has
++ * already inserted the raw record and returns early on a duplicate, so the same call is counted once. The row is
++ * claimed first (an insert that conflicts does nothing), then read under a row lock and written back, so two
++ * deliveries for the same group cannot lose an increment.
++ */
++async function recordPerformance(t, payload, day) {
++ const p = payload && typeof payload === 'object' ? payload : {};
++ const perfDay = day || dayOf(p.at || new Date().toISOString());
++ const system = text(p.system || 'ai', 100);
++ const provider = text(p.provider, 128);
++ const model = text(p.model, 200);
++ const operation = text(p.operation, 64);
++ const task = text(p.task, 120);
++ const place = text(p.place, 40);
++ const harness = text(p.harness, 60);
++ const key = [perfDay, system, provider, model, operation, task, place, harness];
++
++ await t.prepare('INSERT INTO report_ai_perf_days (day, system, provider, model, operation, task, place, harness)'
++ + ' VALUES (?::date, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING')
++ .run(...key);
++ const current = (await t.prepare('SELECT calls, succeeded, errors, fallbacks, streamed, cache_hits, tool_calls,'
++ + ' tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, tokens_cache_write, charged_usd, list_cost_usd,'
++ + ' cache_saved_usd, ttft_hist, latency_hist, tps_hist, errors_by_class FROM report_ai_perf_days WHERE '
++ + KEY_WHERE + ' FOR UPDATE').all(...key))[0];
++ if (!current) throw new Error('report_ai_perf_days row vanished inside its transaction');
++
++ const outcome = String(p.outcome || 'succeeded');
++ const succeeded = succeededOutcome(outcome);
++ const failed = failedOutcome(outcome);
++ const ttft = asHist(current.ttft_hist);
++ const latency = asHist(current.latency_hist);
++ const tps = asHist(current.tps_hist);
++ bump(ttft, p.ttft_ms == null ? null : num(p.ttft_ms), HISTOGRAM_BOUNDS.ttft);
++ bump(latency, p.latency_ms == null ? null : num(p.latency_ms), HISTOGRAM_BOUNDS.latency);
++ bump(tps, p.tps == null ? null : num(p.tps), HISTOGRAM_BOUNDS.tps);
++ const errors = asMap(current.errors_by_class);
++ const errorClass = text(p.error_class, 60);
++ if (failed && errorClass) errors[errorClass] = (errors[errorClass] || 0) + 1;
++
++ await t.prepare('UPDATE report_ai_perf_days SET calls = ?, succeeded = ?, errors = ?, fallbacks = ?, streamed = ?,'
++ + ' cache_hits = ?, tool_calls = ?, tokens_input = ?, tokens_output = ?, tokens_reasoning = ?, tokens_cache_read = ?,'
++ + ' tokens_cache_write = ?, charged_usd = ?, list_cost_usd = ?, cache_saved_usd = ?, ttft_hist = ?::jsonb,'
++ + ' latency_hist = ?::jsonb, tps_hist = ?::jsonb, errors_by_class = ?::jsonb, updated_at = now() WHERE ' + KEY_WHERE)
++ .run(int(current.calls) + 1,
++ int(current.succeeded) + (succeeded ? 1 : 0),
++ int(current.errors) + (failed ? 1 : 0),
++ int(current.fallbacks) + (num(p.fallbacks) > 0 ? 1 : 0),
++ int(current.streamed) + (p.streamed === true ? 1 : 0),
++ int(current.cache_hits) + (p.cache_hit === true ? 1 : 0),
++ int(current.tool_calls) + int(p.tool_calls),
++ int(current.tokens_input) + int(p.input),
++ int(current.tokens_output) + int(p.output),
++ int(current.tokens_reasoning) + int(p.reasoning),
++ int(current.tokens_cache_read) + int(p.cache_read),
++ int(current.tokens_cache_write) + int(p.cache_write),
++ money(num(current.charged_usd) + num(p.charged_usd)),
++ money(num(current.list_cost_usd) + num(p.list_cost_usd)),
++ money(num(current.cache_saved_usd) + num(p.cache_saved_usd)),
++ JSON.stringify(ttft), JSON.stringify(latency), JSON.stringify(tps), JSON.stringify(errors),
++ ...key);
++ return true;
++}
++
++/**
++ * Rebuild one (day, task kind, system) row of report_ai_task_days from the raw calls of that day. Called after a
++ * task call is inserted. The task's cost is the sum of its calls' charged_usd and its time is task_ms when any call
++ * gives it, else the sum of the calls' latency_ms.
++ */
++async function recordTask(t, payload, day) {
++ const p = payload && typeof payload === 'object' ? payload : {};
++ const taskId = p.task_id == null ? '' : String(p.task_id).trim();
++ if (!taskId) return false;
++ const taskDay = day || dayOf(p.at || new Date().toISOString());
++ const task = text(p.task, 120);
++ const system = text(p.system || 'ai', 100);
++ const rows = await t.prepare('SELECT COUNT(*)::bigint AS calls, SUM(charged_usd) AS cost,'
++ + ' CASE WHEN MAX(task_ms) IS NOT NULL THEN MAX(task_ms) ELSE SUM(COALESCE(latency_ms, 0)) END AS time'
++ + ' FROM report_ai_usage WHERE day = ?::date AND task = ? AND system = ? AND task_id IS NOT NULL AND task_id <> \'\''
++ + ' GROUP BY task_id').all(taskDay, task, system);
++ const costHist = zeroHist();
++ const timeHist = zeroHist();
++ for (const r of rows) {
++ bump(costHist, num(r.cost), HISTOGRAM_BOUNDS.task_cost_usd);
++ bump(timeHist, num(r.time), HISTOGRAM_BOUNDS.task_time_ms);
++ }
++ await t.prepare('INSERT INTO report_ai_task_days (day, task, system, tasks, cost_hist, time_hist)'
++ + ' VALUES (?::date, ?, ?, ?, ?::jsonb, ?::jsonb)'
++ + ' ON CONFLICT (day, task, system) DO UPDATE SET tasks = excluded.tasks, cost_hist = excluded.cost_hist,'
++ + ' time_hist = excluded.time_hist, updated_at = now()')
++ .run(taskDay, task, system, rows.length, JSON.stringify(costHist), JSON.stringify(timeHist));
++ return true;
++}
++
++/** The day bounds of a from/to pair; missing ones are the last 30 days ending today. */
++function rangeOf(from, to, now) {
++ const today = dayOf(now == null ? Date.now() : now);
++ const toDay = to == null || to === '' ? today : String(to).slice(0, 10);
++ const fromDay = from == null || from === '' ? addDays(toDay, -(DEFAULT_RANGE_DAYS - 1)) : String(from).slice(0, 10);
++ for (const [name, value] of [['from', fromDay], ['to', toDay]]) {
++ const d = new Date(`${value}T00:00:00Z`);
++ if (!dateRegex.test(value) || Number.isNaN(d.getTime()) || d.toISOString().slice(0, 10) !== value) {
++ throw new RangeError(name + ' must be a day (YYYY-MM-DD)');
++ }
++ }
++ if (fromDay > toDay) throw new RangeError('from must be on or before to');
++ if (Date.parse(`${toDay}T00:00:00Z`) - Date.parse(`${fromDay}T00:00:00Z`) > MAX_RANGE_DAYS * DAY_MS) {
++ throw new RangeError('the range must be at most ' + MAX_RANGE_DAYS + ' days');
++ }
++ return { from: fromDay, to: toDay };
++}
++
++const emptyAgg = () => ({
++ calls: 0, succeeded: 0, errors: 0, fallbacks: 0, streamed: 0, cache_hits: 0, tool_calls: 0,
++ tokens_input: 0, tokens_output: 0, tokens_reasoning: 0, tokens_cache_read: 0, tokens_cache_write: 0,
++ charged_usd: 0, list_cost_usd: 0, cache_saved_usd: 0,
++ ttft: zeroHist(), latency: zeroHist(), tps: zeroHist(), errors_by_class: {},
++});
++
++/** Add one report_ai_perf_days row into a group's aggregate. */
++function addPerf(agg, r) {
++ agg.calls += int(r.calls);
++ agg.succeeded += int(r.succeeded);
++ agg.errors += int(r.errors);
++ agg.fallbacks += int(r.fallbacks);
++ agg.streamed += int(r.streamed);
++ agg.cache_hits += int(r.cache_hits);
++ agg.tool_calls += int(r.tool_calls);
++ agg.tokens_input += int(r.tokens_input);
++ agg.tokens_output += int(r.tokens_output);
++ agg.tokens_reasoning += int(r.tokens_reasoning);
++ agg.tokens_cache_read += int(r.tokens_cache_read);
++ agg.tokens_cache_write += int(r.tokens_cache_write);
++ agg.charged_usd += num(r.charged_usd);
++ agg.list_cost_usd += num(r.list_cost_usd);
++ agg.cache_saved_usd += num(r.cache_saved_usd);
++ mergeHist(agg.ttft, asHist(r.ttft_hist));
++ mergeHist(agg.latency, asHist(r.latency_hist));
++ mergeHist(agg.tps, asHist(r.tps_hist));
++ for (const [k, v] of Object.entries(asMap(r.errors_by_class))) agg.errors_by_class[k] = (agg.errors_by_class[k] || 0) + int(v);
++ return agg;
++}
++
++/** The top five error classes of an aggregate, most frequent first, the class name breaking ties. */
++function topErrors(map) {
++ return Object.entries(map)
++ .filter(([, n]) => int(n) > 0)
++ .sort(([ak, an], [bk, bn]) => (int(bn) - int(an)) || (ak < bk ? -1 : ak > bk ? 1 : 0))
++ .slice(0, 5)
++ .map(([error_class, count]) => ({ error_class: error_class.slice(0, 60), count: int(count) }));
++}
++
++/** One row of report.ai-performance@1: the rates, the histograms' percentiles, the tokens, the cost and the errors. */
++function rowOf(group, key, agg, range) {
++ const calls = int(agg.calls);
++ const input = int(agg.tokens_input);
++ const output = int(agg.tokens_output);
++ const cacheRead = int(agg.tokens_cache_read);
++ const cacheWrite = int(agg.tokens_cache_write);
++ const prompt = input + cacheRead + cacheWrite;
++ const charged = money(agg.charged_usd);
++ const row = {
++ group,
++ key: String(key == null ? '' : key).slice(0, 200),
++ range: { from: range.from, to: range.to },
++ calls,
++ succeeded: int(agg.succeeded),
++ errors: int(agg.errors),
++ success_rate: calls ? round(int(agg.succeeded) / calls) : null,
++ fallback_rate: calls ? round(int(agg.fallbacks) / calls) : null,
++ streamed_share: calls ? round(int(agg.streamed) / calls) : null,
++ cache_hit_rate: calls ? round(int(agg.cache_hits) / calls) : null,
++ cache_read_share: prompt ? round(cacheRead / prompt) : null,
++ ttft_ms: summary(agg.ttft, HISTOGRAM_BOUNDS.ttft),
++ latency_ms: summary(agg.latency, HISTOGRAM_BOUNDS.latency),
++ tps: summary(agg.tps, HISTOGRAM_BOUNDS.tps),
++ tokens: { input, output, reasoning: int(agg.tokens_reasoning), cache_read: cacheRead, cache_write: cacheWrite },
++ cost: {
++ charged_usd: charged,
++ list_usd: money(agg.list_cost_usd),
++ cache_saved_usd: money(agg.cache_saved_usd),
++ per_call_usd: calls ? money(agg.charged_usd / calls) : null,
++ per_million_output_usd: output > 0 ? money((agg.charged_usd / output) * 1e6) : null,
++ },
++ top_errors: topErrors(agg.errors_by_class),
++ };
++ if (group === 'provider') row.provider = row.key ? row.key.slice(0, 120) : null;
++ return row;
++}
++
++/** The task roll-up rows of a group (task or system), merged across days. */
++async function taskAggregates(store, { group, range, filters }) {
++ const column = GROUPS[group];
++ const where = ['day >= ?::date', 'day <= ?::date'];
++ const args = [range.from, range.to];
++ if (filters && filters.system != null && filters.system !== '') { where.push('system = ?'); args.push(String(filters.system)); }
++ if (filters && filters.task != null && filters.task !== '') { where.push('task = ?'); args.push(String(filters.task)); }
++ const rows = await store.db.prepare('SELECT ' + column + ' AS bucket, tasks, cost_hist, time_hist FROM report_ai_task_days'
++ + ' WHERE ' + where.join(' AND ')).all(...args);
++ const map = new Map();
++ for (const r of rows) {
++ const key = String(r.bucket == null ? '' : r.bucket).slice(0, 200);
++ let agg = map.get(key);
++ if (!agg) { agg = { count: 0, cost: zeroHist(), time: zeroHist() }; map.set(key, agg); }
++ agg.count += int(r.tasks);
++ mergeHist(agg.cost, asHist(r.cost_hist));
++ mergeHist(agg.time, asHist(r.time_hist));
++ }
++ return map;
++}
++
++/** One task aggregate as report.ai-performance@1's tasks object. */
++function tasksOf(agg) {
++ return {
++ count: int(agg && agg.count),
++ cost_per_task_usd: summary(agg ? agg.cost : zeroHist(), HISTOGRAM_BOUNDS.task_cost_usd),
++ time_per_task_ms: summary(agg ? agg.time : zeroHist(), HISTOGRAM_BOUNDS.task_time_ms),
++ };
++}
++
++/**
++ * The performance rows of one group. `filters` is an object of group columns to exact values (the route allows the
++ * public three for anyone and all of them for staff). `minCalls` drops a group below the floor (the public route
++ * passes 20). Rows come biggest first. Task cost/time is added when the group is task or system and no filter
++ * reaches a column the task table does not keep.
++ */
++async function performance(store, { group = 'model', from = null, to = null, filters = {}, minCalls = 0, now } = {}) {
++ const column = GROUPS[group];
++ if (!column) throw new RangeError('group must be one of ' + PERFORMANCE_GROUPS.join(', '));
++ const range = rangeOf(from, to, now);
++ const where = ['day >= ?::date', 'day <= ?::date'];
++ const args = [range.from, range.to];
++ for (const [key, raw] of Object.entries(filters || {})) {
++ if (!FILTERS.includes(key)) throw new RangeError('filter must be one of ' + FILTERS.join(', '));
++ if (raw == null || raw === '') continue;
++ const value = String(raw);
++ if (value.length > 200) throw new RangeError('a filter is too long');
++ where.push(key + ' = ?');
++ args.push(value);
++ }
++ const rows = await store.db.prepare('SELECT ' + column + ' AS bucket, calls, succeeded, errors, fallbacks, streamed,'
++ + ' cache_hits, tool_calls, tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, tokens_cache_write,'
++ + ' charged_usd, list_cost_usd, cache_saved_usd, ttft_hist, latency_hist, tps_hist, errors_by_class'
++ + ' FROM report_ai_perf_days WHERE ' + where.join(' AND ')).all(...args);
++ const aggregates = new Map();
++ for (const r of rows) {
++ const key = String(r.bucket == null ? '' : r.bucket).slice(0, 200);
++ let agg = aggregates.get(key);
++ if (!agg) { agg = emptyAgg(); aggregates.set(key, agg); }
++ addPerf(agg, r);
++ }
++ const taskGroups = (group === 'task' || group === 'system')
++ && Object.keys(filters || {}).every((k) => k === 'system' || k === 'task')
++ ? await taskAggregates(store, { group, range, filters: filters || {} })
++ : null;
++ const out = [];
++ for (const [key, agg] of aggregates) {
++ if (int(agg.calls) < Math.max(0, int(minCalls))) continue;
++ const row = rowOf(group, key, agg, range);
++ if (taskGroups) row.tasks = tasksOf(taskGroups.get(key));
++ out.push(row);
++ }
++ out.sort((a, b) => (b.calls - a.calls) || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
++ return out;
++}
++
++module.exports = {
++ BUCKET_COUNT, HISTOGRAM_BOUNDS, PUBLIC_CALL_FLOOR, GROUPS, PERFORMANCE_GROUPS, PUBLIC_GROUPS, STAFF_GROUPS,
++ FILTERS, PUBLIC_FILTERS, logBounds, bucketOf, zeroHist, bump, asHist, mergeHist, asMap, percentile, summary,
++ recordPerformance, recordTask, performance, rangeOf, rowOf, tasksOf,
++};
ddiff --git a/server/analytics/ai-usage.js b/server/analytics/ai-usage.js
iindex f2b46d7..5b5edba 100644
---- a/server/analytics/ai-usage.js
++++ b/server/analytics/ai-usage.js
@@@ -17,6 +17,8 @@
* A group that fewer than ten people share reads 'fewer than 10' instead of counting them.
*/
++const { recordPerformance, recordTask, performance } = require('./ai-performance');
++
const GROUPS = { day: 'day', site: 'attribution', model: 'model', system: 'system', paid_by: 'paid_by' };
/** The groups a person may ask for their own rows. */
const PERSON_GROUPS = Object.freeze(['day', 'week', 'month', 'provider', 'model', 'surface', 'paid_by']);
@@@ -50,26 +52,37 @@ const weekOf = (day) => {
/**
* Write one record inside the delivery's own transaction. The same call twice is one row: the key is the system
-- * and the run id (or the turn id).
++ * and the run id (or the turn id), and only the insert that claimed the row adds to the performance and task
++ * roll-ups, so a replay changes nothing.
*/
async function recordUsage(t, payload, { occurredAt = null, source = 'events' } = {}) {
const p = payload && typeof payload === 'object' ? payload : {};
const at = p.at || occurredAt || new Date().toISOString();
const key = recordKey(p);
-- const sql = 'INSERT INTO report_ai_usage (record_key, system, run_id, turn_id, subject, attribution, surface, provider, model, effort,'
-- + ' paid_by, paid_by_detail, tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, tokens_cache_write, cache_write_ttl,'
++ const day = dayOf(at);
++ const text = (v, max) => (v == null ? '' : String(v).slice(0, max));
++ const sql = 'INSERT INTO report_ai_usage (record_key, system, run_id, turn_id, subject, attribution, surface, operation, task, task_id, task_ms,'
++ + ' provider, model, effort, paid_by, paid_by_detail, tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, tokens_cache_write, cache_write_ttl,'
+ ' images, audio_seconds, video_seconds, list_cost_usd, charged_usd, fee_usd, cache_saved_usd, price_source, price_date,'
-- + ' latency_ms, ttft_ms, attempts, outcome, at, day, source)'
-- + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
++ + ' latency_ms, ttft_ms, attempts, outcome, streamed, tps, queue_ms, harness, place, error_class, fallbacks, cache_hit, tool_calls, region, at, day, source)'
++ + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+ ' ON CONFLICT (record_key) DO NOTHING';
const r = await t.prepare(sql)
.run(key, String(p.system || 'ai'), p.run_id || null, p.turn_id || null, p.subject || null, String(p.attribution || EMPTY),
-- String(p.surface || EMPTY), p.provider || null, p.model || null, p.effort || null, String(p.paid_by || EMPTY), p.paid_by_detail || null,
++ String(p.surface || EMPTY), text(p.operation, 64), text(p.task, 120), p.task_id || null, p.task_ms == null ? null : int(p.task_ms),
++ p.provider || null, p.model || null, p.effort || null, String(p.paid_by || EMPTY), p.paid_by_detail || null,
int(p.input), int(p.output), int(p.reasoning), int(p.cache_read), int(p.cache_write), p.cache_write_ttl || null,
int(p.images), num(p.audio_seconds), num(p.video_seconds), money(p.list_cost_usd), money(p.charged_usd), money(p.fee_usd), money(p.cache_saved_usd),
String(p.price_source || EMPTY), p.price_date || null, p.latency_ms == null ? null : int(p.latency_ms), p.ttft_ms == null ? null : int(p.ttft_ms),
-- int(p.attempts) || 1, String(p.outcome || 'succeeded'), at, dayOf(at), String(source || 'events'));
-- return Number(r && r.changes) ? 'usage:recorded' : 'usage:duplicate';
++ int(p.attempts) || 1, String(p.outcome || 'succeeded'), p.streamed === true ? true : p.streamed === false ? false : null,
++ p.tps == null ? null : num(p.tps), p.queue_ms == null ? null : int(p.queue_ms), text(p.harness, 60), text(p.place, 40),
++ p.error_class == null ? null : text(p.error_class, 60), p.fallbacks == null ? null : int(p.fallbacks),
++ p.cache_hit === true ? true : p.cache_hit === false ? false : null, p.tool_calls == null ? null : int(p.tool_calls),
++ p.region == null ? null : text(p.region, 40), at, day, String(source || 'events'));
++ if (!Number(r && r.changes)) return 'usage:duplicate';
++ await recordPerformance(t, p, day);
++ await recordTask(t, p, day);
++ return 'usage:recorded';
}
/** The rows of one day, recomputed from the raw records (delete-then-insert, so a second pass counts nothing twice). */
@@@ -456,6 +469,7 @@ function createAiUsage({ store, now = () => Date.now(), log = console } = {}) {
groups: PERSON_GROUPS,
forPerson: (opts = {}) => personUsage(store, { now: now(), ...opts }),
usage: (opts = {}) => usage(store, opts),
++ performance: (opts = {}) => performance(store, { now: now(), ...opts }),
openTotals: (opts = {}) => openTotals(store, opts),
totalsOf,
rollup: (opts = {}) => rollup(store, opts),
@@@ -468,5 +482,5 @@ function createAiUsage({ store, now = () => Date.now(), log = console } = {}) {
module.exports = {
GROUPS, PERSON_GROUPS, RAW_RETENTION_DAYS, DEFAULT_RANGE_DAYS, MAX_RANGE_DAYS, PEOPLE_FLOOR,
recordKey, recordUsage, rollup, rollupDay, prune, usage, personUsage, personRange, totalsOf, daily, openTotals,
-- exportFor, eraseFor, createAiUsage, weekOf, dayOf, money, usdText, periodText, personInsights,
++ performance, exportFor, eraseFor, createAiUsage, weekOf, dayOf, money, usdText, periodText, personInsights,
};
ddiff --git a/server/http/api.js b/server/http/api.js
iindex 99aade4..ee367ba 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -26,6 +26,7 @@ const contracts = require('openvibe-contracts');
const { normaliseDays, isSlug, isStreamId, isUsername, streamIdOf } = require('../data/source');
const { createServiceAuth, CAPS } = require('../auth');
const { GROUPS: AI_USAGE_GROUPS, PERSON_GROUPS } = require('../analytics/ai-usage');
++const { PERFORMANCE_GROUPS, PUBLIC_GROUPS, FILTERS: PERFORMANCE_FILTERS, PUBLIC_FILTERS, PUBLIC_CALL_FLOOR } = require('../analytics/ai-performance');
const GROUPS = Object.keys(AI_USAGE_GROUPS);
const designHealth = require('../data/design-health');
const sharing = require('../sharing');
@@@ -452,6 +453,60 @@ function createApi({ network, live, bench = null, aiCatalogue = null, analytics
}
});
++ // The AI performance rows (plan L40/L41, report.ai-performance@1): public for model, provider and operation,
++ // with a floor of 20 calls and no system, task or person in the answer. System, task, place and harness are for
++ // staff: a signed-in person with staff.site.view (the HTML staff gate) or a Network service token with
++ // report.ai_usage.read (the JSON operator gate), and those answers are never cached.
++ r.get('/ai/performance', async (req, res) => {
++ if (!aiUsage) return problem(req, res, 503, 'report.no_database', 'Report has no database in this process.');
++ const group = String(req.query.group || 'model');
++ if (!PERFORMANCE_GROUPS.includes(group)) {
++ return problem(req, res, 422, 'ai.invalid', 'group must be one of ' + PERFORMANCE_GROUPS.join(', '));
++ }
++ const staffGroup = !PUBLIC_GROUPS.includes(group);
++ const roleStaff = Boolean(req.user && contracts.staff.can(req.user, 'staff.site.view'));
++ let staff = roleStaff;
++ let denied = null;
++ if (!staff) {
++ const v = await serviceAuth.authenticate(req);
++ if (v.ok && serviceAuth.hasCap(v.claims, CAPS.aiUsageRead)) staff = true;
++ else if (v.ok) denied = { status: 403, code: 'capability.denied', detail: 'report.ai_usage.read is not granted to this token.' };
++ else if (!req.user && v && v.status && v.status !== 401) denied = v;
++ }
++ if (staffGroup && !staff) {
++ res.set('Cache-Control', 'private, no-store');
++ if (req.user) return problem(req, res, 403, 'capability.denied', 'staff.site.view is not granted to this account.');
++ if (denied) return problem(req, res, denied.status, denied.code, denied.detail);
++ return problem(req, res, 401, 'auth.sign_in_required', 'Sign in with a staff account, or use a service token with report.ai_usage.read, for this AI performance group.');
++ }
++ const days = Number(req.query.days == null ? 30 : req.query.days);
++ if (![7, 30, 90].includes(days)) return problem(req, res, 422, 'ai.invalid', 'days must be 7, 30 or 90');
++ const filters = {};
++ for (const key of PERFORMANCE_FILTERS) {
++ const raw = req.query[key];
++ if (raw === undefined) continue;
++ if (!staff && !PUBLIC_FILTERS.includes(key)) {
++ res.set('Cache-Control', 'private, no-store');
++ return problem(req, res, 403, 'capability.denied', key + ' is a staff filter.');
++ }
++ if (typeof raw !== 'string' || raw.length === 0 || raw.length > 200) {
++ return problem(req, res, 422, 'ai.invalid', 'filter ' + key + ' must be one non-empty value of at most 200 characters');
++ }
++ filters[key] = raw;
++ }
++ const to = new Date(now()).toISOString().slice(0, 10);
++ const from = new Date(Date.parse(`${to}T00:00:00Z`) - (days - 1) * 86400000).toISOString().slice(0, 10);
++ try {
++ const rows = await aiUsage.performance({ group, from, to, filters, minCalls: staffGroup ? 0 : PUBLIC_CALL_FLOOR });
++ if (staff) res.set('Cache-Control', 'private, no-store');
++ return res.json({ group, range: { from, to }, rows });
++ } catch (err) {
++ if (err instanceof RangeError) return problem(req, res, 422, 'ai.invalid', err.message);
++ log.error('[OpenVibe.Report] the AI performance rows failed:', err && err.message ? String(err.message).slice(0, 200) : err);
++ return problem(req, res, 500, 'report.ai_usage_failed', 'The AI performance rows could not be read.');
++ }
++ });
++
// The person's own rows (plan L41 R1). The subject is the signed-in person from Report's sso session (cookie or
// bearer), never the query; signed out is 401. The last 30 days by default, 400 at most; raw records answer a
// range inside the 90-day retention and the roll-ups the part beyond it. Private: the /me middleware above sets
ddiff --git a/test/ai-performance.test.js b/test/ai-performance.test.js
nnew file mode 100644
iindex 0000000..8cc5366
---- /dev/null
++++ b/test/ai-performance.test.js
@@@ -0,0 +1,249 @@
++'use strict';
++/**
++ * AI performance (plan L40/L41, ai-metrics.md): the per-call roll-ups the ingest adds, the time/cost histograms, the
++ * 20-call public floor, the staff groups, the task cost and time, and every row against report.ai-performance@1.
++ */
++const assert = require('assert');
++const crypto = require('crypto');
++const contracts = require('openvibe-contracts');
++const { signDeliveryV2 } = require('openvibe-sdk/events');
++const { serviceAuth } = require('openvibe-contracts');
++const { createSsoClient } = require('openvibe-sdk/sso');
++const { boot, check, done } = require('./helpers/boot');
++const perf = require('../server/analytics/ai-performance');
++
++const SECRET = 'test-report-events-secret-32-characters-long';
++const ISSUER = 'https://openvibe.network';
++const NOW = Date.parse('2026-10-31T12:00:00.000Z');
++const AT = '2026-10-31T10:00:00.000Z';
++const PERSON = 'usr_01JTESTPERF00000000000000';
++
++const { privateKey, publicKey } = crypto.generateKeyPairSync('rsa', {
++ modulusLength: 2048,
++ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
++ publicKeyEncoding: { type: 'spki', format: 'pem' },
++});
++
++const b64 = (o) => Buffer.from(JSON.stringify(o)).toString('base64url');
++/** A person's access token, the way OpenVibe.Network signs one (real time: the SSO verifier reads the clock). */
++function personToken(claims) {
++ const now = Math.floor(Date.now() / 1000);
++ const head = b64({ alg: 'RS256', typ: 'JWT', kid: 'test' });
++ const body = b64({ iss: ISSUER, aud: ['openvibe.network', 'openvibe.report'], iat: now, exp: now + 3600, ...claims });
++ const sig = crypto.sign('RSA-SHA256', Buffer.from(`${head}.${body}`), privateKey).toString('base64url');
++ return `${head}.${body}.${sig}`;
++}
++const STAFF = personToken({ sub: '1', subject_id: PERSON, username: 'staffer', display_name: 'Staffer', role: 'admin' });
++const PLAIN = personToken({ sub: '2', subject_id: 'usr_01JTESTPERF00000000000001', username: 'visitor', display_name: 'Visitor', role: 'user' });
++
++/** A Network service token (the JSON operator gate). */
++function serviceToken(cap) {
++ const now = Math.floor(NOW / 1000);
++ return serviceAuth.signServiceToken({
++ iss: ISSUER, sub: 'svc:operator', actor_type: 'service', aud: ['openvibe.report'], cap, ns: ['*'],
++ iat: now, exp: now + 300, jti: 'tok_aiperf_' + String(now).slice(-6),
++ }, privateKey);
++}
++
++const sso = createSsoClient({
++ site: 'report', baseUrl: 'https://openvibe.report', clientId: 'report', clientSecret: 'x',
++ networkUrl: ISSUER, publicKey,
++ log: { warn() {}, info() {}, log() {}, error() {} },
++});
++
++let seq = 0;
++/** One ai.call.recorded@1 event with the 1.1.0 fields. */
++function usageEvent(over = {}) {
++ seq += 1;
++ return {
++ event_id: 'evt_AIPERF_' + String(seq).padStart(6, '0'),
++ event_type: 'ai.call.recorded',
++ source: 'ai',
++ occurred_at: AT,
++ payload: {
++ subject: null, attribution: 'site:design', surface: 'ai.generate', system: 'ai',
++ provider: 'anthropic', model: 'public-model', effort: null,
++ paid_by: 'openvibe_key', paid_by_detail: null,
++ input: 200, output: 100, reasoning: 0, cache_read: 800, cache_write: 0, cache_write_ttl: null,
++ images: 0, audio_seconds: 0, video_seconds: 0,
++ list_cost_usd: 0.001, charged_usd: 0.001, fee_usd: 0, cache_saved_usd: 0.0005,
++ price_source: 'models', price_date: '2026-10-31',
++ latency_ms: 1000, ttft_ms: 300, attempts: 1, outcome: 'succeeded',
++ run_id: 'run_aiperf_' + String(seq), turn_id: null, at: AT,
++ operation: 'chat', streamed: true, tps: 50, queue_ms: 5, task: 'chat.reply', task_id: null,
++ task_ms: null, harness: 'openvibe', place: 'provider', error_class: null, fallbacks: 0,
++ cache_hit: false, tool_calls: 1, region: 'us-east',
++ ...over,
++ },
++ };
++}
++
++function delivery(event) {
++ const raw = JSON.stringify({ event });
++ const ts = Math.floor(NOW / 1000);
++ return { raw, headers: { 'content-type': 'application/json', 'x-openvibe-timestamp': String(ts), 'x-openvibe-signature-v2': signDeliveryV2(raw, SECRET, ts) } };
++}
++
++const upper = (hist, bounds, p) => perf.percentile(hist, bounds, p);
++
++(async () => {
++ const t = await boot({ db: true, now: () => NOW, env: { REPORT_EVENTS_SECRET: SECRET, OV_NETWORK_PUBLIC_KEY: publicKey }, sso });
++ const db = t.ctx.store.db;
++ try {
++ await check('a percentile is the upper bound of the bucket the call lands in, and empty is null', async () => {
++ const hist = perf.zeroHist();
++ hist[perf.bucketOf(300, perf.HISTOGRAM_BOUNDS.ttft)] = 10;
++ assert.strictEqual(perf.percentile(hist, perf.HISTOGRAM_BOUNDS.ttft, 50), perf.HISTOGRAM_BOUNDS.ttft[perf.bucketOf(300, perf.HISTOGRAM_BOUNDS.ttft)]);
++ assert.strictEqual(perf.percentile(perf.zeroHist(), perf.HISTOGRAM_BOUNDS.ttft, 50), null);
++ const spread = perf.zeroHist();
++ spread[perf.bucketOf(100, perf.HISTOGRAM_BOUNDS.latency)] = 90;
++ spread[perf.bucketOf(5000, perf.HISTOGRAM_BOUNDS.latency)] = 10;
++ assert.strictEqual(perf.percentile(spread, perf.HISTOGRAM_BOUNDS.latency, 90), perf.HISTOGRAM_BOUNDS.latency[perf.bucketOf(100, perf.HISTOGRAM_BOUNDS.latency)]);
++ assert.strictEqual(perf.percentile(spread, perf.HISTOGRAM_BOUNDS.latency, 95), perf.HISTOGRAM_BOUNDS.latency[perf.bucketOf(5000, perf.HISTOGRAM_BOUNDS.latency)]);
++ });
++
++ await check('the ingest adds to the performance roll-up in the same transaction, and a replay changes nothing', async () => {
++ const events = [];
++ for (let i = 0; i < 20; i += 1) {
++ const failed = i < 2;
++ events.push(usageEvent(failed
++ ? { outcome: 'failed', error_class: i === 0 ? 'timeout' : 'rate_limited', ttft_ms: null, tps: null, latency_ms: 2000, streamed: true }
++ : { cache_hit: i === 19 }));
++ }
++ for (const e of events) {
++ const d = delivery(e);
++ const r = await t.get('/internal/events', { method: 'POST', headers: d.headers, body: d.raw });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().outcome, 'usage:recorded', r.text);
++ }
++ const row = (await db.query("SELECT * FROM report_ai_perf_days WHERE model = 'public-model'")).rows[0];
++ assert.strictEqual(Number(row.calls), 20);
++ assert.strictEqual(Number(row.succeeded), 18);
++ assert.strictEqual(Number(row.errors), 2);
++ assert.strictEqual(Number(row.streamed), 20);
++ assert.strictEqual(Number(row.cache_hits), 1);
++ assert.strictEqual(Number(row.tool_calls), 20);
++ assert.strictEqual(Number(row.tokens_input), 4000);
++ assert.strictEqual(Number(row.tokens_output), 2000);
++ assert.deepStrictEqual(row.errors_by_class, { timeout: 1, rate_limited: 1 });
++ const ttft = perf.asHist(row.ttft_hist);
++ assert.strictEqual(ttft.reduce((a, b) => a + b, 0), 18, 'the two failures measured no TTFT');
++ assert.strictEqual(ttft[perf.bucketOf(300, perf.HISTOGRAM_BOUNDS.ttft)], 18);
++ const replay = delivery(events[0]);
++ const again = await t.get('/internal/events', { method: 'POST', headers: replay.headers, body: replay.raw });
++ assert.strictEqual(again.json().duplicate, true);
++ assert.strictEqual(Number((await db.query("SELECT calls FROM report_ai_perf_days WHERE model = 'public-model'")).rows[0].calls), 20, 'the replay counted nothing twice');
++ assert.strictEqual(await db.value('SELECT COUNT(*)::int AS n FROM report_ai_usage'), 20); });
++
++ await check('19 calls are below the public floor, 20 are shown and every row matches report.ai-performance@1', async () => {
++ for (let i = 0; i < 19; i += 1) {
++ const e = usageEvent({ model: 'floor-model' });
++ const d = delivery(e);
++ const r = await t.get('/internal/events', { method: 'POST', headers: d.headers, body: d.raw });
++ assert.strictEqual(r.status, 200, r.text);
++ }
++ const publicRows = (await t.get('/api/v1/ai/performance?group=model&days=7')).json();
++ assert.strictEqual(publicRows.group, 'model');
++ assert.strictEqual(publicRows.rows.length, 1, 'only the 20-call model is public');
++ assert.strictEqual(publicRows.rows[0].key, 'public-model');
++ assert.strictEqual(publicRows.rows[0].calls, 20);
++ assert.strictEqual(publicRows.rows[0].succeeded, 18);
++ assert.strictEqual(publicRows.rows[0].success_rate, 0.9);
++ assert.strictEqual(publicRows.rows[0].cache_read_share, 0.8);
++ assert.strictEqual(publicRows.rows[0].ttft_ms.p50, perf.HISTOGRAM_BOUNDS.ttft[perf.bucketOf(300, perf.HISTOGRAM_BOUNDS.ttft)]);
++ assert.strictEqual(publicRows.rows[0].latency_ms.p95, perf.HISTOGRAM_BOUNDS.latency[perf.bucketOf(2000, perf.HISTOGRAM_BOUNDS.latency)]);
++ assert.deepStrictEqual(publicRows.rows[0].top_errors.map((e) => e.error_class).sort(), ['rate_limited', 'timeout']);
++ for (const row of publicRows.rows) {
++ const v = contracts.validate('report.ai-performance@1', row);
++ assert.strictEqual(v.valid, true, JSON.stringify(v.errors));
++ assert.ok(!('system' in row) && !('task' in row) && !('subject' in row), 'a public row names no system, task or person');
++ }
++ for (const group of ['provider', 'operation']) {
++ const answer = await t.get(`/api/v1/ai/performance?group=${group}&days=7`);
++ assert.strictEqual(answer.status, 200, answer.text);
++ assert.ok(answer.json().rows.length >= 1, group);
++ for (const row of answer.json().rows) {
++ const v = contracts.validate('report.ai-performance@1', row);
++ assert.strictEqual(v.valid, true, group + ' ' + JSON.stringify(v.errors));
++ assert.ok(!('system' in row) && !('task' in row) && !('subject' in row), group + ' names no system, task or person');
++ }
++ }
++ });
++
++ await check('the staff groups are refused to the public and the service gate', async () => {
++ const none = await t.get('/api/v1/ai/performance?group=system&days=7');
++ assert.strictEqual(none.status, 401, none.text);
++ assert.strictEqual(none.json().code, 'auth.sign_in_required');
++ assert.strictEqual(none.headers.get('cache-control'), 'private, no-store');
++ const plain = await t.get('/api/v1/ai/performance?group=system&days=7', { headers: { cookie: `ov_token=${PLAIN}` } });
++ assert.strictEqual(plain.status, 403, plain.text);
++ assert.strictEqual(plain.json().code, 'capability.denied');
++ const wrong = await t.get('/api/v1/ai/performance?group=system&days=7', { headers: { authorization: 'Bearer ' + serviceToken(['report.other.read']) } });
++ assert.strictEqual(wrong.status, 403, wrong.text);
++ assert.strictEqual(wrong.json().code, 'capability.denied');
++ const publicFilter = await t.get('/api/v1/ai/performance?group=model&days=7&system=ai');
++ assert.strictEqual(publicFilter.status, 403, 'system is a staff filter');
++ assert.strictEqual((await t.get('/api/v1/ai/performance?group=nope')).status, 422);
++ assert.strictEqual((await t.get('/api/v1/ai/performance?group=model&days=8')).status, 422);
++ });
++
++ await check('staff see system, place and harness, each row valid, and the service token reads them too', async () => {
++ const cookie = { headers: { cookie: `ov_token=${STAFF}` } };
++ for (const group of ['system', 'place', 'harness']) {
++ const r = await t.get(`/api/v1/ai/performance?group=${group}&days=30`, cookie);
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.headers.get('cache-control'), 'private, no-store');
++ assert.ok(r.json().rows.length >= 1, group);
++ for (const row of r.json().rows) {
++ const v = contracts.validate('report.ai-performance@1', row);
++ assert.strictEqual(v.valid, true, group + ' ' + JSON.stringify(v.errors));
++ }
++ }
++ const systems = (await t.get('/api/v1/ai/performance?group=system&days=30', cookie)).json().rows;
++ const ai = systems.find((r) => r.key === 'ai');
++ assert.strictEqual(ai.calls, 39, 'the 20 public-model and 19 floor-model calls are in the ai system');
++ const service = await t.get('/api/v1/ai/performance?group=place&days=30', { headers: { authorization: 'Bearer ' + serviceToken(['report.ai_usage.read']) } });
++ assert.strictEqual(service.status, 200, service.text);
++ assert.strictEqual(service.headers.get('cache-control'), 'private, no-store');
++ assert.ok(service.json().rows.some((r) => r.key === 'provider'));
++ });
++
++ await check('a task is its calls: count, cost and time from task_ms when given, else the summed latency', async () => {
++ const add = async (over) => {
++ const e = usageEvent(over);
++ const d = delivery(e);
++ const r = await t.get('/internal/events', { method: 'POST', headers: d.headers, body: d.raw });
++ assert.strictEqual(r.status, 200, r.text);
++ };
++ await add({ model: 'task-model', system: 'harness:codex', harness: 'codex', task: 'code.turn', task_id: 'task_A', task_ms: null, latency_ms: 700, charged_usd: 0.01 });
++ await add({ model: 'task-model', system: 'harness:codex', harness: 'codex', task: 'code.turn', task_id: 'task_A', task_ms: 1500, latency_ms: 800, charged_usd: 0.02 });
++ await add({ model: 'task-model', system: 'harness:codex', harness: 'codex', task: 'code.turn', task_id: 'task_B', task_ms: 900, latency_ms: 500, charged_usd: 0.005 });
++ const stored = (await db.query("SELECT * FROM report_ai_task_days WHERE task = 'code.turn'")).rows[0];
++ assert.strictEqual(Number(stored.tasks), 2, 'two task ids');
++ const cost = perf.asHist(stored.cost_hist);
++ const time = perf.asHist(stored.time_hist);
++ assert.strictEqual(cost.reduce((a, b) => a + b, 0), 2);
++ assert.strictEqual(time.reduce((a, b) => a + b, 0), 2);
++ assert.strictEqual(cost[perf.bucketOf(0.005, perf.HISTOGRAM_BOUNDS.task_cost_usd)], 1);
++ assert.strictEqual(cost[perf.bucketOf(0.03, perf.HISTOGRAM_BOUNDS.task_cost_usd)], 1);
++ assert.strictEqual(time[perf.bucketOf(900, perf.HISTOGRAM_BOUNDS.task_time_ms)], 1);
++ assert.strictEqual(time[perf.bucketOf(1500, perf.HISTOGRAM_BOUNDS.task_time_ms)], 1);
++ const r = await t.get('/api/v1/ai/performance?group=task&days=30', { headers: { cookie: `ov_token=${STAFF}` } });
++ assert.strictEqual(r.status, 200, r.text);
++ const row = r.json().rows.find((x) => x.key === 'code.turn');
++ assert.ok(row, 'the task kind is answered');
++ assert.strictEqual(row.tasks.count, 2);
++ assert.strictEqual(row.tasks.time_per_task_ms.p50, perf.HISTOGRAM_BOUNDS.task_time_ms[perf.bucketOf(900, perf.HISTOGRAM_BOUNDS.task_time_ms)], 'the middle task is 900 ms, answered as its bucket upper bound');
++ assert.strictEqual(row.tasks.time_per_task_ms.p95, perf.HISTOGRAM_BOUNDS.task_time_ms[perf.bucketOf(1500, perf.HISTOGRAM_BOUNDS.task_time_ms)]);
++ assert.strictEqual(row.tasks.cost_per_task_usd.p50, perf.HISTOGRAM_BOUNDS.task_cost_usd[perf.bucketOf(0.005, perf.HISTOGRAM_BOUNDS.task_cost_usd)]);
++ const v = contracts.validate('report.ai-performance@1', row);
++ assert.strictEqual(v.valid, true, JSON.stringify(v.errors));
++ const systems = (await t.get('/api/v1/ai/performance?group=system&days=30', { headers: { cookie: `ov_token=${STAFF}` } })).json().rows;
++ const codex = systems.find((x) => x.key === 'harness:codex');
++ assert.strictEqual(codex.calls, 3);
++ assert.strictEqual(codex.tasks.count, 2);
++ assert.strictEqual(codex.tasks.time_per_task_ms.p50, perf.HISTOGRAM_BOUNDS.task_time_ms[perf.bucketOf(900, perf.HISTOGRAM_BOUNDS.task_time_ms)]);
++ });
++ } finally { await t.close(); }
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });