openvibers/openvibe.ai Commits

Add audited owner GPU supply policy (#482)

OP OpenVibers committed 1 days ago · df7bd0eeef · 5 files, +223 −1

Changes

ddiff --git a/migrations/0078_gpu_supply_policy.sql b/migrations/0078_gpu_supply_policy.sql
nnew file mode 100644
iindex 0000000..19879dc
---- /dev/null
++++ b/migrations/0078_gpu_supply_policy.sql
@@@ -0,0 +1,9 @@
++-- phase: expand
++-- Owner GPU supply policy. Rows start absent and therefore disabled and quote-only.
++-- Credentials stay in the sealed ai.gpus config namespace, never in this table.
++CREATE TABLE IF NOT EXISTS gpu_supply_policy (
++ scope text COLLATE "C" PRIMARY KEY,
++ settings text COLLATE "C" NOT NULL,
++ updated_by text COLLATE "C" NOT NULL,
++ updated_at text COLLATE "C" NOT NULL
++);
ddiff --git a/server/api/console-data.js b/server/api/console-data.js
iindex 9d252df..6e53f1f 100644
---- a/server/api/console-data.js
++++ b/server/api/console-data.js
@@@ -16,6 +16,7 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
const providerOrder = require('../providers/order').createProviderOrder({ registry });
const supplyManager = supply || require('../platform/supply').createSupply({ db, registry, clock });
const ownerConfig = require('../owner-config').createOwnerConfig({ db, registry, clock });
++ const gpuPolicy = settings?.gpus ? require('../gpu/policy').createGpuPolicy({ db, settings, clock }) : null;
// The public marketplace readings and the per-token cost they imply, added to both console sections. Data only:
// Exchange renders it with its existing console classes.
const marketPayload = async () => {
@@@ -38,6 +39,13 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
const subject = String(req.get('x-ov-subject') || '');
return /^usr_[0-9A-HJKMNP-TV-Z]{26}$/.test(subject) ? subject : req.principal.sub;
};
++ // Exchange verifies the person's Network role before proxying this owner action. Only its service
++ // principal may cross this boundary, and it must name the owner whose audit row receives the change.
++ const gpuOwner = (req, res, next) => {
++ if (req.principal?.sub !== 'svc:exchange' || req.get('x-ov-role') !== 'owner' || !/^usr_[0-9A-HJKMNP-TV-Z]{26}$/.test(String(req.get('x-ov-subject') || '')))
++ return res.status(403).json({ code: 'gpu.owner_only', error: 'Only the owner may change GPU supply settings.' });
++ return next();
++ };
const wrap = (fn) => async (req, res) => {
try { res.set('Cache-Control', 'no-store').json(await fn(req)); }
catch (err) { log.error(`[console-json] ${req.method} ${req.path} failed (${err && err.code || 'ai.internal'})`); res.status(err.status || 500).json({ error: 'AI admin data is unavailable.', code: err.code || 'ai.internal' }); }
@@@ -52,6 +60,7 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
}));
r.get('/api/v1/admin/console/:section', read, wrap(async (req) => {
const section = req.params.section;
++ if (section === 'gpu-policy') return gpuPolicy ? gpuPolicy.read() : { global: null, providers: [] };
if (section === 'supply') return { ...await supplyManager.report(), ...await marketPayload() };
if (section === 'overview') {
const since = new Date(clock.now() - 86400000).toISOString();
@@@ -155,6 +164,14 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
}));
r.post('/api/v1/admin/console/supply/quotes', json, write, wrap(async req => ({ recorded: await require('../pricebook/gpus').recordGpuQuotes(db, JSON.stringify(req.body.quotes)), dry_run: true })));
r.post('/api/v1/admin/console/supply', json, write, wrap(req => supplyManager.save(req.body || {}, actor(req))));
++ r.post('/api/v1/admin/console/gpu-policy/:scope', json, write, gpuOwner, wrap(req => {
++ if (!gpuPolicy) throw new AiError(503, 'gpu.unavailable', 'GPU supply settings are unavailable');
++ return gpuPolicy.save(req.params.scope, req.body || {}, actor(req), req.ov?.traceId);
++ }));
++ r.post('/api/v1/admin/console/gpu-policy/:scope/key', json, write, gpuOwner, wrap(req => {
++ if (!gpuPolicy) throw new AiError(503, 'gpu.unavailable', 'GPU supply settings are unavailable');
++ return gpuPolicy.setKey(req.params.scope, req.body?.api_key, actor(req), req.ov?.traceId);
++ }));
r.post('/api/v1/admin/console/orders/:capability', json, write, wrap(req => providerOrder.save(req.params.capability, req.body, { actor: actor(req), trace: req.traceId })));
r.post('/api/v1/admin/console/owner-inference/spend-caps', json, write, wrap(async (req) => ({ cap: await ownerConfig.setCap(req.body || {}, { actor: actor(req), trace: req.ov && req.ov.traceId }) })));
r.post('/api/v1/admin/console/owner-inference/routing', json, write, wrap(async (req) => ({ routing: await ownerConfig.setRouting(req.body || {}, { actor: actor(req), trace: req.ov && req.ov.traceId }) })));
ddiff --git a/server/gpu/policy.js b/server/gpu/policy.js
nnew file mode 100644
iindex 0000000..77006cc
---- /dev/null
++++ b/server/gpu/policy.js
@@@ -0,0 +1,120 @@
++'use strict';
++
++const { AiError } = require('../util');
++const money = require('./money');
++
++const PROVIDERS = Object.freeze(['vast', 'runpod', 'salad', 'tensordock', 'shadeform', 'verda', 'thunder', 'runpod_serverless', 'reserved']);
++const DEFAULTS = Object.freeze({
++ enabled: false, order: 50, classes: ['interactive', 'batch'], interruptible_allowed: false,
++ max_hourly_nano: null, max_replicas: 1, regions: [], monthly_cap_share_percent: 100,
++});
++const GLOBAL = Object.freeze({ quote_only: true, interactive_cap_nano: null, batch_cap_nano: null });
++const OWNED = new Set(['svc:exchange']);
++
++function invalid(message) { throw new AiError(422, 'gpu.policy_invalid', message); }
++function nano(value, name) {
++ if (value == null || value === '') return null;
++ const n = money.nanoCount(value, name);
++ if (n < 0n || n > 1000000000000000n) invalid(name + ' is out of range');
++ return n.toString();
++}
++function providerInput(body, current, provider) {
++ const out = { ...current };
++ const allowed = new Set(Object.keys(DEFAULTS));
++ for (const [k, v] of Object.entries(body)) {
++ if (!allowed.has(k)) invalid('unknown provider setting ' + k);
++ if (k === 'enabled' || k === 'interruptible_allowed') {
++ if (typeof v !== 'boolean') invalid(k + ' must be true or false');
++ out[k] = v;
++ } else if (k === 'order') {
++ if (!Number.isInteger(v) || v < 0 || v > 1000) invalid('order must be 0-1000');
++ out[k] = v;
++ } else if (k === 'max_replicas') {
++ if (!Number.isInteger(v) || v < 1 || v > 100) invalid('max_replicas must be 1-100');
++ out[k] = v;
++ } else if (k === 'monthly_cap_share_percent') {
++ if (!Number.isInteger(v) || v < 0 || v > 100) invalid('monthly_cap_share_percent must be 0-100');
++ out[k] = v;
++ } else if (k === 'max_hourly_nano') out[k] = nano(v, k);
++ else if (k === 'classes') {
++ if (!Array.isArray(v) || !v.length || v.length > 2 || v.some(x => !['interactive', 'batch'].includes(x)) || new Set(v).size !== v.length) invalid('classes must contain interactive or batch');
++ out[k] = [...v];
++ } else if (k === 'regions') {
++ if (!Array.isArray(v) || v.length > 20 || v.some(x => typeof x !== 'string' || !/^[A-Za-z0-9_-]{2,32}$/.test(x))) invalid('regions must be region codes');
++ out[k] = [...new Set(v.map(x => x.toUpperCase()))];
++ }
++ }
++ if (provider === 'salad' && !Object.hasOwn(body, 'max_replicas') && current === DEFAULTS) out.max_replicas = 3;
++ if (provider === 'salad' && out.classes.includes('interactive')) invalid('SaladCloud is batch only');
++ if (provider === 'reserved' && out.interruptible_allowed) invalid('a reserved server cannot be interruptible');
++ return out;
++}
++function globalInput(body, current) {
++ const out = { ...current };
++ for (const [k, v] of Object.entries(body)) {
++ if (k === 'quote_only') { if (typeof v !== 'boolean') invalid('quote_only must be true or false'); out[k] = v; }
++ else if (['monthly_cap_nano', 'interactive_cap_nano', 'batch_cap_nano'].includes(k)) out[k] = nano(v, k);
++ else invalid('unknown global setting ' + k);
++ }
++ const cap = BigInt(out.monthly_cap_nano || 0);
++ for (const key of ['interactive_cap_nano', 'batch_cap_nano']) if (out[key] != null && BigInt(out[key]) > cap) invalid(key + ' exceeds the monthly cap');
++ return out;
++}
++
++function createGpuPolicy({ db, settings, clock = { now: () => Date.now() } }) {
++ async function row(scope) {
++ const found = await db.prepare('SELECT settings FROM gpu_supply_policy WHERE scope = ?').get(scope);
++ return found ? JSON.parse(found.settings) : null;
++ }
++ async function read() {
++ const values = settings.gpus.values();
++ const summary = await settings.gpus.store.summary();
++ const global = { ...GLOBAL, ...(await row('global') || {}), monthly_cap_nano: values.monthly_cap };
++ const providers = [];
++ for (const key of PROVIDERS) {
++ const policy = { ...DEFAULTS, ...(key === 'salad' ? { classes: ['batch'], max_replicas: 3 } : {}), ...(await row(key) || {}) };
++ const marker = summary.secrets && summary.secrets[`${key === 'runpod_serverless' ? 'runpod' : key}.key`];
++ providers.push({ key, ...policy, key_state: key === 'reserved' ? { required: false } : { required: true, present: Boolean(marker), last_four: marker?.hint || null } });
++ }
++ return { global, providers };
++ }
++ async function save(scope, body, actor, trace = null) {
++ if (scope !== 'global' && !PROVIDERS.includes(scope)) invalid('unknown GPU provider');
++ if (!body || typeof body !== 'object' || Array.isArray(body)) invalid('settings must be an object');
++ const before = scope === 'global'
++ ? { ...GLOBAL, monthly_cap_nano: settings.gpus.values().monthly_cap, ...(await row('global') || {}) }
++ : { ...DEFAULTS, ...(scope === 'salad' ? { classes: ['batch'], max_replicas: 3 } : {}), ...(await row(scope) || {}) };
++ const after = scope === 'global' ? globalInput(body, before) : providerInput(body, before, scope);
++ const changes = Object.keys(after).filter(k => JSON.stringify(before[k]) !== JSON.stringify(after[k]));
++ if (!changes.length) return scope === 'global' ? (await read()).global : (await read()).providers.find(x => x.key === scope);
++ // The existing GPU budget remains the one cap the provisioner checks.
++ if (scope === 'global' && changes.includes('monthly_cap_nano')) {
++ await settings.gpus.store.apply({ monthly_cap: Number(after.monthly_cap_nano || 0) }, { merge: true, actor: { type: 'user', id: actor }, reason: 'GPU supply monthly cap' });
++ }
++ if (scope === 'global' && changes.includes('quote_only')) {
++ await settings.gpus.store.apply({ dry_run: after.quote_only }, { merge: true, actor: { type: 'user', id: actor }, reason: 'GPU supply quote-only switch' });
++ }
++ const persist = { ...after };
++ if (scope === 'global') delete persist.monthly_cap_nano;
++ const at = new Date(clock.now()).toISOString();
++ await db.tx(async tdb => {
++ await tdb.prepare('INSERT INTO gpu_supply_policy(scope,settings,updated_by,updated_at) VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET settings=excluded.settings,updated_by=excluded.updated_by,updated_at=excluded.updated_at').run(scope, JSON.stringify(persist), actor, at);
++ for (const setting of changes) await tdb.prepare('INSERT INTO audit_log(at,actor,action,target_type,target_id,trace_id,metadata) VALUES(?,?,?,?,?,?,?)')
++ .run(at, actor, 'gpu.policy.change', 'gpu_supply_policy', scope, trace, JSON.stringify({ setting: `${scope}.${setting}`, old: before[setting] ?? null, new: after[setting] ?? null }));
++ });
++ return scope === 'global' ? (await read()).global : (await read()).providers.find(x => x.key === scope);
++ }
++ async function setKey(provider, value, actor, trace = null) {
++ if (!PROVIDERS.includes(provider) || provider === 'reserved') invalid('unknown keyed GPU provider');
++ const key = provider === 'runpod_serverless' ? 'runpod' : provider;
++ if (typeof value !== 'string' || value.length < 8 || value.length > 400 || /\s/.test(value)) invalid('key must be 8-400 characters without spaces');
++ const old = Boolean((await settings.gpus.store.summary()).secrets?.[`${key}.key`]);
++ const marker = await settings.gpus.store.setSecret(`${key}.key`, value, { actor: { type: 'user', id: actor }, reason: 'GPU provider key' });
++ await db.prepare('INSERT INTO audit_log(at,actor,action,target_type,target_id,trace_id,metadata) VALUES(?,?,?,?,?,?,?)')
++ .run(new Date(clock.now()).toISOString(), actor, 'gpu.policy.change', 'gpu_supply_policy', provider, trace, JSON.stringify({ setting: `${provider}.key`, old, new: true }));
++ return { present: true, last_four: marker.hint || null };
++ }
++ return { read, save, setKey };
++}
++
++module.exports = { createGpuPolicy, PROVIDERS, DEFAULTS, GLOBAL };
ddiff --git a/server/settings.js b/server/settings.js
iindex 86da0aa..bc7fa9d 100644
---- a/server/settings.js
++++ b/server/settings.js
@@@ -30,7 +30,12 @@ const PROVIDER_GROUP = 'Provider keys';
const GPU_GROUP = 'OpenVibe GPUs';
// The marketplaces the GPU provisioner rents from. AI owns these origins and writes them into the
// namespace, so the owner's key is sealed to the marketplace it was entered for.
--const GPU_ORIGINS = Object.freeze({ vast: 'https://console.vast.ai', runpod: 'https://api.runpod.io' });
++const GPU_ORIGINS = Object.freeze({
++ vast: 'https://console.vast.ai', runpod: 'https://api.runpod.io',
++ salad: 'https://api.salad.com', tensordock: 'https://dashboard.tensordock.com',
++ shadeform: 'https://api.shadeform.ai', verda: 'https://api.verda.com',
++ thunder: 'https://api.thundercompute.com:8443',
++});
/**
* Bounds from server/config.js (load()), so the schema and the code agree. Every field has a title in plain words and
@@@ -233,6 +238,12 @@ const GPU_SCHEMA = Object.freeze({
},
'x-required-for': 'Renting GPUs and own endpoints',
},
++ ...Object.fromEntries(Object.entries(GPU_ORIGINS).filter(([name]) => !['vast', 'runpod'].includes(name)).flatMap(([name]) => [
++ [`${name}.origin`, { type: 'string', title: `${name} API origin`, 'x-group': GPU_GROUP, 'x-advanced': true,
++ description: `AI locks the ${name} key to this provider-owned API origin.` }],
++ [`${name}.key`, { type: 'string', title: `${name} API key`, 'x-group': GPU_GROUP, 'x-bind': `${name}.origin`,
++ description: `The owner's write-only ${name} API key. Only its last four characters are shown after saving.` }],
++ ])),
monthly_cap: {
type: 'integer', title: 'Most the GPUs may cost in a month', 'x-group': GPU_GROUP, 'x-unit': 'nano_usd', 'x-danger': true,
minimum: 0, 'x-example': 50000000000,
@@@ -475,6 +486,7 @@ async function createGpuSettings({ db, env = process.env, log = console, onChang
vast_key: String(v['vast.key'] || ''),
runpod_origin: String(v['runpod.origin'] || ''),
runpod_key: String(v['runpod.key'] || ''),
++ provider_keys: Object.fromEntries(Object.keys(GPU_ORIGINS).map(name => [name, String(v[`${name}.key`] || '')])),
monthly_cap: v.monthly_cap == null ? null : String(v.monthly_cap),
idle_minutes: v.idle_minutes == null ? null : Number(v.idle_minutes),
max_runtime_minutes: v.max_runtime_minutes == null ? null : Number(v.max_runtime_minutes),
ddiff --git a/test/gpu-policy.test.js b/test/gpu-policy.test.js
nnew file mode 100644
iindex 0000000..ca20692
---- /dev/null
++++ b/test/gpu-policy.test.js
@@@ -0,0 +1,64 @@
++'use strict';
++
++const assert = require('assert');
++const express = require('express');
++const { createServer } = require('http');
++const { boot, suite } = require('./helpers');
++const { createGpuPolicy } = require('../server/gpu/policy');
++const { GPU_SCHEMA } = require('../server/settings');
++const { consoleDataRouter } = require('../server/api/console-data');
++
++const t = suite('gpu-policy');
++const owner = 'usr_' + 'A'.repeat(26);
++
++t.test('owner policy saves every changed field with an old and new audit value and keeps keys sealed', async () => {
++ const h = await boot();
++ try {
++ const policy = createGpuPolicy({ db: h.db, settings: h.settings, clock: { now: () => Date.parse('2026-11-11T12:00:00Z') } });
++ const first = await policy.read();
++ assert.ok(GPU_SCHEMA.properties['salad.key'] && GPU_SCHEMA.properties['thunder.key'], 'new provider keys use the sealed config schema');
++ assert.strictEqual(first.global.quote_only, true);
++ assert.strictEqual(first.providers.find(p => p.key === 'salad').max_replicas, 3);
++ assert.strictEqual(first.providers.find(p => p.key === 'vast').enabled, false);
++ await policy.save('global', { monthly_cap_nano: '50000000000', interactive_cap_nano: '20000000000', batch_cap_nano: '30000000000' }, owner);
++ await policy.save('global', { quote_only: false }, owner);
++ assert.strictEqual(h.settings.gpus.values().dry_run, false);
++ await policy.save('vast', { enabled: true, order: 3, classes: ['batch'], interruptible_allowed: true,
++ max_hourly_nano: '500000000', max_replicas: 2, regions: ['US'], monthly_cap_share_percent: 40 }, owner);
++ const marker = await policy.setKey('vast', 'test-provider-key-4321', owner);
++ assert.strictEqual(marker.last_four, '4321');
++ const after = await policy.read();
++ assert.strictEqual(after.global.monthly_cap_nano, '50000000000');
++ assert.strictEqual(after.providers.find(p => p.key === 'vast').key_state.last_four, '4321');
++ assert.strictEqual(JSON.stringify(after).includes('test-provider-key-4321'), false);
++ const rows = await h.db.prepare("SELECT actor,at,metadata FROM audit_log WHERE action = 'gpu.policy.change' ORDER BY id").all();
++ assert.ok(rows.length >= 12);
++ assert.ok(rows.every(r => r.actor === owner && r.at === '2026-11-11T12:00:00.000Z'));
++ assert.ok(rows.some(r => JSON.parse(r.metadata).setting === 'vast.order' && JSON.parse(r.metadata).old === 50 && JSON.parse(r.metadata).new === 3));
++ assert.strictEqual(JSON.stringify(rows).includes('test-provider-key-4321'), false);
++ await assert.rejects(() => policy.save('salad', { classes: ['interactive'] }, owner), /batch only/);
++ } finally { await h.stop(); }
++});
++
++t.test('AI rejects non-owner roles and other service principals', async () => {
++ const app = express();
++ const auth = { requireCap: () => (req, _res, next) => { req.principal = { sub: req.get('x-test-service') || 'svc:exchange' }; req.ov = { traceId: 'test' }; next(); } };
++ app.use(consoleDataRouter({ db: {}, registry: {}, pool: {}, quotas: {}, cache: {}, runs: {}, ops: {}, auth }));
++ const server = createServer(app);
++ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
++ try {
++ const base = `http://127.0.0.1:${server.address().port}`;
++ for (const role of ['admin', 'global_mod', 'user', '']) {
++ const res = await fetch(base + '/api/v1/admin/console/gpu-policy/vast', { method: 'POST', headers: {
++ 'content-type': 'application/json', 'x-ov-subject': owner, 'x-ov-role': role,
++ }, body: JSON.stringify({ enabled: true }) });
++ assert.strictEqual(res.status, 403, role);
++ }
++ const other = await fetch(base + '/api/v1/admin/console/gpu-policy/vast', { method: 'POST', headers: {
++ 'content-type': 'application/json', 'x-ov-subject': owner, 'x-ov-role': 'owner', 'x-test-service': 'svc:other',
++ }, body: JSON.stringify({ enabled: true }) });
++ assert.strictEqual(other.status, 403);
++ } finally { await new Promise(resolve => server.close(resolve)); }
++});
++
++t.run();