openvibers/openvibe.host Commits
Add pinned llama.cpp inference worker and honest cost model (#279)
Changes
nnew file mode 100755@@@ -0,0 +1,280 @@++#!/usr/bin/env node++'use strict';++/**++ * Inference cost model (PLAN.md §E): cost per request and cost per successful task for++ *++ * rented-warm one GPU kept for the billing window; idle time is allocated++ * scale-to-zero serverless per-second execution plus a cold start++ * api-wholesale per-token wholesale API rates, all-in (no storage or egress line)++ *++ * The inputs are a planning estimate, not a quote. This script never reads a provider API and++ * never invents a benchmark: every number comes from the JSON file the operator passes in.++ * Customer liability is Billing's integer nano-USD ledger (D1); this script uses floats only++ * for forecast arithmetic and says so in its output.++ *++ * node scripts/inference-cost-model.js [input.json] [--json]++ *++ * The default input is the clearly labelled example in++ * workers/llm-inference/examples/inference-cost-model.example.json.++ */++const fs = require('fs');++const path = require('path');++++const HOURS_PER_MONTH = 730;++++function fail(where, message) {++ throw new Error(`${where}: ${message}`);++}++++function object(input, key) {++ const value = input[key];++ if (value === null || typeof value !== 'object' || Array.isArray(value)) fail(key, 'must be an object');++ return value;++}++++function number(input, where, key, { required = true, min = 0, max = Infinity, integer = false, fallback } = {}) {++ const value = input == null ? undefined : input[key];++ if (value === undefined || value === null) {++ if (!required) return fallback;++ fail(where, 'is required');++ }++ if (typeof value !== 'number' || !Number.isFinite(value)) fail(where, 'must be a finite number');++ if (integer && !Number.isInteger(value)) fail(where, 'must be an integer');++ if (value < min) fail(where, `must be >= ${min}`);++ if (value > max) fail(where, `must be <= ${max}`);++ return value;++}++++function rate(input, key, fallback) {++ return number(input, key, key, { min: 0, required: fallback === undefined, fallback });++}++++/**++ * Validate an input document against workers/llm-inference/... and++ * scripts/inference-cost-model.schema.json. Throws on the first problem.++ */++function validate(input) {++ if (input === null || typeof input !== 'object' || Array.isArray(input)) throw new Error('input: must be a JSON object');++++ const window = object(input, 'window');++ number(window, 'window.hours', 'hours', { min: Number.MIN_VALUE });++ number(window, 'window.requests', 'requests', { min: 1, integer: true });++++ const request = object(input, 'request');++ for (const key of ['input_tokens', 'cached_read_tokens', 'cache_write_tokens', 'output_tokens']) {++ number(request, `request.${key}`, key, { min: 0, integer: true });++ }++ number(request, 'request.success_rate', 'success_rate', { min: Number.MIN_VALUE, max: 1 });++++ const throughput = object(input, 'throughput');++ number(throughput, 'throughput.prefill_tokens_per_sec', 'prefill_tokens_per_sec', { min: Number.MIN_VALUE });++ number(throughput, 'throughput.decode_tokens_per_sec', 'decode_tokens_per_sec', { min: Number.MIN_VALUE });++ number(throughput, 'throughput.cached_read_tokens_per_sec', 'cached_read_tokens_per_sec', {++ required: false, min: Number.MIN_VALUE, fallback: throughput.prefill_tokens_per_sec,++ });++ number(throughput, 'throughput.concurrency', 'concurrency', { required: false, min: 1, integer: true, fallback: 1 });++++ const gpu = object(input, 'gpu');++ number(gpu, 'gpu.hourly_usd', 'hourly_usd', { min: 0 });++ number(gpu, 'gpu.occupancy', 'occupancy', { min: Number.MIN_VALUE, max: 1 });++ const policy = gpu.idle_policy;++ if (!['billed', 'standby', 'stopped'].includes(policy)) fail('gpu.idle_policy', "must be 'billed', 'standby' or 'stopped'");++ number(gpu, 'gpu.standby_hourly_usd', 'standby_hourly_usd', { required: policy === 'standby', min: 0, fallback: 0 });++ number(gpu, 'gpu.startup_seconds', 'startup_seconds', { min: 0 });++ number(gpu, 'gpu.storage_gb', 'storage_gb', { min: 0 });++ number(gpu, 'gpu.storage_usd_per_gb_month', 'storage_usd_per_gb_month', { min: 0 });++ number(gpu, 'gpu.egress_gb_per_request', 'egress_gb_per_request', { min: 0 });++ number(gpu, 'gpu.egress_usd_per_gb', 'egress_usd_per_gb', { min: 0 });++++ const serverless = object(input, 'serverless');++ number(serverless, 'serverless.hourly_usd', 'hourly_usd', { min: 0 });++ number(serverless, 'serverless.startup_seconds', 'startup_seconds', { min: 0 });++ number(serverless, 'serverless.requests_per_cold_start', 'requests_per_cold_start', { min: 1, fallback: 1, required: false });++ number(serverless, 'serverless.storage_gb', 'storage_gb', { min: 0 });++ number(serverless, 'serverless.storage_usd_per_gb_month', 'storage_usd_per_gb_month', { min: 0 });++ number(serverless, 'serverless.egress_gb_per_request', 'egress_gb_per_request', { min: 0 });++ number(serverless, 'serverless.egress_usd_per_gb', 'egress_usd_per_gb', { min: 0 });++++ const api = object(input, 'api_wholesale');++ for (const key of ['input_usd_per_million', 'cached_read_usd_per_million', 'cache_write_usd_per_million', 'output_usd_per_million', 'per_request_usd']) {++ number(api, `api_wholesale.${key}`, key, { min: 0 });++ }++ number(api, 'api_wholesale.success_rate', 'success_rate', { required: false, min: Number.MIN_VALUE, max: 1, fallback: request.success_rate });++++ return input;++}++++function storagePerRequest(section, window) {++ return section.storage_gb * section.storage_usd_per_gb_month * (window.hours / HOURS_PER_MONTH) / window.requests;++}++++function egressPerRequest(section) {++ return section.egress_gb_per_request * section.egress_usd_per_gb;++}++++function round(value) {++ // Strip binary noise from the report without pretending the estimate is exact.++ return Number(value.toPrecision(12));++}++++/**++ * Compute the three deployment costs. Every returned amount is USD.++ *++ * Prompt seconds: fresh input + cache-write tokens run at prefill throughput; cached-read tokens++ * run at the measured cached-read throughput when one is supplied, otherwise (conservatively) at++ * prefill throughput. Decode seconds: output tokens / decode throughput.++ *++ * rented-warm: steady-state busy seconds / occupancy are billed (the rest is idle), idle at the++ * policy's rate, plus one startup per window. scale-to-zero: busy + startup/requests_per_cold_start at the serverless++ * hourly rate, no idle. api-wholesale: rate * tokens / 1e6 + per-request fee.++ *++ * Cost per successful task divides the per-request cost by the success rate (expected attempts).++ */++function compute(raw) {++ const input = validate(raw);++ const { window, request, throughput, gpu, serverless, api_wholesale: api } = input;++++ const cachedTps = throughput.cached_read_tokens_per_sec === undefined++ ? throughput.prefill_tokens_per_sec++ : throughput.cached_read_tokens_per_sec;++ const prefillSeconds = (request.input_tokens + request.cache_write_tokens) / throughput.prefill_tokens_per_sec++ + request.cached_read_tokens / cachedTps;++ const decodeSeconds = request.output_tokens / throughput.decode_tokens_per_sec;++ const busySeconds = prefillSeconds + decodeSeconds;++++ // rented-warm++ const idleSeconds = busySeconds * ((1 - gpu.occupancy) / gpu.occupancy);++ const idleHourly = gpu.idle_policy === 'billed'++ ? gpu.hourly_usd++ : gpu.idle_policy === 'standby' ? gpu.standby_hourly_usd : 0;++ const rentedParts = {++ gpu_busy: busySeconds * gpu.hourly_usd / 3600,++ gpu_idle: idleSeconds * idleHourly / 3600,++ startup: gpu.startup_seconds * gpu.hourly_usd / 3600 / window.requests,++ storage: storagePerRequest(gpu, window),++ egress: egressPerRequest(gpu),++ };++ const rentedPerRequest = Object.values(rentedParts).reduce((sum, value) => sum + value, 0);++++ // scale-to-zero serverless++ const coldStartsPerRequest = 1 / serverless.requests_per_cold_start;++ const serverlessParts = {++ gpu_execution: busySeconds * serverless.hourly_usd / 3600,++ startup: serverless.startup_seconds * serverless.hourly_usd / 3600 * coldStartsPerRequest,++ storage: storagePerRequest(serverless, window),++ egress: egressPerRequest(serverless),++ };++ const serverlessPerRequest = Object.values(serverlessParts).reduce((sum, value) => sum + value, 0);++++ // wholesale API: all-in rates, no owned storage or egress++ const apiParts = {++ input: request.input_tokens * api.input_usd_per_million / 1e6,++ cached_read: request.cached_read_tokens * api.cached_read_usd_per_million / 1e6,++ cache_write: request.cache_write_tokens * api.cache_write_usd_per_million / 1e6,++ output: request.output_tokens * api.output_usd_per_million / 1e6,++ per_request: api.per_request_usd,++ };++ const apiPerRequest = Object.values(apiParts).reduce((sum, value) => sum + value, 0);++++ const rows = [++ { mode: 'rented-warm', costPerRequest: rentedPerRequest, successRate: request.success_rate, parts: rentedParts },++ { mode: 'scale-to-zero', costPerRequest: serverlessPerRequest, successRate: request.success_rate, parts: serverlessParts },++ { mode: 'api-wholesale', costPerRequest: apiPerRequest, successRate: api.success_rate, parts: apiParts },++ ].map((row) => ({++ ...row,++ costPerRequest: round(row.costPerRequest),++ costPerSuccessfulTask: round(row.costPerRequest / row.successRate),++ parts: Object.fromEntries(Object.entries(row.parts).map(([key, value]) => [key, round(value)])),++ }));++++ return {++ schema: 'openvibe.inference-cost-model-result@1',++ note: 'Planning estimate only, not a quote. Billing settles in integer nano-USD (D1); these are floats.',++ inputNote: typeof input.note === 'string' ? input.note : null,++ window: { hours: window.hours, requests: window.requests },++ request: {++ input_tokens: request.input_tokens,++ cached_read_tokens: request.cached_read_tokens,++ cache_write_tokens: request.cache_write_tokens,++ output_tokens: request.output_tokens,++ success_rate: request.success_rate,++ },++ seconds: {++ prefill: round(prefillSeconds),++ decode: round(decodeSeconds),++ busy: round(busySeconds),++ rented_idle: round(idleSeconds),++ concurrency: throughput.concurrency === undefined ? 1 : throughput.concurrency,++ },++ rows,++ };++}++++function money(value) {++ if (value === 0) return '$0';++ if (value < 1e-6) return `$${value.toExponential(2)}`;++ return `$${value.toFixed(6)}`;++}++++function formatTable(result) {++ const r = result.request;++ const lines = [++ 'Inference cost model — planning estimate, not a quote',++ `Window: ${result.window.hours} h, ${result.window.requests} requests`,++ `Request mix: I=${r.input_tokens} R=${r.cached_read_tokens} W=${r.cache_write_tokens} O=${r.output_tokens} tokens, success ${(r.success_rate * 100).toFixed(2)}%`,++ `Throughput: ${result.seconds.prefill} s prefill + ${result.seconds.decode} s decode = ${result.seconds.busy} s GPU-busy per request (concurrency ${result.seconds.concurrency})`,++ '',++ 'Mode $/request $/successful task success',++ ];++ for (const row of result.rows) {++ lines.push(`${row.mode.padEnd(16)} ${money(row.costPerRequest).padStart(13)} ${money(row.costPerSuccessfulTask).padStart(18)} ${(row.successRate * 100).toFixed(2)}%`);++ }++ lines.push('', 'Breakdown (USD per request):');++ for (const row of result.rows) {++ const parts = Object.entries(row.parts).map(([key, value]) => `${key}=${money(value)}`).join(' ');++ lines.push(` ${row.mode.padEnd(16)} ${parts}`);++ }++ if (result.inputNote) lines.push('', `Input note: ${result.inputNote}`);++ lines.push('', 'Floats are for forecasting only; Billing settles in integer nano-USD. Verify every rate and measured throughput before quoting.');++ return lines.join('\n');++}++++function usage() {++ return [++ 'Usage: node scripts/inference-cost-model.js [input.json] [--json]',++ '',++ ' input.json defaults to workers/llm-inference/examples/inference-cost-model.example.json',++ ' --json print the computed result as JSON instead of a table',++ '',++ 'The input schema is scripts/inference-cost-model.schema.json.',++ ].join('\n');++}++++function main(argv) {++ let file = null;++ let asJson = false;++ for (const arg of argv) {++ if (arg === '--json') asJson = true;++ else if (arg === '--help' || arg === '-h') { console.log(usage()); return 0; }++ else if (arg.startsWith('-')) throw new Error(`unknown option: ${arg}\n${usage()}`);++ else if (file) throw new Error(`only one input file is accepted\n${usage()}`);++ else file = arg;++ }++ if (!file) file = path.join(__dirname, '..', 'workers', 'llm-inference', 'examples', 'inference-cost-model.example.json');++ const input = JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));++ const result = compute(input);++ if (asJson) console.log(JSON.stringify(result, null, 2));++ else console.log(formatTable(result));++ return 0;++}++++if (require.main === module) {++ try {++ process.exitCode = main(process.argv.slice(2));++ } catch (err) {++ console.error(`inference-cost-model: ${err.message}`);++ process.exitCode = 1;++ }++}++++module.exports = { compute, validate, formatTable, money, HOURS_PER_MONTH };nnew file mode 100644@@@ -0,0 +1,241 @@++{++ "$schema": "https://json-schema.org/draft/2020-12/schema",++ "$id": "https://openvibe.network/schemas/inference-cost-model.v1.json",++ "title": "InferenceCostModelInput",++ "description": "Inputs for scripts/inference-cost-model.js (PLAN.md §E). Every price and throughput is an operator-supplied placeholder or measurement; this schema cannot tell a quote from a guess, so the input must say which it is in `note`.",++ "type": "object",++ "required": ["window", "request", "throughput", "gpu", "serverless", "api_wholesale"],++ "additionalProperties": false,++ "properties": {++ "$schema": {++ "type": "string",++ "description": "Optional editor association with this schema file."++ },++ "note": {++ "type": "string",++ "description": "Provenance, for example 'example, not a quote' or the quote ids and date checked."++ },++ "window": {++ "type": "object",++ "required": ["hours", "requests"],++ "additionalProperties": false,++ "properties": {++ "hours": {++ "type": "number",++ "exclusiveMinimum": 0,++ "description": "Length of the measurement or billing window in hours."++ },++ "requests": {++ "type": "integer",++ "minimum": 1,++ "description": "Requests in the window. Startup and storage are amortized over this count for rented-warm."++ }++ }++ },++ "request": {++ "type": "object",++ "required": ["input_tokens", "cached_read_tokens", "cache_write_tokens", "output_tokens", "success_rate"],++ "additionalProperties": false,++ "properties": {++ "input_tokens": {++ "type": "integer",++ "minimum": 0,++ "description": "I: fresh input tokens per request."++ },++ "cached_read_tokens": {++ "type": "integer",++ "minimum": 0,++ "description": "R: input tokens read from a prompt cache per request."++ },++ "cache_write_tokens": {++ "type": "integer",++ "minimum": 0,++ "description": "W: input tokens written to a prompt cache per request."++ },++ "output_tokens": {++ "type": "integer",++ "minimum": 0,++ "description": "O: generated output tokens per request."++ },++ "success_rate": {++ "type": "number",++ "exclusiveMinimum": 0,++ "maximum": 1,++ "description": "Fraction of requests that produce a successful task (0 < rate <= 1). Cost per successful task divides by this."++ }++ }++ },++ "throughput": {++ "type": "object",++ "required": ["prefill_tokens_per_sec", "decode_tokens_per_sec"],++ "additionalProperties": false,++ "properties": {++ "prefill_tokens_per_sec": {++ "type": "number",++ "exclusiveMinimum": 0,++ "description": "Measured prompt-processing throughput at the stated concurrency."++ },++ "decode_tokens_per_sec": {++ "type": "number",++ "exclusiveMinimum": 0,++ "description": "Measured output throughput at the stated concurrency."++ },++ "cached_read_tokens_per_sec": {++ "type": "number",++ "exclusiveMinimum": 0,++ "description": "Optional measured throughput for cached-read tokens. Omit to charge them at prefill throughput (conservative)."++ },++ "concurrency": {++ "type": "integer",++ "minimum": 1,++ "default": 1,++ "description": "Concurrency at which the two throughput numbers were measured. It is recorded, not used to multiply throughput."++ }++ }++ },++ "gpu": {++ "type": "object",++ "required": ["hourly_usd", "occupancy", "idle_policy", "startup_seconds", "storage_gb", "storage_usd_per_gb_month", "egress_gb_per_request", "egress_usd_per_gb"],++ "additionalProperties": false,++ "properties": {++ "hourly_usd": {++ "type": "number",++ "minimum": 0,++ "description": "Rented GPU rate in USD per GPU-hour, from a complete quote."++ },++ "occupancy": {++ "type": "number",++ "exclusiveMinimum": 0,++ "maximum": 1,++ "description": "Steady-state fraction of paid GPU wall time that is busy, excluding the one-time startup. Billed steady-state time is busy / occupancy."++ },++ "idle_policy": {++ "type": "string",++ "enum": ["billed", "standby", "stopped"],++ "description": "What idle time costs: the GPU rate, standby_hourly_usd, or nothing."++ },++ "standby_hourly_usd": {++ "type": "number",++ "minimum": 0,++ "default": 0,++ "description": "Idle rate when idle_policy is 'standby' (required in that case)."++ },++ "startup_seconds": {++ "type": "number",++ "minimum": 0,++ "description": "Cold start per window: download, load, and warm-up. Charged at the GPU rate once, in addition to the occupancy allocation, and amortized over window.requests."++ },++ "storage_gb": {++ "type": "number",++ "minimum": 0,++ "description": "Model/artifact storage held for the window."++ },++ "storage_usd_per_gb_month": {++ "type": "number",++ "minimum": 0,++ "description": "Storage rate for a 730-hour month."++ },++ "egress_gb_per_request": {++ "type": "number",++ "minimum": 0,++ "description": "Network egress per request."++ },++ "egress_usd_per_gb": {++ "type": "number",++ "minimum": 0,++ "description": "Egress rate."++ }++ },++ "allOf": [++ {++ "if": {++ "properties": { "idle_policy": { "const": "standby" } },++ "required": ["idle_policy"]++ },++ "then": { "required": ["standby_hourly_usd"] }++ }++ ]++ },++ "serverless": {++ "type": "object",++ "required": ["hourly_usd", "startup_seconds", "storage_gb", "storage_usd_per_gb_month", "egress_gb_per_request", "egress_usd_per_gb"],++ "additionalProperties": false,++ "properties": {++ "hourly_usd": {++ "type": "number",++ "minimum": 0,++ "description": "Scale-to-zero per-second rate in USD per GPU-hour. No idle charge is modelled."++ },++ "startup_seconds": {++ "type": "number",++ "minimum": 0,++ "description": "Cold start charged on each scale-up."++ },++ "requests_per_cold_start": {++ "type": "number",++ "minimum": 1,++ "default": 1,++ "description": "Requests served per cold start before scaling down. 1 = every request pays a cold start."++ },++ "storage_gb": {++ "type": "number",++ "minimum": 0,++ "description": "Artifact storage held for the window."++ },++ "storage_usd_per_gb_month": {++ "type": "number",++ "minimum": 0,++ "description": "Storage rate for a 730-hour month."++ },++ "egress_gb_per_request": {++ "type": "number",++ "minimum": 0,++ "description": "Network egress per request."++ },++ "egress_usd_per_gb": {++ "type": "number",++ "minimum": 0,++ "description": "Egress rate."++ }++ }++ },++ "api_wholesale": {++ "type": "object",++ "required": ["input_usd_per_million", "cached_read_usd_per_million", "cache_write_usd_per_million", "output_usd_per_million", "per_request_usd"],++ "additionalProperties": false,++ "properties": {++ "input_usd_per_million": {++ "type": "number",++ "minimum": 0,++ "description": "Wholesale input-token rate per million tokens."++ },++ "cached_read_usd_per_million": {++ "type": "number",++ "minimum": 0,++ "description": "Wholesale cached-read rate per million tokens."++ },++ "cache_write_usd_per_million": {++ "type": "number",++ "minimum": 0,++ "description": "Wholesale cache-write rate per million tokens."++ },++ "output_usd_per_million": {++ "type": "number",++ "minimum": 0,++ "description": "Wholesale output-token rate per million tokens."++ },++ "per_request_usd": {++ "type": "number",++ "minimum": 0,++ "description": "Fixed fee per request, if the agreement has one."++ },++ "success_rate": {++ "type": "number",++ "exclusiveMinimum": 0,++ "maximum": 1,++ "description": "Optional provider-specific success rate; falls back to request.success_rate."++ }++ }++ }++ }++}nnew file mode 100644@@@ -0,0 +1,115 @@++'use strict';++// workers/llm-inference/verify-artifact.sh: a pinned artifact is downloaded once, checked against++// the manifest SHA-256 and refused (without leaving a file behind) on any mismatch or placeholder.++const assert = require('assert');++const crypto = require('crypto');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { spawnSync } = require('child_process');++const { pathToFileURL } = require('url');++++const root = path.join(__dirname, '..');++const verify = path.join(root, 'workers', 'llm-inference', 'verify-artifact.sh');++const entrypoint = path.join(root, 'workers', 'llm-inference', 'entrypoint.sh');++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-artifact-'));++const baseEnv = { ...process.env };++delete baseEnv.ALLOW_LOCAL_ARTIFACT_URL;++++function sha256(file) {++ return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');++}++++function manifest({ sha, revision = 'c9e90669eeb205d5af35c28a3e9983fc9293c2ec', filename = 'model.gguf', url, schema = 'openvibe.ai.worker-artifact@1' }) {++ return {++ schema,++ artifact: {++ id: `example/qwen3-4b:q4_k_m@${revision}`,++ revision,++ sha256: sha,++ license: 'apache-2.0',++ },++ download: { url, filename },++ };++}++++function writeManifest(name, document) {++ const file = path.join(tmp, name);++ fs.writeFileSync(file, JSON.stringify(document, null, 2));++ return file;++}++++function run(manifestFile, dest, { allowLocal = true } = {}) {++ const env = { ...baseEnv };++ if (allowLocal) env.ALLOW_LOCAL_ARTIFACT_URL = '1';++ return spawnSync('sh', [verify, manifestFile, dest], { encoding: 'utf8', env });++}++++const source = path.join(tmp, 'source.gguf');++fs.writeFileSync(source, 'a small GGUF stand-in, not a model');++const goodSha = sha256(source);++++// A correct manifest downloads, verifies and prints the local path on stdout.++const dest1 = path.join(tmp, 'models-1');++const ok = run(writeManifest('good.json', manifest({ sha: goodSha, url: pathToFileURL(source).href })), dest1);++assert.strictEqual(ok.status, 0, ok.stderr);++assert.strictEqual(ok.stdout.trim(), path.join(dest1, 'model.gguf'), 'stdout is only the verified path');++assert.strictEqual(fs.readFileSync(path.join(dest1, 'model.gguf'), 'utf8'), fs.readFileSync(source, 'utf8'));++++// A wrong SHA-256 is refused, and neither the final file nor a partial file is left behind.++const dest2 = path.join(tmp, 'models-2');++const wrong = run(writeManifest('wrong.json', manifest({ sha: '0'.repeat(64), url: pathToFileURL(source).href })), dest2);++assert.notStrictEqual(wrong.status, 0, 'wrong hash must fail');++assert.match(wrong.stderr, /SHA-256 mismatch/);++assert.ok(!fs.existsSync(path.join(dest2, 'model.gguf')), 'mismatched file must not be installed');++assert.deepStrictEqual(fs.readdirSync(dest2), [], 'partial download must be removed');++++// A TODO placeholder (revision or hash) is refused before any download.++const todo = run(writeManifest('todo.json', manifest({ sha: goodSha, revision: 'TODO: pin the commit', url: pathToFileURL(source).href })), path.join(tmp, 'models-todo'));++assert.notStrictEqual(todo.status, 0);++assert.match(todo.stderr, /TODO placeholder/);++++// A manifest cannot write outside its model directory.++const traversal = run(writeManifest('traversal.json', manifest({ sha: goodSha, filename: '../escape.gguf', url: pathToFileURL(source).href })), path.join(tmp, 'models-traversal'));++assert.notStrictEqual(traversal.status, 0);++assert.match(traversal.stderr, /plain file name/);++assert.ok(!fs.existsSync(path.join(tmp, 'escape.gguf')));++++// Local URLs are for smoke tests only; production config has to be https.++const local = run(writeManifest('local.json', manifest({ sha: goodSha, url: pathToFileURL(source).href })), path.join(tmp, 'models-local'), { allowLocal: false });++assert.notStrictEqual(local.status, 0);++assert.match(local.stderr, /ALLOW_LOCAL_ARTIFACT_URL/);++++// A malformed hash is refused even when its length is not 64.++const shortHash = run(writeManifest('short.json', manifest({ sha: 'abc', url: pathToFileURL(source).href })), path.join(tmp, 'models-short'));++assert.notStrictEqual(shortHash.status, 0);++assert.match(shortHash.stderr, /64 lowercase hex/);++++// A cached file that already matches is reused without touching the URL.++const dest4 = path.join(tmp, 'models-cached');++fs.mkdirSync(dest4);++fs.writeFileSync(path.join(dest4, 'model.gguf'), fs.readFileSync(source));++const cached = run(writeManifest('missing-source.json', manifest({ sha: goodSha, url: 'file:///nonexistent/model.gguf' })), dest4);++assert.strictEqual(cached.status, 0, cached.stderr);++assert.strictEqual(cached.stdout.trim(), path.join(dest4, 'model.gguf'));++++// The scripts parse under the shell the image uses, and the entrypoint refuses bounded settings++// before it reaches the model or the binaries.++for (const file of [verify, entrypoint]) {++ const syntax = spawnSync('sh', ['-n', file], { encoding: 'utf8' });++ assert.strictEqual(syntax.status, 0, `${file}: ${syntax.stderr}`);++}++const cases = [++ [{ LLM_CTX: '40000', LLM_CTX_MAX: '32768' }, /exceeds LLM_CTX_MAX/],++ [{ LLM_CTX: '1024', LLM_PARALLEL: '4' }, /at least 512 tokens per slot/],++ [{ LLM_KV_TYPE_K: 'magic' }, /LLM_KV_TYPE_K/],++ [{ DRAIN_SECONDS: '0' }, /DRAIN_SECONDS/],++];++for (const [env, pattern] of cases) {++ const refused = spawnSync('sh', [entrypoint], { encoding: 'utf8', env: { ...baseEnv, ...env } });++ assert.notStrictEqual(refused.status, 0, JSON.stringify(env));++ assert.match(refused.stderr, pattern, JSON.stringify(env));++}++++fs.rmSync(tmp, { recursive: true, force: true });++console.log('inference artifact verification: all checks passed');nnew file mode 100644@@@ -0,0 +1,158 @@++'use strict';++// scripts/inference-cost-model.js: the PLAN.md §E arithmetic on fixed, hand-computed inputs.++// Every expected number below is written out by hand in the comment above it; the test does not++// re-derive it with the implementation.++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { execFileSync } = require('child_process');++const { compute, formatTable } = require('../scripts/inference-cost-model');++++function close(actual, expected, label) {++ assert.ok(Math.abs(actual - expected) < 1e-12, `${label}: expected ${expected}, got ${actual}`);++}++++// 1000 prefill tokens at 1000 tok/s + 100 output tokens at 50 tok/s = 1 s + 2 s = 3 s busy.++// Rented: occupancy 0.5 -> 6 s paid at $1.80/h = 6/3600 * 1.8 = $0.003 per request.++// 0.8 success -> $0.003 / 0.8 = $0.00375 per successful task.++// Serverless: 3 s at $3.60/h = $0.003; no startup.++// API: 1000 * $100/M = $0.1, 100 * $400/M = $0.04, per request $0.001 -> $0.141;++// $0.141 / 0.8 = $0.17625 per successful task.++const fixed = {++ note: 'test fixture',++ window: { hours: 10, requests: 100 },++ request: { input_tokens: 1000, cached_read_tokens: 0, cache_write_tokens: 0, output_tokens: 100, success_rate: 0.8 },++ throughput: { prefill_tokens_per_sec: 1000, decode_tokens_per_sec: 50, concurrency: 1 },++ gpu: {++ hourly_usd: 1.8, occupancy: 0.5, idle_policy: 'billed', standby_hourly_usd: 0, startup_seconds: 0,++ storage_gb: 0, storage_usd_per_gb_month: 0, egress_gb_per_request: 0, egress_usd_per_gb: 0,++ },++ serverless: {++ hourly_usd: 3.6, startup_seconds: 0, requests_per_cold_start: 1,++ storage_gb: 0, storage_usd_per_gb_month: 0, egress_gb_per_request: 0, egress_usd_per_gb: 0,++ },++ api_wholesale: {++ input_usd_per_million: 100, cached_read_usd_per_million: 10, cache_write_usd_per_million: 100,++ output_usd_per_million: 400, per_request_usd: 0.001, success_rate: 0.8,++ },++};++++const result = compute(fixed);++assert.strictEqual(result.rows.length, 3);++assert.deepStrictEqual(result.rows.map((row) => row.mode), ['rented-warm', 'scale-to-zero', 'api-wholesale']);++close(result.seconds.prefill, 1, 'prefill seconds');++close(result.seconds.decode, 2, 'decode seconds');++close(result.seconds.busy, 3, 'busy seconds');++close(result.seconds.rented_idle, 3, 'rented idle seconds');++close(result.rows[0].costPerRequest, 0.003, 'rented-warm per request');++close(result.rows[0].costPerSuccessfulTask, 0.00375, 'rented-warm per success');++close(result.rows[1].costPerRequest, 0.003, 'serverless per request');++close(result.rows[1].costPerSuccessfulTask, 0.00375, 'serverless per success');++close(result.rows[2].costPerRequest, 0.141, 'api per request');++close(result.rows[2].costPerSuccessfulTask, 0.17625, 'api per success');++++// A full month makes storage easy: 10 GB * $1/GB-month = $10, over 1000 requests = $0.01 each.++// Busy 3 s at $3.60/h = $0.003; idle 9 s (occupancy 0.25) at standby $0.36/h = $0.0009;++// startup 36 s at $3.60/h = $0.036 / 1000 requests = $0.000036; egress 0.001 GB * $2 = $0.002.++// Rented total = 0.003 + 0.0009 + 0.000036 + 0.01 + 0.002 = 0.015936; / 0.8 = 0.01992.++// Serverless: 3 s at $7.20/h = $0.006; startup 18 s at $7.20/h / 3 requests = $0.012;++// storage 0.01 + egress 0.002 -> 0.03; / 0.8 = 0.0375.++const withOverheads = JSON.parse(JSON.stringify(fixed));++withOverheads.window = { hours: 730, requests: 1000 };++withOverheads.gpu.hourly_usd = 3.6;++withOverheads.gpu.occupancy = 0.25;++withOverheads.gpu.idle_policy = 'standby';++withOverheads.gpu.standby_hourly_usd = 0.36;++withOverheads.gpu.startup_seconds = 36;++withOverheads.gpu.storage_gb = 10;++withOverheads.gpu.storage_usd_per_gb_month = 1;++withOverheads.gpu.egress_gb_per_request = 0.001;++withOverheads.gpu.egress_usd_per_gb = 2;++withOverheads.serverless.hourly_usd = 7.2;++withOverheads.serverless.startup_seconds = 18;++withOverheads.serverless.requests_per_cold_start = 3;++withOverheads.serverless.storage_gb = 10;++withOverheads.serverless.storage_usd_per_gb_month = 1;++withOverheads.serverless.egress_gb_per_request = 0.001;++withOverheads.serverless.egress_usd_per_gb = 2;++++const overheads = compute(withOverheads);++close(overheads.rows[0].costPerRequest, 0.015936, 'rented-warm with overheads');++close(overheads.rows[0].costPerSuccessfulTask, 0.01992, 'rented-warm with overheads per success');++close(overheads.rows[1].costPerRequest, 0.03, 'serverless with overheads');++close(overheads.rows[1].costPerSuccessfulTask, 0.0375, 'serverless with overheads per success');++++// Cached read: 500 R and 250 W tokens. Without a measured cached-read rate, prefill is++// (1000 + 250)/1000 + 500/1000 = 1.75 s. With 5000 cached tok/s it is 1.25 + 500/5000 = 1.35 s.++const cached = JSON.parse(JSON.stringify(fixed));++cached.request.input_tokens = 1000;++cached.request.cached_read_tokens = 500;++cached.request.cache_write_tokens = 250;++cached.request.output_tokens = 100;++cached.api_wholesale.success_rate = 0.8;++const noCacheRate = compute(cached);++close(noCacheRate.seconds.prefill, 1.75, 'cached tokens at prefill rate');++// API: 1000*100/M + 500*10/M + 250*100/M + 100*400/M + 0.001 = 0.1 + 0.005 + 0.025 + 0.04 + 0.001.++close(noCacheRate.rows[2].costPerRequest, 0.171, 'api cached-read mix');++close(noCacheRate.rows[2].costPerSuccessfulTask, 0.21375, 'api cached-read mix per success');++cached.throughput.cached_read_tokens_per_sec = 5000;++const measuredCache = compute(cached);++close(measuredCache.seconds.prefill, 1.35, 'cached tokens at measured rate');++++// A per-mode API success rate overrides the request success rate.++const apiSuccess = JSON.parse(JSON.stringify(fixed));++apiSuccess.api_wholesale.success_rate = 0.5;++assert.ok(Math.abs(compute(apiSuccess).rows[2].costPerSuccessfulTask - 0.282) < 1e-12);++++// The failure allocation is a division by the success rate, so it is visible in both directions.++const allSuccess = JSON.parse(JSON.stringify(fixed));++allSuccess.request.success_rate = 1;++assert.ok(compute(allSuccess).rows[0].costPerSuccessfulTask === compute(allSuccess).rows[0].costPerRequest);++++// Invalid inputs are refused with the field named.++for (const [label, mutate, pattern] of [++ ['occupancy zero', (x) => { x.gpu.occupancy = 0; }, /gpu\.occupancy/],++ ['success rate zero', (x) => { x.request.success_rate = 0; }, /success_rate/],++ ['negative output', (x) => { x.request.output_tokens = -1; }, /output_tokens/],++ ['missing serverless', (x) => { delete x.serverless; }, /serverless/],++ ['standby rate missing', (x) => { x.gpu.idle_policy = 'standby'; delete x.gpu.standby_hourly_usd; }, /standby_hourly_usd/],++ ['bad policy', (x) => { x.gpu.idle_policy = 'free'; }, /idle_policy/],++ ['zero prefill', (x) => { x.throughput.prefill_tokens_per_sec = 0; }, /prefill_tokens_per_sec/],++ ['fractional requests', (x) => { x.window.requests = 1.5; }, /window\.requests/],++]) {++ const bad = JSON.parse(JSON.stringify(fixed));++ mutate(bad);++ assert.throws(() => compute(bad), pattern, label);++}++++// The checked-in example is clearly a placeholder and still runs through the whole model.++const examplePath = path.join(__dirname, '..', 'workers', 'llm-inference', 'examples', 'inference-cost-model.example.json');++const example = JSON.parse(fs.readFileSync(examplePath, 'utf8'));++assert.match(example.note, /example, not a quote/);++const exampleResult = compute(example);++assert.strictEqual(exampleResult.rows.length, 3);++for (const row of exampleResult.rows) {++ assert.ok(Number.isFinite(row.costPerRequest) && row.costPerRequest > 0, `${row.mode} per request`);++ assert.ok(Number.isFinite(row.costPerSuccessfulTask) && row.costPerSuccessfulTask > 0, `${row.mode} per success`);++}++const table = formatTable(exampleResult);++assert.match(table, /planning estimate, not a quote/);++assert.match(table, /rented-warm/);++assert.match(table, /scale-to-zero/);++assert.match(table, /api-wholesale/);++++// The CLI reads the same file and prints JSON.++const output = execFileSync(process.execPath, [path.join(__dirname, '..', 'scripts', 'inference-cost-model.js'), examplePath, '--json'], { encoding: 'utf8' });++const cli = JSON.parse(output);++assert.deepStrictEqual(cli.rows.map((row) => row.mode), ['rented-warm', 'scale-to-zero', 'api-wholesale']);++++// The schema and the example describe the same fields; this catches drift in either direction.++const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'scripts', 'inference-cost-model.schema.json'), 'utf8'));++for (const key of ['window', 'request', 'throughput', 'gpu', 'serverless', 'api_wholesale']) {++ assert.ok(schema.properties[key], `schema property ${key}`);++ for (const field of Object.keys(example[key])) {++ assert.ok(schema.properties[key].properties[field], `schema property ${key}.${field}`);++ }++}++++console.log('inference cost model: all hand-computed checks passed');nnew file mode 100644@@@ -0,0 +1,43 @@++# Gateway for the OpenVibe LLM inference worker.++#++# llama.cpp listens only on loopback; this is the only public listener. It adds the two endpoints++# llama.cpp does not have as separate paths:++#++# GET /ready the same check as llama.cpp GET /health, so a slot/model that is not loaded is++# not ready. The orchestrator removes the worker from rotation when this is not 2xx.++# GET /health passed through to llama.cpp.++# GET /metrics passed through, Prometheus text.++#++# Everything else (OpenAI-compatible chat/completions, props, slots, ...) is passed through++# unchanged. Streaming responses are flushed immediately. Dynamic LoRA management is not exposed:++# the worker starts no LoRA and the vLLM-style load/unload paths are refused here as well.++#++# DRAIN_SECONDS is also the gateway's shutdown grace period: on SIGTERM Caddy stops accepting new++# connections and waits up to this long for in-flight requests before the entrypoint stops++# llama.cpp.++++{++ admin off++ auto_https off++ persist_config off++ grace_period {$DRAIN_SECONDS}s++}++++:{$PORT} {++ @lora path /lora* /v1/load_lora_adapter* /v1/unload_lora_adapter*++ respond @lora 404++++ @ready path /ready++ handle @ready {++ rewrite * /health++ reverse_proxy 127.0.0.1:{$LLM_PORT} {++ flush_interval -1++ }++ }++++ handle {++ reverse_proxy 127.0.0.1:{$LLM_PORT} {++ flush_interval -1++ }++ }++}nnew file mode 100644@@@ -0,0 +1,65 @@++# syntax=docker/dockerfile:1++#++# OpenVibe LLM inference worker (llama.cpp server, GGUF). Build from this directory:++#++# docker build -t openvibe/llm-inference .++# docker build --build-arg LLAMA_CPP_TAG=server \++# --build-arg LLAMA_CPP_DIGEST=sha256:10a38e6cc91760ab157f0a0a8546218d35f4a314432a70298abfa3d096829ad7 \++# -t openvibe/llm-inference:cpu .++#++# Both llama.cpp bases are pinned by digest, not by tag alone. The tag names the artifact for a++# human reader; the digest is what Docker resolves. Refresh a pin only as a deliberate change:++#++# docker buildx imagetools inspect ghcr.io/ggml-org/llama.cpp:server-cuda++# docker buildx imagetools inspect docker.io/library/caddy:2++#++# Resolved 2026-11-04 (server-cuda and server are the upstream rolling tags at that moment).++ARG LLAMA_CPP_IMAGE=ghcr.io/ggml-org/llama.cpp++ARG LLAMA_CPP_TAG=server-cuda++ARG LLAMA_CPP_DIGEST=sha256:d5ffba7d096ab0236c65dc147d797b4b7658a7e5425634593373635d661bccda++++# The gateway binary is copied from the pinned Caddy image; no package repository is trusted at++# build time for it.++ARG CADDY_IMAGE=docker.io/library/caddy++ARG CADDY_TAG=2++ARG CADDY_DIGEST=sha256:36c24ad91a7ff7bc4c87fe22bfa8df3c4b18da0773d73fc12ee2c52b98c0bb7e++++FROM ${CADDY_IMAGE}:${CADDY_TAG}@${CADDY_DIGEST} AS gateway++++FROM ${LLAMA_CPP_IMAGE}:${LLAMA_CPP_TAG}@${LLAMA_CPP_DIGEST}++++# jq parses the manifest; curl and sha256sum come from the llama.cpp runtime image (curl is part++# of that image, the checksum tool is coreutils). passwd is explicit so groupadd/useradd exist on++# every base variant.++RUN apt-get update \++ && apt-get install -y --no-install-recommends ca-certificates jq passwd \++ && apt-get clean \++ && rm -rf /var/lib/apt/lists/* \++ && groupadd --system openvibe \++ && useradd --system --gid openvibe --home-dir /app --shell /usr/sbin/nologin openvibe \++ && mkdir -p /models /etc/openvibe /etc/caddy /tmp/caddy \++ && chown -R openvibe:openvibe /models /tmp/caddy++++COPY --from=gateway /usr/bin/caddy /usr/local/bin/caddy++COPY entrypoint.sh verify-artifact.sh /usr/local/bin/++COPY Caddyfile /etc/caddy/Caddyfile++++RUN chmod 0755 /usr/local/bin/entrypoint.sh /usr/local/bin/verify-artifact.sh /usr/local/bin/caddy \++ && chown -R openvibe:openvibe /etc/caddy /etc/openvibe++++ENV PORT=8080 \++ LLM_PORT=8081 \++ DRAIN_SECONDS=30 \++ HOME=/tmp/caddy \++ XDG_DATA_HOME=/tmp/caddy \++ XDG_CONFIG_HOME=/tmp/caddy++++USER openvibe++EXPOSE 8080++++# Start period covers the artifact download plus model load; until llama.cpp answers /health the++# gateway's /ready is not 2xx.++HEALTHCHECK --interval=15s --timeout=3s --start-period=900s --retries=3 \++ CMD curl -fsS "http://127.0.0.1:${PORT:-8080}/ready" || exit 1++++ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]nnew file mode 100644@@@ -0,0 +1,232 @@++# LLM inference worker (llama.cpp, GGUF)++++A portable GPU worker image for one pinned GGUF artifact: llama.cpp server on loopback, a small++Caddy gateway in front, checksum-verified model download, bounded context/parallel/KV settings,++`/health` + `/ready`, the Prometheus `/metrics` endpoint passed through, and a graceful drain on++`SIGTERM`. No dynamic LoRA API is exposed.++++## Ownership++++OpenVibe.Host owns worker/deployment orchestration and safe budgeted GPU placement (PLAN.md §B),++so the deployment artifact and its cost model live here. The model/variant registry, catalog++entries, revisions, routing and inference requests are OpenVibe.AI's (PLAN.md §B, D3/D4); this++worker consumes one pinned artifact and never registers or advertises a model. `docs/nodes.md`++describes Host's node-registry reporting, and Host's `workerUnits` inventory hook is for units that++drain and exit on their own (Host never starts, stops or restarts them). This image is that kind++of long-running worker: it can run under a `workerUnits` unit or any GPU orchestrator.++++## Files++++| File | Purpose |++|---|---|++| `Dockerfile` | llama.cpp base pinned by digest, plus a Caddy binary copied from a pinned image |++| `entrypoint.sh` | validates env bounds, verifies the artifact, starts llama.cpp + gateway, drains |++| `verify-artifact.sh` | downloads by URL, checks the manifest SHA-256, refuses a mismatch |++| `Caddyfile` | `/health`, `/ready`, `/metrics` and API passthrough; blocks LoRA paths |++| `examples/qwen3-4b-abliterated.manifest.example.json` | one compact abliterated Qwen 4B artifact, revision and hash **TODO** |++| `examples/inference-cost-model.example.json` | clearly labelled placeholder inputs for the cost model |++++## Build++++From this directory:++++```sh++# GPU image (CUDA base, the default)++docker build -t openvibe/llm-inference:gpu .++++# CPU image for a smoke test (same entrypoint and checks, CPU llama.cpp base)++docker build \++ --build-arg LLAMA_CPP_TAG=server \++ --build-arg LLAMA_CPP_DIGEST=sha256:10a38e6cc91760ab157f0a0a8546218d35f4a314432a70298abfa3d096829ad7 \++ -t openvibe/llm-inference:cpu .++```++++Both bases are pinned by digest. Refresh a pin only as a deliberate change:++++```sh++docker buildx imagetools inspect ghcr.io/ggml-org/llama.cpp:server-cuda++docker buildx imagetools inspect docker.io/library/caddy:2++```++++## The artifact manifest++++Copy `examples/qwen3-4b-abliterated.manifest.example.json` and fill in **both** placeholders before++running:++++- `artifact.revision`: the immutable commit/revision of the artifact repository (for Hugging Face,++ the commit SHA, not a branch).++- `artifact.sha256` and `download.url`: the revision-pinned URL and the SHA-256 published for that++ exact file. The example deliberately contains `TODO` values; the verifier refuses them. Never++ invent a hash and never reuse one from another quantization.++++```json++{++ "schema": "openvibe.ai.worker-artifact@1",++ "artifact": {++ "id": "mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated:q4_k_m@<revision>",++ "revision": "<revision>",++ "sha256": "<64 lowercase hex>",++ "license": "apache-2.0"++ },++ "download": { "url": "https://…/<revision>/<file>.gguf", "filename": "<file>.gguf" }++}++```++++`verify-artifact.sh`:++++1. requires `schema`, `artifact.id`, `artifact.revision`, `artifact.sha256`, `download.url` and a++ plain `download.filename`;++2. refuses any `TODO` placeholder and any hash that is not 64 lowercase hex characters;++3. requires `https://` for a real pin (`file://` and loopback are allowed only with++ `ALLOW_LOCAL_ARTIFACT_URL=1` for a smoke test);++4. reuses an already-downloaded file only when its hash matches, otherwise downloads to a temporary++ file, verifies it and renames it into place;++5. on mismatch deletes the temporary file, prints `SHA-256 mismatch … refusing to start` and exits++ non-zero, so the container never serves unverified weights.++++The manifest is operator-controlled, not user input. The worker does not download LoRA adapters.++++## Run on a GPU++++```sh++cp examples/qwen3-4b-abliterated.manifest.example.json /tmp/artifact.json++# fill revision + sha256 + url in /tmp/artifact.json++++docker run --rm --gpus all -p 8080:8080 \++ -v /tmp/artifact.json:/etc/openvibe/artifact.json:ro \++ -v openvibe-models:/models \++ -e LLM_CTX=8192 \++ -e LLM_PARALLEL=2 \++ -e LLM_KV_TYPE_K=f16 \++ -e LLM_KV_TYPE_V=f16 \++ openvibe/llm-inference:gpu++```++++`/models` is a cache: a later run with the same manifest re-verifies the file and skips the++download. A bind-mounted host directory must be writable by the image's unprivileged `openvibe`++user (named volumes inherit the image ownership), and the bind-mounted manifest must be readable++by that user (uid 995): `chmod 644 /tmp/artifact.json`, or run with `--user "$(id -u):$(id -g)"`++only if your runtime gives that uid access to the GPU devices.++++## Run on CPU for a smoke test++++The CPU image exercises the same entrypoint, manifest verification, gateway and drain path:++++```sh++docker run --rm -p 8080:8080 \++ -v /tmp/artifact.json:/etc/openvibe/artifact.json:ro \++ -v openvibe-models:/models \++ -e LLM_GPU_LAYERS=0 \++ -e LLM_THREADS=4 \++ -e LLM_CTX=2048 \++ -e LLM_PARALLEL=1 \++ openvibe/llm-inference:cpu++```++++A 4B Q4_K_M model is slow on CPU; use it for one request and the endpoint checks, not for++throughput. Any small GGUF works for the smoke test as long as its manifest has the real revision++and hash — do not use a placeholder hash to bypass verification.++++## Endpoints++++The only public listener is the gateway on `$PORT` (default 8080); llama.cpp listens on++`127.0.0.1:$LLM_PORT` (default 8081).++++| Path | Behaviour |++|---|---|++| `GET /health` | llama.cpp health (200 when the model/slot is loaded) |++| `GET /ready` | the same check as `/health`; use this as the orchestrator readiness probe |++| `GET /metrics` | llama.cpp Prometheus metrics, passed through |++| `/v1/chat/completions`, `/props`, `/slots`, … | passed through; streaming responses flush immediately |++| `/lora*`, `/v1/load_lora_adapter*`, `/v1/unload_lora_adapter*` | 404; no dynamic LoRA management |++++The worker has **no authentication** of its own. Expose it only on a private network or behind++OpenVibe.AI's inference gateway; do not publish port 8080 to the internet.++++## Bounded settings++++All values are integers unless stated; a value outside its bound stops the worker before it++downloads or loads anything.++++| Env | Default | Bound | Meaning |++|---|---|---|---|++| `LLM_CTX` | `4096` | `512`–`131072`, ≤ `LLM_CTX_MAX` | total context across slots (`--ctx-size`) |++| `LLM_CTX_MAX` | `32768` | `512`–`131072` | operator ceiling for `LLM_CTX` |++| `LLM_PARALLEL` | `1` | `1`–`16`, ≤ `LLM_PARALLEL_MAX` | slots (`--parallel`) |++| `LLM_PARALLEL_MAX` | `8` | `1`–`16` | operator ceiling for `LLM_PARALLEL` |++| — | — | `LLM_CTX / LLM_PARALLEL ≥ 512` | per-slot context floor |++| `LLM_KV_TYPE_K` | `f16` | `f32 f16 bf16 q8_0 q4_0 q4_1 iq4_nl q5_0 q5_1` | K KV cache type |++| `LLM_KV_TYPE_V` | `f16` | same list | V KV cache type |++| `LLM_GPU_LAYERS` | `99` | `0`–`999` | layers offloaded; use `0` on the CPU image |++| `LLM_THREADS` | unset | `1`–`256` | optional CPU thread count |++| `DRAIN_SECONDS` | `30` | `1`–`300` | gateway drain grace and shutdown budget |++| `PORT` | `8080` | `1`–`65535` | public gateway port |++| `LLM_PORT` | `8081` | `1`–`65535`, ≠ `PORT` | loopback llama.cpp port |++| `ARTIFACT_MANIFEST` | `/etc/openvibe/artifact.json` | — | manifest path |++| `MODEL_DIR` | `/models` | — | download/cache directory |++++`LLM_CTX` is llama.cpp's **total** context, not per slot; with `LLM_PARALLEL=2` and++`LLM_CTX=8192` each slot has about 4096 tokens. Measure VRAM with the real KV type, context,++concurrency and prompt lengths before choosing a GPU — the worker bounds these settings, it does++not model your model's memory.++++## Graceful drain++++On `SIGTERM` the entrypoint first stops the gateway: Caddy stops accepting new connections and++waits up to `DRAIN_SECONDS` for in-flight requests (its `grace_period`). Only after the gateway has++drained does the entrypoint stop llama.cpp, then exit 0. The orchestrator's stop timeout must be at++least `DRAIN_SECONDS` plus the time llama.cpp needs to exit; systemd's `TimeoutStopSec` and++`workerUnits` drain policies are the relevant Host settings.++++## Cost model++++`scripts/inference-cost-model.js` (schema: `scripts/inference-cost-model.schema.json`) computes++cost per request and cost per successful task for rented-warm GPU, scale-to-zero serverless and++wholesale API, following PLAN.md §E:++++```sh++node scripts/inference-cost-model.js workers/llm-inference/examples/inference-cost-model.example.json++node scripts/inference-cost-model.js workers/llm-inference/examples/inference-cost-model.example.json --json++```++++- **rented-warm:** steady-state busy seconds / occupancy at the GPU rate, idle at the++ `idle_policy` rate, one startup per window (a separate line, not part of occupancy), storage and++ egress.++- **scale-to-zero:** busy seconds + (`startup_seconds / requests_per_cold_start`) at the++ serverless rate, no idle, storage and egress.++- **api-wholesale:** `tokens * $/million` for input, cached read, cache write and output, plus a++ per-request fee. Wholesale rates are treated as all-in (no owned storage/egress line).++++Cost per successful task divides by the success rate (expected attempts). Prompt time charges++cache-write tokens at prefill throughput and cached-read tokens at `cached_read_tokens_per_sec`++when measured, otherwise at prefill throughput (conservative). The example file is marked++**example, not a quote**; every price and throughput in it is a placeholder, and the model has no++built-in benchmarks. Replace them with dated complete quotes and measured tok/s first.++++The script uses floats for forecasting. Customer liability is Billing's integer nano-USD ledger++(D1); this is not a settlement or quoting authority.++++## Tests++++```sh++node test/inference-artifact-verify.test.js++node test/inference-cost-model.test.js++# or, through the repository runner:++npm test -- inference++```++++The artifact test covers a correct download, a wrong hash (refused, no file left), a `TODO`++placeholder, a traversal filename, a local URL without the smoke-test opt-in, a cached file, and++the entrypoint's bounded-env refusals. The cost-model test checks the arithmetic against numbers++worked out by hand, invalid inputs and the checked-in example.++++## Not done here++++- No artifact is pinned: the example's revision and hash are deliberately `TODO`. Pinning is an++ operator action with the model license/provenance check and a published hash.++- No model registry entry, catalog route or request routing; those belong to OpenVibe.AI.++- No GPU rental, deployment or benchmark run is performed by this directory.++- No quality, refusal, license or provenance gate is run here. The example artifact is a candidate,++ not an approval: benchmark gates and the upstream license/responsible-use review happen before it++ is served.++- No dynamic LoRA API; only a static `--lora` would be a future, admin-only change.nnew file mode 100755@@@ -0,0 +1,151 @@++#!/bin/sh++# OpenVibe LLM inference worker: verify the pinned GGUF, run llama.cpp server on loopback, and put++# Caddy in front for /health, /ready and the Prometheus /metrics passthrough.++#++# Bounded settings come from the environment; see README.md. The worker never exposes a dynamic++# LoRA API: llama.cpp is started without --lora, and Caddy refuses the LoRA load/unload paths.++set -eu++++log() { printf 'llm-inference: %s\n' "$*" >&2; }++die() { log "$*"; exit 1; }++++is_uint() {++ case "$1" in++ ''|*[!0-9]*) return 1 ;;++ esac++ return 0++}++++# uint_in_range NAME VALUE MIN MAX++uint_in_range() {++ name=$1++ value=$2++ min=$3++ max=$4++ is_uint "$value" || die "$name must be an integer (got '$value')"++ [ "$value" -ge "$min" ] || die "$name must be >= $min (got $value)"++ [ "$value" -le "$max" ] || die "$name must be <= $max (got $value)"++}++++ARTIFACT_MANIFEST=${ARTIFACT_MANIFEST:-/etc/openvibe/artifact.json}++MODEL_DIR=${MODEL_DIR:-/models}++PORT=${PORT:-8080}++LLM_PORT=${LLM_PORT:-8081}++DRAIN_SECONDS=${DRAIN_SECONDS:-30}++LLM_CTX=${LLM_CTX:-4096}++LLM_CTX_MAX=${LLM_CTX_MAX:-32768}++LLM_PARALLEL=${LLM_PARALLEL:-1}++LLM_PARALLEL_MAX=${LLM_PARALLEL_MAX:-8}++LLM_KV_TYPE_K=${LLM_KV_TYPE_K:-f16}++LLM_KV_TYPE_V=${LLM_KV_TYPE_V:-f16}++LLM_GPU_LAYERS=${LLM_GPU_LAYERS:-99}++LLM_THREADS=${LLM_THREADS:-}++++# Caddy reads these from its environment for {$PORT}, {$LLM_PORT} and {$DRAIN_SECONDS}.++export PORT LLM_PORT DRAIN_SECONDS++++uint_in_range PORT "$PORT" 1 65535++uint_in_range LLM_PORT "$LLM_PORT" 1 65535++[ "$PORT" != "$LLM_PORT" ] || die "PORT and LLM_PORT must differ"++++uint_in_range DRAIN_SECONDS "$DRAIN_SECONDS" 1 300++uint_in_range LLM_CTX "$LLM_CTX" 512 131072++uint_in_range LLM_CTX_MAX "$LLM_CTX_MAX" 512 131072++[ "$LLM_CTX" -le "$LLM_CTX_MAX" ] || die "LLM_CTX ($LLM_CTX) exceeds LLM_CTX_MAX ($LLM_CTX_MAX)"++uint_in_range LLM_PARALLEL "$LLM_PARALLEL" 1 16++uint_in_range LLM_PARALLEL_MAX "$LLM_PARALLEL_MAX" 1 16++[ "$LLM_PARALLEL" -le "$LLM_PARALLEL_MAX" ] || die "LLM_PARALLEL ($LLM_PARALLEL) exceeds LLM_PARALLEL_MAX ($LLM_PARALLEL_MAX)"++[ $((LLM_CTX / LLM_PARALLEL)) -ge 512 ] || die "LLM_CTX / LLM_PARALLEL must be at least 512 tokens per slot"++uint_in_range LLM_GPU_LAYERS "$LLM_GPU_LAYERS" 0 999++if [ -n "$LLM_THREADS" ]; then++ uint_in_range LLM_THREADS "$LLM_THREADS" 1 256++fi++++allowed_kv=' f32 f16 bf16 q8_0 q4_0 q4_1 iq4_nl q5_0 q5_1 '++case "$allowed_kv" in++ *" $LLM_KV_TYPE_K "*) ;;++ *) die "LLM_KV_TYPE_K must be one of:$allowed_kv(got '$LLM_KV_TYPE_K')" ;;++esac++case "$allowed_kv" in++ *" $LLM_KV_TYPE_V "*) ;;++ *) die "LLM_KV_TYPE_V must be one of:$allowed_kv(got '$LLM_KV_TYPE_V')" ;;++esac++++[ -x /app/llama-server ] || die "/app/llama-server is missing"++command -v caddy >/dev/null 2>&1 || die "caddy is missing"++++log "verifying artifact from $ARTIFACT_MANIFEST"++MODEL_PATH=$(/usr/local/bin/verify-artifact.sh "$ARTIFACT_MANIFEST" "$MODEL_DIR") || exit 1++MODEL_ID=$(jq -r '.artifact.id' "$ARTIFACT_MANIFEST") || die "cannot read artifact.id"++log "artifact verified: $MODEL_PATH"++++set -- /app/llama-server \++ --model "$MODEL_PATH" \++ --alias "$MODEL_ID" \++ --host 127.0.0.1 \++ --port "$LLM_PORT" \++ --ctx-size "$LLM_CTX" \++ --parallel "$LLM_PARALLEL" \++ --n-gpu-layers "$LLM_GPU_LAYERS" \++ --cache-type-k "$LLM_KV_TYPE_K" \++ --cache-type-v "$LLM_KV_TYPE_V" \++ --no-webui \++ --metrics++if [ -n "$LLM_THREADS" ]; then++ set -- "$@" --threads "$LLM_THREADS"++fi++++log "starting llama-server on 127.0.0.1:$LLM_PORT (ctx $LLM_CTX total, $((LLM_CTX / LLM_PARALLEL)) per slot, parallel $LLM_PARALLEL, KV $LLM_KV_TYPE_K/$LLM_KV_TYPE_V)"++"$@" &++LLAMA_PID=$!++++log "starting gateway on :$PORT (ready maps to llama.cpp /health)"++caddy run --config /etc/caddy/Caddyfile --adapter caddyfile &++CADDY_PID=$!++++wait_pid() {++ pid=$1++ while kill -0 "$pid" 2>/dev/null; do++ wait "$pid" 2>/dev/null || true++ done++}++++DRAINING=0++on_signal() {++ [ "$DRAINING" = 1 ] && return++ DRAINING=1++ log "SIGTERM: not ready; draining in-flight requests for up to ${DRAIN_SECONDS}s"++ # Caddy stops accepting at SIGTERM and waits for in-flight requests (grace_period).++ if kill -0 "$CADDY_PID" 2>/dev/null; then++ kill -TERM "$CADDY_PID" 2>/dev/null || true++ wait_pid "$CADDY_PID"++ fi++ log "gateway drained; stopping llama-server"++ if kill -0 "$LLAMA_PID" 2>/dev/null; then++ kill -TERM "$LLAMA_PID" 2>/dev/null || true++ wait_pid "$LLAMA_PID"++ fi++ log "drained"++ exit 0++}++trap on_signal TERM INT++++while :; do++ if ! kill -0 "$LLAMA_PID" 2>/dev/null; then++ status=0++ wait "$LLAMA_PID" 2>/dev/null || status=$?++ log "llama-server exited (status $status); stopping gateway"++ kill -TERM "$CADDY_PID" 2>/dev/null || true++ wait_pid "$CADDY_PID"++ exit "$status"++ fi++ if ! kill -0 "$CADDY_PID" 2>/dev/null; then++ status=0++ wait "$CADDY_PID" 2>/dev/null || status=$?++ log "gateway exited (status $status); stopping llama-server"++ kill -TERM "$LLAMA_PID" 2>/dev/null || true++ wait_pid "$LLAMA_PID"++ exit "$status"++ fi++ sleep 1 || true++donennew file mode 100644@@@ -0,0 +1,48 @@++{++ "$schema": "../../../scripts/inference-cost-model.schema.json",++ "note": "example, not a quote — every price and throughput below is a placeholder for arithmetic only. Replace each one with a dated complete quote and a measured tok/s run before using this model for a decision.",++ "window": {++ "hours": 24,++ "requests": 1000++ },++ "request": {++ "input_tokens": 1000,++ "cached_read_tokens": 0,++ "cache_write_tokens": 0,++ "output_tokens": 500,++ "success_rate": 0.95++ },++ "throughput": {++ "prefill_tokens_per_sec": 800,++ "decode_tokens_per_sec": 40,++ "concurrency": 1++ },++ "gpu": {++ "hourly_usd": 0.1,++ "occupancy": 0.35,++ "idle_policy": "standby",++ "standby_hourly_usd": 0,++ "startup_seconds": 90,++ "storage_gb": 4,++ "storage_usd_per_gb_month": 0.1,++ "egress_gb_per_request": 0.0002,++ "egress_usd_per_gb": 0.09++ },++ "serverless": {++ "hourly_usd": 0.75,++ "startup_seconds": 45,++ "requests_per_cold_start": 1,++ "storage_gb": 4,++ "storage_usd_per_gb_month": 0.1,++ "egress_gb_per_request": 0.0002,++ "egress_usd_per_gb": 0.09++ },++ "api_wholesale": {++ "input_usd_per_million": 0.1,++ "cached_read_usd_per_million": 0.02,++ "cache_write_usd_per_million": 0.1,++ "output_usd_per_million": 0.4,++ "per_request_usd": 0.000005,++ "success_rate": 0.98++ }++}nnew file mode 100644@@@ -0,0 +1,17 @@++{++ "schema": "openvibe.ai.worker-artifact@1",++ "artifact": {++ "id": "mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated:q4_k_m@TODO-REVISION",++ "family": "qwen3",++ "variant": "abliterated",++ "quant": "q4_k_m",++ "revision": "TODO: pin the Hugging Face commit sha of mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF",++ "sha256": "TODO: copy the published SHA-256 of the Q4_K_M file; never invent or reuse a hash",++ "license": "apache-2.0",++ "source_url": "https://huggingface.co/mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF"++ },++ "download": {++ "url": "https://huggingface.co/mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF/resolve/TODO-REVISION/Huihui-Qwen3-4B-Instruct-2507-abliterated.Q4_K_M.gguf",++ "filename": "Huihui-Qwen3-4B-Instruct-2507-abliterated.Q4_K_M.gguf"++ }++}nnew file mode 100755@@@ -0,0 +1,98 @@++#!/bin/sh++# Download one GGUF artifact and verify its SHA-256 against the pinned manifest before it is used.++#++# verify-artifact.sh <manifest.json> <destination-dir>++#++# On success prints the verified local path (and nothing else) on stdout. Logs and failures go to++# stderr. Exit 1 means the worker must not start: a wrong hash is never repaired or retried with++# different bytes.++#++# The manifest is operator-supplied, not user content. Local URLs (file://, loopback http) are++# accepted only with ALLOW_LOCAL_ARTIFACT_URL=1 so a production manifest cannot be pointed at a++# local file by accident; a real pin is https:// plus a published SHA-256.++set -eu++++die() {++ printf 'artifact verification: %s\n' "$*" >&2++ exit 1++}++++[ "$#" -eq 2 ] || die "usage: verify-artifact.sh <manifest.json> <destination-dir>"++manifest=$1++dest=$2++++[ -f "$manifest" ] || die "manifest not found: $manifest"++[ -r "$manifest" ] || die "manifest is not readable by uid $(id -u): $manifest"++for command in jq curl sha256sum; do++ command -v "$command" >/dev/null 2>&1 || die "$command is required"++done++++schema=$(jq -r '.schema // empty' "$manifest" 2>/dev/null) || die "cannot parse manifest: $manifest"++[ "$schema" = "openvibe.ai.worker-artifact@1" ] || die "manifest schema must be openvibe.ai.worker-artifact@1 (got '${schema:-missing}')"++++field() {++ value=$(jq -r "$1" "$manifest" 2>/dev/null) || die "cannot read $2 from $manifest"++ case "$value" in++ ''|null) die "$2 is missing or empty in $manifest" ;;++ esac++ printf '%s' "$value"++}++++model_id=$(field '.artifact.id' 'artifact.id')++revision=$(field '.artifact.revision' 'artifact.revision')++sha=$(field '.artifact.sha256' 'artifact.sha256')++url=$(field '.download.url' 'download.url')++filename=$(field '.download.filename' 'download.filename')++++# A TODO is a promise the operator has not kept yet. Refuse it instead of downloading something++# that happens to be at a guessed URL, and never invent a hash.++case "$model_id $revision $sha $url $filename" in++ *TODO*) die "manifest still has a TODO placeholder; pin the revision and the published SHA-256 first" ;;++esac++++case "$sha" in++ *[!0-9a-f]*) die "artifact.sha256 must be 64 lowercase hex characters" ;;++esac++[ "${#sha}" -eq 64 ] || die "artifact.sha256 must be 64 lowercase hex characters"++++case "$filename" in++ ''|.|..|*/*) die "download.filename must be a plain file name" ;;++esac++++case "$url" in++ https://*) ;;++ file://*|http://127.0.0.1:*|http://localhost:*)++ [ "${ALLOW_LOCAL_ARTIFACT_URL:-0}" = "1" ] || die "local artifact URLs require ALLOW_LOCAL_ARTIFACT_URL=1 (use https:// for a real pin)"++ ;;++ *) die "download.url must be https:// (local URLs only in a smoke test)" ;;++esac++++mkdir -p "$dest" || die "cannot create $dest"++target="$dest/$filename"++part="$target.part.$$"++++if [ -f "$target" ]; then++ actual=$(sha256sum "$target" | awk '{print $1}')++ if [ "$actual" = "$sha" ]; then++ printf '%s\n' "$target"++ exit 0++ fi++ printf 'artifact verification: existing %s does not match the manifest; downloading again\n' "$target" >&2++ rm -f "$target"++fi++++rm -f "$part"++# --fail makes an HTTP error a non-zero exit; the file is never treated as an artifact.++if ! curl -fsSL --retry 3 --retry-connrefused --connect-timeout 20 -o "$part" "$url"; then++ rm -f "$part"++ die "download failed: $url"++fi++++actual=$(sha256sum "$part" | awk '{print $1}')++if [ "$actual" != "$sha" ]; then++ rm -f "$part"++ die "SHA-256 mismatch for $filename: manifest $sha, downloaded $actual; refusing to start"++fi++++mv "$part" "$target" || die "cannot move verified artifact into place: $target"++printf '%s\n' "$target"
new file mode 100755
new file mode 100755
@@ -0,0 +1,280 @@
@@ -0,0 +1,280 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Inference cost model (PLAN.md §E): cost per request and cost per successful task for
+ *
+ * rented-warm one GPU kept for the billing window; idle time is allocated
+ * scale-to-zero serverless per-second execution plus a cold start
+ * api-wholesale per-token wholesale API rates, all-in (no storage or egress line)
+ *
+ * The inputs are a planning estimate, not a quote. This script never reads a provider API and
+ * never invents a benchmark: every number comes from the JSON file the operator passes in.
+ * Customer liability is Billing's integer nano-USD ledger (D1); this script uses floats only
+ * for forecast arithmetic and says so in its output.
+ *
+ * node scripts/inference-cost-model.js [input.json] [--json]
+ *
+ * The default input is the clearly labelled example in
+ * workers/llm-inference/examples/inference-cost-model.example.json.
+ */
+const fs = require('fs');
+const path = require('path');
+
+const HOURS_PER_MONTH = 730;
+
+function fail(where, message) {
+ throw new Error(`${where}: ${message}`);
+}
+
+function object(input, key) {
+ const value = input[key];
+ if (value === null || typeof value !== 'object' || Array.isArray(value)) fail(key, 'must be an object');
+ return value;
+}
+
+function number(input, where, key, { required = true, min = 0, max = Infinity, integer = false, fallback } = {}) {
+ const value = input == null ? undefined : input[key];
+ if (value === undefined || value === null) {
+ if (!required) return fallback;
+ fail(where, 'is required');
+ }
+ if (typeof value !== 'number' || !Number.isFinite(value)) fail(where, 'must be a finite number');
+ if (integer && !Number.isInteger(value)) fail(where, 'must be an integer');
+ if (value < min) fail(where, `must be >= ${min}`);
+ if (value > max) fail(where, `must be <= ${max}`);
+ return value;
+}
+
+function rate(input, key, fallback) {
+ return number(input, key, key, { min: 0, required: fallback === undefined, fallback });
+}
+
+/**
+ * Validate an input document against workers/llm-inference/... and
+ * scripts/inference-cost-model.schema.json. Throws on the first problem.
+ */
+function validate(input) {
+ if (input === null || typeof input !== 'object' || Array.isArray(input)) throw new Error('input: must be a JSON object');
+
+ const window = object(input, 'window');
+ number(window, 'window.hours', 'hours', { min: Number.MIN_VALUE });
+ number(window, 'window.requests', 'requests', { min: 1, integer: true });
+
+ const request = object(input, 'request');
+ for (const key of ['input_tokens', 'cached_read_tokens', 'cache_write_tokens', 'output_tokens']) {
+ number(request, `request.${key}`, key, { min: 0, integer: true });
+ }
+ number(request, 'request.success_rate', 'success_rate', { min: Number.MIN_VALUE, max: 1 });
+
+ const throughput = object(input, 'throughput');
+ number(throughput, 'throughput.prefill_tokens_per_sec', 'prefill_tokens_per_sec', { min: Number.MIN_VALUE });
+ number(throughput, 'throughput.decode_tokens_per_sec', 'decode_tokens_per_sec', { min: Number.MIN_VALUE });
+ number(throughput, 'throughput.cached_read_tokens_per_sec', 'cached_read_tokens_per_sec', {
+ required: false, min: Number.MIN_VALUE, fallback: throughput.prefill_tokens_per_sec,
+ });
+ number(throughput, 'throughput.concurrency', 'concurrency', { required: false, min: 1, integer: true, fallback: 1 });
+
+ const gpu = object(input, 'gpu');
+ number(gpu, 'gpu.hourly_usd', 'hourly_usd', { min: 0 });
+ number(gpu, 'gpu.occupancy', 'occupancy', { min: Number.MIN_VALUE, max: 1 });
+ const policy = gpu.idle_policy;
+ if (!['billed', 'standby', 'stopped'].includes(policy)) fail('gpu.idle_policy', "must be 'billed', 'standby' or 'stopped'");
+ number(gpu, 'gpu.standby_hourly_usd', 'standby_hourly_usd', { required: policy === 'standby', min: 0, fallback: 0 });
+ number(gpu, 'gpu.startup_seconds', 'startup_seconds', { min: 0 });
+ number(gpu, 'gpu.storage_gb', 'storage_gb', { min: 0 });
+ number(gpu, 'gpu.storage_usd_per_gb_month', 'storage_usd_per_gb_month', { min: 0 });
+ number(gpu, 'gpu.egress_gb_per_request', 'egress_gb_per_request', { min: 0 });
+ number(gpu, 'gpu.egress_usd_per_gb', 'egress_usd_per_gb', { min: 0 });
+
+ const serverless = object(input, 'serverless');
+ number(serverless, 'serverless.hourly_usd', 'hourly_usd', { min: 0 });
+ number(serverless, 'serverless.startup_seconds', 'startup_seconds', { min: 0 });
+ number(serverless, 'serverless.requests_per_cold_start', 'requests_per_cold_start', { min: 1, fallback: 1, required: false });
+ number(serverless, 'serverless.storage_gb', 'storage_gb', { min: 0 });
+ number(serverless, 'serverless.storage_usd_per_gb_month', 'storage_usd_per_gb_month', { min: 0 });
+ number(serverless, 'serverless.egress_gb_per_request', 'egress_gb_per_request', { min: 0 });
+ number(serverless, 'serverless.egress_usd_per_gb', 'egress_usd_per_gb', { min: 0 });
+
+ const api = object(input, 'api_wholesale');
+ for (const key of ['input_usd_per_million', 'cached_read_usd_per_million', 'cache_write_usd_per_million', 'output_usd_per_million', 'per_request_usd']) {
+ number(api, `api_wholesale.${key}`, key, { min: 0 });
+ }
+ number(api, 'api_wholesale.success_rate', 'success_rate', { required: false, min: Number.MIN_VALUE, max: 1, fallback: request.success_rate });
+
+ return input;
+}
+
+function storagePerRequest(section, window) {
+ return section.storage_gb * section.storage_usd_per_gb_month * (window.hours / HOURS_PER_MONTH) / window.requests;
+}
+
+function egressPerRequest(section) {
+ return section.egress_gb_per_request * section.egress_usd_per_gb;
+}
+
+function round(value) {
+ // Strip binary noise from the report without pretending the estimate is exact.
+ return Number(value.toPrecision(12));
+}
+
+/**
+ * Compute the three deployment costs. Every returned amount is USD.
+ *
+ * Prompt seconds: fresh input + cache-write tokens run at prefill throughput; cached-read tokens
+ * run at the measured cached-read throughput when one is supplied, otherwise (conservatively) at
+ * prefill throughput. Decode seconds: output tokens / decode throughput.
+ *
+ * rented-warm: steady-state busy seconds / occupancy are billed (the rest is idle), idle at the
+ * policy's rate, plus one startup per window. scale-to-zero: busy + startup/requests_per_cold_start at the serverless
+ * hourly rate, no idle. api-wholesale: rate * tokens / 1e6 + per-request fee.
+ *
+ * Cost per successful task divides the per-request cost by the success rate (expected attempts).
+ */
+function compute(raw) {
+ const input = validate(raw);
+ const { window, request, throughput, gpu, serverless, api_wholesale: api } = input;
+
+ const cachedTps = throughput.cached_read_tokens_per_sec === undefined
+ ? throughput.prefill_tokens_per_sec
+ : throughput.cached_read_tokens_per_sec;
+ const prefillSeconds = (request.input_tokens + request.cache_write_tokens) / throughput.prefill_tokens_per_sec
+ + request.cached_read_tokens / cachedTps;
+ const decodeSeconds = request.output_tokens / throughput.decode_tokens_per_sec;
+ const busySeconds = prefillSeconds + decodeSeconds;
+
+ // rented-warm
+ const idleSeconds = busySeconds * ((1 - gpu.occupancy) / gpu.occupancy);
+ const idleHourly = gpu.idle_policy === 'billed'
+ ? gpu.hourly_usd
+ : gpu.idle_policy === 'standby' ? gpu.standby_hourly_usd : 0;
+ const rentedParts = {
+ gpu_busy: busySeconds * gpu.hourly_usd / 3600,
+ gpu_idle: idleSeconds * idleHourly / 3600,
+ startup: gpu.startup_seconds * gpu.hourly_usd / 3600 / window.requests,
+ storage: storagePerRequest(gpu, window),
+ egress: egressPerRequest(gpu),
+ };
+ const rentedPerRequest = Object.values(rentedParts).reduce((sum, value) => sum + value, 0);
+
+ // scale-to-zero serverless
+ const coldStartsPerRequest = 1 / serverless.requests_per_cold_start;
+ const serverlessParts = {
+ gpu_execution: busySeconds * serverless.hourly_usd / 3600,
+ startup: serverless.startup_seconds * serverless.hourly_usd / 3600 * coldStartsPerRequest,
+ storage: storagePerRequest(serverless, window),
+ egress: egressPerRequest(serverless),
+ };
+ const serverlessPerRequest = Object.values(serverlessParts).reduce((sum, value) => sum + value, 0);
+
+ // wholesale API: all-in rates, no owned storage or egress
+ const apiParts = {
+ input: request.input_tokens * api.input_usd_per_million / 1e6,
+ cached_read: request.cached_read_tokens * api.cached_read_usd_per_million / 1e6,
+ cache_write: request.cache_write_tokens * api.cache_write_usd_per_million / 1e6,
+ output: request.output_tokens * api.output_usd_per_million / 1e6,
+ per_request: api.per_request_usd,
+ };
+ const apiPerRequest = Object.values(apiParts).reduce((sum, value) => sum + value, 0);
+
+ const rows = [
+ { mode: 'rented-warm', costPerRequest: rentedPerRequest, successRate: request.success_rate, parts: rentedParts },
+ { mode: 'scale-to-zero', costPerRequest: serverlessPerRequest, successRate: request.success_rate, parts: serverlessParts },
+ { mode: 'api-wholesale', costPerRequest: apiPerRequest, successRate: api.success_rate, parts: apiParts },
+ ].map((row) => ({
+ ...row,
+ costPerRequest: round(row.costPerRequest),
+ costPerSuccessfulTask: round(row.costPerRequest / row.successRate),
+ parts: Object.fromEntries(Object.entries(row.parts).map(([key, value]) => [key, round(value)])),
+ }));
+
+ return {
+ schema: 'openvibe.inference-cost-model-result@1',
+ note: 'Planning estimate only, not a quote. Billing settles in integer nano-USD (D1); these are floats.',
+ inputNote: typeof input.note === 'string' ? input.note : null,
+ window: { hours: window.hours, requests: window.requests },
+ request: {
+ input_tokens: request.input_tokens,
+ cached_read_tokens: request.cached_read_tokens,
+ cache_write_tokens: request.cache_write_tokens,
+ output_tokens: request.output_tokens,
+ success_rate: request.success_rate,
+ },
+ seconds: {
+ prefill: round(prefillSeconds),
+ decode: round(decodeSeconds),
+ busy: round(busySeconds),
+ rented_idle: round(idleSeconds),
+ concurrency: throughput.concurrency === undefined ? 1 : throughput.concurrency,
+ },
+ rows,
+ };
+}
+
+function money(value) {
+ if (value === 0) return '$0';
+ if (value < 1e-6) return `$${value.toExponential(2)}`;
+ return `$${value.toFixed(6)}`;
+}
+
+function formatTable(result) {
+ const r = result.request;
+ const lines = [
+ 'Inference cost model — planning estimate, not a quote',
+ `Window: ${result.window.hours} h, ${result.window.requests} requests`,
+ `Request mix: I=${r.input_tokens} R=${r.cached_read_tokens} W=${r.cache_write_tokens} O=${r.output_tokens} tokens, success ${(r.success_rate * 100).toFixed(2)}%`,
+ `Throughput: ${result.seconds.prefill} s prefill + ${result.seconds.decode} s decode = ${result.seconds.busy} s GPU-busy per request (concurrency ${result.seconds.concurrency})`,
+ '',
+ 'Mode $/request $/successful task success',
+ ];
+ for (const row of result.rows) {
+ lines.push(`${row.mode.padEnd(16)} ${money(row.costPerRequest).padStart(13)} ${money(row.costPerSuccessfulTask).padStart(18)} ${(row.successRate * 100).toFixed(2)}%`);
+ }
+ lines.push('', 'Breakdown (USD per request):');
+ for (const row of result.rows) {
+ const parts = Object.entries(row.parts).map(([key, value]) => `${key}=${money(value)}`).join(' ');
+ lines.push(` ${row.mode.padEnd(16)} ${parts}`);
+ }
+ if (result.inputNote) lines.push('', `Input note: ${result.inputNote}`);
+ lines.push('', 'Floats are for forecasting only; Billing settles in integer nano-USD. Verify every rate and measured throughput before quoting.');
+ return lines.join('\n');
+}
+
+function usage() {
+ return [
+ 'Usage: node scripts/inference-cost-model.js [input.json] [--json]',
+ '',
+ ' input.json defaults to workers/llm-inference/examples/inference-cost-model.example.json',
+ ' --json print the computed result as JSON instead of a table',
+ '',
+ 'The input schema is scripts/inference-cost-model.schema.json.',
+ ].join('\n');
+}
+
+function main(argv) {
+ let file = null;
+ let asJson = false;
+ for (const arg of argv) {
+ if (arg === '--json') asJson = true;
+ else if (arg === '--help' || arg === '-h') { console.log(usage()); return 0; }
+ else if (arg.startsWith('-')) throw new Error(`unknown option: ${arg}\n${usage()}`);
+ else if (file) throw new Error(`only one input file is accepted\n${usage()}`);
+ else file = arg;
+ }
+ if (!file) file = path.join(__dirname, '..', 'workers', 'llm-inference', 'examples', 'inference-cost-model.example.json');
+ const input = JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));
+ const result = compute(input);
+ if (asJson) console.log(JSON.stringify(result, null, 2));
+ else console.log(formatTable(result));
+ return 0;
+}
+
+if (require.main === module) {
+ try {
+ process.exitCode = main(process.argv.slice(2));
+ } catch (err) {
+ console.error(`inference-cost-model: ${err.message}`);
+ process.exitCode = 1;
+ }
+}
+
+module.exports = { compute, validate, formatTable, money, HOURS_PER_MONTH };
new file mode 100644
new file mode 100644
@@ -0,0 +1,241 @@
@@ -0,0 +1,241 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://openvibe.network/schemas/inference-cost-model.v1.json",
+ "title": "InferenceCostModelInput",
+ "description": "Inputs for scripts/inference-cost-model.js (PLAN.md §E). Every price and throughput is an operator-supplied placeholder or measurement; this schema cannot tell a quote from a guess, so the input must say which it is in `note`.",
+ "type": "object",
+ "required": ["window", "request", "throughput", "gpu", "serverless", "api_wholesale"],
+ "additionalProperties": false,
+ "properties": {
+ "$schema": {
+ "type": "string",
+ "description": "Optional editor association with this schema file."
+ },
+ "note": {
+ "type": "string",
+ "description": "Provenance, for example 'example, not a quote' or the quote ids and date checked."
+ },
+ "window": {
+ "type": "object",
+ "required": ["hours", "requests"],
+ "additionalProperties": false,
+ "properties": {
+ "hours": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "description": "Length of the measurement or billing window in hours."
+ },
+ "requests": {
+ "type": "integer",
+ "minimum": 1,
+ "description": "Requests in the window. Startup and storage are amortized over this count for rented-warm."
+ }
+ }
+ },
+ "request": {
+ "type": "object",
+ "required": ["input_tokens", "cached_read_tokens", "cache_write_tokens", "output_tokens", "success_rate"],
+ "additionalProperties": false,
+ "properties": {
+ "input_tokens": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "I: fresh input tokens per request."
+ },
+ "cached_read_tokens": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "R: input tokens read from a prompt cache per request."
+ },
+ "cache_write_tokens": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "W: input tokens written to a prompt cache per request."
+ },
+ "output_tokens": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "O: generated output tokens per request."
+ },
+ "success_rate": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1,
+ "description": "Fraction of requests that produce a successful task (0 < rate <= 1). Cost per successful task divides by this."
+ }
+ }
+ },
+ "throughput": {
+ "type": "object",
+ "required": ["prefill_tokens_per_sec", "decode_tokens_per_sec"],
+ "additionalProperties": false,
+ "properties": {
+ "prefill_tokens_per_sec": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "description": "Measured prompt-processing throughput at the stated concurrency."
+ },
+ "decode_tokens_per_sec": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "description": "Measured output throughput at the stated concurrency."
+ },
+ "cached_read_tokens_per_sec": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "description": "Optional measured throughput for cached-read tokens. Omit to charge them at prefill throughput (conservative)."
+ },
+ "concurrency": {
+ "type": "integer",
+ "minimum": 1,
+ "default": 1,
+ "description": "Concurrency at which the two throughput numbers were measured. It is recorded, not used to multiply throughput."
+ }
+ }
+ },
+ "gpu": {
+ "type": "object",
+ "required": ["hourly_usd", "occupancy", "idle_policy", "startup_seconds", "storage_gb", "storage_usd_per_gb_month", "egress_gb_per_request", "egress_usd_per_gb"],
+ "additionalProperties": false,
+ "properties": {
+ "hourly_usd": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Rented GPU rate in USD per GPU-hour, from a complete quote."
+ },
+ "occupancy": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1,
+ "description": "Steady-state fraction of paid GPU wall time that is busy, excluding the one-time startup. Billed steady-state time is busy / occupancy."
+ },
+ "idle_policy": {
+ "type": "string",
+ "enum": ["billed", "standby", "stopped"],
+ "description": "What idle time costs: the GPU rate, standby_hourly_usd, or nothing."
+ },
+ "standby_hourly_usd": {
+ "type": "number",
+ "minimum": 0,
+ "default": 0,
+ "description": "Idle rate when idle_policy is 'standby' (required in that case)."
+ },
+ "startup_seconds": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Cold start per window: download, load, and warm-up. Charged at the GPU rate once, in addition to the occupancy allocation, and amortized over window.requests."
+ },
+ "storage_gb": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Model/artifact storage held for the window."
+ },
+ "storage_usd_per_gb_month": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Storage rate for a 730-hour month."
+ },
+ "egress_gb_per_request": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Network egress per request."
+ },
+ "egress_usd_per_gb": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Egress rate."
+ }
+ },
+ "allOf": [
+ {
+ "if": {
+ "properties": { "idle_policy": { "const": "standby" } },
+ "required": ["idle_policy"]
+ },
+ "then": { "required": ["standby_hourly_usd"] }
+ }
+ ]
+ },
+ "serverless": {
+ "type": "object",
+ "required": ["hourly_usd", "startup_seconds", "storage_gb", "storage_usd_per_gb_month", "egress_gb_per_request", "egress_usd_per_gb"],
+ "additionalProperties": false,
+ "properties": {
+ "hourly_usd": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Scale-to-zero per-second rate in USD per GPU-hour. No idle charge is modelled."
+ },
+ "startup_seconds": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Cold start charged on each scale-up."
+ },
+ "requests_per_cold_start": {
+ "type": "number",
+ "minimum": 1,
+ "default": 1,
+ "description": "Requests served per cold start before scaling down. 1 = every request pays a cold start."
+ },
+ "storage_gb": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Artifact storage held for the window."
+ },
+ "storage_usd_per_gb_month": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Storage rate for a 730-hour month."
+ },
+ "egress_gb_per_request": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Network egress per request."
+ },
+ "egress_usd_per_gb": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Egress rate."
+ }
+ }
+ },
+ "api_wholesale": {
+ "type": "object",
+ "required": ["input_usd_per_million", "cached_read_usd_per_million", "cache_write_usd_per_million", "output_usd_per_million", "per_request_usd"],
+ "additionalProperties": false,
+ "properties": {
+ "input_usd_per_million": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Wholesale input-token rate per million tokens."
+ },
+ "cached_read_usd_per_million": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Wholesale cached-read rate per million tokens."
+ },
+ "cache_write_usd_per_million": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Wholesale cache-write rate per million tokens."
+ },
+ "output_usd_per_million": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Wholesale output-token rate per million tokens."
+ },
+ "per_request_usd": {
+ "type": "number",
+ "minimum": 0,
+ "description": "Fixed fee per request, if the agreement has one."
+ },
+ "success_rate": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1,
+ "description": "Optional provider-specific success rate; falls back to request.success_rate."
+ }
+ }
+ }
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+'use strict';
+// workers/llm-inference/verify-artifact.sh: a pinned artifact is downloaded once, checked against
+// the manifest SHA-256 and refused (without leaving a file behind) on any mismatch or placeholder.
+const assert = require('assert');
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { pathToFileURL } = require('url');
+
+const root = path.join(__dirname, '..');
+const verify = path.join(root, 'workers', 'llm-inference', 'verify-artifact.sh');
+const entrypoint = path.join(root, 'workers', 'llm-inference', 'entrypoint.sh');
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-artifact-'));
+const baseEnv = { ...process.env };
+delete baseEnv.ALLOW_LOCAL_ARTIFACT_URL;
+
+function sha256(file) {
+ return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+}
+
+function manifest({ sha, revision = 'c9e90669eeb205d5af35c28a3e9983fc9293c2ec', filename = 'model.gguf', url, schema = 'openvibe.ai.worker-artifact@1' }) {
+ return {
+ schema,
+ artifact: {
+ id: `example/qwen3-4b:q4_k_m@${revision}`,
+ revision,
+ sha256: sha,
+ license: 'apache-2.0',
+ },
+ download: { url, filename },
+ };
+}
+
+function writeManifest(name, document) {
+ const file = path.join(tmp, name);
+ fs.writeFileSync(file, JSON.stringify(document, null, 2));
+ return file;
+}
+
+function run(manifestFile, dest, { allowLocal = true } = {}) {
+ const env = { ...baseEnv };
+ if (allowLocal) env.ALLOW_LOCAL_ARTIFACT_URL = '1';
+ return spawnSync('sh', [verify, manifestFile, dest], { encoding: 'utf8', env });
+}
+
+const source = path.join(tmp, 'source.gguf');
+fs.writeFileSync(source, 'a small GGUF stand-in, not a model');
+const goodSha = sha256(source);
+
+// A correct manifest downloads, verifies and prints the local path on stdout.
+const dest1 = path.join(tmp, 'models-1');
+const ok = run(writeManifest('good.json', manifest({ sha: goodSha, url: pathToFileURL(source).href })), dest1);
+assert.strictEqual(ok.status, 0, ok.stderr);
+assert.strictEqual(ok.stdout.trim(), path.join(dest1, 'model.gguf'), 'stdout is only the verified path');
+assert.strictEqual(fs.readFileSync(path.join(dest1, 'model.gguf'), 'utf8'), fs.readFileSync(source, 'utf8'));
+
+// A wrong SHA-256 is refused, and neither the final file nor a partial file is left behind.
+const dest2 = path.join(tmp, 'models-2');
+const wrong = run(writeManifest('wrong.json', manifest({ sha: '0'.repeat(64), url: pathToFileURL(source).href })), dest2);
+assert.notStrictEqual(wrong.status, 0, 'wrong hash must fail');
+assert.match(wrong.stderr, /SHA-256 mismatch/);
+assert.ok(!fs.existsSync(path.join(dest2, 'model.gguf')), 'mismatched file must not be installed');
+assert.deepStrictEqual(fs.readdirSync(dest2), [], 'partial download must be removed');
+
+// A TODO placeholder (revision or hash) is refused before any download.
+const todo = run(writeManifest('todo.json', manifest({ sha: goodSha, revision: 'TODO: pin the commit', url: pathToFileURL(source).href })), path.join(tmp, 'models-todo'));
+assert.notStrictEqual(todo.status, 0);
+assert.match(todo.stderr, /TODO placeholder/);
+
+// A manifest cannot write outside its model directory.
+const traversal = run(writeManifest('traversal.json', manifest({ sha: goodSha, filename: '../escape.gguf', url: pathToFileURL(source).href })), path.join(tmp, 'models-traversal'));
+assert.notStrictEqual(traversal.status, 0);
+assert.match(traversal.stderr, /plain file name/);
+assert.ok(!fs.existsSync(path.join(tmp, 'escape.gguf')));
+
+// Local URLs are for smoke tests only; production config has to be https.
+const local = run(writeManifest('local.json', manifest({ sha: goodSha, url: pathToFileURL(source).href })), path.join(tmp, 'models-local'), { allowLocal: false });
+assert.notStrictEqual(local.status, 0);
+assert.match(local.stderr, /ALLOW_LOCAL_ARTIFACT_URL/);
+
+// A malformed hash is refused even when its length is not 64.
+const shortHash = run(writeManifest('short.json', manifest({ sha: 'abc', url: pathToFileURL(source).href })), path.join(tmp, 'models-short'));
+assert.notStrictEqual(shortHash.status, 0);
+assert.match(shortHash.stderr, /64 lowercase hex/);
+
+// A cached file that already matches is reused without touching the URL.
+const dest4 = path.join(tmp, 'models-cached');
+fs.mkdirSync(dest4);
+fs.writeFileSync(path.join(dest4, 'model.gguf'), fs.readFileSync(source));
+const cached = run(writeManifest('missing-source.json', manifest({ sha: goodSha, url: 'file:///nonexistent/model.gguf' })), dest4);
+assert.strictEqual(cached.status, 0, cached.stderr);
+assert.strictEqual(cached.stdout.trim(), path.join(dest4, 'model.gguf'));
+
+// The scripts parse under the shell the image uses, and the entrypoint refuses bounded settings
+// before it reaches the model or the binaries.
+for (const file of [verify, entrypoint]) {
+ const syntax = spawnSync('sh', ['-n', file], { encoding: 'utf8' });
+ assert.strictEqual(syntax.status, 0, `${file}: ${syntax.stderr}`);
+}
+const cases = [
+ [{ LLM_CTX: '40000', LLM_CTX_MAX: '32768' }, /exceeds LLM_CTX_MAX/],
+ [{ LLM_CTX: '1024', LLM_PARALLEL: '4' }, /at least 512 tokens per slot/],
+ [{ LLM_KV_TYPE_K: 'magic' }, /LLM_KV_TYPE_K/],
+ [{ DRAIN_SECONDS: '0' }, /DRAIN_SECONDS/],
+];
+for (const [env, pattern] of cases) {
+ const refused = spawnSync('sh', [entrypoint], { encoding: 'utf8', env: { ...baseEnv, ...env } });
+ assert.notStrictEqual(refused.status, 0, JSON.stringify(env));
+ assert.match(refused.stderr, pattern, JSON.stringify(env));
+}
+
+fs.rmSync(tmp, { recursive: true, force: true });
+console.log('inference artifact verification: all checks passed');
new file mode 100644
new file mode 100644
@@ -0,0 +1,158 @@
@@ -0,0 +1,158 @@
+'use strict';
+// scripts/inference-cost-model.js: the PLAN.md §E arithmetic on fixed, hand-computed inputs.
+// Every expected number below is written out by hand in the comment above it; the test does not
+// re-derive it with the implementation.
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const { compute, formatTable } = require('../scripts/inference-cost-model');
+
+function close(actual, expected, label) {
+ assert.ok(Math.abs(actual - expected) < 1e-12, `${label}: expected ${expected}, got ${actual}`);
+}
+
+// 1000 prefill tokens at 1000 tok/s + 100 output tokens at 50 tok/s = 1 s + 2 s = 3 s busy.
+// Rented: occupancy 0.5 -> 6 s paid at $1.80/h = 6/3600 * 1.8 = $0.003 per request.
+// 0.8 success -> $0.003 / 0.8 = $0.00375 per successful task.
+// Serverless: 3 s at $3.60/h = $0.003; no startup.
+// API: 1000 * $100/M = $0.1, 100 * $400/M = $0.04, per request $0.001 -> $0.141;
+// $0.141 / 0.8 = $0.17625 per successful task.
+const fixed = {
+ note: 'test fixture',
+ window: { hours: 10, requests: 100 },
+ request: { input_tokens: 1000, cached_read_tokens: 0, cache_write_tokens: 0, output_tokens: 100, success_rate: 0.8 },
+ throughput: { prefill_tokens_per_sec: 1000, decode_tokens_per_sec: 50, concurrency: 1 },
+ gpu: {
+ hourly_usd: 1.8, occupancy: 0.5, idle_policy: 'billed', standby_hourly_usd: 0, startup_seconds: 0,
+ storage_gb: 0, storage_usd_per_gb_month: 0, egress_gb_per_request: 0, egress_usd_per_gb: 0,
+ },
+ serverless: {
+ hourly_usd: 3.6, startup_seconds: 0, requests_per_cold_start: 1,
+ storage_gb: 0, storage_usd_per_gb_month: 0, egress_gb_per_request: 0, egress_usd_per_gb: 0,
+ },
+ api_wholesale: {
+ input_usd_per_million: 100, cached_read_usd_per_million: 10, cache_write_usd_per_million: 100,
+ output_usd_per_million: 400, per_request_usd: 0.001, success_rate: 0.8,
+ },
+};
+
+const result = compute(fixed);
+assert.strictEqual(result.rows.length, 3);
+assert.deepStrictEqual(result.rows.map((row) => row.mode), ['rented-warm', 'scale-to-zero', 'api-wholesale']);
+close(result.seconds.prefill, 1, 'prefill seconds');
+close(result.seconds.decode, 2, 'decode seconds');
+close(result.seconds.busy, 3, 'busy seconds');
+close(result.seconds.rented_idle, 3, 'rented idle seconds');
+close(result.rows[0].costPerRequest, 0.003, 'rented-warm per request');
+close(result.rows[0].costPerSuccessfulTask, 0.00375, 'rented-warm per success');
+close(result.rows[1].costPerRequest, 0.003, 'serverless per request');
+close(result.rows[1].costPerSuccessfulTask, 0.00375, 'serverless per success');
+close(result.rows[2].costPerRequest, 0.141, 'api per request');
+close(result.rows[2].costPerSuccessfulTask, 0.17625, 'api per success');
+
+// A full month makes storage easy: 10 GB * $1/GB-month = $10, over 1000 requests = $0.01 each.
+// Busy 3 s at $3.60/h = $0.003; idle 9 s (occupancy 0.25) at standby $0.36/h = $0.0009;
+// startup 36 s at $3.60/h = $0.036 / 1000 requests = $0.000036; egress 0.001 GB * $2 = $0.002.
+// Rented total = 0.003 + 0.0009 + 0.000036 + 0.01 + 0.002 = 0.015936; / 0.8 = 0.01992.
+// Serverless: 3 s at $7.20/h = $0.006; startup 18 s at $7.20/h / 3 requests = $0.012;
+// storage 0.01 + egress 0.002 -> 0.03; / 0.8 = 0.0375.
+const withOverheads = JSON.parse(JSON.stringify(fixed));
+withOverheads.window = { hours: 730, requests: 1000 };
+withOverheads.gpu.hourly_usd = 3.6;
+withOverheads.gpu.occupancy = 0.25;
+withOverheads.gpu.idle_policy = 'standby';
+withOverheads.gpu.standby_hourly_usd = 0.36;
+withOverheads.gpu.startup_seconds = 36;
+withOverheads.gpu.storage_gb = 10;
+withOverheads.gpu.storage_usd_per_gb_month = 1;
+withOverheads.gpu.egress_gb_per_request = 0.001;
+withOverheads.gpu.egress_usd_per_gb = 2;
+withOverheads.serverless.hourly_usd = 7.2;
+withOverheads.serverless.startup_seconds = 18;
+withOverheads.serverless.requests_per_cold_start = 3;
+withOverheads.serverless.storage_gb = 10;
+withOverheads.serverless.storage_usd_per_gb_month = 1;
+withOverheads.serverless.egress_gb_per_request = 0.001;
+withOverheads.serverless.egress_usd_per_gb = 2;
+
+const overheads = compute(withOverheads);
+close(overheads.rows[0].costPerRequest, 0.015936, 'rented-warm with overheads');
+close(overheads.rows[0].costPerSuccessfulTask, 0.01992, 'rented-warm with overheads per success');
+close(overheads.rows[1].costPerRequest, 0.03, 'serverless with overheads');
+close(overheads.rows[1].costPerSuccessfulTask, 0.0375, 'serverless with overheads per success');
+
+// Cached read: 500 R and 250 W tokens. Without a measured cached-read rate, prefill is
+// (1000 + 250)/1000 + 500/1000 = 1.75 s. With 5000 cached tok/s it is 1.25 + 500/5000 = 1.35 s.
+const cached = JSON.parse(JSON.stringify(fixed));
+cached.request.input_tokens = 1000;
+cached.request.cached_read_tokens = 500;
+cached.request.cache_write_tokens = 250;
+cached.request.output_tokens = 100;
+cached.api_wholesale.success_rate = 0.8;
+const noCacheRate = compute(cached);
+close(noCacheRate.seconds.prefill, 1.75, 'cached tokens at prefill rate');
+// API: 1000*100/M + 500*10/M + 250*100/M + 100*400/M + 0.001 = 0.1 + 0.005 + 0.025 + 0.04 + 0.001.
+close(noCacheRate.rows[2].costPerRequest, 0.171, 'api cached-read mix');
+close(noCacheRate.rows[2].costPerSuccessfulTask, 0.21375, 'api cached-read mix per success');
+cached.throughput.cached_read_tokens_per_sec = 5000;
+const measuredCache = compute(cached);
+close(measuredCache.seconds.prefill, 1.35, 'cached tokens at measured rate');
+
+// A per-mode API success rate overrides the request success rate.
+const apiSuccess = JSON.parse(JSON.stringify(fixed));
+apiSuccess.api_wholesale.success_rate = 0.5;
+assert.ok(Math.abs(compute(apiSuccess).rows[2].costPerSuccessfulTask - 0.282) < 1e-12);
+
+// The failure allocation is a division by the success rate, so it is visible in both directions.
+const allSuccess = JSON.parse(JSON.stringify(fixed));
+allSuccess.request.success_rate = 1;
+assert.ok(compute(allSuccess).rows[0].costPerSuccessfulTask === compute(allSuccess).rows[0].costPerRequest);
+
+// Invalid inputs are refused with the field named.
+for (const [label, mutate, pattern] of [
+ ['occupancy zero', (x) => { x.gpu.occupancy = 0; }, /gpu\.occupancy/],
+ ['success rate zero', (x) => { x.request.success_rate = 0; }, /success_rate/],
+ ['negative output', (x) => { x.request.output_tokens = -1; }, /output_tokens/],
+ ['missing serverless', (x) => { delete x.serverless; }, /serverless/],
+ ['standby rate missing', (x) => { x.gpu.idle_policy = 'standby'; delete x.gpu.standby_hourly_usd; }, /standby_hourly_usd/],
+ ['bad policy', (x) => { x.gpu.idle_policy = 'free'; }, /idle_policy/],
+ ['zero prefill', (x) => { x.throughput.prefill_tokens_per_sec = 0; }, /prefill_tokens_per_sec/],
+ ['fractional requests', (x) => { x.window.requests = 1.5; }, /window\.requests/],
+]) {
+ const bad = JSON.parse(JSON.stringify(fixed));
+ mutate(bad);
+ assert.throws(() => compute(bad), pattern, label);
+}
+
+// The checked-in example is clearly a placeholder and still runs through the whole model.
+const examplePath = path.join(__dirname, '..', 'workers', 'llm-inference', 'examples', 'inference-cost-model.example.json');
+const example = JSON.parse(fs.readFileSync(examplePath, 'utf8'));
+assert.match(example.note, /example, not a quote/);
+const exampleResult = compute(example);
+assert.strictEqual(exampleResult.rows.length, 3);
+for (const row of exampleResult.rows) {
+ assert.ok(Number.isFinite(row.costPerRequest) && row.costPerRequest > 0, `${row.mode} per request`);
+ assert.ok(Number.isFinite(row.costPerSuccessfulTask) && row.costPerSuccessfulTask > 0, `${row.mode} per success`);
+}
+const table = formatTable(exampleResult);
+assert.match(table, /planning estimate, not a quote/);
+assert.match(table, /rented-warm/);
+assert.match(table, /scale-to-zero/);
+assert.match(table, /api-wholesale/);
+
+// The CLI reads the same file and prints JSON.
+const output = execFileSync(process.execPath, [path.join(__dirname, '..', 'scripts', 'inference-cost-model.js'), examplePath, '--json'], { encoding: 'utf8' });
+const cli = JSON.parse(output);
+assert.deepStrictEqual(cli.rows.map((row) => row.mode), ['rented-warm', 'scale-to-zero', 'api-wholesale']);
+
+// The schema and the example describe the same fields; this catches drift in either direction.
+const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'scripts', 'inference-cost-model.schema.json'), 'utf8'));
+for (const key of ['window', 'request', 'throughput', 'gpu', 'serverless', 'api_wholesale']) {
+ assert.ok(schema.properties[key], `schema property ${key}`);
+ for (const field of Object.keys(example[key])) {
+ assert.ok(schema.properties[key].properties[field], `schema property ${key}.${field}`);
+ }
+}
+
+console.log('inference cost model: all hand-computed checks passed');
new file mode 100644
new file mode 100644
@@ -0,0 +1,43 @@
@@ -0,0 +1,43 @@
+# Gateway for the OpenVibe LLM inference worker.
+#
+# llama.cpp listens only on loopback; this is the only public listener. It adds the two endpoints
+# llama.cpp does not have as separate paths:
+#
+# GET /ready the same check as llama.cpp GET /health, so a slot/model that is not loaded is
+# not ready. The orchestrator removes the worker from rotation when this is not 2xx.
+# GET /health passed through to llama.cpp.
+# GET /metrics passed through, Prometheus text.
+#
+# Everything else (OpenAI-compatible chat/completions, props, slots, ...) is passed through
+# unchanged. Streaming responses are flushed immediately. Dynamic LoRA management is not exposed:
+# the worker starts no LoRA and the vLLM-style load/unload paths are refused here as well.
+#
+# DRAIN_SECONDS is also the gateway's shutdown grace period: on SIGTERM Caddy stops accepting new
+# connections and waits up to this long for in-flight requests before the entrypoint stops
+# llama.cpp.
+
+{
+ admin off
+ auto_https off
+ persist_config off
+ grace_period {$DRAIN_SECONDS}s
+}
+
+:{$PORT} {
+ @lora path /lora* /v1/load_lora_adapter* /v1/unload_lora_adapter*
+ respond @lora 404
+
+ @ready path /ready
+ handle @ready {
+ rewrite * /health
+ reverse_proxy 127.0.0.1:{$LLM_PORT} {
+ flush_interval -1
+ }
+ }
+
+ handle {
+ reverse_proxy 127.0.0.1:{$LLM_PORT} {
+ flush_interval -1
+ }
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,65 @@
@@ -0,0 +1,65 @@
+# syntax=docker/dockerfile:1
+#
+# OpenVibe LLM inference worker (llama.cpp server, GGUF). Build from this directory:
+#
+# docker build -t openvibe/llm-inference .
+# docker build --build-arg LLAMA_CPP_TAG=server \
+# --build-arg LLAMA_CPP_DIGEST=sha256:10a38e6cc91760ab157f0a0a8546218d35f4a314432a70298abfa3d096829ad7 \
+# -t openvibe/llm-inference:cpu .
+#
+# Both llama.cpp bases are pinned by digest, not by tag alone. The tag names the artifact for a
+# human reader; the digest is what Docker resolves. Refresh a pin only as a deliberate change:
+#
+# docker buildx imagetools inspect ghcr.io/ggml-org/llama.cpp:server-cuda
+# docker buildx imagetools inspect docker.io/library/caddy:2
+#
+# Resolved 2026-11-04 (server-cuda and server are the upstream rolling tags at that moment).
+ARG LLAMA_CPP_IMAGE=ghcr.io/ggml-org/llama.cpp
+ARG LLAMA_CPP_TAG=server-cuda
+ARG LLAMA_CPP_DIGEST=sha256:d5ffba7d096ab0236c65dc147d797b4b7658a7e5425634593373635d661bccda
+
+# The gateway binary is copied from the pinned Caddy image; no package repository is trusted at
+# build time for it.
+ARG CADDY_IMAGE=docker.io/library/caddy
+ARG CADDY_TAG=2
+ARG CADDY_DIGEST=sha256:36c24ad91a7ff7bc4c87fe22bfa8df3c4b18da0773d73fc12ee2c52b98c0bb7e
+
+FROM ${CADDY_IMAGE}:${CADDY_TAG}@${CADDY_DIGEST} AS gateway
+
+FROM ${LLAMA_CPP_IMAGE}:${LLAMA_CPP_TAG}@${LLAMA_CPP_DIGEST}
+
+# jq parses the manifest; curl and sha256sum come from the llama.cpp runtime image (curl is part
+# of that image, the checksum tool is coreutils). passwd is explicit so groupadd/useradd exist on
+# every base variant.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates jq passwd \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && groupadd --system openvibe \
+ && useradd --system --gid openvibe --home-dir /app --shell /usr/sbin/nologin openvibe \
+ && mkdir -p /models /etc/openvibe /etc/caddy /tmp/caddy \
+ && chown -R openvibe:openvibe /models /tmp/caddy
+
+COPY --from=gateway /usr/bin/caddy /usr/local/bin/caddy
+COPY entrypoint.sh verify-artifact.sh /usr/local/bin/
+COPY Caddyfile /etc/caddy/Caddyfile
+
+RUN chmod 0755 /usr/local/bin/entrypoint.sh /usr/local/bin/verify-artifact.sh /usr/local/bin/caddy \
+ && chown -R openvibe:openvibe /etc/caddy /etc/openvibe
+
+ENV PORT=8080 \
+ LLM_PORT=8081 \
+ DRAIN_SECONDS=30 \
+ HOME=/tmp/caddy \
+ XDG_DATA_HOME=/tmp/caddy \
+ XDG_CONFIG_HOME=/tmp/caddy
+
+USER openvibe
+EXPOSE 8080
+
+# Start period covers the artifact download plus model load; until llama.cpp answers /health the
+# gateway's /ready is not 2xx.
+HEALTHCHECK --interval=15s --timeout=3s --start-period=900s --retries=3 \
+ CMD curl -fsS "http://127.0.0.1:${PORT:-8080}/ready" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
new file mode 100644
new file mode 100644
@@ -0,0 +1,232 @@
@@ -0,0 +1,232 @@
+# LLM inference worker (llama.cpp, GGUF)
+
+A portable GPU worker image for one pinned GGUF artifact: llama.cpp server on loopback, a small
+Caddy gateway in front, checksum-verified model download, bounded context/parallel/KV settings,
+`/health` + `/ready`, the Prometheus `/metrics` endpoint passed through, and a graceful drain on
+`SIGTERM`. No dynamic LoRA API is exposed.
+
+## Ownership
+
+OpenVibe.Host owns worker/deployment orchestration and safe budgeted GPU placement (PLAN.md §B),
+so the deployment artifact and its cost model live here. The model/variant registry, catalog
+entries, revisions, routing and inference requests are OpenVibe.AI's (PLAN.md §B, D3/D4); this
+worker consumes one pinned artifact and never registers or advertises a model. `docs/nodes.md`
+describes Host's node-registry reporting, and Host's `workerUnits` inventory hook is for units that
+drain and exit on their own (Host never starts, stops or restarts them). This image is that kind
+of long-running worker: it can run under a `workerUnits` unit or any GPU orchestrator.
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `Dockerfile` | llama.cpp base pinned by digest, plus a Caddy binary copied from a pinned image |
+| `entrypoint.sh` | validates env bounds, verifies the artifact, starts llama.cpp + gateway, drains |
+| `verify-artifact.sh` | downloads by URL, checks the manifest SHA-256, refuses a mismatch |
+| `Caddyfile` | `/health`, `/ready`, `/metrics` and API passthrough; blocks LoRA paths |
+| `examples/qwen3-4b-abliterated.manifest.example.json` | one compact abliterated Qwen 4B artifact, revision and hash **TODO** |
+| `examples/inference-cost-model.example.json` | clearly labelled placeholder inputs for the cost model |
+
+## Build
+
+From this directory:
+
+```sh
+# GPU image (CUDA base, the default)
+docker build -t openvibe/llm-inference:gpu .
+
+# CPU image for a smoke test (same entrypoint and checks, CPU llama.cpp base)
+docker build \
+ --build-arg LLAMA_CPP_TAG=server \
+ --build-arg LLAMA_CPP_DIGEST=sha256:10a38e6cc91760ab157f0a0a8546218d35f4a314432a70298abfa3d096829ad7 \
+ -t openvibe/llm-inference:cpu .
+```
+
+Both bases are pinned by digest. Refresh a pin only as a deliberate change:
+
+```sh
+docker buildx imagetools inspect ghcr.io/ggml-org/llama.cpp:server-cuda
+docker buildx imagetools inspect docker.io/library/caddy:2
+```
+
+## The artifact manifest
+
+Copy `examples/qwen3-4b-abliterated.manifest.example.json` and fill in **both** placeholders before
+running:
+
+- `artifact.revision`: the immutable commit/revision of the artifact repository (for Hugging Face,
+ the commit SHA, not a branch).
+- `artifact.sha256` and `download.url`: the revision-pinned URL and the SHA-256 published for that
+ exact file. The example deliberately contains `TODO` values; the verifier refuses them. Never
+ invent a hash and never reuse one from another quantization.
+
+```json
+{
+ "schema": "openvibe.ai.worker-artifact@1",
+ "artifact": {
+ "id": "mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated:q4_k_m@<revision>",
+ "revision": "<revision>",
+ "sha256": "<64 lowercase hex>",
+ "license": "apache-2.0"
+ },
+ "download": { "url": "https://…/<revision>/<file>.gguf", "filename": "<file>.gguf" }
+}
+```
+
+`verify-artifact.sh`:
+
+1. requires `schema`, `artifact.id`, `artifact.revision`, `artifact.sha256`, `download.url` and a
+ plain `download.filename`;
+2. refuses any `TODO` placeholder and any hash that is not 64 lowercase hex characters;
+3. requires `https://` for a real pin (`file://` and loopback are allowed only with
+ `ALLOW_LOCAL_ARTIFACT_URL=1` for a smoke test);
+4. reuses an already-downloaded file only when its hash matches, otherwise downloads to a temporary
+ file, verifies it and renames it into place;
+5. on mismatch deletes the temporary file, prints `SHA-256 mismatch … refusing to start` and exits
+ non-zero, so the container never serves unverified weights.
+
+The manifest is operator-controlled, not user input. The worker does not download LoRA adapters.
+
+## Run on a GPU
+
+```sh
+cp examples/qwen3-4b-abliterated.manifest.example.json /tmp/artifact.json
+# fill revision + sha256 + url in /tmp/artifact.json
+
+docker run --rm --gpus all -p 8080:8080 \
+ -v /tmp/artifact.json:/etc/openvibe/artifact.json:ro \
+ -v openvibe-models:/models \
+ -e LLM_CTX=8192 \
+ -e LLM_PARALLEL=2 \
+ -e LLM_KV_TYPE_K=f16 \
+ -e LLM_KV_TYPE_V=f16 \
+ openvibe/llm-inference:gpu
+```
+
+`/models` is a cache: a later run with the same manifest re-verifies the file and skips the
+download. A bind-mounted host directory must be writable by the image's unprivileged `openvibe`
+user (named volumes inherit the image ownership), and the bind-mounted manifest must be readable
+by that user (uid 995): `chmod 644 /tmp/artifact.json`, or run with `--user "$(id -u):$(id -g)"`
+only if your runtime gives that uid access to the GPU devices.
+
+## Run on CPU for a smoke test
+
+The CPU image exercises the same entrypoint, manifest verification, gateway and drain path:
+
+```sh
+docker run --rm -p 8080:8080 \
+ -v /tmp/artifact.json:/etc/openvibe/artifact.json:ro \
+ -v openvibe-models:/models \
+ -e LLM_GPU_LAYERS=0 \
+ -e LLM_THREADS=4 \
+ -e LLM_CTX=2048 \
+ -e LLM_PARALLEL=1 \
+ openvibe/llm-inference:cpu
+```
+
+A 4B Q4_K_M model is slow on CPU; use it for one request and the endpoint checks, not for
+throughput. Any small GGUF works for the smoke test as long as its manifest has the real revision
+and hash — do not use a placeholder hash to bypass verification.
+
+## Endpoints
+
+The only public listener is the gateway on `$PORT` (default 8080); llama.cpp listens on
+`127.0.0.1:$LLM_PORT` (default 8081).
+
+| Path | Behaviour |
+|---|---|
+| `GET /health` | llama.cpp health (200 when the model/slot is loaded) |
+| `GET /ready` | the same check as `/health`; use this as the orchestrator readiness probe |
+| `GET /metrics` | llama.cpp Prometheus metrics, passed through |
+| `/v1/chat/completions`, `/props`, `/slots`, … | passed through; streaming responses flush immediately |
+| `/lora*`, `/v1/load_lora_adapter*`, `/v1/unload_lora_adapter*` | 404; no dynamic LoRA management |
+
+The worker has **no authentication** of its own. Expose it only on a private network or behind
+OpenVibe.AI's inference gateway; do not publish port 8080 to the internet.
+
+## Bounded settings
+
+All values are integers unless stated; a value outside its bound stops the worker before it
+downloads or loads anything.
+
+| Env | Default | Bound | Meaning |
+|---|---|---|---|
+| `LLM_CTX` | `4096` | `512`–`131072`, ≤ `LLM_CTX_MAX` | total context across slots (`--ctx-size`) |
+| `LLM_CTX_MAX` | `32768` | `512`–`131072` | operator ceiling for `LLM_CTX` |
+| `LLM_PARALLEL` | `1` | `1`–`16`, ≤ `LLM_PARALLEL_MAX` | slots (`--parallel`) |
+| `LLM_PARALLEL_MAX` | `8` | `1`–`16` | operator ceiling for `LLM_PARALLEL` |
+| — | — | `LLM_CTX / LLM_PARALLEL ≥ 512` | per-slot context floor |
+| `LLM_KV_TYPE_K` | `f16` | `f32 f16 bf16 q8_0 q4_0 q4_1 iq4_nl q5_0 q5_1` | K KV cache type |
+| `LLM_KV_TYPE_V` | `f16` | same list | V KV cache type |
+| `LLM_GPU_LAYERS` | `99` | `0`–`999` | layers offloaded; use `0` on the CPU image |
+| `LLM_THREADS` | unset | `1`–`256` | optional CPU thread count |
+| `DRAIN_SECONDS` | `30` | `1`–`300` | gateway drain grace and shutdown budget |
+| `PORT` | `8080` | `1`–`65535` | public gateway port |
+| `LLM_PORT` | `8081` | `1`–`65535`, ≠ `PORT` | loopback llama.cpp port |
+| `ARTIFACT_MANIFEST` | `/etc/openvibe/artifact.json` | — | manifest path |
+| `MODEL_DIR` | `/models` | — | download/cache directory |
+
+`LLM_CTX` is llama.cpp's **total** context, not per slot; with `LLM_PARALLEL=2` and
+`LLM_CTX=8192` each slot has about 4096 tokens. Measure VRAM with the real KV type, context,
+concurrency and prompt lengths before choosing a GPU — the worker bounds these settings, it does
+not model your model's memory.
+
+## Graceful drain
+
+On `SIGTERM` the entrypoint first stops the gateway: Caddy stops accepting new connections and
+waits up to `DRAIN_SECONDS` for in-flight requests (its `grace_period`). Only after the gateway has
+drained does the entrypoint stop llama.cpp, then exit 0. The orchestrator's stop timeout must be at
+least `DRAIN_SECONDS` plus the time llama.cpp needs to exit; systemd's `TimeoutStopSec` and
+`workerUnits` drain policies are the relevant Host settings.
+
+## Cost model
+
+`scripts/inference-cost-model.js` (schema: `scripts/inference-cost-model.schema.json`) computes
+cost per request and cost per successful task for rented-warm GPU, scale-to-zero serverless and
+wholesale API, following PLAN.md §E:
+
+```sh
+node scripts/inference-cost-model.js workers/llm-inference/examples/inference-cost-model.example.json
+node scripts/inference-cost-model.js workers/llm-inference/examples/inference-cost-model.example.json --json
+```
+
+- **rented-warm:** steady-state busy seconds / occupancy at the GPU rate, idle at the
+ `idle_policy` rate, one startup per window (a separate line, not part of occupancy), storage and
+ egress.
+- **scale-to-zero:** busy seconds + (`startup_seconds / requests_per_cold_start`) at the
+ serverless rate, no idle, storage and egress.
+- **api-wholesale:** `tokens * $/million` for input, cached read, cache write and output, plus a
+ per-request fee. Wholesale rates are treated as all-in (no owned storage/egress line).
+
+Cost per successful task divides by the success rate (expected attempts). Prompt time charges
+cache-write tokens at prefill throughput and cached-read tokens at `cached_read_tokens_per_sec`
+when measured, otherwise at prefill throughput (conservative). The example file is marked
+**example, not a quote**; every price and throughput in it is a placeholder, and the model has no
+built-in benchmarks. Replace them with dated complete quotes and measured tok/s first.
+
+The script uses floats for forecasting. Customer liability is Billing's integer nano-USD ledger
+(D1); this is not a settlement or quoting authority.
+
+## Tests
+
+```sh
+node test/inference-artifact-verify.test.js
+node test/inference-cost-model.test.js
+# or, through the repository runner:
+npm test -- inference
+```
+
+The artifact test covers a correct download, a wrong hash (refused, no file left), a `TODO`
+placeholder, a traversal filename, a local URL without the smoke-test opt-in, a cached file, and
+the entrypoint's bounded-env refusals. The cost-model test checks the arithmetic against numbers
+worked out by hand, invalid inputs and the checked-in example.
+
+## Not done here
+
+- No artifact is pinned: the example's revision and hash are deliberately `TODO`. Pinning is an
+ operator action with the model license/provenance check and a published hash.
+- No model registry entry, catalog route or request routing; those belong to OpenVibe.AI.
+- No GPU rental, deployment or benchmark run is performed by this directory.
+- No quality, refusal, license or provenance gate is run here. The example artifact is a candidate,
+ not an approval: benchmark gates and the upstream license/responsible-use review happen before it
+ is served.
+- No dynamic LoRA API; only a static `--lora` would be a future, admin-only change.
new file mode 100755
new file mode 100755
@@ -0,0 +1,151 @@
@@ -0,0 +1,151 @@
+#!/bin/sh
+# OpenVibe LLM inference worker: verify the pinned GGUF, run llama.cpp server on loopback, and put
+# Caddy in front for /health, /ready and the Prometheus /metrics passthrough.
+#
+# Bounded settings come from the environment; see README.md. The worker never exposes a dynamic
+# LoRA API: llama.cpp is started without --lora, and Caddy refuses the LoRA load/unload paths.
+set -eu
+
+log() { printf 'llm-inference: %s\n' "$*" >&2; }
+die() { log "$*"; exit 1; }
+
+is_uint() {
+ case "$1" in
+ ''|*[!0-9]*) return 1 ;;
+ esac
+ return 0
+}
+
+# uint_in_range NAME VALUE MIN MAX
+uint_in_range() {
+ name=$1
+ value=$2
+ min=$3
+ max=$4
+ is_uint "$value" || die "$name must be an integer (got '$value')"
+ [ "$value" -ge "$min" ] || die "$name must be >= $min (got $value)"
+ [ "$value" -le "$max" ] || die "$name must be <= $max (got $value)"
+}
+
+ARTIFACT_MANIFEST=${ARTIFACT_MANIFEST:-/etc/openvibe/artifact.json}
+MODEL_DIR=${MODEL_DIR:-/models}
+PORT=${PORT:-8080}
+LLM_PORT=${LLM_PORT:-8081}
+DRAIN_SECONDS=${DRAIN_SECONDS:-30}
+LLM_CTX=${LLM_CTX:-4096}
+LLM_CTX_MAX=${LLM_CTX_MAX:-32768}
+LLM_PARALLEL=${LLM_PARALLEL:-1}
+LLM_PARALLEL_MAX=${LLM_PARALLEL_MAX:-8}
+LLM_KV_TYPE_K=${LLM_KV_TYPE_K:-f16}
+LLM_KV_TYPE_V=${LLM_KV_TYPE_V:-f16}
+LLM_GPU_LAYERS=${LLM_GPU_LAYERS:-99}
+LLM_THREADS=${LLM_THREADS:-}
+
+# Caddy reads these from its environment for {$PORT}, {$LLM_PORT} and {$DRAIN_SECONDS}.
+export PORT LLM_PORT DRAIN_SECONDS
+
+uint_in_range PORT "$PORT" 1 65535
+uint_in_range LLM_PORT "$LLM_PORT" 1 65535
+[ "$PORT" != "$LLM_PORT" ] || die "PORT and LLM_PORT must differ"
+
+uint_in_range DRAIN_SECONDS "$DRAIN_SECONDS" 1 300
+uint_in_range LLM_CTX "$LLM_CTX" 512 131072
+uint_in_range LLM_CTX_MAX "$LLM_CTX_MAX" 512 131072
+[ "$LLM_CTX" -le "$LLM_CTX_MAX" ] || die "LLM_CTX ($LLM_CTX) exceeds LLM_CTX_MAX ($LLM_CTX_MAX)"
+uint_in_range LLM_PARALLEL "$LLM_PARALLEL" 1 16
+uint_in_range LLM_PARALLEL_MAX "$LLM_PARALLEL_MAX" 1 16
+[ "$LLM_PARALLEL" -le "$LLM_PARALLEL_MAX" ] || die "LLM_PARALLEL ($LLM_PARALLEL) exceeds LLM_PARALLEL_MAX ($LLM_PARALLEL_MAX)"
+[ $((LLM_CTX / LLM_PARALLEL)) -ge 512 ] || die "LLM_CTX / LLM_PARALLEL must be at least 512 tokens per slot"
+uint_in_range LLM_GPU_LAYERS "$LLM_GPU_LAYERS" 0 999
+if [ -n "$LLM_THREADS" ]; then
+ uint_in_range LLM_THREADS "$LLM_THREADS" 1 256
+fi
+
+allowed_kv=' f32 f16 bf16 q8_0 q4_0 q4_1 iq4_nl q5_0 q5_1 '
+case "$allowed_kv" in
+ *" $LLM_KV_TYPE_K "*) ;;
+ *) die "LLM_KV_TYPE_K must be one of:$allowed_kv(got '$LLM_KV_TYPE_K')" ;;
+esac
+case "$allowed_kv" in
+ *" $LLM_KV_TYPE_V "*) ;;
+ *) die "LLM_KV_TYPE_V must be one of:$allowed_kv(got '$LLM_KV_TYPE_V')" ;;
+esac
+
+[ -x /app/llama-server ] || die "/app/llama-server is missing"
+command -v caddy >/dev/null 2>&1 || die "caddy is missing"
+
+log "verifying artifact from $ARTIFACT_MANIFEST"
+MODEL_PATH=$(/usr/local/bin/verify-artifact.sh "$ARTIFACT_MANIFEST" "$MODEL_DIR") || exit 1
+MODEL_ID=$(jq -r '.artifact.id' "$ARTIFACT_MANIFEST") || die "cannot read artifact.id"
+log "artifact verified: $MODEL_PATH"
+
+set -- /app/llama-server \
+ --model "$MODEL_PATH" \
+ --alias "$MODEL_ID" \
+ --host 127.0.0.1 \
+ --port "$LLM_PORT" \
+ --ctx-size "$LLM_CTX" \
+ --parallel "$LLM_PARALLEL" \
+ --n-gpu-layers "$LLM_GPU_LAYERS" \
+ --cache-type-k "$LLM_KV_TYPE_K" \
+ --cache-type-v "$LLM_KV_TYPE_V" \
+ --no-webui \
+ --metrics
+if [ -n "$LLM_THREADS" ]; then
+ set -- "$@" --threads "$LLM_THREADS"
+fi
+
+log "starting llama-server on 127.0.0.1:$LLM_PORT (ctx $LLM_CTX total, $((LLM_CTX / LLM_PARALLEL)) per slot, parallel $LLM_PARALLEL, KV $LLM_KV_TYPE_K/$LLM_KV_TYPE_V)"
+"$@" &
+LLAMA_PID=$!
+
+log "starting gateway on :$PORT (ready maps to llama.cpp /health)"
+caddy run --config /etc/caddy/Caddyfile --adapter caddyfile &
+CADDY_PID=$!
+
+wait_pid() {
+ pid=$1
+ while kill -0 "$pid" 2>/dev/null; do
+ wait "$pid" 2>/dev/null || true
+ done
+}
+
+DRAINING=0
+on_signal() {
+ [ "$DRAINING" = 1 ] && return
+ DRAINING=1
+ log "SIGTERM: not ready; draining in-flight requests for up to ${DRAIN_SECONDS}s"
+ # Caddy stops accepting at SIGTERM and waits for in-flight requests (grace_period).
+ if kill -0 "$CADDY_PID" 2>/dev/null; then
+ kill -TERM "$CADDY_PID" 2>/dev/null || true
+ wait_pid "$CADDY_PID"
+ fi
+ log "gateway drained; stopping llama-server"
+ if kill -0 "$LLAMA_PID" 2>/dev/null; then
+ kill -TERM "$LLAMA_PID" 2>/dev/null || true
+ wait_pid "$LLAMA_PID"
+ fi
+ log "drained"
+ exit 0
+}
+trap on_signal TERM INT
+
+while :; do
+ if ! kill -0 "$LLAMA_PID" 2>/dev/null; then
+ status=0
+ wait "$LLAMA_PID" 2>/dev/null || status=$?
+ log "llama-server exited (status $status); stopping gateway"
+ kill -TERM "$CADDY_PID" 2>/dev/null || true
+ wait_pid "$CADDY_PID"
+ exit "$status"
+ fi
+ if ! kill -0 "$CADDY_PID" 2>/dev/null; then
+ status=0
+ wait "$CADDY_PID" 2>/dev/null || status=$?
+ log "gateway exited (status $status); stopping llama-server"
+ kill -TERM "$LLAMA_PID" 2>/dev/null || true
+ wait_pid "$LLAMA_PID"
+ exit "$status"
+ fi
+ sleep 1 || true
+done
new file mode 100644
new file mode 100644
@@ -0,0 +1,48 @@
@@ -0,0 +1,48 @@
+{
+ "$schema": "../../../scripts/inference-cost-model.schema.json",
+ "note": "example, not a quote — every price and throughput below is a placeholder for arithmetic only. Replace each one with a dated complete quote and a measured tok/s run before using this model for a decision.",
+ "window": {
+ "hours": 24,
+ "requests": 1000
+ },
+ "request": {
+ "input_tokens": 1000,
+ "cached_read_tokens": 0,
+ "cache_write_tokens": 0,
+ "output_tokens": 500,
+ "success_rate": 0.95
+ },
+ "throughput": {
+ "prefill_tokens_per_sec": 800,
+ "decode_tokens_per_sec": 40,
+ "concurrency": 1
+ },
+ "gpu": {
+ "hourly_usd": 0.1,
+ "occupancy": 0.35,
+ "idle_policy": "standby",
+ "standby_hourly_usd": 0,
+ "startup_seconds": 90,
+ "storage_gb": 4,
+ "storage_usd_per_gb_month": 0.1,
+ "egress_gb_per_request": 0.0002,
+ "egress_usd_per_gb": 0.09
+ },
+ "serverless": {
+ "hourly_usd": 0.75,
+ "startup_seconds": 45,
+ "requests_per_cold_start": 1,
+ "storage_gb": 4,
+ "storage_usd_per_gb_month": 0.1,
+ "egress_gb_per_request": 0.0002,
+ "egress_usd_per_gb": 0.09
+ },
+ "api_wholesale": {
+ "input_usd_per_million": 0.1,
+ "cached_read_usd_per_million": 0.02,
+ "cache_write_usd_per_million": 0.1,
+ "output_usd_per_million": 0.4,
+ "per_request_usd": 0.000005,
+ "success_rate": 0.98
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,17 @@
@@ -0,0 +1,17 @@
+{
+ "schema": "openvibe.ai.worker-artifact@1",
+ "artifact": {
+ "id": "mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated:q4_k_m@TODO-REVISION",
+ "family": "qwen3",
+ "variant": "abliterated",
+ "quant": "q4_k_m",
+ "revision": "TODO: pin the Hugging Face commit sha of mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF",
+ "sha256": "TODO: copy the published SHA-256 of the Q4_K_M file; never invent or reuse a hash",
+ "license": "apache-2.0",
+ "source_url": "https://huggingface.co/mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF"
+ },
+ "download": {
+ "url": "https://huggingface.co/mradermacher/Huihui-Qwen3-4B-Instruct-2507-abliterated-GGUF/resolve/TODO-REVISION/Huihui-Qwen3-4B-Instruct-2507-abliterated.Q4_K_M.gguf",
+ "filename": "Huihui-Qwen3-4B-Instruct-2507-abliterated.Q4_K_M.gguf"
+ }
+}
new file mode 100755
new file mode 100755
@@ -0,0 +1,98 @@
@@ -0,0 +1,98 @@
+#!/bin/sh
+# Download one GGUF artifact and verify its SHA-256 against the pinned manifest before it is used.
+#
+# verify-artifact.sh <manifest.json> <destination-dir>
+#
+# On success prints the verified local path (and nothing else) on stdout. Logs and failures go to
+# stderr. Exit 1 means the worker must not start: a wrong hash is never repaired or retried with
+# different bytes.
+#
+# The manifest is operator-supplied, not user content. Local URLs (file://, loopback http) are
+# accepted only with ALLOW_LOCAL_ARTIFACT_URL=1 so a production manifest cannot be pointed at a
+# local file by accident; a real pin is https:// plus a published SHA-256.
+set -eu
+
+die() {
+ printf 'artifact verification: %s\n' "$*" >&2
+ exit 1
+}
+
+[ "$#" -eq 2 ] || die "usage: verify-artifact.sh <manifest.json> <destination-dir>"
+manifest=$1
+dest=$2
+
+[ -f "$manifest" ] || die "manifest not found: $manifest"
+[ -r "$manifest" ] || die "manifest is not readable by uid $(id -u): $manifest"
+for command in jq curl sha256sum; do
+ command -v "$command" >/dev/null 2>&1 || die "$command is required"
+done
+
+schema=$(jq -r '.schema // empty' "$manifest" 2>/dev/null) || die "cannot parse manifest: $manifest"
+[ "$schema" = "openvibe.ai.worker-artifact@1" ] || die "manifest schema must be openvibe.ai.worker-artifact@1 (got '${schema:-missing}')"
+
+field() {
+ value=$(jq -r "$1" "$manifest" 2>/dev/null) || die "cannot read $2 from $manifest"
+ case "$value" in
+ ''|null) die "$2 is missing or empty in $manifest" ;;
+ esac
+ printf '%s' "$value"
+}
+
+model_id=$(field '.artifact.id' 'artifact.id')
+revision=$(field '.artifact.revision' 'artifact.revision')
+sha=$(field '.artifact.sha256' 'artifact.sha256')
+url=$(field '.download.url' 'download.url')
+filename=$(field '.download.filename' 'download.filename')
+
+# A TODO is a promise the operator has not kept yet. Refuse it instead of downloading something
+# that happens to be at a guessed URL, and never invent a hash.
+case "$model_id $revision $sha $url $filename" in
+ *TODO*) die "manifest still has a TODO placeholder; pin the revision and the published SHA-256 first" ;;
+esac
+
+case "$sha" in
+ *[!0-9a-f]*) die "artifact.sha256 must be 64 lowercase hex characters" ;;
+esac
+[ "${#sha}" -eq 64 ] || die "artifact.sha256 must be 64 lowercase hex characters"
+
+case "$filename" in
+ ''|.|..|*/*) die "download.filename must be a plain file name" ;;
+esac
+
+case "$url" in
+ https://*) ;;
+ file://*|http://127.0.0.1:*|http://localhost:*)
+ [ "${ALLOW_LOCAL_ARTIFACT_URL:-0}" = "1" ] || die "local artifact URLs require ALLOW_LOCAL_ARTIFACT_URL=1 (use https:// for a real pin)"
+ ;;
+ *) die "download.url must be https:// (local URLs only in a smoke test)" ;;
+esac
+
+mkdir -p "$dest" || die "cannot create $dest"
+target="$dest/$filename"
+part="$target.part.$$"
+
+if [ -f "$target" ]; then
+ actual=$(sha256sum "$target" | awk '{print $1}')
+ if [ "$actual" = "$sha" ]; then
+ printf '%s\n' "$target"
+ exit 0
+ fi
+ printf 'artifact verification: existing %s does not match the manifest; downloading again\n' "$target" >&2
+ rm -f "$target"
+fi
+
+rm -f "$part"
+# --fail makes an HTTP error a non-zero exit; the file is never treated as an artifact.
+if ! curl -fsSL --retry 3 --retry-connrefused --connect-timeout 20 -o "$part" "$url"; then
+ rm -f "$part"
+ die "download failed: $url"
+fi
+
+actual=$(sha256sum "$part" | awk '{print $1}')
+if [ "$actual" != "$sha" ]; then
+ rm -f "$part"
+ die "SHA-256 mismatch for $filename: manifest $sha, downloaded $actual; refusing to start"
+fi
+
+mv "$part" "$target" || die "cannot move verified artifact into place: $target"
+printf '%s\n' "$target"