openvibers/openvibe.ai Commits
Serve default.embedding from a free local embedding model when…
Changes
@@@ -102,6 +102,11 @@ AI_MODEL=#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct#AI_LOCAL_LLM_TIMEOUT_MS=180000++# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free++# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a++# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions++# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.++#AI_LOCAL_EMBED_MODEL=nomic-embed-text# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296@@@ -368,6 +368,17 @@ streams always win: 2 threads, at most 1.5 CPUs, nice 19, 3 GB. It uses about 0.28 tokens/s. Without it, `media.analyze` writes an extractive overview and says so. To swap the model, put thefile beside it, change `--model`/`--alias` and `AI_LOCAL_LLM_MODEL`, and restart both units.++Local embeddings are off unless `AI_LOCAL_EMBED_MODEL` names an embedding model the server at `AI_LOCAL_LLM_URL`++answers at `/embeddings`. Then `local` advertises `embed`, the model is registered as an `embedding` model, and++`default.embedding` (a capability pool) offers it beside the shared provider's `AI_EMBEDDING_MODEL`; `local` is++billing profile `free`, so its readings carry no cost. llama.cpp's `llama-server` answers `/embeddings` only when++started with `--embedding` and a GGUF that supports embeddings (a chat-only model answers 400, and every embedding++call then fails over to the shared provider with an error in the request log). That flag restricts llama-server++to embeddings, so the Qwen chat unit above runs without it: embeddings need a server that serves both on one URL++(Ollama's `/v1`, for example), or `AI_LOCAL_LLM_URL` given over to an embedding-only server. **Vector dimensions++differ per upstream**: a run says which provider and model made its vectors, and a consumer must never compare or++mix vectors from different providers (re-embed when the provider changes).++## Acceptance: guarantees and where they are tested`npm test` runs every `test/*.test.js` with the stub provider, temp databases and local servers only.@@@ -5,6 +5,9 @@ Description=OpenVibe local model server — llama.cpp llama-server, OpenAI API o# 3 GB of memory. A stream's RTMP ingest and the web services always win.# Install (README "Local model"): llama.cpp's CPU build under /opt/llama.cpp/<build> with /opt/llama.cpp/current,# the model in /var/lib/openvibe-llm/models, both checked against their published sha256.++# Embeddings (AI_LOCAL_EMBED_MODEL, README "Local model") need llama-server started with --embedding and a GGUF that++# supports embeddings (a chat-only model answers 400 and AI falls over to the shared provider, noisily). --embedding++# restricts the server to embeddings, so this chat unit runs without it.After=network.target[Service]@@@ -102,8 +102,9 @@ function load(env = process.env) {// Configurable local HTTP seam: POST {operation, request} JSON to this URL.httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,-- // route media.local (roadmap WS-O task 5). No key; never a paid provider.-- localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },++ // route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the++ // server serves at /embeddings) makes `local` a member of the default.embedding pool.++ localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },// media.analyze (server/workflows/media-analysis.js): the file cap (also bounded by free disk minus a reserve),// the download timeout, the scene-change threshold (scdet, 0-100) and the tools.mediaAnalysis: {@@@ -53,7 +53,10 @@ function providerRecords(config, env) {];if (config.localLlm && config.localLlm.url) {// A local model server (llama.cpp, Ollama) speaking the OpenAI API: no key, never paid (WS-O task 5).-- out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: ['chat', 'generate', 'summarize', 'classify', 'extract'], timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });++ // JSON goes through the adapter's json_schema -> json_object -> prose step-down; embed only with an embedding model.++ const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract', 'json'];++ if (config.localLlm.embedModel) localCaps.push('embed');++ out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });}if (config.httpSeamUrl) {out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });@@@ -79,6 +82,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {if (!await registry.getModel('whisper', 'whisper.cpp')) await registry.upsertModel({ provider_key: 'whisper', model_key: 'whisper.cpp', type: 'stt', supports: { json: false } }, { actor: 'seed' });const embeddingModel = env.AI_EMBEDDING_MODEL || 'text-embedding-3-small';if (!await registry.getModel('shared', embeddingModel)) await registry.upsertModel({ provider_key: 'shared', model_key: embeddingModel, type: 'embedding' }, { actor: 'seed' });++ // The local embedding model joins default.embedding as its free member (vectors differ per upstream).++ const localEmbed = config.localLlm && config.localLlm.url && config.localLlm.embedModel;++ if (localEmbed && !await registry.getModel('local', localEmbed)) await registry.upsertModel({ provider_key: 'local', model_key: localEmbed, type: 'embedding' }, { actor: 'seed' });// Routes. A capability pool covers every provider that can serve the capability, ordered by// openvibe-sdk/placement (brief §4/§5): no static primary/fallbacks and no AI_MODEL_<ROLE> pins.@@@ -92,8 +98,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {for (const role of live.ROLES) {await registry.seedVersioned('route', `live.${role}`, pool(ROLE_TIMEOUT[role]));}-- // Speech and embeddings only admit registered models of the matching type. Until O9 adds another-- // upstream, an unavailable sole provider still answers the explicit 503.++ // Speech and embeddings only admit registered models of the matching type. Embeddings fall over to the local++ // embedding model when AI_LOCAL_EMBED_MODEL registers one; until O9 adds another speech upstream (and for++ // embeddings without a local model), an unavailable sole provider still answers the explicit 503.await registry.seedVersioned('route', 'live.stt', pool(600000, { 'capability': 'transcribe', constraints: { objective: 'balanced' } }));await registry.seedVersioned('route', 'default.chat', pool(30000, { max_output_tokens: 800 }));await registry.seedVersioned('route', 'default.json', pool(60000, { options: { temperature: 0.3 }, max_output_tokens: 2400, response_format: 'json' }));@@@ -79,4 +79,15 @@ t.test('a candidate that is no longer eligible fails over immediately (no minGaiassert.ok(r.reasons.some((x) => /no longer eligible/.test(x)));});++t.test('embeddings: cheapest picks the free local model over the metered shared one', () => {++ // As the router builds them: a free provider is offered as prepaid with no rate card, a metered one on its card.++ const offers = [++ offer('shared:text-embedding-3-small', { capabilities: ['ai:embed'], pricing: { model: 'per-operation', rate_card: 'shared:text-embedding-3-small' } }),++ offer('local:nomic-embed-text', { capabilities: ['ai:embed'], trust: 'community', pricing: { model: 'prepaid' } }),++ ];++ const r = plan(req({ objective: 'cheapest', capabilities: ['ai:embed'] }), offers, { rateCards: [card('shared:text-embedding-3-small', 'shared', 0.02)] });++ assert.strictEqual(r.selected, 'local:nomic-embed-text', r.reasons.join('; '));++ assert.ok(r.candidates.every((c) => c.eligible), 'both stay eligible: the shared one remains the fallback');++});++t.run();@@@ -78,6 +78,46 @@ t.test('forced failure on every route: a logged fallback, or an explicit 503 (neawait h.stop(); await ok.close();});++t.test('a local embedding model makes default.embedding a pool: the failing paid upstream falls to the free local one', async () => {++ // Both upstreams speak the OpenAI API; the shared one is paid (metered), the local one free.++ const paid = await seamServer(() => ({ status: 500, body: { error: { message: 'upstream down' } } }));++ const local = await seamServer((body, req) => (req.url.endsWith('/embeddings')++ ? { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } }++ : { status: 404, body: { error: { message: 'not here' } } }));++ h = await boot({ env: {++ AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50',++ AI_PROVIDER: 'openai', AI_BASE_URL: `${paid.url}/v1`, AI_LOCAL_LLM_URL: `${local.url}/v1`, AI_LOCAL_EMBED_MODEL: 'nomic-embed-text',++ } });++ const lp = await h.registry.getProvider('local');++ assert.ok(lp.capabilities.includes('embed'), 'local advertises embed with an embedding model configured');++ const lm = await h.registry.getModel('local', 'nomic-embed-text');++ assert.strictEqual(lm && lm.type, 'embedding', 'the local embedding model is registered');++++ // The seeded pool (objective balanced): the free local model answers, so default.embedding no longer 503s.++ const embed = async (rr, logs) => h.pool.execute(rr, 'embed', { input: ['x'], timeoutMs: 2000 }, { routeKey: rr.key, routeVersion: rr.version, logRequest: (e) => logs.push(e) });++ const seeded = await h.registry.resolveRoute('default.embedding');++ const direct = await embed(seeded, []);++ assert.strictEqual(direct.provider, 'local');++ assert.deepStrictEqual(direct.result.vectors, [[0.3, 0.4, 0.5]]);++ const placed = await h.pool.candidates(seeded, ['embed'], 'embed');++ assert.strictEqual(placed.placement.selected, 'local:nomic-embed-text', JSON.stringify(placed.explain));++ assert.deepStrictEqual(placed.explain.candidates.map((c) => `${c.provider}:${c.model}`).sort(), ['local:nomic-embed-text', 'shared:text-embedding-3-small']);++++ // With the paid upstream as the authority, it is tried first; it fails and the run falls to local.++ const rr = await route('default.embedding', { capability: 'embed', constraints: { objective: 'correctness', authority: 'shared:text-embedding-3-small' }, timeout_ms: 3000 });++ const logs = [];++ const exec = await embed(rr, logs);++ assert.ok(!exec.synthetic, 'never a synthetic answer');++ assert.strictEqual(exec.provider, 'local');++ assert.strictEqual(exec.model, 'nomic-embed-text');++ assert.strictEqual(exec.explain.selected, 'local', JSON.stringify(exec.explain));++ assert.strictEqual(exec.explain.candidates.find((c) => c.provider === 'local').model, 'nomic-embed-text');++ assert.ok(paid.calls >= 1, 'the paid upstream was tried first');++ assert.ok(logs.some((e) => e.provider_key === 'shared' && e.status !== 'ok'), 'the paid failure is logged');++ assert.ok(logs.some((e) => e.provider_key === 'local' && e.model_key === 'nomic-embed-text' && e.status === 'ok' && e.fallback === 1), 'the fallback to local is logged');++ await h.stop(); await paid.close(); await local.close();++});++t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {const ok = await seam();const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));@@@ -13,6 +13,8 @@ const { cleanSegments } = require('../server/providers/whisper');const { render } = require('../server/templates');const { parseJsonLoose } = require('../server/util');const schemas = require('../server/schemas');++const { load } = require('../server/config');++const { providerRecords } = require('../server/workflows/seed');const t = suite('providers');const req = (extra = {}) => ({ system: [{ text: 'sys', cache: true }], messages: [{ role: 'user', content: 'hi' }], maxTokens: 50, temperature: 0.5, timeoutMs: 3000, model: 'gpt-4o-mini', ...extra });@@@ -112,4 +114,17 @@ t.test('templates render values, JSON and sections; loose JSON repair works', ()assert.deepStrictEqual(parseJsonLoose('{“a”: 1,}'), { a: 1 });});++t.test('local provider: json always, embed only with AI_LOCAL_EMBED_MODEL', () => {++ const caps = (env) => {++ const p = providerRecords(load(env), env).find(x => x.key === 'local');++ return p && p.capabilities;++ };++ assert.strictEqual(caps({}), undefined, 'no local provider without AI_LOCAL_LLM_URL');++ const plain = caps({ AI_LOCAL_LLM_URL: 'http://127.0.0.1:8090/v1' });++ assert.deepStrictEqual(plain, ['chat', 'generate', 'summarize', 'classify', 'extract', 'json']);++ const withEmbed = caps({ AI_LOCAL_LLM_URL: 'http://127.0.0.1:8090/v1', AI_LOCAL_EMBED_MODEL: ' nomic-embed-text ' });++ assert.deepStrictEqual(withEmbed, [...plain, 'embed']);++ assert.strictEqual(load({ AI_LOCAL_EMBED_MODEL: ' nomic-embed-text ' }).localLlm.embedModel, 'nomic-embed-text');++});++t.run();
@@ -102,6 +102,11 @@ AI_MODEL=
@@ -102,6 +102,11 @@ AI_MODEL=
#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media
#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media
#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct
#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct
#AI_LOCAL_LLM_TIMEOUT_MS=180000
#AI_LOCAL_LLM_TIMEOUT_MS=180000
+# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free
+# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a
+# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions
+# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.
+#AI_LOCAL_EMBED_MODEL=nomic-embed-text
# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold
# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold
#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)
#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)
#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296
#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296
@@ -368,6 +368,17 @@ streams always win: 2 threads, at most 1.5 CPUs, nice 19, 3 GB. It uses about 0.
@@ -368,6 +368,17 @@ streams always win: 2 threads, at most 1.5 CPUs, nice 19, 3 GB. It uses about 0.
28 tokens/s. Without it, `media.analyze` writes an extractive overview and says so. To swap the model, put the
28 tokens/s. Without it, `media.analyze` writes an extractive overview and says so. To swap the model, put the
file beside it, change `--model`/`--alias` and `AI_LOCAL_LLM_MODEL`, and restart both units.
file beside it, change `--model`/`--alias` and `AI_LOCAL_LLM_MODEL`, and restart both units.
+Local embeddings are off unless `AI_LOCAL_EMBED_MODEL` names an embedding model the server at `AI_LOCAL_LLM_URL`
+answers at `/embeddings`. Then `local` advertises `embed`, the model is registered as an `embedding` model, and
+`default.embedding` (a capability pool) offers it beside the shared provider's `AI_EMBEDDING_MODEL`; `local` is
+billing profile `free`, so its readings carry no cost. llama.cpp's `llama-server` answers `/embeddings` only when
+started with `--embedding` and a GGUF that supports embeddings (a chat-only model answers 400, and every embedding
+call then fails over to the shared provider with an error in the request log). That flag restricts llama-server
+to embeddings, so the Qwen chat unit above runs without it: embeddings need a server that serves both on one URL
+(Ollama's `/v1`, for example), or `AI_LOCAL_LLM_URL` given over to an embedding-only server. **Vector dimensions
+differ per upstream**: a run says which provider and model made its vectors, and a consumer must never compare or
+mix vectors from different providers (re-embed when the provider changes).
+
## Acceptance: guarantees and where they are tested
## Acceptance: guarantees and where they are tested
`npm test` runs every `test/*.test.js` with the stub provider, temp databases and local servers only.
`npm test` runs every `test/*.test.js` with the stub provider, temp databases and local servers only.
@@ -5,6 +5,9 @@ Description=OpenVibe local model server — llama.cpp llama-server, OpenAI API o
@@ -5,6 +5,9 @@ Description=OpenVibe local model server — llama.cpp llama-server, OpenAI API o
# 3 GB of memory. A stream's RTMP ingest and the web services always win.
# 3 GB of memory. A stream's RTMP ingest and the web services always win.
# Install (README "Local model"): llama.cpp's CPU build under /opt/llama.cpp/<build> with /opt/llama.cpp/current,
# Install (README "Local model"): llama.cpp's CPU build under /opt/llama.cpp/<build> with /opt/llama.cpp/current,
# the model in /var/lib/openvibe-llm/models, both checked against their published sha256.
# the model in /var/lib/openvibe-llm/models, both checked against their published sha256.
+# Embeddings (AI_LOCAL_EMBED_MODEL, README "Local model") need llama-server started with --embedding and a GGUF that
+# supports embeddings (a chat-only model answers 400 and AI falls over to the shared provider, noisily). --embedding
+# restricts the server to embeddings, so this chat unit runs without it.
After=network.target
After=network.target
[Service]
[Service]
@@ -102,8 +102,9 @@ function load(env = process.env) {
@@ -102,8 +102,9 @@ function load(env = process.env) {
// Configurable local HTTP seam: POST {operation, request} JSON to this URL.
// Configurable local HTTP seam: POST {operation, request} JSON to this URL.
httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),
httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),
// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,
// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,
- // route media.local (roadmap WS-O task 5). No key; never a paid provider.
- localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },
+ // route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the
+ // server serves at /embeddings) makes `local` a member of the default.embedding pool.
+ localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },
// media.analyze (server/workflows/media-analysis.js): the file cap (also bounded by free disk minus a reserve),
// media.analyze (server/workflows/media-analysis.js): the file cap (also bounded by free disk minus a reserve),
// the download timeout, the scene-change threshold (scdet, 0-100) and the tools.
// the download timeout, the scene-change threshold (scdet, 0-100) and the tools.
mediaAnalysis: {
mediaAnalysis: {
@@ -53,7 +53,10 @@ function providerRecords(config, env) {
@@ -53,7 +53,10 @@ function providerRecords(config, env) {
];
];
if (config.localLlm && config.localLlm.url) {
if (config.localLlm && config.localLlm.url) {
// A local model server (llama.cpp, Ollama) speaking the OpenAI API: no key, never paid (WS-O task 5).
// A local model server (llama.cpp, Ollama) speaking the OpenAI API: no key, never paid (WS-O task 5).
- out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: ['chat', 'generate', 'summarize', 'classify', 'extract'], timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });
+ // JSON goes through the adapter's json_schema -> json_object -> prose step-down; embed only with an embedding model.
+ const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract', 'json'];
+ if (config.localLlm.embedModel) localCaps.push('embed');
+ out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });
}
}
if (config.httpSeamUrl) {
if (config.httpSeamUrl) {
out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });
out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });
@@ -79,6 +82,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
@@ -79,6 +82,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
if (!await registry.getModel('whisper', 'whisper.cpp')) await registry.upsertModel({ provider_key: 'whisper', model_key: 'whisper.cpp', type: 'stt', supports: { json: false } }, { actor: 'seed' });
if (!await registry.getModel('whisper', 'whisper.cpp')) await registry.upsertModel({ provider_key: 'whisper', model_key: 'whisper.cpp', type: 'stt', supports: { json: false } }, { actor: 'seed' });
const embeddingModel = env.AI_EMBEDDING_MODEL || 'text-embedding-3-small';
const embeddingModel = env.AI_EMBEDDING_MODEL || 'text-embedding-3-small';
if (!await registry.getModel('shared', embeddingModel)) await registry.upsertModel({ provider_key: 'shared', model_key: embeddingModel, type: 'embedding' }, { actor: 'seed' });
if (!await registry.getModel('shared', embeddingModel)) await registry.upsertModel({ provider_key: 'shared', model_key: embeddingModel, type: 'embedding' }, { actor: 'seed' });
+ // The local embedding model joins default.embedding as its free member (vectors differ per upstream).
+ const localEmbed = config.localLlm && config.localLlm.url && config.localLlm.embedModel;
+ if (localEmbed && !await registry.getModel('local', localEmbed)) await registry.upsertModel({ provider_key: 'local', model_key: localEmbed, type: 'embedding' }, { actor: 'seed' });
// Routes. A capability pool covers every provider that can serve the capability, ordered by
// Routes. A capability pool covers every provider that can serve the capability, ordered by
// openvibe-sdk/placement (brief §4/§5): no static primary/fallbacks and no AI_MODEL_<ROLE> pins.
// openvibe-sdk/placement (brief §4/§5): no static primary/fallbacks and no AI_MODEL_<ROLE> pins.
@@ -92,8 +98,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
@@ -92,8 +98,9 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
for (const role of live.ROLES) {
for (const role of live.ROLES) {
await registry.seedVersioned('route', `live.${role}`, pool(ROLE_TIMEOUT[role]));
await registry.seedVersioned('route', `live.${role}`, pool(ROLE_TIMEOUT[role]));
}
}
- // Speech and embeddings only admit registered models of the matching type. Until O9 adds another
- // upstream, an unavailable sole provider still answers the explicit 503.
+ // Speech and embeddings only admit registered models of the matching type. Embeddings fall over to the local
+ // embedding model when AI_LOCAL_EMBED_MODEL registers one; until O9 adds another speech upstream (and for
+ // embeddings without a local model), an unavailable sole provider still answers the explicit 503.
await registry.seedVersioned('route', 'live.stt', pool(600000, { 'capability': 'transcribe', constraints: { objective: 'balanced' } }));
await registry.seedVersioned('route', 'live.stt', pool(600000, { 'capability': 'transcribe', constraints: { objective: 'balanced' } }));
await registry.seedVersioned('route', 'default.chat', pool(30000, { max_output_tokens: 800 }));
await registry.seedVersioned('route', 'default.chat', pool(30000, { max_output_tokens: 800 }));
await registry.seedVersioned('route', 'default.json', pool(60000, { options: { temperature: 0.3 }, max_output_tokens: 2400, response_format: 'json' }));
await registry.seedVersioned('route', 'default.json', pool(60000, { options: { temperature: 0.3 }, max_output_tokens: 2400, response_format: 'json' }));
@@ -79,4 +79,15 @@ t.test('a candidate that is no longer eligible fails over immediately (no minGai
@@ -79,4 +79,15 @@ t.test('a candidate that is no longer eligible fails over immediately (no minGai
assert.ok(r.reasons.some((x) => /no longer eligible/.test(x)));
assert.ok(r.reasons.some((x) => /no longer eligible/.test(x)));
});
});
+t.test('embeddings: cheapest picks the free local model over the metered shared one', () => {
+ // As the router builds them: a free provider is offered as prepaid with no rate card, a metered one on its card.
+ const offers = [
+ offer('shared:text-embedding-3-small', { capabilities: ['ai:embed'], pricing: { model: 'per-operation', rate_card: 'shared:text-embedding-3-small' } }),
+ offer('local:nomic-embed-text', { capabilities: ['ai:embed'], trust: 'community', pricing: { model: 'prepaid' } }),
+ ];
+ const r = plan(req({ objective: 'cheapest', capabilities: ['ai:embed'] }), offers, { rateCards: [card('shared:text-embedding-3-small', 'shared', 0.02)] });
+ assert.strictEqual(r.selected, 'local:nomic-embed-text', r.reasons.join('; '));
+ assert.ok(r.candidates.every((c) => c.eligible), 'both stay eligible: the shared one remains the fallback');
+});
+
t.run();
t.run();
@@ -78,6 +78,46 @@ t.test('forced failure on every route: a logged fallback, or an explicit 503 (ne
@@ -78,6 +78,46 @@ t.test('forced failure on every route: a logged fallback, or an explicit 503 (ne
await h.stop(); await ok.close();
await h.stop(); await ok.close();
});
});
+t.test('a local embedding model makes default.embedding a pool: the failing paid upstream falls to the free local one', async () => {
+ // Both upstreams speak the OpenAI API; the shared one is paid (metered), the local one free.
+ const paid = await seamServer(() => ({ status: 500, body: { error: { message: 'upstream down' } } }));
+ const local = await seamServer((body, req) => (req.url.endsWith('/embeddings')
+ ? { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } }
+ : { status: 404, body: { error: { message: 'not here' } } }));
+ h = await boot({ env: {
+ AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50',
+ AI_PROVIDER: 'openai', AI_BASE_URL: `${paid.url}/v1`, AI_LOCAL_LLM_URL: `${local.url}/v1`, AI_LOCAL_EMBED_MODEL: 'nomic-embed-text',
+ } });
+ const lp = await h.registry.getProvider('local');
+ assert.ok(lp.capabilities.includes('embed'), 'local advertises embed with an embedding model configured');
+ const lm = await h.registry.getModel('local', 'nomic-embed-text');
+ assert.strictEqual(lm && lm.type, 'embedding', 'the local embedding model is registered');
+
+ // The seeded pool (objective balanced): the free local model answers, so default.embedding no longer 503s.
+ const embed = async (rr, logs) => h.pool.execute(rr, 'embed', { input: ['x'], timeoutMs: 2000 }, { routeKey: rr.key, routeVersion: rr.version, logRequest: (e) => logs.push(e) });
+ const seeded = await h.registry.resolveRoute('default.embedding');
+ const direct = await embed(seeded, []);
+ assert.strictEqual(direct.provider, 'local');
+ assert.deepStrictEqual(direct.result.vectors, [[0.3, 0.4, 0.5]]);
+ const placed = await h.pool.candidates(seeded, ['embed'], 'embed');
+ assert.strictEqual(placed.placement.selected, 'local:nomic-embed-text', JSON.stringify(placed.explain));
+ assert.deepStrictEqual(placed.explain.candidates.map((c) => `${c.provider}:${c.model}`).sort(), ['local:nomic-embed-text', 'shared:text-embedding-3-small']);
+
+ // With the paid upstream as the authority, it is tried first; it fails and the run falls to local.
+ const rr = await route('default.embedding', { capability: 'embed', constraints: { objective: 'correctness', authority: 'shared:text-embedding-3-small' }, timeout_ms: 3000 });
+ const logs = [];
+ const exec = await embed(rr, logs);
+ assert.ok(!exec.synthetic, 'never a synthetic answer');
+ assert.strictEqual(exec.provider, 'local');
+ assert.strictEqual(exec.model, 'nomic-embed-text');
+ assert.strictEqual(exec.explain.selected, 'local', JSON.stringify(exec.explain));
+ assert.strictEqual(exec.explain.candidates.find((c) => c.provider === 'local').model, 'nomic-embed-text');
+ assert.ok(paid.calls >= 1, 'the paid upstream was tried first');
+ assert.ok(logs.some((e) => e.provider_key === 'shared' && e.status !== 'ok'), 'the paid failure is logged');
+ assert.ok(logs.some((e) => e.provider_key === 'local' && e.model_key === 'nomic-embed-text' && e.status === 'ok' && e.fallback === 1), 'the fallback to local is logged');
+ await h.stop(); await paid.close(); await local.close();
+});
+
t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {
t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {
const ok = await seam();
const ok = await seam();
const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));
const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));
@@ -13,6 +13,8 @@ const { cleanSegments } = require('../server/providers/whisper');
@@ -13,6 +13,8 @@ const { cleanSegments } = require('../server/providers/whisper');
const { render } = require('../server/templates');
const { render } = require('../server/templates');
const { parseJsonLoose } = require('../server/util');
const { parseJsonLoose } = require('../server/util');
const schemas = require('../server/schemas');
const schemas = require('../server/schemas');
+const { load } = require('../server/config');
+const { providerRecords } = require('../server/workflows/seed');
const t = suite('providers');
const t = suite('providers');
const req = (extra = {}) => ({ system: [{ text: 'sys', cache: true }], messages: [{ role: 'user', content: 'hi' }], maxTokens: 50, temperature: 0.5, timeoutMs: 3000, model: 'gpt-4o-mini', ...extra });
const req = (extra = {}) => ({ system: [{ text: 'sys', cache: true }], messages: [{ role: 'user', content: 'hi' }], maxTokens: 50, temperature: 0.5, timeoutMs: 3000, model: 'gpt-4o-mini', ...extra });
@@ -112,4 +114,17 @@ t.test('templates render values, JSON and sections; loose JSON repair works', ()
@@ -112,4 +114,17 @@ t.test('templates render values, JSON and sections; loose JSON repair works', ()
assert.deepStrictEqual(parseJsonLoose('{“a”: 1,}'), { a: 1 });
assert.deepStrictEqual(parseJsonLoose('{“a”: 1,}'), { a: 1 });
});
});
+t.test('local provider: json always, embed only with AI_LOCAL_EMBED_MODEL', () => {
+ const caps = (env) => {
+ const p = providerRecords(load(env), env).find(x => x.key === 'local');
+ return p && p.capabilities;
+ };
+ assert.strictEqual(caps({}), undefined, 'no local provider without AI_LOCAL_LLM_URL');
+ const plain = caps({ AI_LOCAL_LLM_URL: 'http://127.0.0.1:8090/v1' });
+ assert.deepStrictEqual(plain, ['chat', 'generate', 'summarize', 'classify', 'extract', 'json']);
+ const withEmbed = caps({ AI_LOCAL_LLM_URL: 'http://127.0.0.1:8090/v1', AI_LOCAL_EMBED_MODEL: ' nomic-embed-text ' });
+ assert.deepStrictEqual(withEmbed, [...plain, 'embed']);
+ assert.strictEqual(load({ AI_LOCAL_EMBED_MODEL: ' nomic-embed-text ' }).localLlm.embedModel, 'nomic-embed-text');
+});
+
t.run();
t.run();