openvibers/openvibe.ai Commits
AI_LOCAL_EMBED_URL: the local provider's embed op can use its own llama.cpp server (one model per process), defaulting to AI_LOCAL_LLM_URL (#183)
Changes
@@@ -105,11 +105,14 @@ AI_MODEL=#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct#AI_LOCAL_LLM_TIMEOUT_MS=180000++# A second local OpenAI-compatible server for embeddings, when the chat model's llama.cpp process cannot also++# serve the embedding GGUF (llama.cpp serves one model per process). Default: AI_LOCAL_LLM_URL.++#AI_LOCAL_EMBED_URL=http://127.0.0.1:8091/v1# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.--#AI_LOCAL_EMBED_MODEL=nomic-embed-text++#AI_LOCAL_EMBED_MODEL=bge-small-en-v1.5# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296@@@ -103,8 +103,15 @@ function load(env = process.env) {httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,// route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the-- // server serves at /embeddings) makes `local` a member of the default.embedding pool.-- localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },++ // server serves at /embeddings) makes `local` a member of the default.embedding pool. llama.cpp serves one++ // model per process, so embedUrl may name a second local server; it defaults to the chat model's URL.++ localLlm: {++ url: trimUrl(env.AI_LOCAL_LLM_URL),++ embedUrl: trimUrl(env.AI_LOCAL_EMBED_URL) || trimUrl(env.AI_LOCAL_LLM_URL),++ model: String(env.AI_LOCAL_LLM_MODEL || '').trim(),++ embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(),++ timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000),++ },// Image generation (POST /api/v1/image, workflow ai.image): off until AI_IMAGE_MODEL names the shared// provider's image model (dall-e-3, gpt-image-1); then boot seeds that model and route default.image.image: { model: String(env.AI_IMAGE_MODEL || '').trim(), size: String(env.AI_IMAGE_SIZE || '').trim() || '1024x1024' },@@@ -51,6 +51,9 @@ function buildBody({ model, system, messages, image, json, maxTokens, temperaturfunction createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetch } = {}) {const baseUrl = String(record.base_url || DEFAULT_BASE_URL).replace(/\/+$/, '');++ // A local llama.cpp process serves one model, so the embedding endpoint may live on another local++ // server (AI_LOCAL_EMBED_URL, seeded as metadata.embed_url). Every other operation stays on base_url.++ const embedBaseUrl = String((record.metadata && record.metadata.embed_url) || baseUrl).replace(/\/+$/, '');const headers = () => (apiKey ? { Authorization: `Bearer ${apiKey}` } : {});const caps = new Set(record.capabilities && record.capabilities.length ? record.capabilities : [...TEXT_OPS, 'json', 'vision', 'embed', 'transcribe', 'image']);@@@ -77,7 +80,7 @@ function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetc}async function embed(req) {-- const j = await postJson(`${baseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });++ const j = await postJson(`${embedBaseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });const vectors = (j.data || []).sort((a, b) => a.index - b.index).map(d => d.embedding);const u = j.usage || {};return { vectors, model: j.model || req.model, usage: { input: u.prompt_tokens || u.total_tokens || 0, output: 0, cached: 0 } };@@@ -60,7 +60,7 @@ function providerRecords(config, env) {// No json: the small local model must not be offered the product JSON routes. Embed only with an embedding model.const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract'];if (config.localLlm.embedModel) localCaps.push('embed');-- out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });++ out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', embed_url: config.localLlm.embedUrl || config.localLlm.url, local: true, paid: false, billing_profile: 'free' } });}if (config.httpSeamUrl) {out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });@@@ -119,6 +119,37 @@ t.test('a local embedding model makes default.embedding a pool: the failing paidawait h.stop(); await paid.close(); await local.close();});++t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {++ // llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.++ const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));++ const embedSrv = await seamServer((body, req) => {++ assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);++ return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };++ });++ h = await boot({ env: {++ AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50',++ AI_PROVIDER: 'openai', AI_BASE_URL: `${chat.url}/v1`,++ AI_LOCAL_LLM_URL: `${chat.url}/v1`, AI_LOCAL_LLM_MODEL: 'chat-model',++ AI_LOCAL_EMBED_URL: `${embedSrv.url}/v1`, AI_LOCAL_EMBED_MODEL: 'bge-small-en-v1.5',++ } });++ const local = await h.registry.getProvider('local');++ assert.strictEqual(local.metadata.embed_url, `${embedSrv.url}/v1`, 'the seeded local provider carries the embedding URL');++++ const media = await h.registry.resolveRoute('media.local');++ const chatExec = await h.pool.execute(media, 'summarize', { messages: [{ role: 'user', content: 'hello' }], timeoutMs: 2000 }, { routeKey: media.key, routeVersion: media.version });++ assert.strictEqual(chatExec.provider, 'local');++ assert.strictEqual(chat.calls, 1, 'the chat model stays on AI_LOCAL_LLM_URL');++ assert.strictEqual(embedSrv.calls, 0);++++ const embedding = await h.registry.resolveRoute('default.embedding');++ const embedExec = await h.pool.execute(embedding, 'embed', { input: ['x'], timeoutMs: 2000 }, { routeKey: embedding.key, routeVersion: embedding.version });++ assert.strictEqual(embedExec.provider, 'local');++ assert.deepStrictEqual(embedExec.result.vectors, [[0.3, 0.4, 0.5]]);++ assert.strictEqual(embedSrv.calls, 1, 'the embedding model is called on AI_LOCAL_EMBED_URL');++ assert.strictEqual(chat.calls, 1, 'and never on the chat server');++ await h.stop(); await chat.close(); await embedSrv.close();++});++t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {const ok = await seam();const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));
@@ -105,11 +105,14 @@ AI_MODEL=
@@ -105,11 +105,14 @@ AI_MODEL=
#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media
#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media
#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct
#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct
#AI_LOCAL_LLM_TIMEOUT_MS=180000
#AI_LOCAL_LLM_TIMEOUT_MS=180000
+# A second local OpenAI-compatible server for embeddings, when the chat model's llama.cpp process cannot also
+# serve the embedding GGUF (llama.cpp serves one model per process). Default: AI_LOCAL_LLM_URL.
+#AI_LOCAL_EMBED_URL=http://127.0.0.1:8091/v1
# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free
# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free
# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a
# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a
# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions
# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions
# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.
# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.
-#AI_LOCAL_EMBED_MODEL=nomic-embed-text
+#AI_LOCAL_EMBED_MODEL=bge-small-en-v1.5
# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold
# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold
#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)
#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)
#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296
#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296
@@ -103,8 +103,15 @@ function load(env = process.env) {
@@ -103,8 +103,15 @@ function load(env = process.env) {
httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),
httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),
// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,
// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,
// route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the
// route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the
- // server serves at /embeddings) makes `local` a member of the default.embedding pool.
- localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },
+ // server serves at /embeddings) makes `local` a member of the default.embedding pool. llama.cpp serves one
+ // model per process, so embedUrl may name a second local server; it defaults to the chat model's URL.
+ localLlm: {
+ url: trimUrl(env.AI_LOCAL_LLM_URL),
+ embedUrl: trimUrl(env.AI_LOCAL_EMBED_URL) || trimUrl(env.AI_LOCAL_LLM_URL),
+ model: String(env.AI_LOCAL_LLM_MODEL || '').trim(),
+ embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(),
+ timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000),
+ },
// Image generation (POST /api/v1/image, workflow ai.image): off until AI_IMAGE_MODEL names the shared
// Image generation (POST /api/v1/image, workflow ai.image): off until AI_IMAGE_MODEL names the shared
// provider's image model (dall-e-3, gpt-image-1); then boot seeds that model and route default.image.
// provider's image model (dall-e-3, gpt-image-1); then boot seeds that model and route default.image.
image: { model: String(env.AI_IMAGE_MODEL || '').trim(), size: String(env.AI_IMAGE_SIZE || '').trim() || '1024x1024' },
image: { model: String(env.AI_IMAGE_MODEL || '').trim(), size: String(env.AI_IMAGE_SIZE || '').trim() || '1024x1024' },
@@ -51,6 +51,9 @@ function buildBody({ model, system, messages, image, json, maxTokens, temperatur
@@ -51,6 +51,9 @@ function buildBody({ model, system, messages, image, json, maxTokens, temperatur
function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetch } = {}) {
function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetch } = {}) {
const baseUrl = String(record.base_url || DEFAULT_BASE_URL).replace(/\/+$/, '');
const baseUrl = String(record.base_url || DEFAULT_BASE_URL).replace(/\/+$/, '');
+ // A local llama.cpp process serves one model, so the embedding endpoint may live on another local
+ // server (AI_LOCAL_EMBED_URL, seeded as metadata.embed_url). Every other operation stays on base_url.
+ const embedBaseUrl = String((record.metadata && record.metadata.embed_url) || baseUrl).replace(/\/+$/, '');
const headers = () => (apiKey ? { Authorization: `Bearer ${apiKey}` } : {});
const headers = () => (apiKey ? { Authorization: `Bearer ${apiKey}` } : {});
const caps = new Set(record.capabilities && record.capabilities.length ? record.capabilities : [...TEXT_OPS, 'json', 'vision', 'embed', 'transcribe', 'image']);
const caps = new Set(record.capabilities && record.capabilities.length ? record.capabilities : [...TEXT_OPS, 'json', 'vision', 'embed', 'transcribe', 'image']);
@@ -77,7 +80,7 @@ function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetc
@@ -77,7 +80,7 @@ function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetc
}
}
async function embed(req) {
async function embed(req) {
- const j = await postJson(`${baseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });
+ const j = await postJson(`${embedBaseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });
const vectors = (j.data || []).sort((a, b) => a.index - b.index).map(d => d.embedding);
const vectors = (j.data || []).sort((a, b) => a.index - b.index).map(d => d.embedding);
const u = j.usage || {};
const u = j.usage || {};
return { vectors, model: j.model || req.model, usage: { input: u.prompt_tokens || u.total_tokens || 0, output: 0, cached: 0 } };
return { vectors, model: j.model || req.model, usage: { input: u.prompt_tokens || u.total_tokens || 0, output: 0, cached: 0 } };
@@ -60,7 +60,7 @@ function providerRecords(config, env) {
@@ -60,7 +60,7 @@ function providerRecords(config, env) {
// No json: the small local model must not be offered the product JSON routes. Embed only with an embedding model.
// No json: the small local model must not be offered the product JSON routes. Embed only with an embedding model.
const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract'];
const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract'];
if (config.localLlm.embedModel) localCaps.push('embed');
if (config.localLlm.embedModel) localCaps.push('embed');
- out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });
+ out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', embed_url: config.localLlm.embedUrl || config.localLlm.url, local: true, paid: false, billing_profile: 'free' } });
}
}
if (config.httpSeamUrl) {
if (config.httpSeamUrl) {
out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });
out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });
@@ -119,6 +119,37 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
@@ -119,6 +119,37 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
await h.stop(); await paid.close(); await local.close();
await h.stop(); await paid.close(); await local.close();
});
});
+t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {
+ // llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.
+ const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
+ const embedSrv = await seamServer((body, req) => {
+ assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);
+ return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };
+ });
+ h = await boot({ env: {
+ AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50',
+ AI_PROVIDER: 'openai', AI_BASE_URL: `${chat.url}/v1`,
+ AI_LOCAL_LLM_URL: `${chat.url}/v1`, AI_LOCAL_LLM_MODEL: 'chat-model',
+ AI_LOCAL_EMBED_URL: `${embedSrv.url}/v1`, AI_LOCAL_EMBED_MODEL: 'bge-small-en-v1.5',
+ } });
+ const local = await h.registry.getProvider('local');
+ assert.strictEqual(local.metadata.embed_url, `${embedSrv.url}/v1`, 'the seeded local provider carries the embedding URL');
+
+ const media = await h.registry.resolveRoute('media.local');
+ const chatExec = await h.pool.execute(media, 'summarize', { messages: [{ role: 'user', content: 'hello' }], timeoutMs: 2000 }, { routeKey: media.key, routeVersion: media.version });
+ assert.strictEqual(chatExec.provider, 'local');
+ assert.strictEqual(chat.calls, 1, 'the chat model stays on AI_LOCAL_LLM_URL');
+ assert.strictEqual(embedSrv.calls, 0);
+
+ const embedding = await h.registry.resolveRoute('default.embedding');
+ const embedExec = await h.pool.execute(embedding, 'embed', { input: ['x'], timeoutMs: 2000 }, { routeKey: embedding.key, routeVersion: embedding.version });
+ assert.strictEqual(embedExec.provider, 'local');
+ assert.deepStrictEqual(embedExec.result.vectors, [[0.3, 0.4, 0.5]]);
+ assert.strictEqual(embedSrv.calls, 1, 'the embedding model is called on AI_LOCAL_EMBED_URL');
+ assert.strictEqual(chat.calls, 1, 'and never on the chat server');
+ await h.stop(); await chat.close(); await embedSrv.close();
+});
+
t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {
t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {
const ok = await seam();
const ok = await seam();
const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));
const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));