openvibers/openvibe.ai Commits

AI_LOCAL_EMBED_URL: the local provider's embed op can use its own llama.cpp server (one model per process), defaulting to AI_LOCAL_LLM_URL (#183)

OP OpenVibers committed 16 days ago · f6b063ab4b · 5 files, +49 −5

Changes

ddiff --git a/.env.example b/.env.example
iindex 29305c8..2aa4c10 100644
---- a/.env.example
++++ b/.env.example
@@@ -105,11 +105,14 @@ AI_MODEL=
#AI_MEDIA_STORAGE_HOSTS=*.backblazeb2.com,*.r2.cloudflarestorage.com,edge.openvibe.media
#AI_LOCAL_LLM_MODEL=qwen2.5-1.5b-instruct
#AI_LOCAL_LLM_TIMEOUT_MS=180000
++# A second local OpenAI-compatible server for embeddings, when the chat model's llama.cpp process cannot also
++# serve the embedding GGUF (llama.cpp serves one model per process). Default: AI_LOCAL_LLM_URL.
++#AI_LOCAL_EMBED_URL=http://127.0.0.1:8091/v1
# An embedding model the local server answers at /embeddings: `local` then serves default.embedding as its free
# member. The server must really serve embeddings (llama-server only with --embedding and an embedding GGUF; a
# chat-only model answers 400 and every embedding call falls over to the shared provider). Vector dimensions
# differ per upstream: never compare or mix vectors from different providers. Unset: no local embeddings.
--#AI_LOCAL_EMBED_MODEL=nomic-embed-text
++#AI_LOCAL_EMBED_MODEL=bge-small-en-v1.5
# media.analyze: file cap (also bounded by free temp disk minus the reserve), download timeout, scene threshold
#AI_MEDIA_ANALYSIS_DIR=/var/lib/openvibe-ai/media-tmp (default: data/media-tmp; never a tmpfs)
#AI_MEDIA_ANALYSIS_MAX_BYTES=4294967296
ddiff --git a/server/config.js b/server/config.js
iindex 1cbb50f..bf7b2f2 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -103,8 +103,15 @@ function load(env = process.env) {
httpSeamUrl: trimUrl(env.AI_HTTP_SEAM_URL),
// A local model server speaking the OpenAI API (llama.cpp's llama-server, Ollama's /v1): the provider `local`,
// route media.local (roadmap WS-O task 5). No key; never a paid provider. embedModel (an embedding GGUF the
-- // server serves at /embeddings) makes `local` a member of the default.embedding pool.
-- localLlm: { url: trimUrl(env.AI_LOCAL_LLM_URL), model: String(env.AI_LOCAL_LLM_MODEL || '').trim(), embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(), timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000) },
++ // server serves at /embeddings) makes `local` a member of the default.embedding pool. llama.cpp serves one
++ // model per process, so embedUrl may name a second local server; it defaults to the chat model's URL.
++ localLlm: {
++ url: trimUrl(env.AI_LOCAL_LLM_URL),
++ embedUrl: trimUrl(env.AI_LOCAL_EMBED_URL) || trimUrl(env.AI_LOCAL_LLM_URL),
++ model: String(env.AI_LOCAL_LLM_MODEL || '').trim(),
++ embedModel: String(env.AI_LOCAL_EMBED_MODEL || '').trim(),
++ timeoutMs: int(env.AI_LOCAL_LLM_TIMEOUT_MS, 180000),
++ },
// Image generation (POST /api/v1/image, workflow ai.image): off until AI_IMAGE_MODEL names the shared
// provider's image model (dall-e-3, gpt-image-1); then boot seeds that model and route default.image.
image: { model: String(env.AI_IMAGE_MODEL || '').trim(), size: String(env.AI_IMAGE_SIZE || '').trim() || '1024x1024' },
ddiff --git a/server/providers/openai.js b/server/providers/openai.js
iindex 218833a..9279ab5 100644
---- a/server/providers/openai.js
++++ b/server/providers/openai.js
@@@ -51,6 +51,9 @@ function buildBody({ model, system, messages, image, json, maxTokens, temperatur
function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetch } = {}) {
const baseUrl = String(record.base_url || DEFAULT_BASE_URL).replace(/\/+$/, '');
++ // A local llama.cpp process serves one model, so the embedding endpoint may live on another local
++ // server (AI_LOCAL_EMBED_URL, seeded as metadata.embed_url). Every other operation stays on base_url.
++ const embedBaseUrl = String((record.metadata && record.metadata.embed_url) || baseUrl).replace(/\/+$/, '');
const headers = () => (apiKey ? { Authorization: `Bearer ${apiKey}` } : {});
const caps = new Set(record.capabilities && record.capabilities.length ? record.capabilities : [...TEXT_OPS, 'json', 'vision', 'embed', 'transcribe', 'image']);
@@@ -77,7 +80,7 @@ function createOpenAiProvider(record, { apiKey = '', fetchImpl = globalThis.fetc
}
async function embed(req) {
-- const j = await postJson(`${baseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });
++ const j = await postJson(`${embedBaseUrl}/embeddings`, headers(), { model: req.model, input: req.input }, { signal: req.signal, timeoutMs: req.timeoutMs, fetchImpl });
const vectors = (j.data || []).sort((a, b) => a.index - b.index).map(d => d.embedding);
const u = j.usage || {};
return { vectors, model: j.model || req.model, usage: { input: u.prompt_tokens || u.total_tokens || 0, output: 0, cached: 0 } };
ddiff --git a/server/workflows/seed.js b/server/workflows/seed.js
iindex 61976f9..4622d5a 100644
---- a/server/workflows/seed.js
++++ b/server/workflows/seed.js
@@@ -60,7 +60,7 @@ function providerRecords(config, env) {
// No json: the small local model must not be offered the product JSON routes. Embed only with an embedding model.
const localCaps = ['chat', 'generate', 'summarize', 'classify', 'extract'];
if (config.localLlm.embedModel) localCaps.push('embed');
-- out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', local: true, paid: false, billing_profile: 'free' } });
++ out.push({ key: 'local', display_name: 'Local model (OpenAI-compatible server on this host)', kind: 'openai', status: 'active', base_url: config.localLlm.url, auth_mode: 'none', secret_ref: null, default_model: config.localLlm.model || 'local', capabilities: localCaps, timeout_ms: config.localLlm.timeoutMs, priority: 5, metadata: { configured_by: 'AI_LOCAL_LLM_URL', embed_url: config.localLlm.embedUrl || config.localLlm.url, local: true, paid: false, billing_profile: 'free' } });
}
if (config.httpSeamUrl) {
out.push({ key: 'http-seam', display_name: 'Local HTTP seam', kind: 'http', status: 'active', base_url: config.httpSeamUrl, auth_mode: 'none', default_model: null, capabilities: [], timeout_ms: 30000, priority: 60, metadata: { configured_by: 'AI_HTTP_SEAM_URL', billing_profile: 'metered' } });
ddiff --git a/test/provider-router.test.js b/test/provider-router.test.js
iindex 1b8f9fa..7224e44 100644
---- a/test/provider-router.test.js
++++ b/test/provider-router.test.js
@@@ -119,6 +119,37 @@ t.test('a local embedding model makes default.embedding a pool: the failing paid
await h.stop(); await paid.close(); await local.close();
});
++t.test('the local provider embeds on AI_LOCAL_EMBED_URL while chat stays on AI_LOCAL_LLM_URL', async () => {
++ // llama.cpp serves one model per process: the chat GGUF and the embedding GGUF may be two local servers.
++ const chat = await seamServer(() => ({ text: 'local chat answer', model: 'chat-model', usage: { prompt_tokens: 3, completion_tokens: 2 } }));
++ const embedSrv = await seamServer((body, req) => {
++ assert.ok(req.url.endsWith('/embeddings'), `except embedding requests, nothing reaches the embed server (${req.url})`);
++ return { data: [{ index: 0, embedding: [0.3, 0.4, 0.5] }], model: body.model, usage: { prompt_tokens: 2, total_tokens: 2 } };
++ });
++ h = await boot({ env: {
++ AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50',
++ AI_PROVIDER: 'openai', AI_BASE_URL: `${chat.url}/v1`,
++ AI_LOCAL_LLM_URL: `${chat.url}/v1`, AI_LOCAL_LLM_MODEL: 'chat-model',
++ AI_LOCAL_EMBED_URL: `${embedSrv.url}/v1`, AI_LOCAL_EMBED_MODEL: 'bge-small-en-v1.5',
++ } });
++ const local = await h.registry.getProvider('local');
++ assert.strictEqual(local.metadata.embed_url, `${embedSrv.url}/v1`, 'the seeded local provider carries the embedding URL');
++
++ const media = await h.registry.resolveRoute('media.local');
++ const chatExec = await h.pool.execute(media, 'summarize', { messages: [{ role: 'user', content: 'hello' }], timeoutMs: 2000 }, { routeKey: media.key, routeVersion: media.version });
++ assert.strictEqual(chatExec.provider, 'local');
++ assert.strictEqual(chat.calls, 1, 'the chat model stays on AI_LOCAL_LLM_URL');
++ assert.strictEqual(embedSrv.calls, 0);
++
++ const embedding = await h.registry.resolveRoute('default.embedding');
++ const embedExec = await h.pool.execute(embedding, 'embed', { input: ['x'], timeoutMs: 2000 }, { routeKey: embedding.key, routeVersion: embedding.version });
++ assert.strictEqual(embedExec.provider, 'local');
++ assert.deepStrictEqual(embedExec.result.vectors, [[0.3, 0.4, 0.5]]);
++ assert.strictEqual(embedSrv.calls, 1, 'the embedding model is called on AI_LOCAL_EMBED_URL');
++ assert.strictEqual(chat.calls, 1, 'and never on the chat server');
++ await h.stop(); await chat.close(); await embedSrv.close();
++});
++
t.test('two providers on one capability: the cheaper is chosen; a circuit-open one is excluded', async () => {
const ok = await seam();
const flaky = await seamServer(() => ({ status: 500, body: { error: { message: 'boom' } } }));