openvibers/openvibe.ai Commits
AI: a route's longer timeout wins over the provider's chat timeout (#110)
Changes
@@@ -441,8 +441,11 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.noif (ctx.logRequest) ctx.logRequest({ ...base, status: 'skipped', skip_reason: why });continue;}-- // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins.-- const cap = operation === 'image' ? 600000 : (p.timeout_ms || 600000);++ // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins. So does a++ // route that says so (options.outlast_provider_timeout: space.build writes a whole app, a minute or two). Otherwise++ // the provider's timeout stays strict, so a slow provider fails over quickly.++ const outlast = operation === 'image' || Boolean(route.options && route.options.outlast_provider_timeout);++ const cap = outlast ? Math.max(p.timeout_ms || 0, Number(route.timeout_ms) || 0, operation === 'image' ? 600000 : 0) : (p.timeout_ms || 600000);const timeoutMs = Math.max(1000, Math.min(req.timeoutMs || route.timeout_ms || p.timeout_ms || 30000, cap));const retries = config.providerRetries;let lastErr = null;@@@ -136,7 +136,7 @@ async function seed({ registry, quotas, config, env = process.env, db }) {// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.// Space's spaces (server/workflows/space.js): the builder writes whole apps, so it gets the creative model and room to// finish; a space's own AI is conversation text, an alias of the chat pool.-- await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));++ await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, options: { outlast_provider_timeout: true }, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));await registry.seedVersioned('route', 'space.ask', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 800, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });nnew file mode 100644@@@ -0,0 +1,30 @@++'use strict';++// A route that says so (options.outlast_provider_timeout) outlasts the shared provider's chat timeout: space.build++// (the space builder writes a whole app, 180 s) outlasts a provider timeout of 1 s, as image generation always did;++// an explicit shorter request timeout still ends the call, and other routes keep the provider's timeout strict.++const assert = require('assert');++const { boot, suite, seamServer } = require('./helpers');++++const t = suite('route-timeout');++const ENV = { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50', AI_PROVIDER: 'openai', AI_MODEL: 'gpt-4o-mini' };++++t.test('space.build outlasts the provider\'s 1 s chat timeout; a 1 s request timeout does not', async () => {++ const up = await seamServer(async () => { await new Promise((r) => setTimeout(r, 1500)); return { choices: [{ message: { role: 'assistant', content: 'built' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1 } }; });++ const h = await boot({ env: { ...ENV, AI_BASE_URL: `${up.url}/v1`, AI_PROVIDER_TIMEOUT_MS: '1000' } });++ try {++ assert.strictEqual((await h.registry.getProvider('shared')).timeout_ms, 1000);++ const route = await h.registry.resolveRoute('space.build');++ assert.ok(route.timeout_ms >= 180000, `space.build asks for ${route.timeout_ms} ms`);++ const ok = await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50 }, { routeKey: route.key });++ assert.strictEqual(ok.result.text, 'built');++ let err = null;++ try { await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50, timeoutMs: 1000 }, { routeKey: route.key }); } catch (e) { err = e; }++ assert.ok(err, 'an explicit 1 s request timeout still ends the call');++ const chat = await h.registry.resolveRoute('default.chat');++ let strict = null;++ try { await h.pool.execute(chat, 'chat', { messages: [{ role: 'user', content: 'hi' }], maxTokens: 50 }, { routeKey: chat.key }); } catch (e) { strict = e; }++ assert.ok(strict, 'default.chat keeps the provider\'s 1 s timeout');++ } finally { await h.stop(); await up.close(); }++});++++t.run();
@@ -441,8 +441,11 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
@@ -441,8 +441,11 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
if (ctx.logRequest) ctx.logRequest({ ...base, status: 'skipped', skip_reason: why });
if (ctx.logRequest) ctx.logRequest({ ...base, status: 'skipped', skip_reason: why });
continue;
continue;
}
}
- // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins.
- const cap = operation === 'image' ? 600000 : (p.timeout_ms || 600000);
+ // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins. So does a
+ // route that says so (options.outlast_provider_timeout: space.build writes a whole app, a minute or two). Otherwise
+ // the provider's timeout stays strict, so a slow provider fails over quickly.
+ const outlast = operation === 'image' || Boolean(route.options && route.options.outlast_provider_timeout);
+ const cap = outlast ? Math.max(p.timeout_ms || 0, Number(route.timeout_ms) || 0, operation === 'image' ? 600000 : 0) : (p.timeout_ms || 600000);
const timeoutMs = Math.max(1000, Math.min(req.timeoutMs || route.timeout_ms || p.timeout_ms || 30000, cap));
const timeoutMs = Math.max(1000, Math.min(req.timeoutMs || route.timeout_ms || p.timeout_ms || 30000, cap));
const retries = config.providerRetries;
const retries = config.providerRetries;
let lastErr = null;
let lastErr = null;
@@ -136,7 +136,7 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
@@ -136,7 +136,7 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
// Space's spaces (server/workflows/space.js): the builder writes whole apps, so it gets the creative model and room to
// Space's spaces (server/workflows/space.js): the builder writes whole apps, so it gets the creative model and room to
// finish; a space's own AI is conversation text, an alias of the chat pool.
// finish; a space's own AI is conversation text, an alias of the chat pool.
- await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));
+ await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, options: { outlast_provider_timeout: true }, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));
await registry.seedVersioned('route', 'space.ask', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 800, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
await registry.seedVersioned('route', 'space.ask', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 800, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
new file mode 100644
new file mode 100644
@@ -0,0 +1,30 @@
@@ -0,0 +1,30 @@
+'use strict';
+// A route that says so (options.outlast_provider_timeout) outlasts the shared provider's chat timeout: space.build
+// (the space builder writes a whole app, 180 s) outlasts a provider timeout of 1 s, as image generation always did;
+// an explicit shorter request timeout still ends the call, and other routes keep the provider's timeout strict.
+const assert = require('assert');
+const { boot, suite, seamServer } = require('./helpers');
+
+const t = suite('route-timeout');
+const ENV = { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50', AI_PROVIDER: 'openai', AI_MODEL: 'gpt-4o-mini' };
+
+t.test('space.build outlasts the provider\'s 1 s chat timeout; a 1 s request timeout does not', async () => {
+ const up = await seamServer(async () => { await new Promise((r) => setTimeout(r, 1500)); return { choices: [{ message: { role: 'assistant', content: 'built' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1 } }; });
+ const h = await boot({ env: { ...ENV, AI_BASE_URL: `${up.url}/v1`, AI_PROVIDER_TIMEOUT_MS: '1000' } });
+ try {
+ assert.strictEqual((await h.registry.getProvider('shared')).timeout_ms, 1000);
+ const route = await h.registry.resolveRoute('space.build');
+ assert.ok(route.timeout_ms >= 180000, `space.build asks for ${route.timeout_ms} ms`);
+ const ok = await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50 }, { routeKey: route.key });
+ assert.strictEqual(ok.result.text, 'built');
+ let err = null;
+ try { await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50, timeoutMs: 1000 }, { routeKey: route.key }); } catch (e) { err = e; }
+ assert.ok(err, 'an explicit 1 s request timeout still ends the call');
+ const chat = await h.registry.resolveRoute('default.chat');
+ let strict = null;
+ try { await h.pool.execute(chat, 'chat', { messages: [{ role: 'user', content: 'hi' }], maxTokens: 50 }, { routeKey: chat.key }); } catch (e) { strict = e; }
+ assert.ok(strict, 'default.chat keeps the provider\'s 1 s timeout');
+ } finally { await h.stop(); await up.close(); }
+});
+
+t.run();