openvibers/openvibe.ai Commits

AI: a route's longer timeout wins over the provider's chat timeout (#110)

OP OpenVibers committed 28 days ago · 74eb61bfd1 · 3 files, +36 −3

* AI: a route that asks for longer than the shared provider's chat timeout gets it (space.build writes a whole app; it was cut at 30 s)

* AI: outlasting the provider's timeout is the route's choice (options.outlast_provider_timeout, set on space.build); other routes keep it strict so a slow provider fails over quickly

Changes

ddiff --git a/server/providers/index.js b/server/providers/index.js
iindex e73cc92..841f44d 100644
---- a/server/providers/index.js
++++ b/server/providers/index.js
@@@ -441,8 +441,11 @@ function createProviderPool({ db, registry, config, clock = { now: () => Date.no
if (ctx.logRequest) ctx.logRequest({ ...base, status: 'skipped', skip_reason: why });
continue;
}
-- // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins.
-- const cap = operation === 'image' ? 600000 : (p.timeout_ms || 600000);
++ // An image generation outlasts the shared provider's chat timeout (AI_PROVIDER_TIMEOUT_MS): the route's wins. So does a
++ // route that says so (options.outlast_provider_timeout: space.build writes a whole app, a minute or two). Otherwise
++ // the provider's timeout stays strict, so a slow provider fails over quickly.
++ const outlast = operation === 'image' || Boolean(route.options && route.options.outlast_provider_timeout);
++ const cap = outlast ? Math.max(p.timeout_ms || 0, Number(route.timeout_ms) || 0, operation === 'image' ? 600000 : 0) : (p.timeout_ms || 600000);
const timeoutMs = Math.max(1000, Math.min(req.timeoutMs || route.timeout_ms || p.timeout_ms || 30000, cap));
const retries = config.providerRetries;
let lastErr = null;
ddiff --git a/server/workflows/seed.js b/server/workflows/seed.js
iindex 73dd89a..d1d3d9d 100644
---- a/server/workflows/seed.js
++++ b/server/workflows/seed.js
@@@ -136,7 +136,7 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
// Space's spaces (server/workflows/space.js): the builder writes whole apps, so it gets the creative model and room to
// finish; a space's own AI is conversation text, an alias of the chat pool.
-- await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));
++ await registry.seedVersioned('route', 'space.build', pool(180000, { max_output_tokens: 16000, options: { outlast_provider_timeout: true }, ...(creativeModel ? { constraints: { objective: 'correctness', latency_class: 'interactive', model: creativeModel, authority: `shared:${creativeModel}` } } : {}) }));
await registry.seedVersioned('route', 'space.ask', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 800, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
ddiff --git a/test/route-timeout.test.js b/test/route-timeout.test.js
nnew file mode 100644
iindex 0000000..ea9bb67
---- /dev/null
++++ b/test/route-timeout.test.js
@@@ -0,0 +1,30 @@
++'use strict';
++// A route that says so (options.outlast_provider_timeout) outlasts the shared provider's chat timeout: space.build
++// (the space builder writes a whole app, 180 s) outlasts a provider timeout of 1 s, as image generation always did;
++// an explicit shorter request timeout still ends the call, and other routes keep the provider's timeout strict.
++const assert = require('assert');
++const { boot, suite, seamServer } = require('./helpers');
++
++const t = suite('route-timeout');
++const ENV = { AI_STUB_FALLBACK: 'false', AI_PROVIDER_RETRIES: '0', AI_BREAKER_FAILURES: '50', AI_PROVIDER: 'openai', AI_MODEL: 'gpt-4o-mini' };
++
++t.test('space.build outlasts the provider\'s 1 s chat timeout; a 1 s request timeout does not', async () => {
++ const up = await seamServer(async () => { await new Promise((r) => setTimeout(r, 1500)); return { choices: [{ message: { role: 'assistant', content: 'built' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1 } }; });
++ const h = await boot({ env: { ...ENV, AI_BASE_URL: `${up.url}/v1`, AI_PROVIDER_TIMEOUT_MS: '1000' } });
++ try {
++ assert.strictEqual((await h.registry.getProvider('shared')).timeout_ms, 1000);
++ const route = await h.registry.resolveRoute('space.build');
++ assert.ok(route.timeout_ms >= 180000, `space.build asks for ${route.timeout_ms} ms`);
++ const ok = await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50 }, { routeKey: route.key });
++ assert.strictEqual(ok.result.text, 'built');
++ let err = null;
++ try { await h.pool.execute(route, 'chat', { messages: [{ role: 'user', content: 'build it' }], maxTokens: 50, timeoutMs: 1000 }, { routeKey: route.key }); } catch (e) { err = e; }
++ assert.ok(err, 'an explicit 1 s request timeout still ends the call');
++ const chat = await h.registry.resolveRoute('default.chat');
++ let strict = null;
++ try { await h.pool.execute(chat, 'chat', { messages: [{ role: 'user', content: 'hi' }], maxTokens: 50 }, { routeKey: chat.key }); } catch (e) { strict = e; }
++ assert.ok(strict, 'default.chat keeps the provider\'s 1 s timeout');
++ } finally { await h.stop(); await up.close(); }
++});
++
++t.run();