openvibers/openvibe.chat Commits

TTS via Exchange behind tts_exchange (off) (#501)

OP OpenVibers committed 3 days ago · bdfc0acd00 · 3 files, +109 −1

With the tts_exchange site setting on, google-cloud and amazon-polly voices
synthesize through AI's Exchange speech route instead of the providers
directly: the Chat voice id travels unchanged (one to one), and no provider
host is called. An Exchange failure falls back to local espeak, never to the
providers directly. Off by default: today's direct synthesis is unchanged.

Chat's own service token for AI (server/ai/client bearer) authenticates the
call; the SDK has no Exchange speech client, so this uses the same token
client over plain HTTPS.

Checked: test/tts-via-exchange.test.js (a Google and a Polly voice speak
through a fake Exchange with their ids unchanged; fails if a provider host
is called); tts-voices, sounds-tts, tts-person and chat-ai suites pass.

Changes

ddiff --git a/server/ai/client.js b/server/ai/client.js
iindex 9bb58cf..033d45e 100644
---- a/server/ai/client.js
++++ b/server/ai/client.js
@@@ -47,6 +47,15 @@ function enabled() {
return !!(config.ai.enabled && config.oauth.clientSecret);
}
++/**
++ * Chat's bearer token for AI (its own service token; no CHAT_AI_ENABLED gate — TTS has its own
++ * setting). For calls the SDK's AI client has no method for (the Exchange speech route).
++ */
++async function bearer() {
++ if (!_tokens) ai();
++ return _tokens.getToken();
++}
++
/**
* Run a workflow and return its `output` object, or null: disabled, a refusal, a failed run, or a
* transport error all answer null — exactly Live's ai-service.structured() contract. `force` runs it even with
@@@ -176,5 +185,5 @@ async function catalog() {
return res.json();
}
--module.exports = { structured, stream, complete, catalog, enabled, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },
++module.exports = { structured, stream, complete, catalog, enabled, bearer, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },
_setProblem(p) { _lastProblem = p; }, _reset() { _ai = null; _http = null; _tokens = null; _complete = null; _lastProblem = null; } };
ddiff --git a/server/chat/tts-engine.js b/server/chat/tts-engine.js
iindex 07887b3..1f21e09 100644
---- a/server/chat/tts-engine.js
++++ b/server/chat/tts-engine.js
@@@ -160,6 +160,9 @@ function getTTSSettings() {
if (_settingsCache && (now - _settingsCacheTime) < SETTINGS_CACHE_TTL) return _settingsCache;
_settingsCache = {
enabled: ctx.getSetting('tts_enabled') !== false,
++ // Exchange TTS (off): cloud voices synthesize through AI's POST /v1/audio/speech instead of
++ // the providers directly, so no provider host is called with the setting on.
++ exchange: (() => { const v = ctx.getSetting('tts_exchange'); return v === true || v === 'true' || v === 1 || v === '1'; })(),
provider: ctx.getSetting('tts_provider') || 'espeak-ng',
googleApiKey: ctx.getSetting('tts_google_api_key') || '',
googleServiceAccount: owner.value('google_cloud_service_account_json', 'CHAT_GOOGLE_SERVICE_ACCOUNT_JSON', ctx.getSetting('tts_google_service_account') || ''),
@@@ -553,6 +556,28 @@ function sanitize(text, maxLen, username) {
return clean;
}
++// ── Exchange TTS (behind tts_exchange, off) ───────────────────
++// Cloud voices synthesize through AI's POST /v1/audio/speech instead of the providers directly:
++// Chat's voice id travels as the Exchange voice, one to one, so nobody hears a difference. With the
++// setting on no provider host is ever called (a failure falls back to local espeak, never direct).
++async function synthesizeViaExchange(text, voiceDef, voiceId) {
++ const client = require('../ai/client');
++ const token = await client.bearer();
++ const { ai } = require('../config');
++ const res = await fetch(`${String(ai.internalUrl).replace(/\/+$/, '')}/v1/audio/speech`, {
++ method: 'POST',
++ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'audio/mpeg, audio/ogg, audio/wav' },
++ // The voice id travels unchanged: Chat's voice ids map one to one onto Exchange voices.
++ body: JSON.stringify({ model: 'tts-1', input: text, voice: voiceId, response_format: 'mp3' }),
++ signal: AbortSignal.timeout(60000),
++ });
++ if (!res.ok) throw new Error(`exchange TTS answered ${res.status}`);
++ const buf = Buffer.from(await res.arrayBuffer());
++ if (!buf.length) throw new Error('exchange TTS returned no audio');
++ return { audio: buf.toString('base64'), mimeType: res.headers.get('content-type') || 'audio/mpeg',
++ engine: voiceDef.engine, voiceName: (voiceDef.cloudVoice && voiceDef.cloudVoice.name) || (voiceDef.pollyVoice && voiceDef.pollyVoice.voiceId) || 'exchange' };
++}
++
// ── Main synthesis function ───────────────────────────────────
/**
* Synthesize TTS audio for a chat message.
@@@ -608,6 +633,17 @@ async function _synthesizeWithDef(text, voiceDef, voiceId) {
}
}
++ // Behind tts_exchange a cloud voice never touches a provider host: it speaks through Exchange,
++ // and an Exchange failure falls back to local espeak, never to the providers directly.
++ if (settings.exchange && (voiceDef.engine === 'google-cloud' || voiceDef.engine === 'amazon-polly')) {
++ try {
++ return { ...(await synthesizeViaExchange(text, voiceDef, voiceId)), voiceId };
++ } catch (err) {
++ console.error(`[TTS] Exchange synthesis failed (${voiceId}):`, err.message);
++ return _espeakFallback(text, voiceId);
++ }
++ }
++
try {
switch (voiceDef.engine) {
case 'google-cloud': {
ddiff --git a/test/tts-via-exchange.test.js b/test/tts-via-exchange.test.js
nnew file mode 100644
iindex 0000000..affe5a2
---- /dev/null
++++ b/test/tts-via-exchange.test.js
@@@ -0,0 +1,63 @@
++'use strict';
++/**
++ * Behind tts_exchange (off by default) cloud voices synthesize through AI's POST /v1/audio/speech
++ * instead of the providers directly:
++ * - a Google voice and a Polly voice speak through the fake Exchange with their Chat voice id
++ * unchanged (the one-to-one map), and the same shape as direct synthesis;
++ * - no request reaches a provider host (googleapis.com, amazonaws.com) with the setting on.
++ *
++ * The network is a stand-in: fetch answers for Exchange, https.request records (and would fail)
++ * provider calls. The AI token client is stubbed; nothing reaches AI or the providers.
++ */
++const assert = require('assert');
++const https = require('https');
++
++const ctx = require('../server/live-context');
++const SETTINGS = { tts_enabled: 'true', tts_exchange: 'true', tts_default_voice: 'gary' };
++ctx.getSetting = (k) => (Object.prototype.hasOwnProperty.call(SETTINGS, k) ? SETTINGS[k] : null);
++
++const aiClient = require('../server/ai/client');
++aiClient._setTokens({ getToken: async () => 'test-token' });
++
++const askedFetch = [];
++const realFetch = globalThis.fetch;
++globalThis.fetch = async (url, opts) => {
++ askedFetch.push({ url: String(url), auth: opts && opts.headers && opts.headers.Authorization, body: JSON.parse(opts.body) });
++ assert.match(String(url), /\/v1\/audio\/speech$/, 'chat calls the Exchange speech route');
++ return {
++ ok: true, status: 200,
++ headers: { get: (k) => (String(k).toLowerCase() === 'content-type' ? 'audio/mpeg' : null) },
++ arrayBuffer: async () => Buffer.from(`mp3 of ${JSON.parse(opts.body).voice}:${JSON.parse(opts.body).input}`),
++ };
++};
++
++const askedProviders = [];
++https.request = (url, opts, cb) => {
++ if (typeof url === 'object') { cb = opts; url = `https://${url.hostname || url.host}${url.path || '/'}`; }
++ askedProviders.push(String(url));
++ throw new Error(`provider host must not be called with tts_exchange on: ${url}`);
++};
++console.warn = () => {};
++console.error = () => {};
++
++const tts = require('../server/chat/tts-engine');
++
++(async () => {
++ const google = await tts.synthesize('Ladies and gentlemen', 'voice_vegas', 'josh');
++ assert.strictEqual(google.voiceId, 'voice_vegas');
++ assert.strictEqual(google.engine, 'google-cloud', 'the voice keeps its engine label');
++ assert.strictEqual(Buffer.from(google.audio, 'base64').toString(), 'mp3 of voice_vegas:Ladies and gentlemen');
++ assert.strictEqual(askedFetch.length, 1);
++ assert.strictEqual(askedFetch[0].auth, 'Bearer test-token', 'chat uses its own service token');
++ assert.deepStrictEqual(askedFetch[0].body, { model: 'tts-1', input: 'Ladies and gentlemen', voice: 'voice_vegas', response_format: 'mp3' },
++ 'the Chat voice id travels unchanged');
++
++ const polly = await tts.synthesize('hello there', 'pl_matthew_neural', 'ann');
++ assert.strictEqual(polly.voiceId, 'pl_matthew_neural');
++ assert.strictEqual(polly.engine, 'amazon-polly', 'the voice keeps its engine label');
++ assert.strictEqual(Buffer.from(polly.audio, 'base64').toString(), 'mp3 of pl_matthew_neural:hello there');
++ assert.strictEqual(askedFetch.length, 2);
++
++ assert.deepStrictEqual(askedProviders, [], 'no provider host was called with the setting on');
++ console.log('tts-via-exchange: ok');
++})().catch((err) => { console.log('FAIL', err); process.exit(err ? 1 : 0); }).finally(() => { globalThis.fetch = realFetch; });