openvibers/openvibe.chat Commits
TTS via Exchange behind tts_exchange (off) (#501)
Changes
@@@ -47,6 +47,15 @@ function enabled() {return !!(config.ai.enabled && config.oauth.clientSecret);}++/**++ * Chat's bearer token for AI (its own service token; no CHAT_AI_ENABLED gate — TTS has its own++ * setting). For calls the SDK's AI client has no method for (the Exchange speech route).++ */++async function bearer() {++ if (!_tokens) ai();++ return _tokens.getToken();++}++/*** Run a workflow and return its `output` object, or null: disabled, a refusal, a failed run, or a* transport error all answer null — exactly Live's ai-service.structured() contract. `force` runs it even with@@@ -176,5 +185,5 @@ async function catalog() {return res.json();}--module.exports = { structured, stream, complete, catalog, enabled, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },++module.exports = { structured, stream, complete, catalog, enabled, bearer, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },_setProblem(p) { _lastProblem = p; }, _reset() { _ai = null; _http = null; _tokens = null; _complete = null; _lastProblem = null; } };@@@ -160,6 +160,9 @@ function getTTSSettings() {if (_settingsCache && (now - _settingsCacheTime) < SETTINGS_CACHE_TTL) return _settingsCache;_settingsCache = {enabled: ctx.getSetting('tts_enabled') !== false,++ // Exchange TTS (off): cloud voices synthesize through AI's POST /v1/audio/speech instead of++ // the providers directly, so no provider host is called with the setting on.++ exchange: (() => { const v = ctx.getSetting('tts_exchange'); return v === true || v === 'true' || v === 1 || v === '1'; })(),provider: ctx.getSetting('tts_provider') || 'espeak-ng',googleApiKey: ctx.getSetting('tts_google_api_key') || '',googleServiceAccount: owner.value('google_cloud_service_account_json', 'CHAT_GOOGLE_SERVICE_ACCOUNT_JSON', ctx.getSetting('tts_google_service_account') || ''),@@@ -553,6 +556,28 @@ function sanitize(text, maxLen, username) {return clean;}++// ── Exchange TTS (behind tts_exchange, off) ───────────────────++// Cloud voices synthesize through AI's POST /v1/audio/speech instead of the providers directly:++// Chat's voice id travels as the Exchange voice, one to one, so nobody hears a difference. With the++// setting on no provider host is ever called (a failure falls back to local espeak, never direct).++async function synthesizeViaExchange(text, voiceDef, voiceId) {++ const client = require('../ai/client');++ const token = await client.bearer();++ const { ai } = require('../config');++ const res = await fetch(`${String(ai.internalUrl).replace(/\/+$/, '')}/v1/audio/speech`, {++ method: 'POST',++ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'audio/mpeg, audio/ogg, audio/wav' },++ // The voice id travels unchanged: Chat's voice ids map one to one onto Exchange voices.++ body: JSON.stringify({ model: 'tts-1', input: text, voice: voiceId, response_format: 'mp3' }),++ signal: AbortSignal.timeout(60000),++ });++ if (!res.ok) throw new Error(`exchange TTS answered ${res.status}`);++ const buf = Buffer.from(await res.arrayBuffer());++ if (!buf.length) throw new Error('exchange TTS returned no audio');++ return { audio: buf.toString('base64'), mimeType: res.headers.get('content-type') || 'audio/mpeg',++ engine: voiceDef.engine, voiceName: (voiceDef.cloudVoice && voiceDef.cloudVoice.name) || (voiceDef.pollyVoice && voiceDef.pollyVoice.voiceId) || 'exchange' };++}++// ── Main synthesis function ───────────────────────────────────/*** Synthesize TTS audio for a chat message.@@@ -608,6 +633,17 @@ async function _synthesizeWithDef(text, voiceDef, voiceId) {}}++ // Behind tts_exchange a cloud voice never touches a provider host: it speaks through Exchange,++ // and an Exchange failure falls back to local espeak, never to the providers directly.++ if (settings.exchange && (voiceDef.engine === 'google-cloud' || voiceDef.engine === 'amazon-polly')) {++ try {++ return { ...(await synthesizeViaExchange(text, voiceDef, voiceId)), voiceId };++ } catch (err) {++ console.error(`[TTS] Exchange synthesis failed (${voiceId}):`, err.message);++ return _espeakFallback(text, voiceId);++ }++ }++try {switch (voiceDef.engine) {case 'google-cloud': {nnew file mode 100644@@@ -0,0 +1,63 @@++'use strict';++/**++ * Behind tts_exchange (off by default) cloud voices synthesize through AI's POST /v1/audio/speech++ * instead of the providers directly:++ * - a Google voice and a Polly voice speak through the fake Exchange with their Chat voice id++ * unchanged (the one-to-one map), and the same shape as direct synthesis;++ * - no request reaches a provider host (googleapis.com, amazonaws.com) with the setting on.++ *++ * The network is a stand-in: fetch answers for Exchange, https.request records (and would fail)++ * provider calls. The AI token client is stubbed; nothing reaches AI or the providers.++ */++const assert = require('assert');++const https = require('https');++++const ctx = require('../server/live-context');++const SETTINGS = { tts_enabled: 'true', tts_exchange: 'true', tts_default_voice: 'gary' };++ctx.getSetting = (k) => (Object.prototype.hasOwnProperty.call(SETTINGS, k) ? SETTINGS[k] : null);++++const aiClient = require('../server/ai/client');++aiClient._setTokens({ getToken: async () => 'test-token' });++++const askedFetch = [];++const realFetch = globalThis.fetch;++globalThis.fetch = async (url, opts) => {++ askedFetch.push({ url: String(url), auth: opts && opts.headers && opts.headers.Authorization, body: JSON.parse(opts.body) });++ assert.match(String(url), /\/v1\/audio\/speech$/, 'chat calls the Exchange speech route');++ return {++ ok: true, status: 200,++ headers: { get: (k) => (String(k).toLowerCase() === 'content-type' ? 'audio/mpeg' : null) },++ arrayBuffer: async () => Buffer.from(`mp3 of ${JSON.parse(opts.body).voice}:${JSON.parse(opts.body).input}`),++ };++};++++const askedProviders = [];++https.request = (url, opts, cb) => {++ if (typeof url === 'object') { cb = opts; url = `https://${url.hostname || url.host}${url.path || '/'}`; }++ askedProviders.push(String(url));++ throw new Error(`provider host must not be called with tts_exchange on: ${url}`);++};++console.warn = () => {};++console.error = () => {};++++const tts = require('../server/chat/tts-engine');++++(async () => {++ const google = await tts.synthesize('Ladies and gentlemen', 'voice_vegas', 'josh');++ assert.strictEqual(google.voiceId, 'voice_vegas');++ assert.strictEqual(google.engine, 'google-cloud', 'the voice keeps its engine label');++ assert.strictEqual(Buffer.from(google.audio, 'base64').toString(), 'mp3 of voice_vegas:Ladies and gentlemen');++ assert.strictEqual(askedFetch.length, 1);++ assert.strictEqual(askedFetch[0].auth, 'Bearer test-token', 'chat uses its own service token');++ assert.deepStrictEqual(askedFetch[0].body, { model: 'tts-1', input: 'Ladies and gentlemen', voice: 'voice_vegas', response_format: 'mp3' },++ 'the Chat voice id travels unchanged');++++ const polly = await tts.synthesize('hello there', 'pl_matthew_neural', 'ann');++ assert.strictEqual(polly.voiceId, 'pl_matthew_neural');++ assert.strictEqual(polly.engine, 'amazon-polly', 'the voice keeps its engine label');++ assert.strictEqual(Buffer.from(polly.audio, 'base64').toString(), 'mp3 of pl_matthew_neural:hello there');++ assert.strictEqual(askedFetch.length, 2);++++ assert.deepStrictEqual(askedProviders, [], 'no provider host was called with the setting on');++ console.log('tts-via-exchange: ok');++})().catch((err) => { console.log('FAIL', err); process.exit(err ? 1 : 0); }).finally(() => { globalThis.fetch = realFetch; });
@@ -47,6 +47,15 @@ function enabled() {
@@ -47,6 +47,15 @@ function enabled() {
return !!(config.ai.enabled && config.oauth.clientSecret);
return !!(config.ai.enabled && config.oauth.clientSecret);
}
}
+/**
+ * Chat's bearer token for AI (its own service token; no CHAT_AI_ENABLED gate — TTS has its own
+ * setting). For calls the SDK's AI client has no method for (the Exchange speech route).
+ */
+async function bearer() {
+ if (!_tokens) ai();
+ return _tokens.getToken();
+}
+
/**
/**
* Run a workflow and return its `output` object, or null: disabled, a refusal, a failed run, or a
* Run a workflow and return its `output` object, or null: disabled, a refusal, a failed run, or a
* transport error all answer null — exactly Live's ai-service.structured() contract. `force` runs it even with
* transport error all answer null — exactly Live's ai-service.structured() contract. `force` runs it even with
@@ -176,5 +185,5 @@ async function catalog() {
@@ -176,5 +185,5 @@ async function catalog() {
return res.json();
return res.json();
}
}
-module.exports = { structured, stream, complete, catalog, enabled, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },
+module.exports = { structured, stream, complete, catalog, enabled, bearer, http, lastProblem, _setClient(c) { _ai = c; }, _setComplete(c) { _complete = c; }, _setHttp(h) { _http = h; }, _setTokens(t) { _tokens = t; },
_setProblem(p) { _lastProblem = p; }, _reset() { _ai = null; _http = null; _tokens = null; _complete = null; _lastProblem = null; } };
_setProblem(p) { _lastProblem = p; }, _reset() { _ai = null; _http = null; _tokens = null; _complete = null; _lastProblem = null; } };
@@ -160,6 +160,9 @@ function getTTSSettings() {
@@ -160,6 +160,9 @@ function getTTSSettings() {
if (_settingsCache && (now - _settingsCacheTime) < SETTINGS_CACHE_TTL) return _settingsCache;
if (_settingsCache && (now - _settingsCacheTime) < SETTINGS_CACHE_TTL) return _settingsCache;
_settingsCache = {
_settingsCache = {
enabled: ctx.getSetting('tts_enabled') !== false,
enabled: ctx.getSetting('tts_enabled') !== false,
+ // Exchange TTS (off): cloud voices synthesize through AI's POST /v1/audio/speech instead of
+ // the providers directly, so no provider host is called with the setting on.
+ exchange: (() => { const v = ctx.getSetting('tts_exchange'); return v === true || v === 'true' || v === 1 || v === '1'; })(),
provider: ctx.getSetting('tts_provider') || 'espeak-ng',
provider: ctx.getSetting('tts_provider') || 'espeak-ng',
googleApiKey: ctx.getSetting('tts_google_api_key') || '',
googleApiKey: ctx.getSetting('tts_google_api_key') || '',
googleServiceAccount: owner.value('google_cloud_service_account_json', 'CHAT_GOOGLE_SERVICE_ACCOUNT_JSON', ctx.getSetting('tts_google_service_account') || ''),
googleServiceAccount: owner.value('google_cloud_service_account_json', 'CHAT_GOOGLE_SERVICE_ACCOUNT_JSON', ctx.getSetting('tts_google_service_account') || ''),
@@ -553,6 +556,28 @@ function sanitize(text, maxLen, username) {
@@ -553,6 +556,28 @@ function sanitize(text, maxLen, username) {
return clean;
return clean;
}
}
+// ── Exchange TTS (behind tts_exchange, off) ───────────────────
+// Cloud voices synthesize through AI's POST /v1/audio/speech instead of the providers directly:
+// Chat's voice id travels as the Exchange voice, one to one, so nobody hears a difference. With the
+// setting on no provider host is ever called (a failure falls back to local espeak, never direct).
+async function synthesizeViaExchange(text, voiceDef, voiceId) {
+ const client = require('../ai/client');
+ const token = await client.bearer();
+ const { ai } = require('../config');
+ const res = await fetch(`${String(ai.internalUrl).replace(/\/+$/, '')}/v1/audio/speech`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'audio/mpeg, audio/ogg, audio/wav' },
+ // The voice id travels unchanged: Chat's voice ids map one to one onto Exchange voices.
+ body: JSON.stringify({ model: 'tts-1', input: text, voice: voiceId, response_format: 'mp3' }),
+ signal: AbortSignal.timeout(60000),
+ });
+ if (!res.ok) throw new Error(`exchange TTS answered ${res.status}`);
+ const buf = Buffer.from(await res.arrayBuffer());
+ if (!buf.length) throw new Error('exchange TTS returned no audio');
+ return { audio: buf.toString('base64'), mimeType: res.headers.get('content-type') || 'audio/mpeg',
+ engine: voiceDef.engine, voiceName: (voiceDef.cloudVoice && voiceDef.cloudVoice.name) || (voiceDef.pollyVoice && voiceDef.pollyVoice.voiceId) || 'exchange' };
+}
+
// ── Main synthesis function ───────────────────────────────────
// ── Main synthesis function ───────────────────────────────────
/**
/**
* Synthesize TTS audio for a chat message.
* Synthesize TTS audio for a chat message.
@@ -608,6 +633,17 @@ async function _synthesizeWithDef(text, voiceDef, voiceId) {
@@ -608,6 +633,17 @@ async function _synthesizeWithDef(text, voiceDef, voiceId) {
}
}
}
}
+ // Behind tts_exchange a cloud voice never touches a provider host: it speaks through Exchange,
+ // and an Exchange failure falls back to local espeak, never to the providers directly.
+ if (settings.exchange && (voiceDef.engine === 'google-cloud' || voiceDef.engine === 'amazon-polly')) {
+ try {
+ return { ...(await synthesizeViaExchange(text, voiceDef, voiceId)), voiceId };
+ } catch (err) {
+ console.error(`[TTS] Exchange synthesis failed (${voiceId}):`, err.message);
+ return _espeakFallback(text, voiceId);
+ }
+ }
+
try {
try {
switch (voiceDef.engine) {
switch (voiceDef.engine) {
case 'google-cloud': {
case 'google-cloud': {
new file mode 100644
new file mode 100644
@@ -0,0 +1,63 @@
@@ -0,0 +1,63 @@
+'use strict';
+/**
+ * Behind tts_exchange (off by default) cloud voices synthesize through AI's POST /v1/audio/speech
+ * instead of the providers directly:
+ * - a Google voice and a Polly voice speak through the fake Exchange with their Chat voice id
+ * unchanged (the one-to-one map), and the same shape as direct synthesis;
+ * - no request reaches a provider host (googleapis.com, amazonaws.com) with the setting on.
+ *
+ * The network is a stand-in: fetch answers for Exchange, https.request records (and would fail)
+ * provider calls. The AI token client is stubbed; nothing reaches AI or the providers.
+ */
+const assert = require('assert');
+const https = require('https');
+
+const ctx = require('../server/live-context');
+const SETTINGS = { tts_enabled: 'true', tts_exchange: 'true', tts_default_voice: 'gary' };
+ctx.getSetting = (k) => (Object.prototype.hasOwnProperty.call(SETTINGS, k) ? SETTINGS[k] : null);
+
+const aiClient = require('../server/ai/client');
+aiClient._setTokens({ getToken: async () => 'test-token' });
+
+const askedFetch = [];
+const realFetch = globalThis.fetch;
+globalThis.fetch = async (url, opts) => {
+ askedFetch.push({ url: String(url), auth: opts && opts.headers && opts.headers.Authorization, body: JSON.parse(opts.body) });
+ assert.match(String(url), /\/v1\/audio\/speech$/, 'chat calls the Exchange speech route');
+ return {
+ ok: true, status: 200,
+ headers: { get: (k) => (String(k).toLowerCase() === 'content-type' ? 'audio/mpeg' : null) },
+ arrayBuffer: async () => Buffer.from(`mp3 of ${JSON.parse(opts.body).voice}:${JSON.parse(opts.body).input}`),
+ };
+};
+
+const askedProviders = [];
+https.request = (url, opts, cb) => {
+ if (typeof url === 'object') { cb = opts; url = `https://${url.hostname || url.host}${url.path || '/'}`; }
+ askedProviders.push(String(url));
+ throw new Error(`provider host must not be called with tts_exchange on: ${url}`);
+};
+console.warn = () => {};
+console.error = () => {};
+
+const tts = require('../server/chat/tts-engine');
+
+(async () => {
+ const google = await tts.synthesize('Ladies and gentlemen', 'voice_vegas', 'josh');
+ assert.strictEqual(google.voiceId, 'voice_vegas');
+ assert.strictEqual(google.engine, 'google-cloud', 'the voice keeps its engine label');
+ assert.strictEqual(Buffer.from(google.audio, 'base64').toString(), 'mp3 of voice_vegas:Ladies and gentlemen');
+ assert.strictEqual(askedFetch.length, 1);
+ assert.strictEqual(askedFetch[0].auth, 'Bearer test-token', 'chat uses its own service token');
+ assert.deepStrictEqual(askedFetch[0].body, { model: 'tts-1', input: 'Ladies and gentlemen', voice: 'voice_vegas', response_format: 'mp3' },
+ 'the Chat voice id travels unchanged');
+
+ const polly = await tts.synthesize('hello there', 'pl_matthew_neural', 'ann');
+ assert.strictEqual(polly.voiceId, 'pl_matthew_neural');
+ assert.strictEqual(polly.engine, 'amazon-polly', 'the voice keeps its engine label');
+ assert.strictEqual(Buffer.from(polly.audio, 'base64').toString(), 'mp3 of pl_matthew_neural:hello there');
+ assert.strictEqual(askedFetch.length, 2);
+
+ assert.deepStrictEqual(askedProviders, [], 'no provider host was called with the setting on');
+ console.log('tts-via-exchange: ok');
+})().catch((err) => { console.log('FAIL', err); process.exit(err ? 1 : 0); }).finally(() => { globalThis.fetch = realFetch; });