openvibers/openvibe.live Commits

A channel's speech is heard in its language everywhere (item 151, C11) (#574)

OP OpenVibers committed 10 days ago · e74006177b · 6 files, +87 −15

Media analysis, AI's media.analyze, VOD and clip transcripts and the live memory decoded every channel as English;
they now take the channel's language (its chat language), as the timeline job already did. Whisper's Japanese
inventions over silence and subtitle credits are dropped like their English twins; Japanese speech is kept whole.

Changes

ddiff --git a/server/ai/ai-analysis.js b/server/ai/ai-analysis.js
iindex 9c9cd5bb..d158e8ad 100644
---- a/server/ai/ai-analysis.js
++++ b/server/ai/ai-analysis.js
@@@ -285,6 +285,7 @@ async function generateVodTranscript(vod) {
// the frequent deploy restarts no longer discard an hour of decoding.
const prog = db.getVodTranscriptProgress ? await db.getVodTranscriptProgress(vod.id) : { progressSec: 0, segments: [] };
r = await require('./media-analysis').transcribeOnly(src, {
++ userId: vod.user_id || null,
resumeFromSec: prog.progressSec,
priorSegments: prog.segments,
onWindow: async (sec, segs) => await db.saveVodTranscriptProgress(vod.id, sec, segs),
@@@ -326,7 +327,7 @@ async function generateClipTranscript(clip) {
}
await db.setClipTranscriptStatus(clip.id, 'processing');
let r = { text: '', segments: [], ok: false, error: 'unknown' };
-- try { r = await require('./media-analysis').transcribeOnly(src); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
++ try { r = await require('./media-analysis').transcribeOnly(src, { userId: clip.user_id || null }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
if (r.text) {
try { await db.setClipTranscript(clip.id, r.text, r.segments || []); } catch { /* */ }
await db.setClipTranscriptStatus(clip.id, 'done');
ddiff --git a/server/ai/ai-media-analyze.js b/server/ai/ai-media-analyze.js
iindex 323a5903..bbe76152 100644
---- a/server/ai/ai-media-analyze.js
++++ b/server/ai/ai-media-analyze.js
@@@ -45,7 +45,9 @@ async function analyze(kind, item) {
attemptNo = Number(state && state.transcript_attempts) || 0;
const mediaUrl = await source(kind, item);
if (!mediaUrl) throw new Error('Media did not provide a readable URL');
-- const run = await aiService.run('media.analyze', { media_url: mediaUrl, language: 'en', allow_paid: false }, {
++ // The channel's language (C11): AI hears a Japanese recording as Japanese.
++ const language = await require('./transcribe').speechLanguage(item.user_id, { local: false });
++ const run = await aiService.run('media.analyze', { media_url: mediaUrl, language, allow_paid: false }, {
target: { service: 'live', type: kind, id: String(item.id) },
attribution: aiService.ownerRef(item.user_id),
idempotencyKey: `live.media.analyze:${kind}:${item.id}:${attemptNo}`,
ddiff --git a/server/ai/media-analysis.js b/server/ai/media-analysis.js
iindex 391ece3c..d1d35ffd 100644
---- a/server/ai/media-analysis.js
++++ b/server/ai/media-analysis.js
@@@ -216,10 +216,10 @@ function _ffprobeHasAudio(src) {
* recording where it lies (range requests, nothing downloaded), loudness-normalises it and runs whisper.cpp with the
* same models; low power while streams are live. → { ok, segments, error }
*/
--async function _aiWindow(src, start, len) {
++async function _aiWindow(src, start, len, language = 'en') {
const aiService = require('./ai-service');
const r = await aiService.run('live.media.transcribe', {
-- media_url: src, language: 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),
++ media_url: src, language: language || 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),
}, { waitMs: 60000, pollMs: 20 * 60000 });
const out = aiService.usable(r);
if (!out) return { ok: false, segments: [], error: (r && r.error && (r.error.code || r.error.detail)) || 'OpenVibe.AI did not transcribe' };
@@@ -230,17 +230,19 @@ async function _aiWindow(src, start, len) {
// waiting out rather than skipping the window.
const PERMANENT_AI_ERRORS = new Set(['media.unreadable', 'media.not_found', 'media.too_long', 'validation.failed']);
--async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null } = {}) {
++async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null, userId = null, language = null } = {}) {
const transcribe = require('./transcribe');
// Recordings on Media are transcribed by OpenVibe.AI; only a local file (legacy) is still decoded here.
const viaAi = /^https?:\/\//i.test(String(src || '')) && require('./ai-service').enabled();
if (!viaAi && !transcribe.available()) return { text: '', segments: [], ok: false, error: 'whisper unavailable' };
++ // The channel's language (C11): a Japanese stream is heard as Japanese, not decoded as English.
++ const lang = language || (userId ? await transcribe.speechLanguage(userId, { local: !viaAi }) : 'en');
if (viaAi && duration > 0 && duration <= 200) {
-- const w = await _aiWindow(src, 0, 0);
++ const w = await _aiWindow(src, 0, 0, lang);
return { text: w.segments.map(g => g.text).join(' ').replace(/\s+/g, ' ').trim(), segments: w.segments, ok: w.ok, error: w.error };
}
if (duration > 0 && duration <= 200) {
-- return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000 });
++ return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000, language: lang });
}
// FULL coverage, not a sample. This used to take three 60-second windows at 10%/45%/80%
// of the recording, so a 4-hour VOD produced 3 minutes of transcript and the rest was
@@@ -272,7 +274,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
for (const start of windows) {
const len = Math.min(WINDOW_SEC, total - start);
if (viaAi) {
-- const w = await _aiWindow(src, start, len);
++ const w = await _aiWindow(src, start, len, lang);
if (w.ok) anyOk = true; else lastErr = w.error || lastErr;
if (w.segments.length) segments.push(...w.segments);
if (w.ok && typeof onWindow === 'function') {
@@@ -287,7 +289,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
const wav = _tmp('wav');
const ffOk = await _runFf('ffmpeg', ['-y', '-ss', String(start), '-i', src, '-t', String(len), '-vn', '-ac', '1', '-ar', '16000', '-af', 'loudnorm=I=-16:TP=-1.5:LRA=11', '-f', 'wav', wav], 180000);
if (ffOk) {
-- const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start });
++ const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start, language: lang });
if (r.ok) anyOk = true; else lastErr = r.error || lastErr;
if (r.segments && r.segments.length) segments.push(...r.segments);
if (r.ok && typeof onWindow === 'function') {
@@@ -334,7 +336,7 @@ async function analyzeMedia(src, { streamId = null, userId = null, numFrames = n
frames = await captureFrameMemories(src, times, { streamId, userId, offsetBase, store: storeMemories });
}
-- const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration);
++ const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration, { userId });
let overview = null;
if ((frames.length || transcript) && await ai.isEnabled() && await ai.withinBudget()) {
ddiff --git a/server/ai/stream-memory-job.js b/server/ai/stream-memory-job.js
iindex abd828e9..c009f30c 100644
---- a/server/ai/stream-memory-job.js
++++ b/server/ai/stream-memory-job.js
@@@ -122,7 +122,8 @@ async function _analyzeOne(stream, { allowFfmpeg = true, reason = 'periodic' } =
} else if (ai.transcriptionEnabled && await ai.transcriptionEnabled()) {
const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;
if (audio) {
-- const tx = await require('./transcribe').transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC) });
++ const transcribe = require('./transcribe');
++ const tx = await transcribe.transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC), language: await transcribe.speechLanguage(stream.user_id) });
heard = tx.text || '';
heardSegments = (tx.segments && tx.segments.length) ? tx.segments : null;
try { require('fs').unlinkSync(audio); } catch { /* */ }
ddiff --git a/server/ai/transcribe.js b/server/ai/transcribe.js
iindex dc6280c9..0b8c9cb7 100644
---- a/server/ai/transcribe.js
++++ b/server/ai/transcribe.js
@@@ -69,6 +69,19 @@ function parakeetAvailable() {
if (process.env.ASR_ENGINE === 'whisper') return false;
try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }
}
++/**
++ * The language a channel's speech is decoded in (item 151, C11): its chat language (set, or detected from its bio),
++ * when a local model can decode it (Parakeet's languages, or a multilingual whisper); else English. OpenVibe.AI's
++ * path takes the channel's language as it is: AI picks its own model.
++ */
++async function speechLanguage(userId, { local = true } = {}) {
++ let lang = 'en';
++ try { lang = await require('../i18n/translate').channelLanguage(userId); } catch { lang = 'en'; }
++ lang = String(lang || 'en').toLowerCase();
++ if (!local || lang === 'en') return lang || 'en';
++ return engineFor(lang) === 'parakeet' || multilingualModel() ? lang : 'en';
++}
++
/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */
function engineFor(language = 'en') {
return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';
@@@ -185,6 +198,10 @@ const HALLUCINATIONS = new Set([
'thanks for watching.', 'please subscribe', 'subscribe', 'like and subscribe',
'thank you for watching', 'thank you very much',
'thank you so much', 'silence', 'music', 'applause',
++ // Whisper's Japanese stock inventions over silence and music (item 151, C11: Japanese streams are now heard in
++ // Japanese, and these came with it): "thanks for watching", "please subscribe".
++ 'ご視聴ありがとうございました', 'ご視聴ありがとうございます', 'ご視聴いただきありがとうございました',
++ 'チャンネル登録よろしくお願いします', 'チャンネル登録お願いします', 'チャンネル登録よろしくね',
]);
// Drop-only-without-VAD: these ARE real words a streamer says constantly. They were on the
// blocklist because, with the decoder run over 87% silence, they were overwhelmingly
@@@ -201,8 +218,10 @@ function _isNoise(t) {
if (/^[\s.\-–—_*]+$/.test(s)) return true;
if (/^[\[(♪].*[\])♪]?$/.test(s)) return true;
if (/^♪/.test(s) || /♪$/.test(s)) return true;
-- const norm = s.toLowerCase().replace(/[.!?,…]+$/g, '').trim();
++ const norm = s.toLowerCase().replace(/[.!?,…。!?、]+$/g, '').trim();
if (HALLUCINATIONS.has(norm)) return true;
++ // Japanese subtitle credits ("字幕: …", "字幕作成者 …") are training-data boilerplate, never stream speech.
++ if (/^(?:字幕|翻訳)(?:作成者?|制作|提供)?[\s::]/.test(norm) || /^字幕\s*by\b/i.test(norm)) return true;
// Whisper's training data was full of subtitle boilerplate, and it emits those exact
// phrases over noise even when VAD says a voice is present — a real live segment came
// back as "For more information, visit www.fema.org." These are recognisable by shape:
@@@ -398,7 +417,7 @@ async function transcribeWav(wavPath, opts = {}) {
* Extract audio from ANY media (local file OR http(s) URL) → temp wav → detailed
* transcription. @returns {Promise<{text, segments}>}
*/
--function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000 } = {}) {
++function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000, language = 'en' } = {}) {
return new Promise((resolve) => {
const isUrl = /^https?:/i.test(mediaPath || '');
if (!available() || !mediaPath || (!isUrl && !fs.existsSync(mediaPath))) return resolve({ text: '', segments: [], ok: false, error: 'source unavailable' });
@@@ -415,7 +434,7 @@ function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeou
clearTimeout(ffTimer);
if (code !== 0) { cleanup(); return resolve({ text: '', segments: [], ok: false, error: `ffmpeg exited ${code}` }); }
let r = { text: '', segments: [], ok: false, error: 'unknown' };
-- try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
++ try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec, language }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
cleanup();
resolve(r);
});
@@@ -463,4 +482,4 @@ function describe() {
function isLowPower() { return _lowPower; }
--module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
++module.exports = { _isNoise, speechLanguage, available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
ddiff --git a/test/channel-language-asr.test.js b/test/channel-language-asr.test.js
nnew file mode 100644
iindex 00000000..66d112be
---- /dev/null
++++ b/test/channel-language-asr.test.js
@@@ -0,0 +1,47 @@
++/**
++ * A channel's speech is heard in its language (item 151, C11): media analysis, clip and VOD transcripts, OpenVibe.AI's
++ * runs and the live memory decode in the channel's language (its chat language, set or detected) when a model can,
++ * and Japanese speech survives the noise filter while whisper's Japanese inventions over silence do not.
++ *
++ * node test/run.js channel-language-asr
++ */
++'use strict';
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++
++const translate = require('../server/i18n/translate');
++const langs = { 7: 'ja', 8: 'de', 9: 'en' };
++translate.channelLanguage = async (userId) => langs[userId] || 'en';
++const transcribe = require('../server/ai/transcribe');
++
++(async () => {
++ // AI's path takes the channel's language as it is.
++ assert.strictEqual(await transcribe.speechLanguage(7, { local: false }), 'ja');
++ assert.strictEqual(await transcribe.speechLanguage(9, { local: false }), 'en');
++ // A local decode: Japanese only with a multilingual whisper; otherwise English (never a model that cannot).
++ const multi = Boolean(transcribe.multilingualModel());
++ assert.strictEqual(await transcribe.speechLanguage(7), multi ? 'ja' : 'en');
++ assert.strictEqual(await transcribe.speechLanguage(null), 'en');
++
++ // Every path passes it on: no hard-coded English left where a channel is known.
++ const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');
++ const ma = src('media-analysis.js');
++ assert.doesNotMatch(ma, /language: 'en', start_sec/, 'AI windows carry the language');
++ assert.match(ma, /_transcribeSpan\(src, duration, \{ userId \}\)/, 'analysis passes the channel');
++ assert.match(ma, /transcribeWavDetailed\(wav, \{[^}]*language: lang/, 'local windows decode in it');
++ assert.doesNotMatch(src('ai-media-analyze.js'), /language: 'en'/, 'AI\'s media.analyze is not forced to English');
++ assert.match(src('ai-analysis.js'), /transcribeOnly\(src, \{ userId: clip\.user_id/, 'clip transcripts know their channel');
++ assert.match(src('ai-analysis.js'), /userId: vod\.user_id/, 'VOD transcripts know their channel');
++ assert.match(src('stream-memory-job.js'), /language: await transcribe\.speechLanguage\(stream\.user_id\)/, 'the live memory hears the channel');
++
++ // Japanese speech is kept whole; whisper's Japanese stock phrases and subtitle credits are noise.
++ assert.strictEqual(transcribe._isNoise('今日はマイクラで家を建てます'), false);
++ assert.strictEqual(transcribe._isNoise('ありがとうございます、みんな!'), false);
++ assert.strictEqual(transcribe._isNoise('ご視聴ありがとうございました。'), true);
++ assert.strictEqual(transcribe._isNoise('チャンネル登録よろしくお願いします!'), true);
++ assert.strictEqual(transcribe._isNoise('字幕:山田'), true);
++ assert.strictEqual(transcribe._isNoise('Thanks for watching!'), true);
++ assert.strictEqual(transcribe._isNoise('Let us build a house'), false);
++ console.log('channel language asr: all checks passed');
++})().catch((e) => { console.error(e); process.exit(1); });