openvibers/openvibe.live Commits
A channel's speech is heard in its language everywhere (item 151, C11) (#574)
Changes
@@@ -285,6 +285,7 @@ async function generateVodTranscript(vod) {// the frequent deploy restarts no longer discard an hour of decoding.const prog = db.getVodTranscriptProgress ? await db.getVodTranscriptProgress(vod.id) : { progressSec: 0, segments: [] };r = await require('./media-analysis').transcribeOnly(src, {++ userId: vod.user_id || null,resumeFromSec: prog.progressSec,priorSegments: prog.segments,onWindow: async (sec, segs) => await db.saveVodTranscriptProgress(vod.id, sec, segs),@@@ -326,7 +327,7 @@ async function generateClipTranscript(clip) {}await db.setClipTranscriptStatus(clip.id, 'processing');let r = { text: '', segments: [], ok: false, error: 'unknown' };-- try { r = await require('./media-analysis').transcribeOnly(src); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }++ try { r = await require('./media-analysis').transcribeOnly(src, { userId: clip.user_id || null }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }if (r.text) {try { await db.setClipTranscript(clip.id, r.text, r.segments || []); } catch { /* */ }await db.setClipTranscriptStatus(clip.id, 'done');@@@ -45,7 +45,9 @@ async function analyze(kind, item) {attemptNo = Number(state && state.transcript_attempts) || 0;const mediaUrl = await source(kind, item);if (!mediaUrl) throw new Error('Media did not provide a readable URL');-- const run = await aiService.run('media.analyze', { media_url: mediaUrl, language: 'en', allow_paid: false }, {++ // The channel's language (C11): AI hears a Japanese recording as Japanese.++ const language = await require('./transcribe').speechLanguage(item.user_id, { local: false });++ const run = await aiService.run('media.analyze', { media_url: mediaUrl, language, allow_paid: false }, {target: { service: 'live', type: kind, id: String(item.id) },attribution: aiService.ownerRef(item.user_id),idempotencyKey: `live.media.analyze:${kind}:${item.id}:${attemptNo}`,@@@ -216,10 +216,10 @@ function _ffprobeHasAudio(src) {* recording where it lies (range requests, nothing downloaded), loudness-normalises it and runs whisper.cpp with the* same models; low power while streams are live. → { ok, segments, error }*/--async function _aiWindow(src, start, len) {++async function _aiWindow(src, start, len, language = 'en') {const aiService = require('./ai-service');const r = await aiService.run('live.media.transcribe', {-- media_url: src, language: 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),++ media_url: src, language: language || 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),}, { waitMs: 60000, pollMs: 20 * 60000 });const out = aiService.usable(r);if (!out) return { ok: false, segments: [], error: (r && r.error && (r.error.code || r.error.detail)) || 'OpenVibe.AI did not transcribe' };@@@ -230,17 +230,19 @@ async function _aiWindow(src, start, len) {// waiting out rather than skipping the window.const PERMANENT_AI_ERRORS = new Set(['media.unreadable', 'media.not_found', 'media.too_long', 'validation.failed']);--async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null } = {}) {++async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null, userId = null, language = null } = {}) {const transcribe = require('./transcribe');// Recordings on Media are transcribed by OpenVibe.AI; only a local file (legacy) is still decoded here.const viaAi = /^https?:\/\//i.test(String(src || '')) && require('./ai-service').enabled();if (!viaAi && !transcribe.available()) return { text: '', segments: [], ok: false, error: 'whisper unavailable' };++ // The channel's language (C11): a Japanese stream is heard as Japanese, not decoded as English.++ const lang = language || (userId ? await transcribe.speechLanguage(userId, { local: !viaAi }) : 'en');if (viaAi && duration > 0 && duration <= 200) {-- const w = await _aiWindow(src, 0, 0);++ const w = await _aiWindow(src, 0, 0, lang);return { text: w.segments.map(g => g.text).join(' ').replace(/\s+/g, ' ').trim(), segments: w.segments, ok: w.ok, error: w.error };}if (duration > 0 && duration <= 200) {-- return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000 });++ return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000, language: lang });}// FULL coverage, not a sample. This used to take three 60-second windows at 10%/45%/80%// of the recording, so a 4-hour VOD produced 3 minutes of transcript and the rest was@@@ -272,7 +274,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegmentsfor (const start of windows) {const len = Math.min(WINDOW_SEC, total - start);if (viaAi) {-- const w = await _aiWindow(src, start, len);++ const w = await _aiWindow(src, start, len, lang);if (w.ok) anyOk = true; else lastErr = w.error || lastErr;if (w.segments.length) segments.push(...w.segments);if (w.ok && typeof onWindow === 'function') {@@@ -287,7 +289,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegmentsconst wav = _tmp('wav');const ffOk = await _runFf('ffmpeg', ['-y', '-ss', String(start), '-i', src, '-t', String(len), '-vn', '-ac', '1', '-ar', '16000', '-af', 'loudnorm=I=-16:TP=-1.5:LRA=11', '-f', 'wav', wav], 180000);if (ffOk) {-- const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start });++ const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start, language: lang });if (r.ok) anyOk = true; else lastErr = r.error || lastErr;if (r.segments && r.segments.length) segments.push(...r.segments);if (r.ok && typeof onWindow === 'function') {@@@ -334,7 +336,7 @@ async function analyzeMedia(src, { streamId = null, userId = null, numFrames = nframes = await captureFrameMemories(src, times, { streamId, userId, offsetBase, store: storeMemories });}-- const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration);++ const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration, { userId });let overview = null;if ((frames.length || transcript) && await ai.isEnabled() && await ai.withinBudget()) {@@@ -122,7 +122,8 @@ async function _analyzeOne(stream, { allowFfmpeg = true, reason = 'periodic' } =} else if (ai.transcriptionEnabled && await ai.transcriptionEnabled()) {const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;if (audio) {-- const tx = await require('./transcribe').transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC) });++ const transcribe = require('./transcribe');++ const tx = await transcribe.transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC), language: await transcribe.speechLanguage(stream.user_id) });heard = tx.text || '';heardSegments = (tx.segments && tx.segments.length) ? tx.segments : null;try { require('fs').unlinkSync(audio); } catch { /* */ }@@@ -69,6 +69,19 @@ function parakeetAvailable() {if (process.env.ASR_ENGINE === 'whisper') return false;try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }}++/**++ * The language a channel's speech is decoded in (item 151, C11): its chat language (set, or detected from its bio),++ * when a local model can decode it (Parakeet's languages, or a multilingual whisper); else English. OpenVibe.AI's++ * path takes the channel's language as it is: AI picks its own model.++ */++async function speechLanguage(userId, { local = true } = {}) {++ let lang = 'en';++ try { lang = await require('../i18n/translate').channelLanguage(userId); } catch { lang = 'en'; }++ lang = String(lang || 'en').toLowerCase();++ if (!local || lang === 'en') return lang || 'en';++ return engineFor(lang) === 'parakeet' || multilingualModel() ? lang : 'en';++}++/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */function engineFor(language = 'en') {return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';@@@ -185,6 +198,10 @@ const HALLUCINATIONS = new Set(['thanks for watching.', 'please subscribe', 'subscribe', 'like and subscribe','thank you for watching', 'thank you very much','thank you so much', 'silence', 'music', 'applause',++ // Whisper's Japanese stock inventions over silence and music (item 151, C11: Japanese streams are now heard in++ // Japanese, and these came with it): "thanks for watching", "please subscribe".++ 'ご視聴ありがとうございました', 'ご視聴ありがとうございます', 'ご視聴いただきありがとうございました',++ 'チャンネル登録よろしくお願いします', 'チャンネル登録お願いします', 'チャンネル登録よろしくね',]);// Drop-only-without-VAD: these ARE real words a streamer says constantly. They were on the// blocklist because, with the decoder run over 87% silence, they were overwhelmingly@@@ -201,8 +218,10 @@ function _isNoise(t) {if (/^[\s.\-–—_*]+$/.test(s)) return true;if (/^[\[(♪].*[\])♪]?$/.test(s)) return true;if (/^♪/.test(s) || /♪$/.test(s)) return true;-- const norm = s.toLowerCase().replace(/[.!?,…]+$/g, '').trim();++ const norm = s.toLowerCase().replace(/[.!?,…。!?、]+$/g, '').trim();if (HALLUCINATIONS.has(norm)) return true;++ // Japanese subtitle credits ("字幕: …", "字幕作成者 …") are training-data boilerplate, never stream speech.++ if (/^(?:字幕|翻訳)(?:作成者?|制作|提供)?[\s::]/.test(norm) || /^字幕\s*by\b/i.test(norm)) return true;// Whisper's training data was full of subtitle boilerplate, and it emits those exact// phrases over noise even when VAD says a voice is present — a real live segment came// back as "For more information, visit www.fema.org." These are recognisable by shape:@@@ -398,7 +417,7 @@ async function transcribeWav(wavPath, opts = {}) {* Extract audio from ANY media (local file OR http(s) URL) → temp wav → detailed* transcription. @returns {Promise<{text, segments}>}*/--function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000 } = {}) {++function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000, language = 'en' } = {}) {return new Promise((resolve) => {const isUrl = /^https?:/i.test(mediaPath || '');if (!available() || !mediaPath || (!isUrl && !fs.existsSync(mediaPath))) return resolve({ text: '', segments: [], ok: false, error: 'source unavailable' });@@@ -415,7 +434,7 @@ function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeouclearTimeout(ffTimer);if (code !== 0) { cleanup(); return resolve({ text: '', segments: [], ok: false, error: `ffmpeg exited ${code}` }); }let r = { text: '', segments: [], ok: false, error: 'unknown' };-- try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }++ try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec, language }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }cleanup();resolve(r);});@@@ -463,4 +482,4 @@ function describe() {function isLowPower() { return _lowPower; }--module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };++module.exports = { _isNoise, speechLanguage, available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };nnew file mode 100644@@@ -0,0 +1,47 @@++/**++ * A channel's speech is heard in its language (item 151, C11): media analysis, clip and VOD transcripts, OpenVibe.AI's++ * runs and the live memory decode in the channel's language (its chat language, set or detected) when a model can,++ * and Japanese speech survives the noise filter while whisper's Japanese inventions over silence do not.++ *++ * node test/run.js channel-language-asr++ */++'use strict';++const assert = require('assert');++const fs = require('fs');++const path = require('path');++++const translate = require('../server/i18n/translate');++const langs = { 7: 'ja', 8: 'de', 9: 'en' };++translate.channelLanguage = async (userId) => langs[userId] || 'en';++const transcribe = require('../server/ai/transcribe');++++(async () => {++ // AI's path takes the channel's language as it is.++ assert.strictEqual(await transcribe.speechLanguage(7, { local: false }), 'ja');++ assert.strictEqual(await transcribe.speechLanguage(9, { local: false }), 'en');++ // A local decode: Japanese only with a multilingual whisper; otherwise English (never a model that cannot).++ const multi = Boolean(transcribe.multilingualModel());++ assert.strictEqual(await transcribe.speechLanguage(7), multi ? 'ja' : 'en');++ assert.strictEqual(await transcribe.speechLanguage(null), 'en');++++ // Every path passes it on: no hard-coded English left where a channel is known.++ const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');++ const ma = src('media-analysis.js');++ assert.doesNotMatch(ma, /language: 'en', start_sec/, 'AI windows carry the language');++ assert.match(ma, /_transcribeSpan\(src, duration, \{ userId \}\)/, 'analysis passes the channel');++ assert.match(ma, /transcribeWavDetailed\(wav, \{[^}]*language: lang/, 'local windows decode in it');++ assert.doesNotMatch(src('ai-media-analyze.js'), /language: 'en'/, 'AI\'s media.analyze is not forced to English');++ assert.match(src('ai-analysis.js'), /transcribeOnly\(src, \{ userId: clip\.user_id/, 'clip transcripts know their channel');++ assert.match(src('ai-analysis.js'), /userId: vod\.user_id/, 'VOD transcripts know their channel');++ assert.match(src('stream-memory-job.js'), /language: await transcribe\.speechLanguage\(stream\.user_id\)/, 'the live memory hears the channel');++++ // Japanese speech is kept whole; whisper's Japanese stock phrases and subtitle credits are noise.++ assert.strictEqual(transcribe._isNoise('今日はマイクラで家を建てます'), false);++ assert.strictEqual(transcribe._isNoise('ありがとうございます、みんな!'), false);++ assert.strictEqual(transcribe._isNoise('ご視聴ありがとうございました。'), true);++ assert.strictEqual(transcribe._isNoise('チャンネル登録よろしくお願いします!'), true);++ assert.strictEqual(transcribe._isNoise('字幕:山田'), true);++ assert.strictEqual(transcribe._isNoise('Thanks for watching!'), true);++ assert.strictEqual(transcribe._isNoise('Let us build a house'), false);++ console.log('channel language asr: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });
@@ -285,6 +285,7 @@ async function generateVodTranscript(vod) {
@@ -285,6 +285,7 @@ async function generateVodTranscript(vod) {
// the frequent deploy restarts no longer discard an hour of decoding.
// the frequent deploy restarts no longer discard an hour of decoding.
const prog = db.getVodTranscriptProgress ? await db.getVodTranscriptProgress(vod.id) : { progressSec: 0, segments: [] };
const prog = db.getVodTranscriptProgress ? await db.getVodTranscriptProgress(vod.id) : { progressSec: 0, segments: [] };
r = await require('./media-analysis').transcribeOnly(src, {
r = await require('./media-analysis').transcribeOnly(src, {
+ userId: vod.user_id || null,
resumeFromSec: prog.progressSec,
resumeFromSec: prog.progressSec,
priorSegments: prog.segments,
priorSegments: prog.segments,
onWindow: async (sec, segs) => await db.saveVodTranscriptProgress(vod.id, sec, segs),
onWindow: async (sec, segs) => await db.saveVodTranscriptProgress(vod.id, sec, segs),
@@ -326,7 +327,7 @@ async function generateClipTranscript(clip) {
@@ -326,7 +327,7 @@ async function generateClipTranscript(clip) {
}
}
await db.setClipTranscriptStatus(clip.id, 'processing');
await db.setClipTranscriptStatus(clip.id, 'processing');
let r = { text: '', segments: [], ok: false, error: 'unknown' };
let r = { text: '', segments: [], ok: false, error: 'unknown' };
- try { r = await require('./media-analysis').transcribeOnly(src); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
+ try { r = await require('./media-analysis').transcribeOnly(src, { userId: clip.user_id || null }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
if (r.text) {
if (r.text) {
try { await db.setClipTranscript(clip.id, r.text, r.segments || []); } catch { /* */ }
try { await db.setClipTranscript(clip.id, r.text, r.segments || []); } catch { /* */ }
await db.setClipTranscriptStatus(clip.id, 'done');
await db.setClipTranscriptStatus(clip.id, 'done');
@@ -45,7 +45,9 @@ async function analyze(kind, item) {
@@ -45,7 +45,9 @@ async function analyze(kind, item) {
attemptNo = Number(state && state.transcript_attempts) || 0;
attemptNo = Number(state && state.transcript_attempts) || 0;
const mediaUrl = await source(kind, item);
const mediaUrl = await source(kind, item);
if (!mediaUrl) throw new Error('Media did not provide a readable URL');
if (!mediaUrl) throw new Error('Media did not provide a readable URL');
- const run = await aiService.run('media.analyze', { media_url: mediaUrl, language: 'en', allow_paid: false }, {
+ // The channel's language (C11): AI hears a Japanese recording as Japanese.
+ const language = await require('./transcribe').speechLanguage(item.user_id, { local: false });
+ const run = await aiService.run('media.analyze', { media_url: mediaUrl, language, allow_paid: false }, {
target: { service: 'live', type: kind, id: String(item.id) },
target: { service: 'live', type: kind, id: String(item.id) },
attribution: aiService.ownerRef(item.user_id),
attribution: aiService.ownerRef(item.user_id),
idempotencyKey: `live.media.analyze:${kind}:${item.id}:${attemptNo}`,
idempotencyKey: `live.media.analyze:${kind}:${item.id}:${attemptNo}`,
@@ -216,10 +216,10 @@ function _ffprobeHasAudio(src) {
@@ -216,10 +216,10 @@ function _ffprobeHasAudio(src) {
* recording where it lies (range requests, nothing downloaded), loudness-normalises it and runs whisper.cpp with the
* recording where it lies (range requests, nothing downloaded), loudness-normalises it and runs whisper.cpp with the
* same models; low power while streams are live. → { ok, segments, error }
* same models; low power while streams are live. → { ok, segments, error }
*/
*/
-async function _aiWindow(src, start, len) {
+async function _aiWindow(src, start, len, language = 'en') {
const aiService = require('./ai-service');
const aiService = require('./ai-service');
const r = await aiService.run('live.media.transcribe', {
const r = await aiService.run('live.media.transcribe', {
- media_url: src, language: 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),
+ media_url: src, language: language || 'en', start_sec: start, seconds: Math.max(0, Math.round(len)), low_power: require('./transcribe').isLowPower(),
}, { waitMs: 60000, pollMs: 20 * 60000 });
}, { waitMs: 60000, pollMs: 20 * 60000 });
const out = aiService.usable(r);
const out = aiService.usable(r);
if (!out) return { ok: false, segments: [], error: (r && r.error && (r.error.code || r.error.detail)) || 'OpenVibe.AI did not transcribe' };
if (!out) return { ok: false, segments: [], error: (r && r.error && (r.error.code || r.error.detail)) || 'OpenVibe.AI did not transcribe' };
@@ -230,17 +230,19 @@ async function _aiWindow(src, start, len) {
@@ -230,17 +230,19 @@ async function _aiWindow(src, start, len) {
// waiting out rather than skipping the window.
// waiting out rather than skipping the window.
const PERMANENT_AI_ERRORS = new Set(['media.unreadable', 'media.not_found', 'media.too_long', 'validation.failed']);
const PERMANENT_AI_ERRORS = new Set(['media.unreadable', 'media.not_found', 'media.too_long', 'validation.failed']);
-async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null } = {}) {
+async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments = null, onWindow = null, userId = null, language = null } = {}) {
const transcribe = require('./transcribe');
const transcribe = require('./transcribe');
// Recordings on Media are transcribed by OpenVibe.AI; only a local file (legacy) is still decoded here.
// Recordings on Media are transcribed by OpenVibe.AI; only a local file (legacy) is still decoded here.
const viaAi = /^https?:\/\//i.test(String(src || '')) && require('./ai-service').enabled();
const viaAi = /^https?:\/\//i.test(String(src || '')) && require('./ai-service').enabled();
if (!viaAi && !transcribe.available()) return { text: '', segments: [], ok: false, error: 'whisper unavailable' };
if (!viaAi && !transcribe.available()) return { text: '', segments: [], ok: false, error: 'whisper unavailable' };
+ // The channel's language (C11): a Japanese stream is heard as Japanese, not decoded as English.
+ const lang = language || (userId ? await transcribe.speechLanguage(userId, { local: !viaAi }) : 'en');
if (viaAi && duration > 0 && duration <= 200) {
if (viaAi && duration > 0 && duration <= 200) {
- const w = await _aiWindow(src, 0, 0);
+ const w = await _aiWindow(src, 0, 0, lang);
return { text: w.segments.map(g => g.text).join(' ').replace(/\s+/g, ' ').trim(), segments: w.segments, ok: w.ok, error: w.error };
return { text: w.segments.map(g => g.text).join(' ').replace(/\s+/g, ' ').trim(), segments: w.segments, ok: w.ok, error: w.error };
}
}
if (duration > 0 && duration <= 200) {
if (duration > 0 && duration <= 200) {
- return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000 });
+ return await transcribe.transcribeMediaDetailed(src, { seconds: 0, timeoutMs: 300000, language: lang });
}
}
// FULL coverage, not a sample. This used to take three 60-second windows at 10%/45%/80%
// FULL coverage, not a sample. This used to take three 60-second windows at 10%/45%/80%
// of the recording, so a 4-hour VOD produced 3 minutes of transcript and the rest was
// of the recording, so a 4-hour VOD produced 3 minutes of transcript and the rest was
@@ -272,7 +274,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
@@ -272,7 +274,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
for (const start of windows) {
for (const start of windows) {
const len = Math.min(WINDOW_SEC, total - start);
const len = Math.min(WINDOW_SEC, total - start);
if (viaAi) {
if (viaAi) {
- const w = await _aiWindow(src, start, len);
+ const w = await _aiWindow(src, start, len, lang);
if (w.ok) anyOk = true; else lastErr = w.error || lastErr;
if (w.ok) anyOk = true; else lastErr = w.error || lastErr;
if (w.segments.length) segments.push(...w.segments);
if (w.segments.length) segments.push(...w.segments);
if (w.ok && typeof onWindow === 'function') {
if (w.ok && typeof onWindow === 'function') {
@@ -287,7 +289,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
@@ -287,7 +289,7 @@ async function _transcribeSpan(src, duration, { resumeFromSec = 0, priorSegments
const wav = _tmp('wav');
const wav = _tmp('wav');
const ffOk = await _runFf('ffmpeg', ['-y', '-ss', String(start), '-i', src, '-t', String(len), '-vn', '-ac', '1', '-ar', '16000', '-af', 'loudnorm=I=-16:TP=-1.5:LRA=11', '-f', 'wav', wav], 180000);
const ffOk = await _runFf('ffmpeg', ['-y', '-ss', String(start), '-i', src, '-t', String(len), '-vn', '-ac', '1', '-ar', '16000', '-af', 'loudnorm=I=-16:TP=-1.5:LRA=11', '-f', 'wav', wav], 180000);
if (ffOk) {
if (ffOk) {
- const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start });
+ const r = await transcribe.transcribeWavDetailed(wav, { timeoutMs: 600000, offsetSec: start, language: lang });
if (r.ok) anyOk = true; else lastErr = r.error || lastErr;
if (r.ok) anyOk = true; else lastErr = r.error || lastErr;
if (r.segments && r.segments.length) segments.push(...r.segments);
if (r.segments && r.segments.length) segments.push(...r.segments);
if (r.ok && typeof onWindow === 'function') {
if (r.ok && typeof onWindow === 'function') {
@@ -334,7 +336,7 @@ async function analyzeMedia(src, { streamId = null, userId = null, numFrames = n
@@ -334,7 +336,7 @@ async function analyzeMedia(src, { streamId = null, userId = null, numFrames = n
frames = await captureFrameMemories(src, times, { streamId, userId, offsetBase, store: storeMemories });
frames = await captureFrameMemories(src, times, { streamId, userId, offsetBase, store: storeMemories });
}
}
- const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration);
+ const { text: transcript, segments, ok: transcriptOk } = await _transcribeSpan(src, duration, { userId });
let overview = null;
let overview = null;
if ((frames.length || transcript) && await ai.isEnabled() && await ai.withinBudget()) {
if ((frames.length || transcript) && await ai.isEnabled() && await ai.withinBudget()) {
@@ -122,7 +122,8 @@ async function _analyzeOne(stream, { allowFfmpeg = true, reason = 'periodic' } =
@@ -122,7 +122,8 @@ async function _analyzeOne(stream, { allowFfmpeg = true, reason = 'periodic' } =
} else if (ai.transcriptionEnabled && await ai.transcriptionEnabled()) {
} else if (ai.transcriptionEnabled && await ai.transcriptionEnabled()) {
const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;
const audio = require('./stream-audio').captureAudioChunk ? await require('./stream-audio').captureAudioChunk(stream, CHUNK_SEC) : null;
if (audio) {
if (audio) {
- const tx = await require('./transcribe').transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC) });
+ const transcribe = require('./transcribe');
+ const tx = await transcribe.transcribeWavDetailed(audio, { offsetSec: Math.max(0, offset - CHUNK_SEC), language: await transcribe.speechLanguage(stream.user_id) });
heard = tx.text || '';
heard = tx.text || '';
heardSegments = (tx.segments && tx.segments.length) ? tx.segments : null;
heardSegments = (tx.segments && tx.segments.length) ? tx.segments : null;
try { require('fs').unlinkSync(audio); } catch { /* */ }
try { require('fs').unlinkSync(audio); } catch { /* */ }
@@ -69,6 +69,19 @@ function parakeetAvailable() {
@@ -69,6 +69,19 @@ function parakeetAvailable() {
if (process.env.ASR_ENGINE === 'whisper') return false;
if (process.env.ASR_ENGINE === 'whisper') return false;
try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }
try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }
}
}
+/**
+ * The language a channel's speech is decoded in (item 151, C11): its chat language (set, or detected from its bio),
+ * when a local model can decode it (Parakeet's languages, or a multilingual whisper); else English. OpenVibe.AI's
+ * path takes the channel's language as it is: AI picks its own model.
+ */
+async function speechLanguage(userId, { local = true } = {}) {
+ let lang = 'en';
+ try { lang = await require('../i18n/translate').channelLanguage(userId); } catch { lang = 'en'; }
+ lang = String(lang || 'en').toLowerCase();
+ if (!local || lang === 'en') return lang || 'en';
+ return engineFor(lang) === 'parakeet' || multilingualModel() ? lang : 'en';
+}
+
/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */
/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */
function engineFor(language = 'en') {
function engineFor(language = 'en') {
return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';
return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';
@@ -185,6 +198,10 @@ const HALLUCINATIONS = new Set([
@@ -185,6 +198,10 @@ const HALLUCINATIONS = new Set([
'thanks for watching.', 'please subscribe', 'subscribe', 'like and subscribe',
'thanks for watching.', 'please subscribe', 'subscribe', 'like and subscribe',
'thank you for watching', 'thank you very much',
'thank you for watching', 'thank you very much',
'thank you so much', 'silence', 'music', 'applause',
'thank you so much', 'silence', 'music', 'applause',
+ // Whisper's Japanese stock inventions over silence and music (item 151, C11: Japanese streams are now heard in
+ // Japanese, and these came with it): "thanks for watching", "please subscribe".
+ 'ご視聴ありがとうございました', 'ご視聴ありがとうございます', 'ご視聴いただきありがとうございました',
+ 'チャンネル登録よろしくお願いします', 'チャンネル登録お願いします', 'チャンネル登録よろしくね',
]);
]);
// Drop-only-without-VAD: these ARE real words a streamer says constantly. They were on the
// Drop-only-without-VAD: these ARE real words a streamer says constantly. They were on the
// blocklist because, with the decoder run over 87% silence, they were overwhelmingly
// blocklist because, with the decoder run over 87% silence, they were overwhelmingly
@@ -201,8 +218,10 @@ function _isNoise(t) {
@@ -201,8 +218,10 @@ function _isNoise(t) {
if (/^[\s.\-–—_*]+$/.test(s)) return true;
if (/^[\s.\-–—_*]+$/.test(s)) return true;
if (/^[\[(♪].*[\])♪]?$/.test(s)) return true;
if (/^[\[(♪].*[\])♪]?$/.test(s)) return true;
if (/^♪/.test(s) || /♪$/.test(s)) return true;
if (/^♪/.test(s) || /♪$/.test(s)) return true;
- const norm = s.toLowerCase().replace(/[.!?,…]+$/g, '').trim();
+ const norm = s.toLowerCase().replace(/[.!?,…。!?、]+$/g, '').trim();
if (HALLUCINATIONS.has(norm)) return true;
if (HALLUCINATIONS.has(norm)) return true;
+ // Japanese subtitle credits ("字幕: …", "字幕作成者 …") are training-data boilerplate, never stream speech.
+ if (/^(?:字幕|翻訳)(?:作成者?|制作|提供)?[\s::]/.test(norm) || /^字幕\s*by\b/i.test(norm)) return true;
// Whisper's training data was full of subtitle boilerplate, and it emits those exact
// Whisper's training data was full of subtitle boilerplate, and it emits those exact
// phrases over noise even when VAD says a voice is present — a real live segment came
// phrases over noise even when VAD says a voice is present — a real live segment came
// back as "For more information, visit www.fema.org." These are recognisable by shape:
// back as "For more information, visit www.fema.org." These are recognisable by shape:
@@ -398,7 +417,7 @@ async function transcribeWav(wavPath, opts = {}) {
@@ -398,7 +417,7 @@ async function transcribeWav(wavPath, opts = {}) {
* Extract audio from ANY media (local file OR http(s) URL) → temp wav → detailed
* Extract audio from ANY media (local file OR http(s) URL) → temp wav → detailed
* transcription. @returns {Promise<{text, segments}>}
* transcription. @returns {Promise<{text, segments}>}
*/
*/
-function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000 } = {}) {
+function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeoutMs = 300000, language = 'en' } = {}) {
return new Promise((resolve) => {
return new Promise((resolve) => {
const isUrl = /^https?:/i.test(mediaPath || '');
const isUrl = /^https?:/i.test(mediaPath || '');
if (!available() || !mediaPath || (!isUrl && !fs.existsSync(mediaPath))) return resolve({ text: '', segments: [], ok: false, error: 'source unavailable' });
if (!available() || !mediaPath || (!isUrl && !fs.existsSync(mediaPath))) return resolve({ text: '', segments: [], ok: false, error: 'source unavailable' });
@@ -415,7 +434,7 @@ function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeou
@@ -415,7 +434,7 @@ function transcribeMediaDetailed(mediaPath, { seconds = 0, offsetSec = 0, timeou
clearTimeout(ffTimer);
clearTimeout(ffTimer);
if (code !== 0) { cleanup(); return resolve({ text: '', segments: [], ok: false, error: `ffmpeg exited ${code}` }); }
if (code !== 0) { cleanup(); return resolve({ text: '', segments: [], ok: false, error: `ffmpeg exited ${code}` }); }
let r = { text: '', segments: [], ok: false, error: 'unknown' };
let r = { text: '', segments: [], ok: false, error: 'unknown' };
- try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
+ try { r = await transcribeWavDetailed(wav, { timeoutMs, offsetSec, language }); } catch (e) { r = { text: '', segments: [], ok: false, error: e.message }; }
cleanup();
cleanup();
resolve(r);
resolve(r);
});
});
@@ -463,4 +482,4 @@ function describe() {
@@ -463,4 +482,4 @@ function describe() {
function isLowPower() { return _lowPower; }
function isLowPower() { return _lowPower; }
-module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
+module.exports = { _isNoise, speechLanguage, available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+/**
+ * A channel's speech is heard in its language (item 151, C11): media analysis, clip and VOD transcripts, OpenVibe.AI's
+ * runs and the live memory decode in the channel's language (its chat language, set or detected) when a model can,
+ * and Japanese speech survives the noise filter while whisper's Japanese inventions over silence do not.
+ *
+ * node test/run.js channel-language-asr
+ */
+'use strict';
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+const translate = require('../server/i18n/translate');
+const langs = { 7: 'ja', 8: 'de', 9: 'en' };
+translate.channelLanguage = async (userId) => langs[userId] || 'en';
+const transcribe = require('../server/ai/transcribe');
+
+(async () => {
+ // AI's path takes the channel's language as it is.
+ assert.strictEqual(await transcribe.speechLanguage(7, { local: false }), 'ja');
+ assert.strictEqual(await transcribe.speechLanguage(9, { local: false }), 'en');
+ // A local decode: Japanese only with a multilingual whisper; otherwise English (never a model that cannot).
+ const multi = Boolean(transcribe.multilingualModel());
+ assert.strictEqual(await transcribe.speechLanguage(7), multi ? 'ja' : 'en');
+ assert.strictEqual(await transcribe.speechLanguage(null), 'en');
+
+ // Every path passes it on: no hard-coded English left where a channel is known.
+ const src = (f) => fs.readFileSync(path.join(__dirname, '..', 'server', 'ai', f), 'utf8');
+ const ma = src('media-analysis.js');
+ assert.doesNotMatch(ma, /language: 'en', start_sec/, 'AI windows carry the language');
+ assert.match(ma, /_transcribeSpan\(src, duration, \{ userId \}\)/, 'analysis passes the channel');
+ assert.match(ma, /transcribeWavDetailed\(wav, \{[^}]*language: lang/, 'local windows decode in it');
+ assert.doesNotMatch(src('ai-media-analyze.js'), /language: 'en'/, 'AI\'s media.analyze is not forced to English');
+ assert.match(src('ai-analysis.js'), /transcribeOnly\(src, \{ userId: clip\.user_id/, 'clip transcripts know their channel');
+ assert.match(src('ai-analysis.js'), /userId: vod\.user_id/, 'VOD transcripts know their channel');
+ assert.match(src('stream-memory-job.js'), /language: await transcribe\.speechLanguage\(stream\.user_id\)/, 'the live memory hears the channel');
+
+ // Japanese speech is kept whole; whisper's Japanese stock phrases and subtitle credits are noise.
+ assert.strictEqual(transcribe._isNoise('今日はマイクラで家を建てます'), false);
+ assert.strictEqual(transcribe._isNoise('ありがとうございます、みんな!'), false);
+ assert.strictEqual(transcribe._isNoise('ご視聴ありがとうございました。'), true);
+ assert.strictEqual(transcribe._isNoise('チャンネル登録よろしくお願いします!'), true);
+ assert.strictEqual(transcribe._isNoise('字幕:山田'), true);
+ assert.strictEqual(transcribe._isNoise('Thanks for watching!'), true);
+ assert.strictEqual(transcribe._isNoise('Let us build a house'), false);
+ console.log('channel language asr: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });