openvibers/openvibe.live Commits

Live speech-to-text: NVIDIA Parakeet TDT 0.6B v3 for English and 24 European languages (more words heard, punctuated, as fast as base.en on a live segment); whisper keeps Japanese and the rest, and takes any audio Parakeet fails on (#144)

OP OpenVibers committed 28 days ago · 7eacb82513 · 5 files, +177 −7

Changes

ddiff --git a/.env.example b/.env.example
iindex 67672402..12f4cd8e 100644
---- a/.env.example
++++ b/.env.example
@@@ -16,6 +16,14 @@ ADMIN_PASSWORD=changeme123
# channels keep using the English model and live captions stay hidden.
# WHISPER_MODEL_MULTI=/home/openvibe/whisper.cpp/models/ggml-small.bin
++# Speech-to-text engine for English and 24 other European languages: NVIDIA Parakeet TDT 0.6B v3 through
++# whisper.cpp's parakeet-cli (more accurate than whisper base.en and fast enough for live on 4 cores). Used
++# when both files exist; other languages, and any Parakeet failure, go to whisper. ASR_ENGINE=whisper forces whisper.
++# Model: https://huggingface.co/ggml-org/parakeet-GGUF (ggml-parakeet-tdt-0.6b-v3-q8_0.bin).
++# PARAKEET_BIN=/home/openvibe/whisper.cpp/build/bin/parakeet-cli
++# PARAKEET_MODEL=/home/openvibe/whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin
++# ASR_ENGINE=whisper
++
# OpenVibe.AI (roadmap Wave 13, WS-O task 2). Every AI call Live makes (chat AI, moments, paste/frame
# analysis, overviews, translation, recaps, slogans, arena, AI viewers) is a workflow run on the AI
# service, with a service token for audience openvibe.ai (grants ai.run.create, ai.run.read and
ddiff --git a/server/ai/transcribe.js b/server/ai/transcribe.js
iindex d818cc85..3005238d 100644
---- a/server/ai/transcribe.js
++++ b/server/ai/transcribe.js
@@@ -1,6 +1,15 @@
/**
-- * transcribe.js — free, LOCAL speech-to-text via whisper.cpp (no API, runs on the
-- * server CPU). Used for live-stream memories and VOD/clip transcripts.
++ * transcribe.js — free, LOCAL speech-to-text (no API, runs on the server CPU). Used for
++ * live-stream memories, the live timeline (which the Arena listens to) and VOD/clip transcripts.
++ *
++ * Two engines, both from the whisper.cpp tree (ggml):
++ * - NVIDIA Parakeet TDT 0.6B v3 (`parakeet-cli`), for English and 24 other European languages.
++ * Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,
++ * punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped
++ * whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,
++ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.
++ * - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when
++ * Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.
*
* Reliability features:
* - JSON output (-oj) → per-segment timestamps (ms) we keep as contextual data.
@@@ -50,6 +59,20 @@ function _modelFor(opts) {
}
const THREADS = Math.max(2, Math.min(8, parseInt(process.env.WHISPER_THREADS, 10) || 4));
++// ── Parakeet (NVIDIA Parakeet TDT, ggml, from the same whisper.cpp build) ──
++const PARAKEET_BIN = process.env.PARAKEET_BIN || path.join(HOME, 'whisper.cpp/build/bin/parakeet-cli');
++const PARAKEET_MODEL = process.env.PARAKEET_MODEL || path.join(HOME, 'whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin');
++// Parakeet TDT 0.6B v3's languages (NVIDIA's model card): every other language goes to whisper.
++const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);
++function parakeetAvailable() {
++ if (process.env.ASR_ENGINE === 'whisper') return false;
++ try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }
++}
++/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */
++function engineFor(language = 'en') {
++ return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';
++}
++
// ── Voice Activity Detection ──────────────────────────────────────────────
// Measured on 60s of real stream audio: VAD cut large-v3-turbo from 51.0s to 24.0s
// AND removed a trailing "Thank you." hallucination. Both effects matter, and the
@@@ -84,6 +107,10 @@ function whisperBin() {
return found;
}
function available() {
++ if (parakeetAvailable()) return true;
++ try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }
++}
++function whisperAvailable() {
try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }
}
@@@ -210,6 +237,62 @@ function _joinSegments(segments) {
* @returns {Promise<{text:string, segments:Array<{start:number,end:number,text:string}>}>}
* start/end in seconds (shifted by opts.offsetSec).
*/
++// ── Parakeet output → timed sentences ──────────────────────────────────────
++// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a
++// word ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s, so each timeline row stays a
++// line someone said, deep-linkable by its start.
++const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;
++// Fillers Parakeet writes out ("uh", "um") carry nothing for a reader or the Arena's quotes.
++const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;
++function parseParakeet(stderr, offsetSec = 0) {
++ const out = [];
++ let cur = null;
++ const close = () => {
++ if (!cur) return;
++ const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();
++ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });
++ cur = null;
++ };
++ for (const line of String(stderr || '').split('\n')) {
++ const m = TOKEN_LINE.exec(line);
++ if (!m) continue;
++ const t0 = Number(m[1]), t1 = Number(m[2]);
++ const wordStart = m[3] === 'true';
++ const raw = m[4];
++ if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens
++ const piece = raw.replace(/\u2581/g, ' ');
++ if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();
++ if (!cur) cur = { t0, t1, text: '' };
++ cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;
++ cur.t1 = Math.max(cur.t1, t1);
++ if (/[.!?]["')\]]?$/.test(piece.trim())) close();
++ }
++ close();
++ return out;
++}
++
++function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
++ return new Promise((resolve) => {
++ if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });
++ const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];
++ let ff;
++ try { ff = _track(spawn('nice', ['-n', '15', PARAKEET_BIN, ...args], { stdio: ['ignore', 'ignore', 'pipe'] })); }
++ catch (e) { return resolve({ text: '', segments: [], ok: false, error: e.message }); }
++ const chunks = [];
++ let size = 0;
++ ff.stderr.on('data', (d) => { if (size < 64 * 1024 * 1024) { chunks.push(d); size += d.length; } });
++ let done = false;
++ const finish = (result) => { if (done) return; done = true; clearTimeout(timer); resolve(result); };
++ const timer = setTimeout(() => { try { ff.kill('SIGKILL'); } catch { /* */ } finish({ text: '', segments: [], ok: false, error: 'timeout' }); }, timeoutMs);
++ ff.on('close', (code) => {
++ if (code !== 0) return finish({ text: '', segments: [], ok: false, error: `parakeet exited ${code}` });
++ const segments = _cleanSegments(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));
++ finish({ text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' });
++ });
++ ff.on('error', (e) => finish({ text: '', segments: [], ok: false, error: e.message }));
++ });
++}
++
// Inner implementation. Wrapped by transcribeWavDetailed, which holds the concurrency
// slot for the whole run so only MAX_CONCURRENT decoders exist at any moment.
function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false, language = 'en' } = {}) {
@@@ -218,7 +301,7 @@ function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live
// ok=false only for genuine FAILURES (missing binary, spawn/exec error, timeout,
// non-zero exit, unparseable output). A clean run that finds no speech is ok=true
// with empty text — the caller must NOT treat that as a failure to retry.
-- if (!bin || !available() || !wavPath || !fs.existsSync(wavPath)) {
++ if (!bin || !whisperAvailable() || !wavPath || !fs.existsSync(wavPath)) {
return resolve({ text: '', segments: [], ok: false, error: 'whisper unavailable' });
}
const outBase = `${wavPath}.out`;
@@@ -311,8 +394,15 @@ async function transcribeMedia(mediaPath, opts = {}) {
async function transcribeWavDetailed(wavPath, opts = {}) {
const lane = opts && opts.live ? 'live' : 'batch';
await _acquire(lane);
-- try { return await _transcribeWavInner(wavPath, opts); }
-- finally { _release(lane); }
++ try {
++ if (engineFor(opts && opts.language) === 'parakeet') {
++ const r = await _parakeetInner(wavPath, opts);
++ // A Parakeet failure (a crash, a bad model file) must not leave a hole: whisper takes the same audio.
++ if (r.ok || !whisperAvailable()) return r;
++ console.warn(`[Transcribe] parakeet failed (${r.error}); whisper takes this audio`);
++ }
++ return await _transcribeWavInner(wavPath, opts);
++ } finally { _release(lane); }
}
/** Human-readable availability report (logged at boot so a broken install is visible). */
@@@ -324,10 +414,11 @@ function describe() {
model: MODEL, modelExists: fs.existsSync(MODEL),
modelLive: MODEL_LIVE, modelLiveExists: fs.existsSync(MODEL_LIVE), modelMulti: MODEL_MULTI, modelMultiExists: !!multilingualModel(),
vadModel: vadModel() || null,
++ parakeet: parakeetAvailable() ? PARAKEET_MODEL : null, engineEn: engineFor('en'),
threads: THREADS, beam: BEAM, maxConcurrent: MAX_CONCURRENT,
};
}
function isLowPower() { return _lowPower; }
--module.exports = { available, describe, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
++module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
ddiff --git a/server/index.js b/server/index.js
iindex 822f69bb..ffc2fc10 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -1381,7 +1381,7 @@ async function start() {
// AI stream-memory job (no-op until enabled in openvibe.network/admin → AI).
try { require('./ai/stream-memory-job').start(); } catch (e) { console.warn('[AI] memory job not started:', e.message); }
-- try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] whisper ${d.available ? 'available' : 'UNAVAILABLE'} — bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();
++ try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] speech-to-text ${d.available ? 'available' : 'UNAVAILABLE'} — English via ${d.engineEn}${d.parakeet ? ` (${d.parakeet})` : ''}; whisper bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();
require('./ai/backfill-job').start(); } catch (e) { console.warn('[AI] backfill job not started:', e.message); }
// AI viewers activity log: keep a week.
setInterval(async () => { try { const n = await db.pruneAiViewerLog(7); if (n) console.log(`[AI-Viewers] pruned ${n} log row(s)`); } catch { /* */ } }, 6 * 3600 * 1000);
ddiff --git a/test/fixtures/parakeet-tokens.txt b/test/fixtures/parakeet-tokens.txt
nnew file mode 100644
iindex 00000000..1d6cbf9b
---- /dev/null
++++ b/test/fixtures/parakeet-tokens.txt
@@@ -0,0 +1,34 @@
++Segments (1):
++Segment 0: [0 -> 601] "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's"
++ [ 0] id= 3108 frame= 4 dur_idx= 2 dur_val= 2 p=0.5552 plog=-5.8166 t0= 32 t1= 48 word_start=true "▁For"
++ [ 1] id= 1111 frame= 6 dur_idx= 2 dur_val= 2 p=0.9792 plog=-7.9274 t0= 48 t1= 64 word_start=true "▁all"
++ [ 2] id= 380 frame= 8 dur_idx= 2 dur_val= 2 p=0.9872 plog=-10.5001 t0= 64 t1= 80 word_start=true "▁I"
++ [ 3] id= 2865 frame= 10 dur_idx= 1 dur_val= 1 p=0.9908 plog=-8.5432 t0= 80 t1= 88 word_start=true "▁know"
++ [ 4] id= 869 frame= 12 dur_idx= 1 dur_val= 1 p=0.9073 plog=-7.9946 t0= 96 t1= 104 word_start=true "▁it"
++ [ 5] id= 5635 frame= 14 dur_idx= 1 dur_val= 1 p=0.7077 plog=-9.1575 t0= 112 t1= 120 word_start=true "▁said"
++ [ 6] id= 7877 frame= 18 dur_idx= 1 dur_val= 1 p=0.4234 plog=-9.9571 t0= 120 t1= 120 word_start=false ","
++ [ 7] id= 2846 frame= 19 dur_idx= 3 dur_val= 3 p=0.6802 plog=-7.0393 t0= 152 t1= 176 word_start=true "▁just"
++ [ 8] id= 869 frame= 24 dur_idx= 1 dur_val= 1 p=0.9056 plog=-7.0782 t0= 192 t1= 200 word_start=true "▁it"
++ [ 9] id= 7931 frame= 25 dur_idx= 1 dur_val= 1 p=0.9562 plog=-9.0419 t0= 200 t1= 200 word_start=false "'"
++ [10] id= 7870 frame= 26 dur_idx= 1 dur_val= 1 p=0.9993 plog=-8.2082 t0= 208 t1= 216 word_start=false "s"
++ [11] id= 279 frame= 27 dur_idx= 2 dur_val= 2 p=0.9903 plog=-13.6674 t0= 216 t1= 232 word_start=true "▁a"
++ [12] id= 1639 frame= 29 dur_idx= 2 dur_val= 2 p=0.9965 plog=-10.3544 t0= 232 t1= 248 word_start=true "▁bl"
++ [13] id= 2192 frame= 31 dur_idx= 3 dur_val= 3 p=0.9997 plog=-11.0735 t0= 248 t1= 272 word_start=false "ank"
++ [14] id= 627 frame= 34 dur_idx= 2 dur_val= 2 p=0.9994 plog=-11.8629 t0= 272 t1= 288 word_start=true "▁ch"
++ [15] id= 2300 frame= 36 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.4228 t0= 288 t1= 304 word_start=false "ip"
++ [16] id= 334 frame= 38 dur_idx= 1 dur_val= 1 p=0.9888 plog=-14.3064 t0= 304 t1= 312 word_start=true "▁in"
++ [17] id= 700 frame= 39 dur_idx= 2 dur_val= 2 p=1.0000 plog=-13.0235 t0= 312 t1= 328 word_start=false "iti"
++ [18] id= 2274 frame= 41 dur_idx= 1 dur_val= 1 p=0.9987 plog=-9.2411 t0= 328 t1= 336 word_start=false "ally"
++ [19] id= 2530 frame= 43 dur_idx= 2 dur_val= 2 p=0.5170 plog=-7.3316 t0= 344 t1= 360 word_start=true "▁they"
++ [20] id= 2617 frame= 45 dur_idx= 1 dur_val= 1 p=0.9922 plog=-8.1131 t0= 360 t1= 368 word_start=true "▁put"
++ [21] id= 334 frame= 48 dur_idx= 2 dur_val= 2 p=0.9799 plog=-10.5362 t0= 384 t1= 400 word_start=true "▁in"
++ [22] id= 575 frame= 51 dur_idx= 3 dur_val= 3 p=0.9582 plog=-6.8221 t0= 408 t1= 432 word_start=true "▁and"
++ [23] id= 869 frame= 54 dur_idx= 1 dur_val= 1 p=0.4425 plog=-7.9854 t0= 432 t1= 440 word_start=true "▁it"
++ [24] id= 618 frame= 58 dur_idx= 2 dur_val= 2 p=0.9823 plog=-7.4988 t0= 464 t1= 480 word_start=true "▁as"
++ [25] id= 547 frame= 60 dur_idx= 1 dur_val= 1 p=0.9914 plog=-11.8962 t0= 480 t1= 488 word_start=true "▁so"
++ [26] id= 289 frame= 61 dur_idx= 1 dur_val= 1 p=0.9996 plog=-9.9585 t0= 488 t1= 496 word_start=false "on"
++ [27] id= 618 frame= 62 dur_idx= 2 dur_val= 2 p=0.9982 plog=-14.3261 t0= 496 t1= 512 word_start=true "▁as"
++ [28] id= 869 frame= 64 dur_idx= 3 dur_val= 3 p=0.9623 plog=-8.3384 t0= 512 t1= 536 word_start=true "▁it"
++ [29] id= 869 frame= 67 dur_idx= 2 dur_val= 2 p=0.6599 plog=-6.3797 t0= 536 t1= 552 word_start=true "▁it"
++ [30] id= 7931 frame= 74 dur_idx= 0 dur_val= 0 p=0.3250 plog=-16.9064 t0= 552 t1= 552 word_start=false "'"
++ [31] id= 7870 frame= 74 dur_idx= 0 dur_val= 0 p=0.9837 plog=-25.8700 t0= 592 t1= 592 word_start=false "s"
ddiff --git a/test/transcribe-parakeet.test.js b/test/transcribe-parakeet.test.js
nnew file mode 100644
iindex 00000000..b7c72944
---- /dev/null
++++ b/test/transcribe-parakeet.test.js
@@@ -0,0 +1,37 @@
++'use strict';
++// Parakeet TDT (server/ai/transcribe.js): parakeet-cli's per-token timing (centiseconds) becomes timed sentences: a
++// sentence ends at . ? !, a 1.5 s pause or 20 s; "uh"/"um" are dropped; control tokens are skipped; the offset shifts
++// every time; languages Parakeet does not know go to whisper, and ASR_ENGINE=whisper forces whisper.
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const t = require('../server/ai/transcribe');
++
++const tok = (t0, t1, start, s) => ` [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1.0 t0=${t0} t1=${t1} word_start=${start} "${s}"`;
++
++// The real output of parakeet-cli -ps on 6 s of stream talk (2026-10-16).
++const real = t.parseParakeet(fs.readFileSync(path.join(__dirname, 'fixtures', 'parakeet-tokens.txt'), 'utf8'), 100);
++assert.strictEqual(real.length, 1);
++assert.strictEqual(real[0].text, "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's");
++assert.deepStrictEqual([real[0].start, real[0].end], [100.32, 105.92]);
++
++const lines = [
++ 'Segments (1):',
++ tok(10, 30, true, '▁So'), tok(30, 40, true, '▁uh'), tok(40, 41, false, ','), tok(41, 60, true, '▁he'), tok(60, 80, true, '▁lost'), tok(80, 81, false, '.'),
++ tok(90, 100, true, '▁What'), tok(100, 110, true, '▁a'), tok(110, 130, true, '▁clown'), tok(130, 131, false, '?'),
++ tok(140, 150, true, '<unk>'),
++ tok(150, 170, true, '▁Then'), tok(170, 190, true, '▁silence'),
++ tok(400, 420, true, '▁Back'), tok(420, 440, true, '▁again'),
++].join('\n');
++const out = t.parseParakeet(lines, 10);
++assert.deepStrictEqual(out.map((s) => s.text), ['So he lost.', 'What a clown?', 'Then silence', 'Back again']);
++assert.deepStrictEqual(out.map((s) => [s.start, s.end]), [[10.1, 10.81], [10.9, 11.31], [11.5, 11.9], [14, 14.4]], 'a 2.1 s pause ends a line');
++
++const long = Array.from({ length: 30 }, (_, i) => tok(i * 100, i * 100 + 50, true, `▁w${i}`)).join('\n');
++assert.ok(t.parseParakeet(long).length >= 2, 'no line runs past 20 s');
++assert.deepStrictEqual(t.parseParakeet(''), []);
++
++// Engines: without the Parakeet files every language is whisper's; Japanese is always whisper's.
++assert.strictEqual(t.engineFor('ja'), 'whisper');
++assert.ok(['parakeet', 'whisper'].includes(t.engineFor('en')));
++console.log('transcribe-parakeet: ok');