openvibers/openvibe.live Commits
Live speech-to-text: NVIDIA Parakeet TDT 0.6B v3 for English and 24 European languages (more words heard, punctuated, as fast as base.en on a live segment); whisper keeps Japanese and the rest, and takes any audio Parakeet fails on (#144)
Changes
@@@ -16,6 +16,14 @@ ADMIN_PASSWORD=changeme123# channels keep using the English model and live captions stay hidden.# WHISPER_MODEL_MULTI=/home/openvibe/whisper.cpp/models/ggml-small.bin++# Speech-to-text engine for English and 24 other European languages: NVIDIA Parakeet TDT 0.6B v3 through++# whisper.cpp's parakeet-cli (more accurate than whisper base.en and fast enough for live on 4 cores). Used++# when both files exist; other languages, and any Parakeet failure, go to whisper. ASR_ENGINE=whisper forces whisper.++# Model: https://huggingface.co/ggml-org/parakeet-GGUF (ggml-parakeet-tdt-0.6b-v3-q8_0.bin).++# PARAKEET_BIN=/home/openvibe/whisper.cpp/build/bin/parakeet-cli++# PARAKEET_MODEL=/home/openvibe/whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin++# ASR_ENGINE=whisper++# OpenVibe.AI (roadmap Wave 13, WS-O task 2). Every AI call Live makes (chat AI, moments, paste/frame# analysis, overviews, translation, recaps, slogans, arena, AI viewers) is a workflow run on the AI# service, with a service token for audience openvibe.ai (grants ai.run.create, ai.run.read and@@@ -1,6 +1,15 @@/**-- * transcribe.js — free, LOCAL speech-to-text via whisper.cpp (no API, runs on the-- * server CPU). Used for live-stream memories and VOD/clip transcripts.++ * transcribe.js — free, LOCAL speech-to-text (no API, runs on the server CPU). Used for++ * live-stream memories, the live timeline (which the Arena listens to) and VOD/clip transcripts.++ *++ * Two engines, both from the whisper.cpp tree (ggml):++ * - NVIDIA Parakeet TDT 0.6B v3 (`parakeet-cli`), for English and 24 other European languages.++ * Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,++ * punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped++ * whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,++ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.++ * - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when++ * Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.** Reliability features:* - JSON output (-oj) → per-segment timestamps (ms) we keep as contextual data.@@@ -50,6 +59,20 @@ function _modelFor(opts) {}const THREADS = Math.max(2, Math.min(8, parseInt(process.env.WHISPER_THREADS, 10) || 4));++// ── Parakeet (NVIDIA Parakeet TDT, ggml, from the same whisper.cpp build) ──++const PARAKEET_BIN = process.env.PARAKEET_BIN || path.join(HOME, 'whisper.cpp/build/bin/parakeet-cli');++const PARAKEET_MODEL = process.env.PARAKEET_MODEL || path.join(HOME, 'whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin');++// Parakeet TDT 0.6B v3's languages (NVIDIA's model card): every other language goes to whisper.++const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);++function parakeetAvailable() {++ if (process.env.ASR_ENGINE === 'whisper') return false;++ try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }++}++/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */++function engineFor(language = 'en') {++ return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';++}++// ── Voice Activity Detection ──────────────────────────────────────────────// Measured on 60s of real stream audio: VAD cut large-v3-turbo from 51.0s to 24.0s// AND removed a trailing "Thank you." hallucination. Both effects matter, and the@@@ -84,6 +107,10 @@ function whisperBin() {return found;}function available() {++ if (parakeetAvailable()) return true;++ try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }++}++function whisperAvailable() {try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }}@@@ -210,6 +237,62 @@ function _joinSegments(segments) {* @returns {Promise<{text:string, segments:Array<{start:number,end:number,text:string}>}>}* start/end in seconds (shifted by opts.offsetSec).*/++// ── Parakeet output → timed sentences ──────────────────────────────────────++// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a++// word ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s, so each timeline row stays a++// line someone said, deep-linkable by its start.++const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;++// Fillers Parakeet writes out ("uh", "um") carry nothing for a reader or the Arena's quotes.++const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;++function parseParakeet(stderr, offsetSec = 0) {++ const out = [];++ let cur = null;++ const close = () => {++ if (!cur) return;++ const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();++ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });++ cur = null;++ };++ for (const line of String(stderr || '').split('\n')) {++ const m = TOKEN_LINE.exec(line);++ if (!m) continue;++ const t0 = Number(m[1]), t1 = Number(m[2]);++ const wordStart = m[3] === 'true';++ const raw = m[4];++ if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens++ const piece = raw.replace(/\u2581/g, ' ');++ if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();++ if (!cur) cur = { t0, t1, text: '' };++ cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;++ cur.t1 = Math.max(cur.t1, t1);++ if (/[.!?]["')\]]?$/.test(piece.trim())) close();++ }++ close();++ return out;++}++++function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {++ return new Promise((resolve) => {++ if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });++ const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];++ let ff;++ try { ff = _track(spawn('nice', ['-n', '15', PARAKEET_BIN, ...args], { stdio: ['ignore', 'ignore', 'pipe'] })); }++ catch (e) { return resolve({ text: '', segments: [], ok: false, error: e.message }); }++ const chunks = [];++ let size = 0;++ ff.stderr.on('data', (d) => { if (size < 64 * 1024 * 1024) { chunks.push(d); size += d.length; } });++ let done = false;++ const finish = (result) => { if (done) return; done = true; clearTimeout(timer); resolve(result); };++ const timer = setTimeout(() => { try { ff.kill('SIGKILL'); } catch { /* */ } finish({ text: '', segments: [], ok: false, error: 'timeout' }); }, timeoutMs);++ ff.on('close', (code) => {++ if (code !== 0) return finish({ text: '', segments: [], ok: false, error: `parakeet exited ${code}` });++ const segments = _cleanSegments(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));++ finish({ text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' });++ });++ ff.on('error', (e) => finish({ text: '', segments: [], ok: false, error: e.message }));++ });++}++// Inner implementation. Wrapped by transcribeWavDetailed, which holds the concurrency// slot for the whole run so only MAX_CONCURRENT decoders exist at any moment.function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false, language = 'en' } = {}) {@@@ -218,7 +301,7 @@ function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live// ok=false only for genuine FAILURES (missing binary, spawn/exec error, timeout,// non-zero exit, unparseable output). A clean run that finds no speech is ok=true// with empty text — the caller must NOT treat that as a failure to retry.-- if (!bin || !available() || !wavPath || !fs.existsSync(wavPath)) {++ if (!bin || !whisperAvailable() || !wavPath || !fs.existsSync(wavPath)) {return resolve({ text: '', segments: [], ok: false, error: 'whisper unavailable' });}const outBase = `${wavPath}.out`;@@@ -311,8 +394,15 @@ async function transcribeMedia(mediaPath, opts = {}) {async function transcribeWavDetailed(wavPath, opts = {}) {const lane = opts && opts.live ? 'live' : 'batch';await _acquire(lane);-- try { return await _transcribeWavInner(wavPath, opts); }-- finally { _release(lane); }++ try {++ if (engineFor(opts && opts.language) === 'parakeet') {++ const r = await _parakeetInner(wavPath, opts);++ // A Parakeet failure (a crash, a bad model file) must not leave a hole: whisper takes the same audio.++ if (r.ok || !whisperAvailable()) return r;++ console.warn(`[Transcribe] parakeet failed (${r.error}); whisper takes this audio`);++ }++ return await _transcribeWavInner(wavPath, opts);++ } finally { _release(lane); }}/** Human-readable availability report (logged at boot so a broken install is visible). */@@@ -324,10 +414,11 @@ function describe() {model: MODEL, modelExists: fs.existsSync(MODEL),modelLive: MODEL_LIVE, modelLiveExists: fs.existsSync(MODEL_LIVE), modelMulti: MODEL_MULTI, modelMultiExists: !!multilingualModel(),vadModel: vadModel() || null,++ parakeet: parakeetAvailable() ? PARAKEET_MODEL : null, engineEn: engineFor('en'),threads: THREADS, beam: BEAM, maxConcurrent: MAX_CONCURRENT,};}function isLowPower() { return _lowPower; }--module.exports = { available, describe, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };++module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };@@@ -1381,7 +1381,7 @@ async function start() {// AI stream-memory job (no-op until enabled in openvibe.network/admin → AI).try { require('./ai/stream-memory-job').start(); } catch (e) { console.warn('[AI] memory job not started:', e.message); }-- try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] whisper ${d.available ? 'available' : 'UNAVAILABLE'} — bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();++ try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] speech-to-text ${d.available ? 'available' : 'UNAVAILABLE'} — English via ${d.engineEn}${d.parakeet ? ` (${d.parakeet})` : ''}; whisper bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();require('./ai/backfill-job').start(); } catch (e) { console.warn('[AI] backfill job not started:', e.message); }// AI viewers activity log: keep a week.setInterval(async () => { try { const n = await db.pruneAiViewerLog(7); if (n) console.log(`[AI-Viewers] pruned ${n} log row(s)`); } catch { /* */ } }, 6 * 3600 * 1000);nnew file mode 100644@@@ -0,0 +1,34 @@++Segments (1):++Segment 0: [0 -> 601] "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's"++ [ 0] id= 3108 frame= 4 dur_idx= 2 dur_val= 2 p=0.5552 plog=-5.8166 t0= 32 t1= 48 word_start=true "▁For"++ [ 1] id= 1111 frame= 6 dur_idx= 2 dur_val= 2 p=0.9792 plog=-7.9274 t0= 48 t1= 64 word_start=true "▁all"++ [ 2] id= 380 frame= 8 dur_idx= 2 dur_val= 2 p=0.9872 plog=-10.5001 t0= 64 t1= 80 word_start=true "▁I"++ [ 3] id= 2865 frame= 10 dur_idx= 1 dur_val= 1 p=0.9908 plog=-8.5432 t0= 80 t1= 88 word_start=true "▁know"++ [ 4] id= 869 frame= 12 dur_idx= 1 dur_val= 1 p=0.9073 plog=-7.9946 t0= 96 t1= 104 word_start=true "▁it"++ [ 5] id= 5635 frame= 14 dur_idx= 1 dur_val= 1 p=0.7077 plog=-9.1575 t0= 112 t1= 120 word_start=true "▁said"++ [ 6] id= 7877 frame= 18 dur_idx= 1 dur_val= 1 p=0.4234 plog=-9.9571 t0= 120 t1= 120 word_start=false ","++ [ 7] id= 2846 frame= 19 dur_idx= 3 dur_val= 3 p=0.6802 plog=-7.0393 t0= 152 t1= 176 word_start=true "▁just"++ [ 8] id= 869 frame= 24 dur_idx= 1 dur_val= 1 p=0.9056 plog=-7.0782 t0= 192 t1= 200 word_start=true "▁it"++ [ 9] id= 7931 frame= 25 dur_idx= 1 dur_val= 1 p=0.9562 plog=-9.0419 t0= 200 t1= 200 word_start=false "'"++ [10] id= 7870 frame= 26 dur_idx= 1 dur_val= 1 p=0.9993 plog=-8.2082 t0= 208 t1= 216 word_start=false "s"++ [11] id= 279 frame= 27 dur_idx= 2 dur_val= 2 p=0.9903 plog=-13.6674 t0= 216 t1= 232 word_start=true "▁a"++ [12] id= 1639 frame= 29 dur_idx= 2 dur_val= 2 p=0.9965 plog=-10.3544 t0= 232 t1= 248 word_start=true "▁bl"++ [13] id= 2192 frame= 31 dur_idx= 3 dur_val= 3 p=0.9997 plog=-11.0735 t0= 248 t1= 272 word_start=false "ank"++ [14] id= 627 frame= 34 dur_idx= 2 dur_val= 2 p=0.9994 plog=-11.8629 t0= 272 t1= 288 word_start=true "▁ch"++ [15] id= 2300 frame= 36 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.4228 t0= 288 t1= 304 word_start=false "ip"++ [16] id= 334 frame= 38 dur_idx= 1 dur_val= 1 p=0.9888 plog=-14.3064 t0= 304 t1= 312 word_start=true "▁in"++ [17] id= 700 frame= 39 dur_idx= 2 dur_val= 2 p=1.0000 plog=-13.0235 t0= 312 t1= 328 word_start=false "iti"++ [18] id= 2274 frame= 41 dur_idx= 1 dur_val= 1 p=0.9987 plog=-9.2411 t0= 328 t1= 336 word_start=false "ally"++ [19] id= 2530 frame= 43 dur_idx= 2 dur_val= 2 p=0.5170 plog=-7.3316 t0= 344 t1= 360 word_start=true "▁they"++ [20] id= 2617 frame= 45 dur_idx= 1 dur_val= 1 p=0.9922 plog=-8.1131 t0= 360 t1= 368 word_start=true "▁put"++ [21] id= 334 frame= 48 dur_idx= 2 dur_val= 2 p=0.9799 plog=-10.5362 t0= 384 t1= 400 word_start=true "▁in"++ [22] id= 575 frame= 51 dur_idx= 3 dur_val= 3 p=0.9582 plog=-6.8221 t0= 408 t1= 432 word_start=true "▁and"++ [23] id= 869 frame= 54 dur_idx= 1 dur_val= 1 p=0.4425 plog=-7.9854 t0= 432 t1= 440 word_start=true "▁it"++ [24] id= 618 frame= 58 dur_idx= 2 dur_val= 2 p=0.9823 plog=-7.4988 t0= 464 t1= 480 word_start=true "▁as"++ [25] id= 547 frame= 60 dur_idx= 1 dur_val= 1 p=0.9914 plog=-11.8962 t0= 480 t1= 488 word_start=true "▁so"++ [26] id= 289 frame= 61 dur_idx= 1 dur_val= 1 p=0.9996 plog=-9.9585 t0= 488 t1= 496 word_start=false "on"++ [27] id= 618 frame= 62 dur_idx= 2 dur_val= 2 p=0.9982 plog=-14.3261 t0= 496 t1= 512 word_start=true "▁as"++ [28] id= 869 frame= 64 dur_idx= 3 dur_val= 3 p=0.9623 plog=-8.3384 t0= 512 t1= 536 word_start=true "▁it"++ [29] id= 869 frame= 67 dur_idx= 2 dur_val= 2 p=0.6599 plog=-6.3797 t0= 536 t1= 552 word_start=true "▁it"++ [30] id= 7931 frame= 74 dur_idx= 0 dur_val= 0 p=0.3250 plog=-16.9064 t0= 552 t1= 552 word_start=false "'"++ [31] id= 7870 frame= 74 dur_idx= 0 dur_val= 0 p=0.9837 plog=-25.8700 t0= 592 t1= 592 word_start=false "s"nnew file mode 100644@@@ -0,0 +1,37 @@++'use strict';++// Parakeet TDT (server/ai/transcribe.js): parakeet-cli's per-token timing (centiseconds) becomes timed sentences: a++// sentence ends at . ? !, a 1.5 s pause or 20 s; "uh"/"um" are dropped; control tokens are skipped; the offset shifts++// every time; languages Parakeet does not know go to whisper, and ASR_ENGINE=whisper forces whisper.++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const t = require('../server/ai/transcribe');++++const tok = (t0, t1, start, s) => ` [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1.0 t0=${t0} t1=${t1} word_start=${start} "${s}"`;++++// The real output of parakeet-cli -ps on 6 s of stream talk (2026-10-16).++const real = t.parseParakeet(fs.readFileSync(path.join(__dirname, 'fixtures', 'parakeet-tokens.txt'), 'utf8'), 100);++assert.strictEqual(real.length, 1);++assert.strictEqual(real[0].text, "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's");++assert.deepStrictEqual([real[0].start, real[0].end], [100.32, 105.92]);++++const lines = [++ 'Segments (1):',++ tok(10, 30, true, '▁So'), tok(30, 40, true, '▁uh'), tok(40, 41, false, ','), tok(41, 60, true, '▁he'), tok(60, 80, true, '▁lost'), tok(80, 81, false, '.'),++ tok(90, 100, true, '▁What'), tok(100, 110, true, '▁a'), tok(110, 130, true, '▁clown'), tok(130, 131, false, '?'),++ tok(140, 150, true, '<unk>'),++ tok(150, 170, true, '▁Then'), tok(170, 190, true, '▁silence'),++ tok(400, 420, true, '▁Back'), tok(420, 440, true, '▁again'),++].join('\n');++const out = t.parseParakeet(lines, 10);++assert.deepStrictEqual(out.map((s) => s.text), ['So he lost.', 'What a clown?', 'Then silence', 'Back again']);++assert.deepStrictEqual(out.map((s) => [s.start, s.end]), [[10.1, 10.81], [10.9, 11.31], [11.5, 11.9], [14, 14.4]], 'a 2.1 s pause ends a line');++++const long = Array.from({ length: 30 }, (_, i) => tok(i * 100, i * 100 + 50, true, `▁w${i}`)).join('\n');++assert.ok(t.parseParakeet(long).length >= 2, 'no line runs past 20 s');++assert.deepStrictEqual(t.parseParakeet(''), []);++++// Engines: without the Parakeet files every language is whisper's; Japanese is always whisper's.++assert.strictEqual(t.engineFor('ja'), 'whisper');++assert.ok(['parakeet', 'whisper'].includes(t.engineFor('en')));++console.log('transcribe-parakeet: ok');
@@ -16,6 +16,14 @@ ADMIN_PASSWORD=changeme123
@@ -16,6 +16,14 @@ ADMIN_PASSWORD=changeme123
# channels keep using the English model and live captions stay hidden.
# channels keep using the English model and live captions stay hidden.
# WHISPER_MODEL_MULTI=/home/openvibe/whisper.cpp/models/ggml-small.bin
# WHISPER_MODEL_MULTI=/home/openvibe/whisper.cpp/models/ggml-small.bin
+# Speech-to-text engine for English and 24 other European languages: NVIDIA Parakeet TDT 0.6B v3 through
+# whisper.cpp's parakeet-cli (more accurate than whisper base.en and fast enough for live on 4 cores). Used
+# when both files exist; other languages, and any Parakeet failure, go to whisper. ASR_ENGINE=whisper forces whisper.
+# Model: https://huggingface.co/ggml-org/parakeet-GGUF (ggml-parakeet-tdt-0.6b-v3-q8_0.bin).
+# PARAKEET_BIN=/home/openvibe/whisper.cpp/build/bin/parakeet-cli
+# PARAKEET_MODEL=/home/openvibe/whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin
+# ASR_ENGINE=whisper
+
# OpenVibe.AI (roadmap Wave 13, WS-O task 2). Every AI call Live makes (chat AI, moments, paste/frame
# OpenVibe.AI (roadmap Wave 13, WS-O task 2). Every AI call Live makes (chat AI, moments, paste/frame
# analysis, overviews, translation, recaps, slogans, arena, AI viewers) is a workflow run on the AI
# analysis, overviews, translation, recaps, slogans, arena, AI viewers) is a workflow run on the AI
# service, with a service token for audience openvibe.ai (grants ai.run.create, ai.run.read and
# service, with a service token for audience openvibe.ai (grants ai.run.create, ai.run.read and
@@ -1,6 +1,15 @@
@@ -1,6 +1,15 @@
/**
/**
- * transcribe.js — free, LOCAL speech-to-text via whisper.cpp (no API, runs on the
- * server CPU). Used for live-stream memories and VOD/clip transcripts.
+ * transcribe.js — free, LOCAL speech-to-text (no API, runs on the server CPU). Used for
+ * live-stream memories, the live timeline (which the Arena listens to) and VOD/clip transcripts.
+ *
+ * Two engines, both from the whisper.cpp tree (ggml):
+ * - NVIDIA Parakeet TDT 0.6B v3 (`parakeet-cli`), for English and 24 other European languages.
+ * Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,
+ * punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped
+ * whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,
+ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.
+ * - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when
+ * Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.
*
*
* Reliability features:
* Reliability features:
* - JSON output (-oj) → per-segment timestamps (ms) we keep as contextual data.
* - JSON output (-oj) → per-segment timestamps (ms) we keep as contextual data.
@@ -50,6 +59,20 @@ function _modelFor(opts) {
@@ -50,6 +59,20 @@ function _modelFor(opts) {
}
}
const THREADS = Math.max(2, Math.min(8, parseInt(process.env.WHISPER_THREADS, 10) || 4));
const THREADS = Math.max(2, Math.min(8, parseInt(process.env.WHISPER_THREADS, 10) || 4));
+// ── Parakeet (NVIDIA Parakeet TDT, ggml, from the same whisper.cpp build) ──
+const PARAKEET_BIN = process.env.PARAKEET_BIN || path.join(HOME, 'whisper.cpp/build/bin/parakeet-cli');
+const PARAKEET_MODEL = process.env.PARAKEET_MODEL || path.join(HOME, 'whisper.cpp/models/ggml-parakeet-tdt-0.6b-v3-q8_0.bin');
+// Parakeet TDT 0.6B v3's languages (NVIDIA's model card): every other language goes to whisper.
+const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);
+function parakeetAvailable() {
+ if (process.env.ASR_ENGINE === 'whisper') return false;
+ try { return fs.existsSync(PARAKEET_BIN) && fs.existsSync(PARAKEET_MODEL); } catch { return false; }
+}
+/** The engine a decode in `language` uses: 'parakeet' or 'whisper'. */
+function engineFor(language = 'en') {
+ return parakeetAvailable() && PARAKEET_LANGS.has(String(language || 'en').slice(0, 2).toLowerCase()) ? 'parakeet' : 'whisper';
+}
+
// ── Voice Activity Detection ──────────────────────────────────────────────
// ── Voice Activity Detection ──────────────────────────────────────────────
// Measured on 60s of real stream audio: VAD cut large-v3-turbo from 51.0s to 24.0s
// Measured on 60s of real stream audio: VAD cut large-v3-turbo from 51.0s to 24.0s
// AND removed a trailing "Thank you." hallucination. Both effects matter, and the
// AND removed a trailing "Thank you." hallucination. Both effects matter, and the
@@ -84,6 +107,10 @@ function whisperBin() {
@@ -84,6 +107,10 @@ function whisperBin() {
return found;
return found;
}
}
function available() {
function available() {
+ if (parakeetAvailable()) return true;
+ try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }
+}
+function whisperAvailable() {
try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }
try { return !!whisperBin() && fs.existsSync(MODEL); } catch { return false; }
}
}
@@ -210,6 +237,62 @@ function _joinSegments(segments) {
@@ -210,6 +237,62 @@ function _joinSegments(segments) {
* @returns {Promise<{text:string, segments:Array<{start:number,end:number,text:string}>}>}
* @returns {Promise<{text:string, segments:Array<{start:number,end:number,text:string}>}>}
* start/end in seconds (shifted by opts.offsetSec).
* start/end in seconds (shifted by opts.offsetSec).
*/
*/
+// ── Parakeet output → timed sentences ──────────────────────────────────────
+// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a
+// word ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s, so each timeline row stays a
+// line someone said, deep-linkable by its start.
+const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;
+// Fillers Parakeet writes out ("uh", "um") carry nothing for a reader or the Arena's quotes.
+const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;
+function parseParakeet(stderr, offsetSec = 0) {
+ const out = [];
+ let cur = null;
+ const close = () => {
+ if (!cur) return;
+ const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();
+ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });
+ cur = null;
+ };
+ for (const line of String(stderr || '').split('\n')) {
+ const m = TOKEN_LINE.exec(line);
+ if (!m) continue;
+ const t0 = Number(m[1]), t1 = Number(m[2]);
+ const wordStart = m[3] === 'true';
+ const raw = m[4];
+ if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens
+ const piece = raw.replace(/\u2581/g, ' ');
+ if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();
+ if (!cur) cur = { t0, t1, text: '' };
+ cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;
+ cur.t1 = Math.max(cur.t1, t1);
+ if (/[.!?]["')\]]?$/.test(piece.trim())) close();
+ }
+ close();
+ return out;
+}
+
+function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
+ return new Promise((resolve) => {
+ if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });
+ const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];
+ let ff;
+ try { ff = _track(spawn('nice', ['-n', '15', PARAKEET_BIN, ...args], { stdio: ['ignore', 'ignore', 'pipe'] })); }
+ catch (e) { return resolve({ text: '', segments: [], ok: false, error: e.message }); }
+ const chunks = [];
+ let size = 0;
+ ff.stderr.on('data', (d) => { if (size < 64 * 1024 * 1024) { chunks.push(d); size += d.length; } });
+ let done = false;
+ const finish = (result) => { if (done) return; done = true; clearTimeout(timer); resolve(result); };
+ const timer = setTimeout(() => { try { ff.kill('SIGKILL'); } catch { /* */ } finish({ text: '', segments: [], ok: false, error: 'timeout' }); }, timeoutMs);
+ ff.on('close', (code) => {
+ if (code !== 0) return finish({ text: '', segments: [], ok: false, error: `parakeet exited ${code}` });
+ const segments = _cleanSegments(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));
+ finish({ text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' });
+ });
+ ff.on('error', (e) => finish({ text: '', segments: [], ok: false, error: e.message }));
+ });
+}
+
// Inner implementation. Wrapped by transcribeWavDetailed, which holds the concurrency
// Inner implementation. Wrapped by transcribeWavDetailed, which holds the concurrency
// slot for the whole run so only MAX_CONCURRENT decoders exist at any moment.
// slot for the whole run so only MAX_CONCURRENT decoders exist at any moment.
function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false, language = 'en' } = {}) {
function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false, language = 'en' } = {}) {
@@ -218,7 +301,7 @@ function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live
@@ -218,7 +301,7 @@ function _transcribeWavInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live
// ok=false only for genuine FAILURES (missing binary, spawn/exec error, timeout,
// ok=false only for genuine FAILURES (missing binary, spawn/exec error, timeout,
// non-zero exit, unparseable output). A clean run that finds no speech is ok=true
// non-zero exit, unparseable output). A clean run that finds no speech is ok=true
// with empty text — the caller must NOT treat that as a failure to retry.
// with empty text — the caller must NOT treat that as a failure to retry.
- if (!bin || !available() || !wavPath || !fs.existsSync(wavPath)) {
+ if (!bin || !whisperAvailable() || !wavPath || !fs.existsSync(wavPath)) {
return resolve({ text: '', segments: [], ok: false, error: 'whisper unavailable' });
return resolve({ text: '', segments: [], ok: false, error: 'whisper unavailable' });
}
}
const outBase = `${wavPath}.out`;
const outBase = `${wavPath}.out`;
@@ -311,8 +394,15 @@ async function transcribeMedia(mediaPath, opts = {}) {
@@ -311,8 +394,15 @@ async function transcribeMedia(mediaPath, opts = {}) {
async function transcribeWavDetailed(wavPath, opts = {}) {
async function transcribeWavDetailed(wavPath, opts = {}) {
const lane = opts && opts.live ? 'live' : 'batch';
const lane = opts && opts.live ? 'live' : 'batch';
await _acquire(lane);
await _acquire(lane);
- try { return await _transcribeWavInner(wavPath, opts); }
- finally { _release(lane); }
+ try {
+ if (engineFor(opts && opts.language) === 'parakeet') {
+ const r = await _parakeetInner(wavPath, opts);
+ // A Parakeet failure (a crash, a bad model file) must not leave a hole: whisper takes the same audio.
+ if (r.ok || !whisperAvailable()) return r;
+ console.warn(`[Transcribe] parakeet failed (${r.error}); whisper takes this audio`);
+ }
+ return await _transcribeWavInner(wavPath, opts);
+ } finally { _release(lane); }
}
}
/** Human-readable availability report (logged at boot so a broken install is visible). */
/** Human-readable availability report (logged at boot so a broken install is visible). */
@@ -324,10 +414,11 @@ function describe() {
@@ -324,10 +414,11 @@ function describe() {
model: MODEL, modelExists: fs.existsSync(MODEL),
model: MODEL, modelExists: fs.existsSync(MODEL),
modelLive: MODEL_LIVE, modelLiveExists: fs.existsSync(MODEL_LIVE), modelMulti: MODEL_MULTI, modelMultiExists: !!multilingualModel(),
modelLive: MODEL_LIVE, modelLiveExists: fs.existsSync(MODEL_LIVE), modelMulti: MODEL_MULTI, modelMultiExists: !!multilingualModel(),
vadModel: vadModel() || null,
vadModel: vadModel() || null,
+ parakeet: parakeetAvailable() ? PARAKEET_MODEL : null, engineEn: engineFor('en'),
threads: THREADS, beam: BEAM, maxConcurrent: MAX_CONCURRENT,
threads: THREADS, beam: BEAM, maxConcurrent: MAX_CONCURRENT,
};
};
}
}
function isLowPower() { return _lowPower; }
function isLowPower() { return _lowPower; }
-module.exports = { available, describe, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
+module.exports = { available, describe, engineFor, parseParakeet, laneStatus, multilingualModel, transcribeWav, transcribeWavDetailed, transcribeMedia, transcribeMediaDetailed, killActive, setLowPower, isLowPower };
@@ -1381,7 +1381,7 @@ async function start() {
@@ -1381,7 +1381,7 @@ async function start() {
// AI stream-memory job (no-op until enabled in openvibe.network/admin → AI).
// AI stream-memory job (no-op until enabled in openvibe.network/admin → AI).
try { require('./ai/stream-memory-job').start(); } catch (e) { console.warn('[AI] memory job not started:', e.message); }
try { require('./ai/stream-memory-job').start(); } catch (e) { console.warn('[AI] memory job not started:', e.message); }
- try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] whisper ${d.available ? 'available' : 'UNAVAILABLE'} — bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();
+ try { (() => { try { const d = require('./ai/transcribe').describe(); console.log(`[AI] speech-to-text ${d.available ? 'available' : 'UNAVAILABLE'} — English via ${d.engineEn}${d.parakeet ? ` (${d.parakeet})` : ''}; whisper bin=${d.bin} model=${d.model}${d.modelExists ? '' : ' (MISSING)'} live=${d.modelLive}${d.modelLiveExists ? '' : ' (MISSING)'} vad=${d.vadModel || 'off'} threads=${d.threads}`); } catch (e) { console.warn('[AI] whisper probe failed:', e.message); } })();
require('./ai/backfill-job').start(); } catch (e) { console.warn('[AI] backfill job not started:', e.message); }
require('./ai/backfill-job').start(); } catch (e) { console.warn('[AI] backfill job not started:', e.message); }
// AI viewers activity log: keep a week.
// AI viewers activity log: keep a week.
setInterval(async () => { try { const n = await db.pruneAiViewerLog(7); if (n) console.log(`[AI-Viewers] pruned ${n} log row(s)`); } catch { /* */ } }, 6 * 3600 * 1000);
setInterval(async () => { try { const n = await db.pruneAiViewerLog(7); if (n) console.log(`[AI-Viewers] pruned ${n} log row(s)`); } catch { /* */ } }, 6 * 3600 * 1000);
new file mode 100644
new file mode 100644
@@ -0,0 +1,34 @@
@@ -0,0 +1,34 @@
+Segments (1):
+Segment 0: [0 -> 601] "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's"
+ [ 0] id= 3108 frame= 4 dur_idx= 2 dur_val= 2 p=0.5552 plog=-5.8166 t0= 32 t1= 48 word_start=true "▁For"
+ [ 1] id= 1111 frame= 6 dur_idx= 2 dur_val= 2 p=0.9792 plog=-7.9274 t0= 48 t1= 64 word_start=true "▁all"
+ [ 2] id= 380 frame= 8 dur_idx= 2 dur_val= 2 p=0.9872 plog=-10.5001 t0= 64 t1= 80 word_start=true "▁I"
+ [ 3] id= 2865 frame= 10 dur_idx= 1 dur_val= 1 p=0.9908 plog=-8.5432 t0= 80 t1= 88 word_start=true "▁know"
+ [ 4] id= 869 frame= 12 dur_idx= 1 dur_val= 1 p=0.9073 plog=-7.9946 t0= 96 t1= 104 word_start=true "▁it"
+ [ 5] id= 5635 frame= 14 dur_idx= 1 dur_val= 1 p=0.7077 plog=-9.1575 t0= 112 t1= 120 word_start=true "▁said"
+ [ 6] id= 7877 frame= 18 dur_idx= 1 dur_val= 1 p=0.4234 plog=-9.9571 t0= 120 t1= 120 word_start=false ","
+ [ 7] id= 2846 frame= 19 dur_idx= 3 dur_val= 3 p=0.6802 plog=-7.0393 t0= 152 t1= 176 word_start=true "▁just"
+ [ 8] id= 869 frame= 24 dur_idx= 1 dur_val= 1 p=0.9056 plog=-7.0782 t0= 192 t1= 200 word_start=true "▁it"
+ [ 9] id= 7931 frame= 25 dur_idx= 1 dur_val= 1 p=0.9562 plog=-9.0419 t0= 200 t1= 200 word_start=false "'"
+ [10] id= 7870 frame= 26 dur_idx= 1 dur_val= 1 p=0.9993 plog=-8.2082 t0= 208 t1= 216 word_start=false "s"
+ [11] id= 279 frame= 27 dur_idx= 2 dur_val= 2 p=0.9903 plog=-13.6674 t0= 216 t1= 232 word_start=true "▁a"
+ [12] id= 1639 frame= 29 dur_idx= 2 dur_val= 2 p=0.9965 plog=-10.3544 t0= 232 t1= 248 word_start=true "▁bl"
+ [13] id= 2192 frame= 31 dur_idx= 3 dur_val= 3 p=0.9997 plog=-11.0735 t0= 248 t1= 272 word_start=false "ank"
+ [14] id= 627 frame= 34 dur_idx= 2 dur_val= 2 p=0.9994 plog=-11.8629 t0= 272 t1= 288 word_start=true "▁ch"
+ [15] id= 2300 frame= 36 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.4228 t0= 288 t1= 304 word_start=false "ip"
+ [16] id= 334 frame= 38 dur_idx= 1 dur_val= 1 p=0.9888 plog=-14.3064 t0= 304 t1= 312 word_start=true "▁in"
+ [17] id= 700 frame= 39 dur_idx= 2 dur_val= 2 p=1.0000 plog=-13.0235 t0= 312 t1= 328 word_start=false "iti"
+ [18] id= 2274 frame= 41 dur_idx= 1 dur_val= 1 p=0.9987 plog=-9.2411 t0= 328 t1= 336 word_start=false "ally"
+ [19] id= 2530 frame= 43 dur_idx= 2 dur_val= 2 p=0.5170 plog=-7.3316 t0= 344 t1= 360 word_start=true "▁they"
+ [20] id= 2617 frame= 45 dur_idx= 1 dur_val= 1 p=0.9922 plog=-8.1131 t0= 360 t1= 368 word_start=true "▁put"
+ [21] id= 334 frame= 48 dur_idx= 2 dur_val= 2 p=0.9799 plog=-10.5362 t0= 384 t1= 400 word_start=true "▁in"
+ [22] id= 575 frame= 51 dur_idx= 3 dur_val= 3 p=0.9582 plog=-6.8221 t0= 408 t1= 432 word_start=true "▁and"
+ [23] id= 869 frame= 54 dur_idx= 1 dur_val= 1 p=0.4425 plog=-7.9854 t0= 432 t1= 440 word_start=true "▁it"
+ [24] id= 618 frame= 58 dur_idx= 2 dur_val= 2 p=0.9823 plog=-7.4988 t0= 464 t1= 480 word_start=true "▁as"
+ [25] id= 547 frame= 60 dur_idx= 1 dur_val= 1 p=0.9914 plog=-11.8962 t0= 480 t1= 488 word_start=true "▁so"
+ [26] id= 289 frame= 61 dur_idx= 1 dur_val= 1 p=0.9996 plog=-9.9585 t0= 488 t1= 496 word_start=false "on"
+ [27] id= 618 frame= 62 dur_idx= 2 dur_val= 2 p=0.9982 plog=-14.3261 t0= 496 t1= 512 word_start=true "▁as"
+ [28] id= 869 frame= 64 dur_idx= 3 dur_val= 3 p=0.9623 plog=-8.3384 t0= 512 t1= 536 word_start=true "▁it"
+ [29] id= 869 frame= 67 dur_idx= 2 dur_val= 2 p=0.6599 plog=-6.3797 t0= 536 t1= 552 word_start=true "▁it"
+ [30] id= 7931 frame= 74 dur_idx= 0 dur_val= 0 p=0.3250 plog=-16.9064 t0= 552 t1= 552 word_start=false "'"
+ [31] id= 7870 frame= 74 dur_idx= 0 dur_val= 0 p=0.9837 plog=-25.8700 t0= 592 t1= 592 word_start=false "s"
new file mode 100644
new file mode 100644
@@ -0,0 +1,37 @@
@@ -0,0 +1,37 @@
+'use strict';
+// Parakeet TDT (server/ai/transcribe.js): parakeet-cli's per-token timing (centiseconds) becomes timed sentences: a
+// sentence ends at . ? !, a 1.5 s pause or 20 s; "uh"/"um" are dropped; control tokens are skipped; the offset shifts
+// every time; languages Parakeet does not know go to whisper, and ASR_ENGINE=whisper forces whisper.
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const t = require('../server/ai/transcribe');
+
+const tok = (t0, t1, start, s) => ` [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1.0 t0=${t0} t1=${t1} word_start=${start} "${s}"`;
+
+// The real output of parakeet-cli -ps on 6 s of stream talk (2026-10-16).
+const real = t.parseParakeet(fs.readFileSync(path.join(__dirname, 'fixtures', 'parakeet-tokens.txt'), 'utf8'), 100);
+assert.strictEqual(real.length, 1);
+assert.strictEqual(real[0].text, "For all I know it said, just it's a blank chip initially they put in and it as soon as it it's");
+assert.deepStrictEqual([real[0].start, real[0].end], [100.32, 105.92]);
+
+const lines = [
+ 'Segments (1):',
+ tok(10, 30, true, '▁So'), tok(30, 40, true, '▁uh'), tok(40, 41, false, ','), tok(41, 60, true, '▁he'), tok(60, 80, true, '▁lost'), tok(80, 81, false, '.'),
+ tok(90, 100, true, '▁What'), tok(100, 110, true, '▁a'), tok(110, 130, true, '▁clown'), tok(130, 131, false, '?'),
+ tok(140, 150, true, '<unk>'),
+ tok(150, 170, true, '▁Then'), tok(170, 190, true, '▁silence'),
+ tok(400, 420, true, '▁Back'), tok(420, 440, true, '▁again'),
+].join('\n');
+const out = t.parseParakeet(lines, 10);
+assert.deepStrictEqual(out.map((s) => s.text), ['So he lost.', 'What a clown?', 'Then silence', 'Back again']);
+assert.deepStrictEqual(out.map((s) => [s.start, s.end]), [[10.1, 10.81], [10.9, 11.31], [11.5, 11.9], [14, 14.4]], 'a 2.1 s pause ends a line');
+
+const long = Array.from({ length: 30 }, (_, i) => tok(i * 100, i * 100 + 50, true, `▁w${i}`)).join('\n');
+assert.ok(t.parseParakeet(long).length >= 2, 'no line runs past 20 s');
+assert.deepStrictEqual(t.parseParakeet(''), []);
+
+// Engines: without the Parakeet files every language is whisper's; Japanese is always whisper's.
+assert.strictEqual(t.engineFor('ja'), 'whisper');
+assert.ok(['parakeet', 'whisper'].includes(t.engineFor('en')));
+console.log('transcribe-parakeet: ok');