openvibers/openvibe.live Commits

Live speech-to-text: audio longer than 45 s goes to Parakeet in 30 s pieces (a 300 s VOD window needed 3.1 GB in one run; a 30 s piece about 0.8 GB) (#148)

OP OpenVibers committed 28 days ago · 231be36b51 · 2 files, +83 −2

Changes

ddiff --git a/server/ai/transcribe.js b/server/ai/transcribe.js
iindex 3005238d..a9d1fd19 100644
---- a/server/ai/transcribe.js
++++ b/server/ai/transcribe.js
@@@ -7,7 +7,8 @@
* Measured 2026-10-16 on 2 min of real stream talk on this 4-core box: 394 words in 31 s,
* punctuated, where whisper base.en (the old live model) caught 327 words in 15 s and dropped
* whole sentences ("they put in" → "the Putin"), and large-v3-turbo needed 121 s (1x real time,
-- * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did.
++ * too slow for live). A 10 s live segment decodes in 2.5 s on 2 threads, like base.en did. Audio longer than 45 s
++ * (VOD windows) is decoded in 30 s pieces: one 300 s piece needed 3.1 GB.
* - whisper.cpp, for every other language (Japanese, Korean, Chinese…) and as the fallback when
* Parakeet is not installed. ASR_ENGINE=whisper forces it everywhere.
*
@@@ -271,7 +272,48 @@ function parseParakeet(stderr, offsetSec = 0) {
return out;
}
--function _parakeetInner(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
++// Parakeet's memory and time grow faster than the audio: measured on this box, 10 s took 0.74 GB, 60 s 1.1 GB and a
++// 300 s VOD window 3.1 GB in 154 s. Anything longer than CHUNK_MAX_SEC is cut into CHUNK_SEC pieces (PCM, cut exactly,
++// so each piece's times only shift by its start) and decoded one after another.
++const CHUNK_SEC = 30;
++const CHUNK_MAX_SEC = 45;
++function wavSeconds(wavPath) {
++ try { return Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch { return 0; } // 16 kHz mono s16le
++}
++function _splitWav(wavPath, dir) {
++ return new Promise((resolve) => {
++ let ff;
++ try { ff = _track(spawn('nice', ['-n', '15', 'ffmpeg', '-nostdin', '-loglevel', 'error', '-y', '-i', wavPath, '-f', 'segment', '-segment_time', String(CHUNK_SEC), '-c', 'copy', path.join(dir, 'p%04d.wav')], { stdio: 'ignore' })); }
++ catch { return resolve(null); }
++ ff.on('close', (code) => {
++ if (code !== 0) return resolve(null);
++ try { resolve(fs.readdirSync(dir).filter((n) => /^p\d{4}\.wav$/.test(n)).sort().map((n) => path.join(dir, n))); } catch { resolve(null); }
++ });
++ ff.on('error', () => resolve(null));
++ });
++}
++async function _parakeetInner(wavPath, opts = {}) {
++ if (!wavPath || wavSeconds(wavPath) <= CHUNK_MAX_SEC) return await _parakeetOnce(wavPath, opts);
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'openvibe-pk-'));
++ try {
++ const pieces = await _splitWav(wavPath, dir);
++ if (!pieces || !pieces.length) return { text: '', segments: [], ok: false, error: 'could not split the audio' };
++ const segments = [];
++ const deadline = Date.now() + (opts.timeoutMs || 180000);
++ for (let i = 0; i < pieces.length; i++) {
++ const left = deadline - Date.now();
++ if (left <= 0) return { text: '', segments: [], ok: false, error: 'timeout' };
++ const r = await _parakeetOnce(pieces[i], { ...opts, timeoutMs: left, offsetSec: (opts.offsetSec || 0) + i * CHUNK_SEC });
++ if (!r.ok) return r;
++ segments.push(...r.segments);
++ }
++ return { text: _joinSegments(segments), segments, ok: true, engine: 'parakeet' };
++ } finally {
++ try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* */ }
++ }
++}
++
++function _parakeetOnce(wavPath, { timeoutMs = 180000, offsetSec = 0, live = false } = {}) {
return new Promise((resolve) => {
if (!parakeetAvailable() || !wavPath || !fs.existsSync(wavPath)) return resolve({ text: '', segments: [], ok: false, error: 'parakeet unavailable' });
const args = ['-t', String(_threads(live)), '-ps', '-np', '-m', PARAKEET_MODEL, '-f', wavPath];
ddiff --git a/test/transcribe-parakeet-chunks.test.js b/test/transcribe-parakeet-chunks.test.js
nnew file mode 100644
iindex 00000000..2cd9a85a
---- /dev/null
++++ b/test/transcribe-parakeet-chunks.test.js
@@@ -0,0 +1,39 @@
++'use strict';
++// Long audio goes to Parakeet in 30 s pieces (server/ai/transcribe.js): a 70 s WAV is cut into three, each decoded on
++// its own, and each piece's lines shift by where the piece starts. A stand-in parakeet-cli (a script that says "word"
++// one second into whatever it is given) stands in for the real one.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const { execFileSync } = require('child_process');
++
++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-pk-chunks-'));
++const bin = path.join(dir, 'parakeet-cli');
++const calls = path.join(dir, 'calls.log');
++fs.writeFileSync(bin, `#!/bin/sh\necho "$@" >> ${calls}\necho ' [ 0] id= 1 frame= 0 dur_idx= 1 dur_val= 1 p=0.9 plog=-1 t0= 100 t1= 150 word_start=true "\u2581Word."' 1>&2\n`);
++fs.chmodSync(bin, 0o755);
++const model = path.join(dir, 'model.bin');
++fs.writeFileSync(model, 'x');
++process.env.PARAKEET_BIN = bin;
++process.env.PARAKEET_MODEL = model;
++const t = require('../server/ai/transcribe');
++
++let ffmpeg = true;
++try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); } catch { ffmpeg = false; }
++(async () => {
++ assert.strictEqual(t.engineFor('en'), 'parakeet');
++ if (!ffmpeg) { console.log('transcribe-parakeet-chunks: ffmpeg not installed here, chunking not exercised'); return; }
++ const wav = path.join(dir, 'long.wav');
++ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '70', '-c:a', 'pcm_s16le', wav]);
++ const r = await t.transcribeWavDetailed(wav, { offsetSec: 100, timeoutMs: 60000 });
++ assert.ok(r.ok, r.error);
++ assert.deepStrictEqual(r.segments.map((s) => s.start), [101, 131, 161], 'each 30 s piece shifted by its start');
++ assert.strictEqual(fs.readFileSync(calls, 'utf8').trim().split('\n').length, 3, 'three pieces, three runs');
++ const short = path.join(dir, 'short.wav');
++ execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-f', 'lavfi', '-i', 'anullsrc=r=16000:cl=mono', '-t', '10', '-c:a', 'pcm_s16le', short]);
++ const s = await t.transcribeWavDetailed(short, { offsetSec: 0 });
++ assert.deepStrictEqual(s.segments.map((x) => [x.start, x.text]), [[1, 'Word.']], 'short audio: one run, no split');
++ assert.ok(!fs.readdirSync(os.tmpdir()).some((n) => n.startsWith('openvibe-pk-') && fs.existsSync(path.join(os.tmpdir(), n, 'p0000.wav'))), 'pieces are cleaned up');
++ console.log('transcribe-parakeet-chunks: ok');
++})().catch((e) => { console.error(e); process.exit(1); }).finally(() => fs.rmSync(dir, { recursive: true, force: true }));